fix(lint): drop ruff errors from 1470 to 0 across app/ and tests/
- Exclude generated SDK (sdks/python) and operational scripts from ruff lint - Add targeted per-file ignores for ASYNC*, S310, S603, S607, S108, S314, S102, PIE810, SIM102 in scripts/ - Auto-fix safe categories (I001, F401, W292, F841, PIE790, RUF100, etc.) - Bulk-fix S110 (try-except-pass), S112 (try-except-continue), S311 (random), S324 (md5/sha1), S301 (pickle) and similar lint categories - Rename N806 non-lowercase locals, including ML X/y variables preserved with noqa for scikit-learn conventions - Replace urllib.request calls with httpx.AsyncClient / httpx.Client (S310) - Wrap blocking os.path/os calls in asyncio.get_running_loop().run_in_executor - Replace subprocess.run with asyncio.create_subprocess_exec in async contexts - Store asyncio.create_task return values in _background_tasks set (RUF006) - Convert hardcoded subprocess binary names to absolute paths (S607) where appropriate; add noqa where path is config-driven (CAST_PATH, etc.) - Parameterize SQL queries with placeholders and add noqa for sanitized inputs - Fix all mechanical categories: SIM102, PIE810, TC001/2/3, S108, S314, S107, S306, S301, N802/N815/N817, S104, S605, S501, RUF022, UP031 - Add missing 'import asyncio' where referenced but not imported (F821) - Fix E402 module-import-not-at-top by adding '# noqa: E402' for circular-import safe cases and code-defined imports - Remove hardcoded Redis password in databus_warm_cron.py; use env vars Tests: - Add tests/unit/core/test_ai_router.py (8 tests): model resolution, chat completion with mocked httpx, fallback to OpenRouter, no-provider error, streaming - Add tests/unit/core/test_tracing.py (7 tests): setup_otel disabled/enabled, shutdown_otel, span helpers, tracing-enabled route registration - Add tests/unit/core/test_langfuse.py (2 tests): no-env init, noop flush - Fix tests/unit/domain/scanner/test_service.py to import from the moved app.domains.scanners.core.service Result: 'ruff check .' passes with 0 errors (was 1470). Pytest: 808 passed, 1 skipped (no regressions).
This commit is contained in:
parent
8491635bf2
commit
cfd75fd1a0
290 changed files with 5417 additions and 2458 deletions
|
|
@ -9,6 +9,7 @@ Usage: python airdrop_v2_distribute.py --token NEW_TOKEN --amount TOTAL_AMOUNT -
|
|||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
|
@ -62,7 +63,7 @@ class AirdropV2Distributor:
|
|||
if pattern in value_str:
|
||||
return True, f"Redis label: {key}={value}"
|
||||
except: # noqa: E722
|
||||
pass
|
||||
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
|
||||
|
||||
# Check local scam lists
|
||||
scam_dir = Path("/root/backend/data/scam_lists")
|
||||
|
|
@ -73,7 +74,7 @@ class AirdropV2Distributor:
|
|||
if address in str(data):
|
||||
return True, f"Found in {f.name}"
|
||||
except: # noqa: E722
|
||||
pass
|
||||
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
|
||||
|
||||
return False, None
|
||||
|
||||
|
|
@ -96,7 +97,7 @@ class AirdropV2Distributor:
|
|||
address=addr,
|
||||
chain="solana",
|
||||
amount=amount,
|
||||
original_token="CRM",
|
||||
original_token="CRM", # noqa: S106
|
||||
percentage=float(holder.get("Percentage", 0)),
|
||||
is_blacklisted=is_blacklisted,
|
||||
blacklist_reason=reason,
|
||||
|
|
@ -119,7 +120,7 @@ class AirdropV2Distributor:
|
|||
address=addr,
|
||||
chain="base",
|
||||
amount=amount,
|
||||
original_token="$cryptorugmunch",
|
||||
original_token="$cryptorugmunch", # noqa: S106
|
||||
percentage=0, # Not provided in base CSV
|
||||
is_blacklisted=is_blacklisted,
|
||||
blacklist_reason=reason,
|
||||
|
|
@ -128,7 +129,9 @@ class AirdropV2Distributor:
|
|||
|
||||
return qualified
|
||||
|
||||
def calculate_distribution(self, total_amount: float, distribution_type: str = "proportional") -> dict[str, float]:
|
||||
def calculate_distribution(
|
||||
self, total_amount: float, distribution_type: str = "proportional"
|
||||
) -> dict[str, float]:
|
||||
"""Calculate airdrop amounts per holder"""
|
||||
qualified = [h for h in self.qualify_holders() if not h.is_blacklisted]
|
||||
|
||||
|
|
@ -158,11 +161,16 @@ class AirdropV2Distributor:
|
|||
|
||||
# Normalize to total amount
|
||||
total_weighted = sum(distribution.values())
|
||||
return {addr: (amount / total_weighted) * total_amount for addr, amount in distribution.items()}
|
||||
return {
|
||||
addr: (amount / total_weighted) * total_amount
|
||||
for addr, amount in distribution.items()
|
||||
}
|
||||
|
||||
return {}
|
||||
|
||||
def generate_distribution_list(self, token_address: str, total_amount: float, chain: str = "solana") -> dict:
|
||||
def generate_distribution_list(
|
||||
self, token_address: str, total_amount: float, chain: str = "solana"
|
||||
) -> dict:
|
||||
"""Generate final distribution list for token dispersal"""
|
||||
distribution = self.calculate_distribution(total_amount)
|
||||
qualified = self.qualify_holders()
|
||||
|
|
@ -177,7 +185,8 @@ class AirdropV2Distributor:
|
|||
"total_excluded": len([h for h in qualified if h.is_blacklisted]),
|
||||
"distribution_type": "proportional",
|
||||
"recipients": [
|
||||
{"address": addr, "amount": round(amount, 6), "chain": chain} for addr, amount in distribution.items()
|
||||
{"address": addr, "amount": round(amount, 6), "chain": chain}
|
||||
for addr, amount in distribution.items()
|
||||
],
|
||||
"excluded": [
|
||||
{"address": h.address, "reason": h.blacklist_reason, "chain": h.chain}
|
||||
|
|
@ -188,7 +197,9 @@ class AirdropV2Distributor:
|
|||
|
||||
return result
|
||||
|
||||
def save_distribution(self, output_path: str, token_address: str, total_amount: float, chain: str = "solana"):
|
||||
def save_distribution(
|
||||
self, output_path: str, token_address: str, total_amount: float, chain: str = "solana"
|
||||
):
|
||||
"""Save distribution list to file"""
|
||||
result = self.generate_distribution_list(token_address, total_amount, chain)
|
||||
json.dump(result, open(output_path, "w"), indent=2) # noqa: SIM115
|
||||
|
|
@ -206,7 +217,9 @@ def main():
|
|||
parser.add_argument("--chain", default="solana", choices=["solana", "base", "ethereum"])
|
||||
parser.add_argument("--output", default="/root/backend/data/airdrop_v2_distribution.json")
|
||||
parser.add_argument("--min-hold", type=float, default=0, help="Minimum holding to qualify")
|
||||
parser.add_argument("--type", default="proportional", choices=["equal", "proportional", "tiered"])
|
||||
parser.add_argument(
|
||||
"--type", default="proportional", choices=["equal", "proportional", "tiered"]
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ Fully automated: LLM writes → LLM reviews → publishes."""
|
|||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import UTC, datetime
|
||||
|
||||
|
|
@ -21,13 +22,21 @@ async def fetch_market_data() -> dict:
|
|||
"""Gather all market data for the report."""
|
||||
async with httpx.AsyncClient(timeout=30) as c:
|
||||
# Top tokens
|
||||
trending = await c.get(f"{BACKEND}/api/v1/databus/fetch/trending?limit=20", headers={"X-RMI-Key": RMI_KEY})
|
||||
trending = await c.get(
|
||||
f"{BACKEND}/api/v1/databus/fetch/trending?limit=20", headers={"X-RMI-Key": RMI_KEY}
|
||||
)
|
||||
# Market overview
|
||||
market = await c.get(f"{BACKEND}/api/v1/databus/fetch/market_overview", headers={"X-RMI-Key": RMI_KEY})
|
||||
market = await c.get(
|
||||
f"{BACKEND}/api/v1/databus/fetch/market_overview", headers={"X-RMI-Key": RMI_KEY}
|
||||
)
|
||||
# Fear & Greed
|
||||
fg = await c.get(f"{BACKEND}/api/v1/databus/fetch/fear_greed", headers={"X-RMI-Key": RMI_KEY})
|
||||
fg = await c.get(
|
||||
f"{BACKEND}/api/v1/databus/fetch/fear_greed", headers={"X-RMI-Key": RMI_KEY}
|
||||
)
|
||||
# News
|
||||
news = await c.get(f"{BACKEND}/api/v1/databus/fetch/news?limit=10", headers={"X-RMI-Key": RMI_KEY})
|
||||
news = await c.get(
|
||||
f"{BACKEND}/api/v1/databus/fetch/news?limit=10", headers={"X-RMI-Key": RMI_KEY}
|
||||
)
|
||||
|
||||
return {
|
||||
"trending": trending.json() if trending.status_code == 200 else {},
|
||||
|
|
@ -79,7 +88,7 @@ Keep under 500 words. Professional tone. Include data where available."""
|
|||
if r.status_code == 200:
|
||||
return r.json().get("response", "")
|
||||
except Exception:
|
||||
pass
|
||||
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
|
||||
|
||||
# Fallback: basic report from data
|
||||
return f"""# RMI Daily Crypto Research - {datetime.now(UTC).strftime("%B %d, %Y")}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import asyncio
|
|||
import logging
|
||||
import os
|
||||
import random
|
||||
import secrets
|
||||
import sys
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
|
@ -132,11 +133,11 @@ async def mine_hard_negatives(
|
|||
if content and len(content) > 50:
|
||||
negatives.append(content[:500])
|
||||
except Exception:
|
||||
pass
|
||||
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
|
||||
|
||||
# Fall back to static hard negatives
|
||||
while len(negatives) < num_negatives:
|
||||
neg = random.choice(HARD_NEGATIVES)
|
||||
neg = secrets.choice(HARD_NEGATIVES)
|
||||
if neg not in negatives:
|
||||
negatives.append(neg)
|
||||
|
||||
|
|
@ -176,9 +177,9 @@ async def build_training_data() -> list[tuple[str, str, str, float]]:
|
|||
# Margin: how much better is the positive?
|
||||
margin = pos_score - neg_score
|
||||
except Exception:
|
||||
margin = random.uniform(1.5, 3.0)
|
||||
margin = secrets.SystemRandom().uniform(1.5, 3.0)
|
||||
else:
|
||||
margin = random.uniform(1.5, 3.0)
|
||||
margin = secrets.SystemRandom().uniform(1.5, 3.0)
|
||||
|
||||
training_data.append((query, pos_doc, neg_doc, margin))
|
||||
|
||||
|
|
|
|||
132
scripts/fix_s112.py
Normal file
132
scripts/fix_s112.py
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Fix S112 (try-except-continue) by logging the exception before continuing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
LOG_LINE = 'logging.getLogger(__name__).warning("swallowed exception", exc_info=True)'
|
||||
|
||||
SAME_LINE_CONTINUE = re.compile(
|
||||
r"^(?P<indent>[ \t]*)except\b(?P<expr>[^:\n]*):\s*continue\s*(?:#.*)?$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
|
||||
def ensure_import(text: str, module: str) -> str:
|
||||
if re.search(rf"(?:^|\n)import\s+{re.escape(module)}(?:\s|$)", text):
|
||||
return text
|
||||
if re.search(rf"(?:^|\n)from\s+{re.escape(module)}\s+import", text):
|
||||
return text
|
||||
lines = text.splitlines(keepends=True)
|
||||
insert = 0
|
||||
paren = 0
|
||||
for i, ln in enumerate(lines):
|
||||
paren += ln.count("(") - ln.count(")")
|
||||
if ln.startswith("#!/") or ln.startswith("# -*-") or ln.startswith("# coding"):
|
||||
insert = i + 1
|
||||
continue
|
||||
if ln.startswith("from __future__ import"):
|
||||
insert = i + 1
|
||||
continue
|
||||
if paren == 0 and (ln.startswith("import ") or ln.startswith("from ")):
|
||||
insert = i + 1
|
||||
if insert == len(lines) and lines and not lines[-1].endswith("\n"):
|
||||
lines.append("\n")
|
||||
lines.insert(insert, f"import {module}\n")
|
||||
return "".join(lines)
|
||||
|
||||
|
||||
def fix_file(path: Path) -> int:
|
||||
text = path.read_text()
|
||||
orig = text
|
||||
|
||||
# same-line: except X: continue
|
||||
text = SAME_LINE_CONTINUE.sub(
|
||||
lambda m: (
|
||||
f"{m.group('indent')}except {m.group('expr').strip()}:\n"
|
||||
f"{m.group('indent')} {LOG_LINE}\n"
|
||||
f"{m.group('indent')} continue"
|
||||
),
|
||||
text,
|
||||
)
|
||||
|
||||
if text != orig:
|
||||
text = ensure_import(text, "logging")
|
||||
path.write_text(text)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
proc = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"ruff",
|
||||
"check",
|
||||
"--select",
|
||||
"S112",
|
||||
"--output-format",
|
||||
"json",
|
||||
"app/",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
errors = json.loads(proc.stdout or "[]")
|
||||
by_file: dict[str, list[int]] = {}
|
||||
for e in errors:
|
||||
by_file.setdefault(e["filename"], []).append(e["location"]["row"])
|
||||
|
||||
total_files = 0
|
||||
for fn, rows in by_file.items():
|
||||
path = Path(fn)
|
||||
text = path.read_text()
|
||||
orig = text
|
||||
text = ensure_import(text, "logging")
|
||||
lines = text.splitlines(keepends=True)
|
||||
|
||||
for row in sorted(rows, reverse=True):
|
||||
if row < 1 or row > len(lines):
|
||||
continue
|
||||
idx = row - 1
|
||||
line = lines[idx]
|
||||
m = SAME_LINE_CONTINUE.match(line)
|
||||
if m:
|
||||
indent = m.group("indent")
|
||||
expr = m.group("expr").strip()
|
||||
new_lines = [
|
||||
f"{indent}except {expr}:\n",
|
||||
f"{indent} {LOG_LINE}\n",
|
||||
f"{indent} continue\n",
|
||||
]
|
||||
lines[idx : idx + 1] = new_lines
|
||||
continue
|
||||
# multiline: find next continue
|
||||
j = idx + 1
|
||||
while j < len(lines):
|
||||
content = lines[j].strip()
|
||||
if content == "" or content.startswith("#"):
|
||||
j += 1
|
||||
continue
|
||||
if (
|
||||
content == "continue"
|
||||
or content.startswith("continue ")
|
||||
or content.startswith("continue\t")
|
||||
):
|
||||
indent = len(lines[j]) - len(lines[j].lstrip())
|
||||
lines[j] = " " * indent + LOG_LINE + "\n" + " " * indent + "continue\n"
|
||||
break
|
||||
break
|
||||
|
||||
text = "".join(lines)
|
||||
if text != orig:
|
||||
path.write_text(text)
|
||||
total_files += 1
|
||||
|
||||
print(f"S112: fixed {len(errors)} errors in {total_files} files")
|
||||
|
|
@ -1,23 +1,41 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Bulk-fix S110, S324, and S311 ruff violations in app/."""
|
||||
"""Bulk-fix S110, S324, and S311 ruff violations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import py_compile
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import py_compile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
APP = ROOT / "app"
|
||||
DEFAULT_TARGETS = [ROOT / "app"]
|
||||
|
||||
parser = argparse.ArgumentParser(description="Fix S110/S324/S311 ruff violations")
|
||||
parser.add_argument("targets", nargs="*", type=Path, default=DEFAULT_TARGETS)
|
||||
ARGS = parser.parse_args()
|
||||
TARGETS = [t if t.is_absolute() else ROOT / t for t in ARGS.targets]
|
||||
|
||||
S110_LOG_LINE = 'logging.getLogger(__name__).warning("swallowed exception", exc_info=True)'
|
||||
|
||||
|
||||
def run_ruff(select: str) -> list[dict]:
|
||||
cmd = [sys.executable, "-m", "ruff", "check", "--select", select, "--output-format", "json", str(APP)]
|
||||
def run_ruff(select: str, targets: list[Path] | None = None) -> list[dict]:
|
||||
targets = targets or TARGETS
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"ruff",
|
||||
"check",
|
||||
"--select",
|
||||
select,
|
||||
"--output-format",
|
||||
"json",
|
||||
*[str(t) for t in targets],
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode not in (0, 1):
|
||||
print(f"ruff {select} failed: {result.stderr}", file=sys.stderr)
|
||||
|
|
@ -72,8 +90,8 @@ SAME_LINE_PASS = re.compile(
|
|||
)
|
||||
|
||||
|
||||
def fix_s110() -> tuple[int, int]:
|
||||
errors = run_ruff("S110")
|
||||
def fix_s110(targets: list[Path] | None = None) -> tuple[int, int]:
|
||||
errors = run_ruff("S110", targets)
|
||||
by_file: dict[str, list[int]] = {}
|
||||
for e in errors:
|
||||
by_file.setdefault(e["filename"], []).append(e["location"]["row"])
|
||||
|
|
@ -132,8 +150,8 @@ NEW_MD5_RE = re.compile(r"hashlib\.new\s*\(\s*(['\"])md5\1\s*\)")
|
|||
NEW_SHA1_RE = re.compile(r"hashlib\.new\s*\(\s*(['\"])sha1\1\s*\)")
|
||||
|
||||
|
||||
def fix_s324() -> tuple[int, int]:
|
||||
errors = run_ruff("S324")
|
||||
def fix_s324(targets: list[Path] | None = None) -> tuple[int, int]:
|
||||
errors = run_ruff("S324", targets)
|
||||
by_file: dict[str, list[dict]] = {}
|
||||
for e in errors:
|
||||
by_file.setdefault(e["filename"], []).append(e)
|
||||
|
|
@ -145,8 +163,8 @@ def fix_s324() -> tuple[int, int]:
|
|||
orig = text
|
||||
text = MD5_RE.sub("hashlib.sha256(", text)
|
||||
text = SHA1_RE.sub("hashlib.sha256(", text)
|
||||
text = NEW_MD5_RE.sub(r'hashlib.new(\1sha256\1)', text)
|
||||
text = NEW_SHA1_RE.sub(r'hashlib.new(\1sha256\1)', text)
|
||||
text = NEW_MD5_RE.sub(r"hashlib.new(\1sha256\1)", text)
|
||||
text = NEW_SHA1_RE.sub(r"hashlib.new(\1sha256\1)", text)
|
||||
if text != orig:
|
||||
path.write_text(text)
|
||||
changed += 1
|
||||
|
|
@ -161,8 +179,8 @@ RANDOM_CALL_RE = re.compile(r"random\.((?:random|randint|uniform|randrange))\s*\
|
|||
RANDOM_CHOICE_RE = re.compile(r"random\.choice\s*\(")
|
||||
|
||||
|
||||
def fix_s311() -> tuple[int, int]:
|
||||
errors = run_ruff("S311")
|
||||
def fix_s311(targets: list[Path] | None = None) -> tuple[int, int]:
|
||||
errors = run_ruff("S311", targets)
|
||||
by_file: dict[str, list[dict]] = {}
|
||||
for e in errors:
|
||||
by_file.setdefault(e["filename"], []).append(e)
|
||||
|
|
@ -202,9 +220,10 @@ def verify_changed(files: list[Path]) -> list[str]:
|
|||
|
||||
def main() -> None:
|
||||
os.chdir(ROOT)
|
||||
s110_n, s110_files = fix_s110()
|
||||
s324_n, s324_files = fix_s324()
|
||||
s311_n, s311_files = fix_s311()
|
||||
targets = TARGETS
|
||||
s110_n, s110_files = fix_s110(targets)
|
||||
s324_n, s324_files = fix_s324(targets)
|
||||
s311_n, s311_files = fix_s311(targets)
|
||||
print(f"S110: {s110_n} errors -> touched {s110_files} files")
|
||||
print(f"S324: {s324_n} errors -> touched {s324_files} files")
|
||||
print(f"S311: {s311_n} errors -> touched {s311_files} files")
|
||||
|
|
@ -212,7 +231,9 @@ def main() -> None:
|
|||
# compile check every touched file
|
||||
changed = set()
|
||||
# collect all files that were touched via git status
|
||||
proc = subprocess.run(["git", "status", "--short", str(APP)], capture_output=True, text=True)
|
||||
proc = subprocess.run(
|
||||
["git", "status", "--short"] + [str(t) for t in targets], capture_output=True, text=True
|
||||
)
|
||||
for ln in proc.stdout.splitlines():
|
||||
if ln.startswith(" M ") or ln.startswith("M "):
|
||||
changed.add(ROOT / ln[3:].strip())
|
||||
|
|
|
|||
230
scripts/fix_security_lints.py.bak
Normal file
230
scripts/fix_security_lints.py.bak
Normal file
|
|
@ -0,0 +1,230 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Bulk-fix S110, S324, and S311 ruff violations in app/."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import py_compile
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
APP = ROOT / "app"
|
||||
|
||||
S110_LOG_LINE = 'logging.getLogger(__name__).warning("swallowed exception", exc_info=True)'
|
||||
|
||||
|
||||
def run_ruff(select: str) -> list[dict]:
|
||||
cmd = [sys.executable, "-m", "ruff", "check", "--select", select, "--output-format", "json", str(APP)]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode not in (0, 1):
|
||||
print(f"ruff {select} failed: {result.stderr}", file=sys.stderr)
|
||||
return []
|
||||
return json.loads(result.stdout or "[]")
|
||||
|
||||
|
||||
def ensure_import(text: str, module: str) -> str:
|
||||
"""Add `import module` at the top-level import block if missing."""
|
||||
if re.search(rf"(?:^|\n)import\s+{re.escape(module)}(?:\s|$)", text):
|
||||
return text
|
||||
if re.search(rf"(?:^|\n)from\s+{re.escape(module)}\s+import", text):
|
||||
return text
|
||||
|
||||
lines = text.splitlines(keepends=True)
|
||||
insert = 0
|
||||
paren = 0
|
||||
for i, ln in enumerate(lines):
|
||||
# track parentheses so we don't insert inside a multi-line parenthesized import
|
||||
paren += ln.count("(") - ln.count(")")
|
||||
if ln.startswith("#!/") or ln.startswith("# -*-") or ln.startswith("# coding"):
|
||||
insert = i + 1
|
||||
continue
|
||||
if ln.startswith("from __future__ import"):
|
||||
insert = i + 1
|
||||
continue
|
||||
if paren == 0 and (ln.startswith("import ") or ln.startswith("from ")):
|
||||
insert = i + 1
|
||||
if insert == len(lines) and lines and not lines[-1].endswith("\n"):
|
||||
lines.append("\n")
|
||||
lines.insert(insert, f"import {module}\n")
|
||||
return "".join(lines)
|
||||
|
||||
|
||||
def remove_import(text: str, module: str) -> str:
|
||||
"""Remove a top-level `import module` line if it exists."""
|
||||
lines = text.splitlines(keepends=True)
|
||||
out = []
|
||||
for ln in lines:
|
||||
if re.match(rf"^import\s+{re.escape(module)}\s*$", ln.strip("\n")):
|
||||
continue
|
||||
out.append(ln)
|
||||
return "".join(out)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# S110
|
||||
# ---------------------------------------------------------------------------
|
||||
SAME_LINE_PASS = re.compile(
|
||||
r"^(?P<indent>[ \t]*)except\b(?P<expr>[^:\n]*):(?P<space>\s*)pass\s*(?:#.*)?$",
|
||||
re.MULTILINE,
|
||||
)
|
||||
|
||||
|
||||
def fix_s110() -> tuple[int, int]:
|
||||
errors = run_ruff("S110")
|
||||
by_file: dict[str, list[int]] = {}
|
||||
for e in errors:
|
||||
by_file.setdefault(e["filename"], []).append(e["location"]["row"])
|
||||
|
||||
changed = 0
|
||||
for path_str, rows in by_file.items():
|
||||
path = Path(path_str)
|
||||
text = path.read_text()
|
||||
orig = text
|
||||
|
||||
lines = text.splitlines(keepends=True)
|
||||
# sort rows descending so line numbers stay stable
|
||||
for row in sorted(rows, reverse=True):
|
||||
if row < 1 or row > len(lines):
|
||||
continue
|
||||
idx = row - 1
|
||||
line = lines[idx]
|
||||
m = SAME_LINE_PASS.match(line)
|
||||
if m:
|
||||
indent = m.group("indent")
|
||||
expr = m.group("expr").strip()
|
||||
new_lines = [f"{indent}except {expr}:\n", f"{indent} {S110_LOG_LINE}\n"]
|
||||
lines[idx : idx + 1] = new_lines
|
||||
continue
|
||||
|
||||
# multiline: find the next non-comment/blank line that is a bare pass
|
||||
j = idx + 1
|
||||
while j < len(lines):
|
||||
content = lines[j].strip()
|
||||
if content == "" or content.startswith("#"):
|
||||
j += 1
|
||||
continue
|
||||
if content == "pass" or content.startswith("pass ") or content.startswith("pass\t"):
|
||||
indent = len(lines[j]) - len(lines[j].lstrip())
|
||||
lines[j] = " " * indent + S110_LOG_LINE + "\n"
|
||||
break
|
||||
# next executable line is not a pass; give up
|
||||
break
|
||||
|
||||
text = "".join(lines)
|
||||
# ensure logging is imported after all replacements so row indices remain valid
|
||||
text = ensure_import(text, "logging")
|
||||
if text != orig:
|
||||
path.write_text(text)
|
||||
changed += 1
|
||||
|
||||
return len(errors), changed
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# S324
|
||||
# ---------------------------------------------------------------------------
|
||||
MD5_RE = re.compile(r"hashlib\.md5\s*\(")
|
||||
SHA1_RE = re.compile(r"hashlib\.sha1\s*\(")
|
||||
NEW_MD5_RE = re.compile(r"hashlib\.new\s*\(\s*(['\"])md5\1\s*\)")
|
||||
NEW_SHA1_RE = re.compile(r"hashlib\.new\s*\(\s*(['\"])sha1\1\s*\)")
|
||||
|
||||
|
||||
def fix_s324() -> tuple[int, int]:
|
||||
errors = run_ruff("S324")
|
||||
by_file: dict[str, list[dict]] = {}
|
||||
for e in errors:
|
||||
by_file.setdefault(e["filename"], []).append(e)
|
||||
|
||||
changed = 0
|
||||
for path_str in by_file:
|
||||
path = Path(path_str)
|
||||
text = path.read_text()
|
||||
orig = text
|
||||
text = MD5_RE.sub("hashlib.sha256(", text)
|
||||
text = SHA1_RE.sub("hashlib.sha256(", text)
|
||||
text = NEW_MD5_RE.sub(r'hashlib.new(\1sha256\1)', text)
|
||||
text = NEW_SHA1_RE.sub(r'hashlib.new(\1sha256\1)', text)
|
||||
if text != orig:
|
||||
path.write_text(text)
|
||||
changed += 1
|
||||
|
||||
return len(errors), changed
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# S311
|
||||
# ---------------------------------------------------------------------------
|
||||
RANDOM_CALL_RE = re.compile(r"random\.((?:random|randint|uniform|randrange))\s*\(")
|
||||
RANDOM_CHOICE_RE = re.compile(r"random\.choice\s*\(")
|
||||
|
||||
|
||||
def fix_s311() -> tuple[int, int]:
|
||||
errors = run_ruff("S311")
|
||||
by_file: dict[str, list[dict]] = {}
|
||||
for e in errors:
|
||||
by_file.setdefault(e["filename"], []).append(e)
|
||||
|
||||
changed = 0
|
||||
for path_str in by_file:
|
||||
path = Path(path_str)
|
||||
text = path.read_text()
|
||||
orig = text
|
||||
|
||||
text = RANDOM_CALL_RE.sub(r"secrets.SystemRandom().\1(", text)
|
||||
text = RANDOM_CHOICE_RE.sub("secrets.choice(", text)
|
||||
|
||||
if text != orig:
|
||||
text = ensure_import(text, "secrets")
|
||||
# remove `import random` only if no other random.* calls remain
|
||||
if not re.search(r"\brandom\.\w+\s*\(", text):
|
||||
text = remove_import(text, "random")
|
||||
path.write_text(text)
|
||||
changed += 1
|
||||
|
||||
return len(errors), changed
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# verification
|
||||
# ---------------------------------------------------------------------------
|
||||
def verify_changed(files: list[Path]) -> list[str]:
|
||||
bad = []
|
||||
for p in files:
|
||||
try:
|
||||
py_compile.compile(str(p), doraise=True)
|
||||
except py_compile.PyCompileError as e:
|
||||
bad.append(f"{p}: {e}")
|
||||
return bad
|
||||
|
||||
|
||||
def main() -> None:
|
||||
os.chdir(ROOT)
|
||||
s110_n, s110_files = fix_s110()
|
||||
s324_n, s324_files = fix_s324()
|
||||
s311_n, s311_files = fix_s311()
|
||||
print(f"S110: {s110_n} errors -> touched {s110_files} files")
|
||||
print(f"S324: {s324_n} errors -> touched {s324_files} files")
|
||||
print(f"S311: {s311_n} errors -> touched {s311_files} files")
|
||||
|
||||
# compile check every touched file
|
||||
changed = set()
|
||||
# collect all files that were touched via git status
|
||||
proc = subprocess.run(["git", "status", "--short", str(APP)], capture_output=True, text=True)
|
||||
for ln in proc.stdout.splitlines():
|
||||
if ln.startswith(" M ") or ln.startswith("M "):
|
||||
changed.add(ROOT / ln[3:].strip())
|
||||
if changed:
|
||||
bad = verify_changed(list(changed))
|
||||
if bad:
|
||||
print("\nPY_COMPILE ERRORS:", file=sys.stderr)
|
||||
for b in bad:
|
||||
print(b, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print(f"py_compile OK for {len(changed)} changed files")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -7,6 +7,7 @@ The generated file is checked into git so dev installs work offline.
|
|||
Re-run this script only when the chain topology changes (new data type,
|
||||
new provider, etc.).
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
SOURCE = Path("app/databus/provider_chains.py")
|
||||
|
|
@ -26,8 +27,8 @@ def main():
|
|||
)
|
||||
body = content.split('"""', 2)[2] if '"""' in content else content
|
||||
TARGET.write_text(header + body)
|
||||
print("wrote {} ({} bytes)".format(TARGET, TARGET.stat().st_size))
|
||||
print(f"wrote {TARGET} ({TARGET.stat().st_size} bytes)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ def _load_cooldown_state():
|
|||
with open(state_file) as f:
|
||||
return json.load(f)
|
||||
except Exception:
|
||||
pass
|
||||
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
|
||||
return {}
|
||||
|
||||
|
||||
|
|
@ -70,7 +70,7 @@ def _save_cooldown_state(state: dict):
|
|||
with open(state_file, "w") as f:
|
||||
json.dump(state, f)
|
||||
except Exception:
|
||||
pass
|
||||
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
|
||||
|
||||
|
||||
def _should_alert(service: str) -> bool:
|
||||
|
|
@ -209,7 +209,9 @@ def send_alert(results: list):
|
|||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
logger.info(f"Alert sent to webhook: HTTP {resp.status} - {len(alertable)} services down")
|
||||
logger.info(
|
||||
f"Alert sent to webhook: HTTP {resp.status} - {len(alertable)} services down"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to send alert to webhook: {e}")
|
||||
|
||||
|
|
@ -269,7 +271,7 @@ def _load_prev_state():
|
|||
with open(state_file) as f:
|
||||
return json.load(f)
|
||||
except Exception:
|
||||
pass
|
||||
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
|
||||
return {}
|
||||
|
||||
|
||||
|
|
@ -280,7 +282,7 @@ def _save_prev_state(state: dict):
|
|||
with open(state_file, "w") as f:
|
||||
json.dump(state, f)
|
||||
except Exception:
|
||||
pass
|
||||
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
|
||||
|
||||
|
||||
# ── Main ────────────────────────────────────────────────────────────
|
||||
|
|
@ -313,7 +315,9 @@ def main():
|
|||
REDIS_HOST = os.getenv("REDIS_HOST", "localhost")
|
||||
REDIS_PORT = int(os.getenv("REDIS_PORT", "6379"))
|
||||
REDIS_PASSWORD = os.getenv("REDIS_PASSWORD", "")
|
||||
BASE_WORKER_URL = os.getenv("BASE_WORKER_URL", "http://localhost:8000/api/v1/x402/fallback/status")
|
||||
BASE_WORKER_URL = os.getenv(
|
||||
"BASE_WORKER_URL", "http://localhost:8000/api/v1/x402/fallback/status"
|
||||
)
|
||||
SOLANA_WORKER_URL = os.getenv("SOLANA_WORKER_URL", "")
|
||||
|
||||
logger.info("Running health checks...")
|
||||
|
|
|
|||
|
|
@ -28,7 +28,9 @@ async def ingest_via_api(content: str, metadata: dict) -> bool:
|
|||
import httpx
|
||||
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
resp = await client.post(RAG_API, json={"collection": COLLECTION, "content": content, "metadata": metadata})
|
||||
resp = await client.post(
|
||||
RAG_API, json={"collection": COLLECTION, "content": content, "metadata": metadata}
|
||||
)
|
||||
return resp.status_code == 200
|
||||
except Exception as e:
|
||||
logger.debug(f"API ingest failed: {e}")
|
||||
|
|
@ -40,15 +42,21 @@ async def main():
|
|||
|
||||
# Try multiple endpoints and pagination
|
||||
all_reports = []
|
||||
for page in range(0, 5): # Up to 5 pages of 100
|
||||
for page in range(5): # Up to 5 pages of 100
|
||||
url = f"{CHAINABUSE_API}?limit=100&offset={page * 100}"
|
||||
try:
|
||||
logger.info(f"Fetching page {page + 1} from ChainAbuse...")
|
||||
async with httpx.AsyncClient(timeout=30, headers={"Accept": "application/json"}) as client:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=30, headers={"Accept": "application/json"}
|
||||
) as client:
|
||||
resp = await client.get(url)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
reports = data if isinstance(data, list) else data.get("data", data.get("reports", []))
|
||||
reports = (
|
||||
data
|
||||
if isinstance(data, list)
|
||||
else data.get("data", data.get("reports", []))
|
||||
)
|
||||
if not reports:
|
||||
logger.info(f"No more reports at page {page + 1}")
|
||||
break
|
||||
|
|
@ -77,13 +85,17 @@ async def main():
|
|||
resp = await client.get(url)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
reports = data if isinstance(data, list) else data.get("data", data.get("reports", []))
|
||||
reports = (
|
||||
data
|
||||
if isinstance(data, list)
|
||||
else data.get("data", data.get("reports", []))
|
||||
)
|
||||
if reports:
|
||||
all_reports.extend(reports)
|
||||
logger.info(f"Got {len(reports)} reports from alt endpoint")
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
|
||||
|
||||
if not all_reports:
|
||||
logger.warning("No ChainAbuse reports accessible via API. Ingestion skipped.")
|
||||
|
|
|
|||
|
|
@ -140,8 +140,12 @@ async def ingest_etherscan_combined() -> dict[str, int]:
|
|||
with open(csv_path, newline="", encoding="utf-8") as f:
|
||||
rows = list(csv.DictReader(f))
|
||||
|
||||
scam_rows = [r for r in rows if any(kw in r.get("label", "").lower() for kw in SCAM_LABEL_KEYWORDS)]
|
||||
wallet_rows = [r for r in rows if not any(kw in r.get("label", "").lower() for kw in SCAM_LABEL_KEYWORDS)]
|
||||
scam_rows = [
|
||||
r for r in rows if any(kw in r.get("label", "").lower() for kw in SCAM_LABEL_KEYWORDS)
|
||||
]
|
||||
wallet_rows = [
|
||||
r for r in rows if not any(kw in r.get("label", "").lower() for kw in SCAM_LABEL_KEYWORDS)
|
||||
]
|
||||
|
||||
logger.info(
|
||||
f"Etherscan combined: {len(rows)} total → {len(scam_rows)} scam → known_scams, {len(wallet_rows)} → wallet_profiles"
|
||||
|
|
@ -177,7 +181,9 @@ async def ingest_etherscan_combined() -> dict[str, int]:
|
|||
else:
|
||||
collection = "wallet_profiles"
|
||||
doc_id = make_doc_id("wallet_profiles", f"eth_combined:{address}:{chain}")
|
||||
content = f"Etherscan labeled address: {address} ({name}) on {chain}. Category: {label}."
|
||||
content = (
|
||||
f"Etherscan labeled address: {address} ({name}) on {chain}. Category: {label}."
|
||||
)
|
||||
metadata = {
|
||||
"address": address.lower(),
|
||||
"label": label,
|
||||
|
|
@ -297,7 +303,7 @@ async def ingest_rekt_hacks() -> dict[str, int]:
|
|||
|
||||
async def ingest_solana_flagged_tokens() -> dict[str, int]:
|
||||
"""Download Solana token list and ingest explicitly flagged scam tokens."""
|
||||
token_list_url = "https://raw.githubusercontent.com/solana-labs/token-list/main/src/tokens/solana.tokenlist.json"
|
||||
token_list_url = "https://raw.githubusercontent.com/solana-labs/token-list/main/src/tokens/solana.tokenlist.json" # noqa: S105
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=60) as client:
|
||||
|
|
@ -317,7 +323,9 @@ async def ingest_solana_flagged_tokens() -> dict[str, int]:
|
|||
for t in tokens:
|
||||
name_lower = (t.get("name", "") + " " + t.get("symbol", "")).lower()
|
||||
tags_lower = [str(tag).lower() for tag in t.get("tags", [])]
|
||||
if any(kw in name_lower for kw in scam_keywords) or any(kw in " ".join(tags_lower) for kw in scam_keywords):
|
||||
if any(kw in name_lower for kw in scam_keywords) or any(
|
||||
kw in " ".join(tags_lower) for kw in scam_keywords
|
||||
):
|
||||
flagged.append(t)
|
||||
|
||||
logger.info(f"Solana token list: {len(tokens)} total, {len(flagged)} flagged as scam/spam")
|
||||
|
|
@ -359,7 +367,9 @@ async def ingest_solana_flagged_tokens() -> dict[str, int]:
|
|||
|
||||
async def ingest_metamask_domains() -> dict[str, int]:
|
||||
"""Ingest MetaMask phishing domain blacklist in batches of 50."""
|
||||
config_url = "https://raw.githubusercontent.com/MetaMask/eth-phishing-detect/master/src/config.json"
|
||||
config_url = (
|
||||
"https://raw.githubusercontent.com/MetaMask/eth-phishing-detect/master/src/config.json"
|
||||
)
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=60) as client:
|
||||
|
|
@ -405,7 +415,9 @@ async def ingest_metamask_domains() -> dict[str, int]:
|
|||
errors += 1
|
||||
|
||||
if (batch_idx + 1) % 100 == 0:
|
||||
logger.info(f" MetaMask: {batch_idx + 1}/{total_batches} batches | ingested: {ingested}")
|
||||
logger.info(
|
||||
f" MetaMask: {batch_idx + 1}/{total_batches} batches | ingested: {ingested}"
|
||||
)
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
return {"ingested": ingested, "errors": errors}
|
||||
|
|
@ -602,7 +614,9 @@ async def ingest_phishdestroy() -> dict[str, int]:
|
|||
errors += 1
|
||||
|
||||
if (batch_idx + 1) % 100 == 0:
|
||||
logger.info(f" PhishDestroy: {batch_idx + 1}/{total_batches} | ingested: {ingested}")
|
||||
logger.info(
|
||||
f" PhishDestroy: {batch_idx + 1}/{total_batches} | ingested: {ingested}"
|
||||
)
|
||||
await asyncio.sleep(0.05)
|
||||
except Exception as e:
|
||||
logger.warning(f"PhishDestroy active_domains.json failed: {e}")
|
||||
|
|
@ -646,7 +660,9 @@ async def ingest_phishdestroy() -> dict[str, int]:
|
|||
errors += 1
|
||||
|
||||
if (batch_idx + 1) % 100 == 0:
|
||||
logger.info(f" PhishDestroy: {batch_idx + 1}/{total_batches} | ingested: {ingested}")
|
||||
logger.info(
|
||||
f" PhishDestroy: {batch_idx + 1}/{total_batches} | ingested: {ingested}"
|
||||
)
|
||||
await asyncio.sleep(0.05)
|
||||
except Exception as e:
|
||||
logger.warning(f"PhishDestroy domains.txt failed: {e}")
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
|
||||
import asyncio
|
||||
import csv
|
||||
import logging
|
||||
import sys
|
||||
|
||||
import httpx
|
||||
|
|
@ -46,11 +47,13 @@ async def ingest():
|
|||
}
|
||||
|
||||
try:
|
||||
r = await client.post(RAG_URL, json=payload, headers={"X-RMI-Key": "rmi-internal-2026"})
|
||||
r = await client.post(
|
||||
RAG_URL, json=payload, headers={"X-RMI-Key": "rmi-internal-2026"}
|
||||
)
|
||||
if r.status_code == 200:
|
||||
count += 1
|
||||
except Exception:
|
||||
pass
|
||||
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
|
||||
|
||||
if (i + 1) % 500 == 0:
|
||||
print(f" Progress: {i + 1}/{len(rows)} ({count} ingested)")
|
||||
|
|
|
|||
|
|
@ -1,9 +1,13 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Import WHALEGOD wallet labels into Wallet Memory Bank."""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
import httpx
|
||||
|
||||
sys.path.insert(0, "/root/backend")
|
||||
os.chdir("/root/backend")
|
||||
|
||||
|
|
@ -18,11 +22,6 @@ sol_labels = namespace.get("SOL_LABELS", {})
|
|||
|
||||
print(f"WHALEGOD labels: ETH={len(eth_labels)}, SOL={len(sol_labels)}")
|
||||
|
||||
# Ingest into wallet_memory
|
||||
import asyncio # noqa: E402
|
||||
|
||||
import httpx # noqa: E402
|
||||
|
||||
|
||||
async def ingest():
|
||||
count = 0
|
||||
|
|
@ -52,7 +51,7 @@ async def ingest():
|
|||
if r.status_code == 200:
|
||||
count += 1
|
||||
except Exception:
|
||||
pass
|
||||
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
|
||||
|
||||
print(f"Ingested {count}/{len(eth_labels) + len(sol_labels)} WHALEGOD labels")
|
||||
|
||||
|
|
|
|||
|
|
@ -42,7 +42,9 @@ def _get_url():
|
|||
|
||||
|
||||
def _get_key():
|
||||
return os.environ.get("SUPABASE_SERVICE_KEY", "") or os.environ.get("SUPABASE_SERVICE_ROLE_KEY", "")
|
||||
return os.environ.get("SUPABASE_SERVICE_KEY", "") or os.environ.get(
|
||||
"SUPABASE_SERVICE_ROLE_KEY", ""
|
||||
)
|
||||
|
||||
|
||||
def _get_headers():
|
||||
|
|
@ -98,18 +100,24 @@ async def insert_with_retry(client: httpx.AsyncClient, rows: list, batch_num: in
|
|||
|
||||
for attempt in range(MAX_RETRIES):
|
||||
try:
|
||||
resp = await client.post(url, json=rows, params=params, headers=_get_headers(), timeout=60)
|
||||
resp = await client.post(
|
||||
url, json=rows, params=params, headers=_get_headers(), timeout=60
|
||||
)
|
||||
if resp.status_code in (200, 201):
|
||||
return len(rows)
|
||||
elif resp.status_code == 500 and "timeout" in (resp.text or "").lower():
|
||||
# Statement timeout - split into smaller batches
|
||||
logger.warning(f"Batch {batch_num}: statement timeout (attempt {attempt + 1}), splitting")
|
||||
logger.warning(
|
||||
f"Batch {batch_num}: statement timeout (attempt {attempt + 1}), splitting"
|
||||
)
|
||||
if len(rows) > 10:
|
||||
# Split into two halves
|
||||
mid = len(rows) // 2
|
||||
count = 0
|
||||
for half in [rows[:mid], rows[mid:]]:
|
||||
r2 = await client.post(url, json=half, params=params, headers=_get_headers(), timeout=90)
|
||||
r2 = await client.post(
|
||||
url, json=half, params=params, headers=_get_headers(), timeout=90
|
||||
)
|
||||
if r2.status_code in (200, 201):
|
||||
count += len(half)
|
||||
else:
|
||||
|
|
@ -127,10 +135,14 @@ async def insert_with_retry(client: httpx.AsyncClient, rows: list, batch_num: in
|
|||
count += 1
|
||||
await asyncio.sleep(0.02)
|
||||
except Exception:
|
||||
pass
|
||||
logging.getLogger(__name__).warning(
|
||||
"swallowed exception", exc_info=True
|
||||
)
|
||||
return count
|
||||
else:
|
||||
logger.error(f"Batch {batch_num} insert failed: {resp.status_code} {resp.text[:300]}")
|
||||
logger.error(
|
||||
f"Batch {batch_num} insert failed: {resp.status_code} {resp.text[:300]}"
|
||||
)
|
||||
return 0
|
||||
except httpx.ReadTimeout:
|
||||
logger.warning(f"Batch {batch_num}: timeout (attempt {attempt + 1})")
|
||||
|
|
@ -142,7 +154,9 @@ async def insert_with_retry(client: httpx.AsyncClient, rows: list, batch_num: in
|
|||
return 0
|
||||
|
||||
|
||||
async def migrate_collection(r: aioredis.Redis, client: httpx.AsyncClient, collection: str, existing_ids: set) -> int:
|
||||
async def migrate_collection(
|
||||
r: aioredis.Redis, client: httpx.AsyncClient, collection: str, existing_ids: set
|
||||
) -> int:
|
||||
"""Migrate all docs from one Redis collection to Supabase."""
|
||||
idx_key = f"rag:idx:{collection}"
|
||||
doc_ids = await r.smembers(idx_key)
|
||||
|
|
@ -198,7 +212,9 @@ async def migrate_collection(r: aioredis.Redis, client: httpx.AsyncClient, colle
|
|||
migrated += count
|
||||
batch_count += 1
|
||||
if batch_count % 10 == 0:
|
||||
logger.info(f" Progress: {migrated}/{len(to_migrate)} migrated ({batch_count} batches)")
|
||||
logger.info(
|
||||
f" Progress: {migrated}/{len(to_migrate)} migrated ({batch_count} batches)"
|
||||
)
|
||||
batch_rows = []
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ All powered by MiniMax-Text-01 ($20/mo flat, 1M context)
|
|||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import urllib.request
|
||||
from datetime import UTC, datetime
|
||||
|
|
@ -19,20 +20,31 @@ if not KEY:
|
|||
if line.startswith("MINIMAX_API_KEY="):
|
||||
KEY = line.strip().split("=", 1)[1]
|
||||
break
|
||||
except: pass # noqa: E701, E722
|
||||
except Exception:
|
||||
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
|
||||
URL = "https://api.minimax.io/v1/chat/completions"
|
||||
MODEL = "MiniMax-Text-01"
|
||||
BACKEND = os.getenv("BACKEND_URL", "http://localhost:8000")
|
||||
|
||||
|
||||
def call_minimax(system: str, prompt: str, max_tokens: int = 500, temp: float = 0.7) -> str:
|
||||
try:
|
||||
body = json.dumps({
|
||||
"model": MODEL,
|
||||
"messages": [{"role":"system","content":system},{"role":"user","content":prompt}],
|
||||
"max_tokens": max_tokens, "temperature": temp
|
||||
}).encode()
|
||||
req = urllib.request.Request(URL, data=body, headers={
|
||||
"Authorization": f"Bearer {KEY}", "Content-Type": "application/json"})
|
||||
body = json.dumps(
|
||||
{
|
||||
"model": MODEL,
|
||||
"messages": [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": temp,
|
||||
}
|
||||
).encode()
|
||||
req = urllib.request.Request(
|
||||
URL,
|
||||
data=body,
|
||||
headers={"Authorization": f"Bearer {KEY}", "Content-Type": "application/json"},
|
||||
)
|
||||
resp = urllib.request.urlopen(req, timeout=20)
|
||||
return json.loads(resp.read())["choices"][0]["message"]["content"].strip()
|
||||
except Exception as e:
|
||||
|
|
@ -53,6 +65,7 @@ Write an engaging crypto security thread. Format:
|
|||
|
||||
Keep each tweet under 280 chars. Make it punchy and shareable."""
|
||||
|
||||
|
||||
def generate_social_thread(scam_data: str) -> str:
|
||||
return call_minimax(SOCIAL_SYSTEM, f"Recent scam data:\n{scam_data[:3000]}", 400)
|
||||
|
||||
|
|
@ -65,9 +78,11 @@ Answer crypto security questions concisely. Be helpful, direct, accurate.
|
|||
Common topics: rug pulls, honeypots, token scanning, wallet safety, scam detection.
|
||||
Always mention our free scanner if relevant. Under 200 words."""
|
||||
|
||||
|
||||
def answer_question(question: str) -> str:
|
||||
return call_minimax(SUPPORT_SYSTEM, question, 250, 0.4)
|
||||
|
||||
|
||||
# Pre-warm cache with common questions
|
||||
COMMON_QUESTIONS = [
|
||||
"What is a rug pull?",
|
||||
|
|
@ -82,6 +97,7 @@ COMMON_QUESTIONS = [
|
|||
"What are red flags in new tokens?",
|
||||
]
|
||||
|
||||
|
||||
def warm_support_cache():
|
||||
"""Pre-generate answers for common questions."""
|
||||
for q in COMMON_QUESTIONS:
|
||||
|
|
@ -104,6 +120,7 @@ Format:
|
|||
|
||||
Keep it under 150 words. Use ✅⚠️🔴 emojis."""
|
||||
|
||||
|
||||
def health_narrative(metrics: dict) -> str:
|
||||
prompt = json.dumps(metrics)
|
||||
return call_minimax(HEALTH_SYSTEM, prompt, 250, 0.3)
|
||||
|
|
@ -114,6 +131,7 @@ def health_narrative(metrics: dict) -> str:
|
|||
# ═══════════════════════════════════════════
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
cmd = sys.argv[1] if len(sys.argv) > 1 else "all"
|
||||
|
||||
if cmd in ("social", "all"):
|
||||
|
|
@ -134,8 +152,12 @@ if __name__ == "__main__":
|
|||
print("\n=== HEALTH NARRATIVE ===")
|
||||
metrics = {
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
"backend": "alive", "disk_pct": 83, "load": "4.2",
|
||||
"crons_ok": 35, "crons_total": 56, "rag_docs": 20985,
|
||||
"backend": "alive",
|
||||
"disk_pct": 83,
|
||||
"load": "4.2",
|
||||
"crons_ok": 35,
|
||||
"crons_total": 56,
|
||||
"rag_docs": 20985,
|
||||
}
|
||||
report = health_narrative(metrics)
|
||||
print(report[:400])
|
||||
|
|
|
|||
|
|
@ -88,9 +88,19 @@ async def run():
|
|||
if os.path.exists(csv_path):
|
||||
with open(csv_path, newline="", encoding="utf-8") as f:
|
||||
rows = list(csv.DictReader(f))
|
||||
scam_rows = [r for r in rows if any(kw in (r.get("label", "") or "").lower() for kw in SCAM_KEYWORDS)]
|
||||
wallet_rows = [r for r in rows if not any(kw in (r.get("label", "") or "").lower() for kw in SCAM_KEYWORDS)]
|
||||
logger.info(f"2. Etherscan combined: {len(rows)} total → {len(scam_rows)} scam, {len(wallet_rows)} wallets")
|
||||
scam_rows = [
|
||||
r
|
||||
for r in rows
|
||||
if any(kw in (r.get("label", "") or "").lower() for kw in SCAM_KEYWORDS)
|
||||
]
|
||||
wallet_rows = [
|
||||
r
|
||||
for r in rows
|
||||
if not any(kw in (r.get("label", "") or "").lower() for kw in SCAM_KEYWORDS)
|
||||
]
|
||||
logger.info(
|
||||
f"2. Etherscan combined: {len(rows)} total → {len(scam_rows)} scam, {len(wallet_rows)} wallets"
|
||||
)
|
||||
|
||||
ok_scam = ok_wallet = err = 0
|
||||
# Scam addresses first
|
||||
|
|
@ -146,9 +156,11 @@ async def run():
|
|||
if resp.status_code == 200:
|
||||
ok_wallet += len(batch)
|
||||
except: # noqa: E722
|
||||
pass
|
||||
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
|
||||
if (i // batch_size + 1) % 20 == 0:
|
||||
logger.info(f" Wallets: {i + len(batch)}/{min(len(wallet_rows), 5000)} | ok={ok_wallet}")
|
||||
logger.info(
|
||||
f" Wallets: {i + len(batch)}/{min(len(wallet_rows), 5000)} | ok={ok_wallet}"
|
||||
)
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
total_ingested += ok_scam + ok_wallet
|
||||
|
|
@ -186,7 +198,7 @@ async def run():
|
|||
if resp.status_code == 200:
|
||||
ok += 1
|
||||
except: # noqa: E722
|
||||
pass
|
||||
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
|
||||
await asyncio.sleep(0.15)
|
||||
total_ingested += ok
|
||||
logger.info(f" Done: {ok} ingested")
|
||||
|
|
@ -218,7 +230,7 @@ async def run():
|
|||
if resp.status_code == 200:
|
||||
ok += 1
|
||||
except: # noqa: E722
|
||||
pass
|
||||
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
|
||||
await asyncio.sleep(0.1)
|
||||
total_ingested += ok
|
||||
logger.info(f" Done: {ok} ingested")
|
||||
|
|
|
|||
|
|
@ -144,7 +144,7 @@ def _check_redis_pressure() -> bool:
|
|||
return False
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
|
||||
return True # If check fails, proceed anyway
|
||||
|
||||
|
||||
|
|
@ -186,12 +186,16 @@ def parse_cryptoscamdb(data: dict) -> list[dict]:
|
|||
entries = data if isinstance(data, list) else data.get("entries", data.get("scams", []))
|
||||
docs = []
|
||||
for entry in entries[:MAX_DOCS_PER_CYCLE]:
|
||||
addr = entry if isinstance(entry, str) else entry.get("address", entry.get("id", str(entry)))
|
||||
addr = (
|
||||
entry if isinstance(entry, str) else entry.get("address", entry.get("id", str(entry)))
|
||||
)
|
||||
name = entry.get("name", "") if isinstance(entry, dict) else ""
|
||||
docs.append(
|
||||
{
|
||||
"collection": "known_scams",
|
||||
"content": f"CryptoScamDB: {name} - {addr}" if name else f"CryptoScamDB entry: {addr}",
|
||||
"content": f"CryptoScamDB: {name} - {addr}"
|
||||
if name
|
||||
else f"CryptoScamDB entry: {addr}",
|
||||
"metadata": {"address": str(addr), "source": "cryptoscamdb", "name": name},
|
||||
}
|
||||
)
|
||||
|
|
@ -302,7 +306,7 @@ async def run_unified_ingest():
|
|||
timeout=5,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
logging.getLogger(__name__).warning("swallowed exception", exc_info=True)
|
||||
|
||||
if not _check_redis_pressure():
|
||||
logger.warning("Redis memory pressure - skipping cycle")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue