c762564d40
- Fix 71 invalid-syntax files (class-body newline-broken assignments) - Add from/None chain to 307 B904 raise-without-from sites - Add B008 ignore to ruff.toml (already in pyproject.toml) - Noqa F401 on __init__.py re-exports (137 sites) - Noqa E402 on deferred imports (63 sites) - Bulk-add stdlib/FastAPI/project imports for F821 (127 sites) - Replace ×→x, –→-, …→... in docstrings (4093 chars) - Manual refactor of 5 SIM103/SIM116 patterns Tests: 791 passed (66 deselected due to pre-existing Redis issues in test_rag.py) Co-authored-by: opencode <opencode@rugmunch.io>
132 lines
4.7 KiB
Python
132 lines
4.7 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Ingest ChainAbuse reports into RAG known_scams collection.
|
|
API: https://api.chainabuse.com/v1/reports
|
|
"""
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import logging
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s")
|
|
logger = logging.getLogger("ingest_chainabuse")
|
|
|
|
CHAINABUSE_API = "https://api.chainabuse.com/v1/reports"
|
|
RAG_API = "http://localhost:8000/api/v1/rag/ingest"
|
|
COLLECTION = "known_scams"
|
|
|
|
|
|
def make_doc_id(address: str) -> str:
|
|
return hashlib.sha256(f"known_scams:{address.lower()}:chainabuse".encode()).hexdigest()[:16]
|
|
|
|
|
|
async def ingest_via_api(content: str, metadata: dict) -> bool:
|
|
try:
|
|
import httpx
|
|
|
|
async with httpx.AsyncClient(timeout=15) as client:
|
|
resp = await client.post(RAG_API, json={"collection": COLLECTION, "content": content, "metadata": metadata})
|
|
return resp.status_code == 200
|
|
except Exception as e:
|
|
logger.debug(f"API ingest failed: {e}")
|
|
return False
|
|
|
|
|
|
async def main():
|
|
import httpx
|
|
|
|
# Try multiple endpoints and pagination
|
|
all_reports = []
|
|
for page in range(0, 5): # Up to 5 pages of 100
|
|
url = f"{CHAINABUSE_API}?limit=100&offset={page * 100}"
|
|
try:
|
|
logger.info(f"Fetching page {page + 1} from ChainAbuse...")
|
|
async with httpx.AsyncClient(timeout=30, headers={"Accept": "application/json"}) as client:
|
|
resp = await client.get(url)
|
|
if resp.status_code == 200:
|
|
data = resp.json()
|
|
reports = data if isinstance(data, list) else data.get("data", data.get("reports", []))
|
|
if not reports:
|
|
logger.info(f"No more reports at page {page + 1}")
|
|
break
|
|
all_reports.extend(reports)
|
|
logger.info(f"Got {len(reports)} reports (total: {len(all_reports)})")
|
|
elif resp.status_code == 401:
|
|
logger.warning("ChainAbuse API requires authentication - trying public feed")
|
|
break
|
|
else:
|
|
logger.warning(f"ChainAbuse returned {resp.status_code}")
|
|
break
|
|
except Exception as e:
|
|
logger.warning(f"ChainAbuse fetch failed: {e}")
|
|
break
|
|
|
|
if not all_reports:
|
|
# Try alternative public endpoints
|
|
alt_urls = [
|
|
"https://api.chainabuse.com/v0/reports?limit=100",
|
|
"https://api.chainabuse.com/reports?limit=100",
|
|
"https://chainabuse.com/api/reports",
|
|
]
|
|
for url in alt_urls:
|
|
try:
|
|
async with httpx.AsyncClient(timeout=15) as client:
|
|
resp = await client.get(url)
|
|
if resp.status_code == 200:
|
|
data = resp.json()
|
|
reports = data if isinstance(data, list) else data.get("data", data.get("reports", []))
|
|
if reports:
|
|
all_reports.extend(reports)
|
|
logger.info(f"Got {len(reports)} reports from alt endpoint")
|
|
break
|
|
except Exception:
|
|
pass
|
|
|
|
if not all_reports:
|
|
logger.warning("No ChainAbuse reports accessible via API. Ingestion skipped.")
|
|
logger.info("ChainAbuse may require API key - visit https://chainabuse.com for access")
|
|
return
|
|
|
|
logger.info(f"Processing {len(all_reports)} ChainAbuse reports")
|
|
|
|
ingested = 0
|
|
errors = 0
|
|
|
|
for i, report in enumerate(all_reports):
|
|
if not isinstance(report, dict):
|
|
continue
|
|
|
|
address = report.get("address", report.get("targetAddress", ""))
|
|
category = report.get("category", report.get("type", "unknown"))
|
|
description = report.get("description", report.get("report", ""))
|
|
status = report.get("status", "unknown")
|
|
reporter = report.get("reporter", "")
|
|
|
|
if not address:
|
|
continue
|
|
|
|
content = f"ChainAbuse report: {address}. Category: {category}. Status: {status}. Description: {description}. Reporter: {reporter}."
|
|
metadata = {
|
|
"source": "chainabuse",
|
|
"address": address.lower(),
|
|
"category": category,
|
|
"status": status,
|
|
"severity": "high",
|
|
}
|
|
|
|
if await ingest_via_api(content, metadata):
|
|
ingested += 1
|
|
else:
|
|
errors += 1
|
|
|
|
if (i + 1) % 50 == 0:
|
|
logger.info(f"Progress: {i + 1}/{len(all_reports)} | ingested: {ingested}")
|
|
|
|
logger.info(f"COMPLETE: ingested={ingested}, errors={errors}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|