merge: chore/cleanup-remove-bloat-and-secrets into main

This commit is contained in:
Crypto Rug Munch 2026-07-02 01:24:22 +07:00
commit bde2f3a97d
1173 changed files with 437609 additions and 0 deletions

View file

@ -0,0 +1,16 @@
"""Source adapters for the federated label API.
Each adapter implements `query(address, chain) -> list[Label]`.
To add a new source:
1. Create a new module in this package (e.g., `my_source.py`)
2. Implement `async def query_my_source(address: str, chain: str) -> list[Label]`
3. Register it in LabelSource enum (models.py)
4. Wire it in federated.py:_query_source dispatcher
All adapters follow these conventions:
- Return [] on no match (not None)
- Raise exceptions on infrastructure errors (handled by caller)
- All addresses are normalized to lowercase before query
- Each label has source=LabelSource.MY_SOURCE
"""

View file

@ -0,0 +1,21 @@
"""Chainbase source — placeholder, requires API key.
TODO: add CHAINBASE_API_KEY to gopass + env. Free tier: 50K calls/month.
For now returns []. Stub.
"""
from __future__ import annotations
import logging
from app.domain.labels.models import Label
log = logging.getLogger(__name__)
async def query_chainbase(address: str, chain: str = "ethereum") -> list[Label]:
"""Query Chainbase for an address's labels.
Returns [] until CHAINBASE_API_KEY is configured.
"""
return []

View file

@ -0,0 +1,41 @@
"""ClickHouse wallet_memory.wallet_labels source — currently empty.
Schema (verified):
address String
chain_id String
label_name String
label_category String
source String
is_sanctioned UInt8 DEFAULT 0
loaded_at DateTime DEFAULT now()
label_subtype String DEFAULT ''
Returns [] until labels are imported. The plan is to bulk-import
the CSVs (etherscan_combined, ethereum labels, solana labels) via
the warehouse_etl jobs. Until then, this adapter is a no-op.
"""
from __future__ import annotations
import logging
from app.domain.labels.models import Label
log = logging.getLogger(__name__)
async def query_clickhouse_labels(address: str, chain: str = "ethereum") -> list[Label]:
"""Query ClickHouse wallet_memory.wallet_labels table.
Currently returns [] table is empty until CSVs are bulk-imported.
Real implementation will use HTTP to clickhouse client over
native protocol via app.core.duckdb_analytics.query_postgres-style attach.
Chain mapping: clickhouse stores chain_id as String. We translate
our 'ethereum'/'solana' names to the chain_id values used in the table.
"""
# TODO: when wallet_memory.wallet_labels is populated, use:
# from app.core.duckdb_analytics import DuckDBAnalytics
# d = DuckDBAnalytics()
# rows = d.query_postgres(...) # if we ever swap CH for Postgres
# OR use httpx against CH HTTP interface at http://localhost:8123
return []

View file

@ -0,0 +1,92 @@
"""eth-labels.db source — 115K EVM labels from local SQLite.
Schema (verified):
accounts(id, chain_id, address, label, name_tag, created_at, updated_at)
chain_id mapping:
1 = ethereum, 42161 = arbitrum, 137 = polygon, etc.
"""
from __future__ import annotations
import asyncio
import logging
from datetime import UTC, datetime
from app.domain.labels.models import Label, LabelSource
log = logging.getLogger(__name__)
DB_PATH = "/home/dev/rmi/eth-labels.db"
# chain_id -> chain name mapping (from EVM chains list)
_CHAIN_ID_MAP = {
1: "ethereum",
42161: "arbitrum",
10: "optimism",
137: "polygon",
56: "bsc",
43114: "avalanche",
250: "fantom",
100: "gnosis",
}
async def query_eth_labels_db(address: str, chain: str = "ethereum") -> list[Label]:
"""Query the local eth-labels.db SQLite for an address.
Args:
address: EVM address (0x...)
chain: Chain name (only EVM chains in this DB)
Returns:
List of labels found (one per chain_id that has a match).
"""
def _query() -> list[Label]:
import sqlite3
try:
conn = sqlite3.connect(DB_PATH, timeout=2.0)
conn.row_factory = sqlite3.Row
try:
cursor = conn.execute(
"SELECT chain_id, label, name_tag, updated_at FROM accounts WHERE address = ? LIMIT 50",
(address.lower(),),
)
labels = []
for row in cursor:
chain_id = row["chain_id"]
chain_name = _CHAIN_ID_MAP.get(chain_id, f"chain-{chain_id}")
label_value = row["name_tag"] or row["label"]
if not label_value:
continue
labels.append(
Label(
address=address.lower(),
chain=chain_name,
label_type="entity",
label_value=label_value,
source=LabelSource.ETH_LABELS_DB,
confidence=0.9, # eth-labels.db is curated
verified_at=_parse_date(row["updated_at"]),
attributes={"chain_id": chain_id, "raw_label": row["label"]},
)
)
return labels
finally:
conn.close()
except Exception as e:
log.warning("eth_labels_db_query_fail address=%s err=%s", address[:10], e)
raise
return await asyncio.to_thread(_query)
def _parse_date(s: str | None) -> datetime:
"""Parse SQLite date string to datetime."""
if not s:
return datetime.now(UTC)
try:
# Format: '2024-06-30 21:58:54'
return datetime.fromisoformat(s).replace(tzinfo=UTC)
except Exception:
return datetime.now(UTC)

View file

@ -0,0 +1,67 @@
"""Ethereum wallet labels CSV source — 51K EVM labels via DuckDB."""
from __future__ import annotations
import asyncio
import logging
from datetime import UTC, datetime
from app.domain.labels.models import Label, LabelSource
log = logging.getLogger(__name__)
CSV_PATH = "/home/dev/rmi/backend/data/wallet-labels-clean/wallet_labels_ethereum.csv"
async def query_ethereum_labels_csv(address: str, chain: str = "ethereum") -> list[Label]:
"""Query the ethereum wallet labels CSV via DuckDB.
Schema:
address,chain,source,name,entity_type,threat_group,risk_level,verified
"""
def _query() -> list[Label]:
from app.core.duckdb_analytics import DuckDBAnalytics
try:
d = DuckDBAnalytics()
try:
rows = d.query_parquet(
CSV_PATH,
"""
SELECT address, chain, source, name, entity_type, threat_group, risk_level, verified
FROM data
WHERE LOWER(address) = ?
LIMIT 20
""",
[address.lower()],
)
labels = []
for row in rows:
label_value = row.get("name") or row.get("entity_type")
if not label_value:
continue
chain_name = row.get("chain", chain).lower()
labels.append(
Label(
address=row["address"].lower(),
chain=chain_name,
label_type="entity",
label_value=label_value,
source=LabelSource.ETHEREUM_LABELS_CSV,
confidence=0.85,
verified_at=datetime.now(UTC),
attributes={
"entity_type": row.get("entity_type"),
"threat_group": row.get("threat_group"),
"risk_level": row.get("risk_level"),
"verified": row.get("verified"),
},
)
)
return labels
finally:
d.close()
except Exception as e:
log.warning("ethereum_labels_csv_query_fail address=%s err=%s", address[:10], e)
raise
return await asyncio.to_thread(_query)

View file

@ -0,0 +1,70 @@
"""Etherscan combined labels CSV source — 51K EVM labels via DuckDB."""
from __future__ import annotations
import asyncio
import logging
from datetime import UTC, datetime
from app.domain.labels.models import Label, LabelSource
log = logging.getLogger(__name__)
CSV_PATH = "/home/dev/rmi/backend/data/etherscan_combined_labels.csv"
async def query_etherscan_csv(address: str, chain: str = "ethereum") -> list[Label]:
"""Query the Etherscan combined labels CSV via DuckDB.
Schema:
address,chain,source,name,label_type,label_subtype,project,entity_type,entity_types
Returns:
List of labels (usually 1 per matching row).
"""
def _query() -> list[Label]:
from app.core.duckdb_analytics import DuckDBAnalytics
try:
d = DuckDBAnalytics()
try:
rows = d.query_parquet(
CSV_PATH,
"""
SELECT address, chain, source, name, label_type, label_subtype, project, entity_type
FROM data
WHERE LOWER(address) = ?
LIMIT 20
""",
[address.lower()],
)
labels = []
for row in rows:
label_value = row.get("name") or row.get("label_type") or row.get("source")
if not label_value:
continue
chain_name = row.get("chain", chain).lower()
labels.append(
Label(
address=row["address"].lower(),
chain=chain_name,
label_type="entity",
label_value=label_value,
source=LabelSource.ETHERSCAN_CSV,
confidence=0.85,
verified_at=datetime.now(UTC),
attributes={
"raw_label_type": row.get("label_type"),
"label_subtype": row.get("label_subtype"),
"project": row.get("project"),
"entity_type": row.get("entity_type"),
},
)
)
return labels
finally:
d.close()
except Exception as e:
log.warning("etherscan_csv_query_fail address=%s err=%s", address[:10], e)
raise
return await asyncio.to_thread(_query)

View file

@ -0,0 +1,22 @@
"""Internal labels source — our own growing set in Postgres/Redis.
Currently returns []. The schema for internal labels (community
contributions, RMI-curated tags) is being designed separately.
TODO: query Postgres internal_labels table once schema lands.
"""
from __future__ import annotations
import logging
from app.domain.labels.models import Label
log = logging.getLogger(__name__)
async def query_internal(address: str, chain: str = "ethereum") -> list[Label]:
"""Query internal labels database.
Returns [] until internal_labels table is provisioned.
"""
return []

View file

@ -0,0 +1,31 @@
"""
MBAL query adapter for the federated label API.
Implements the MBAL source API for the federated label system.
Usage (called by federated.py):
labels = await query_mbal("0x...", "ethereum")
Pattern: async function that returns list[Label]
"""
from typing import List
from app.domain.labels.models import Label
from app.domain.labels.sources.mbal_source import get_mbalsy_service
async def query_mbal(address: str, chain: str) -> List[Label]:
"""
Query MBAL for labels for an address.
Args:
address: The wallet address to look up
chain: The blockchain network (ethereum, bitcoin, etc.)
Returns:
List of labels from MBAL
"""
# Get or initialize the MBAL service
service = get_mbalsy_service()
return await service.get_labels(address, chain)

View file

@ -0,0 +1,234 @@
"""
MBAL (Multi-blockchain Anti-Laundering) Label Integration
Integrates with MBAL's 10M+ wallet labels via the MBAL API or direct ClickHouse access.
MBAL provides comprehensive blockchain attribution, wallet clustering, and risk scoring
across Bitcoin, Ethereum, and 15+ major chains.
API Documentation: https://mbal.io/api
Rate Limits: Free tier - 100 queries/day, Pro - 1000/day, Enterprise - Unlimited
"""
import asyncio
import logging
from datetime import datetime
from typing import List, Optional
from urllib.parse import urlencode
import httpx
from app.domain.labels.models import Label, LabelSource
logger = logging.getLogger(__name__)
class MBALService:
"""MBAL API Integration for multi-blockchain wallet labels"""
def __init__(self, api_key: Optional[str] = None):
"""
Initialize MBAL service.
Args:
api_key: MBAL API key. If None, uses free tier with limitations.
"""
self.base_url = "https://api.mbal.io/v1"
self.api_key = api_key
self.client = httpx.AsyncClient(
timeout=httpx.Timeout(30.0),
limits=httpx.Limits(max_connections=10, max_keepalive_connections=5),
)
# Cache for rate limiting
self.last_request_time = 0
self.request_count = 0
self.reset_time = 0 # Unix timestamp when quota resets
async def close(self):
"""Close the HTTP client."""
await self.client.aclose()
async def get_labels(self, address: str, chain: str) -> List[Label]:
"""
Retrieve labels for a blockchain address from MBAL.
Args:
address: The blockchain address to label
chain: The blockchain (ethereum, bitcoin, tron, etc.)
Returns:
List of labels with metadata
"""
try:
# MBAL requires different endpoints per chain
labels = []
# Prepare query payload
params = {
"address": address,
"chain": chain.lower(),
}
# Rate limiting - max 100 requests per day for free
await self._rate_limit()
headers = {}
if self.api_key:
headers["Authorization"] = f"Bearer {self.api_key}"
# Query MBAL's address endpoint
response = await self.client.get(
f"{self.base_url}/address",
params=params,
headers=headers
)
response.raise_for_status()
data = response.json()
# Process the result according to MBAL's schema
if "entities" in data:
for entity in data["entities"]:
label_value = entity.get("category", "unknown")
entity_type = entity.get("type", "address")
confidence = entity.get("confidence", 0.8) # MBAL typically confident
# Map MBAL categories to our standardized label types
label_type = self._map_category_to_type(label_value)
labels.append(Label(
address=address,
chain=chain,
label_type=label_type,
label_value=label_value,
source=LabelSource.MBAL, # MBAL source
verified_at=datetime.now(),
confidence=confidence,
attributes={
"mbal_entity_id": entity.get("entityId"),
"mbal_tags": entity.get("tags", []),
"mbal_risk_score": entity.get("riskScore", 0),
"mbal_cluster_id": entity.get("clusterId"),
}
))
if "behaviors" in data and len(labels) == 0:
# If no main entities, fall back to behavioral labels
for behavior in data["behaviors"]:
label_value = behavior["type"]
label_type = self._map_behavior_to_type(label_value)
confidence = behavior.get("confidence", 0.7)
labels.append(Label(
address=address,
chain=chain,
label_type=label_type,
label_value=label_value,
source=LabelSource.MBAL, # MBAL API source
verified_at=datetime.now(),
confidence=confidence,
attributes={
"behavior_description": behavior.get("description"),
"behavior_risk_level": behavior.get("riskLevel", "medium"),
}
))
return labels
except httpx.HTTPStatusError as e:
if e.response.status_code == 429:
logger.warning(f"MBAL rate limit exceeded: {e}")
return []
logger.error(f"MBAL API error: {e} for {chain}:{address}")
return []
except Exception as e:
logger.error(f"Unexpected error querying MBAL: {e}")
return []
def _map_category_to_type(self, category: str) -> str:
"""Map MBAL category strings to our standard label type strings."""
category_lower = category.lower()
# Exchange mappings
if any(exchange in category_lower for exchange in ["binance", "coinbase", "kraken", "kucoin",
"huobi", "gate.io", "okx", "bybit", "gemini"]):
return "exchange"
# Service mappings
if any(service in category_lower for service in ["miner", "validator", "staking", "liquidity",
"amm", "dex", "bridge", "cex", "dexe"]):
return "service"
# Actor mappings
if any(actor in category_lower for actor in ["blacklist", "scammer", "hacker", "tornado",
"mixer", "launderer", "crime", "malicious"]):
return "actor"
# Default to entity for unknown but potentially identifiable categories
return "actor" if "blacklist" in category_lower else "entity"
def _map_behavior_to_type(self, behavior: str) -> str:
"""Map MBAL behaviors to our standard label type strings."""
behavior_lower = behavior.lower()
if any(risky in behavior_lower for risky in ["malware", "ransom", "exploit", "theft",
"scam", "hacking", "malicious"]):
return "actor"
if any(behavior in behavior_lower for behavior in ["mining", "block generation", "transaction fee", "gas"]):
return "service"
return "entity" # Default
async def _rate_limit(self):
"""Implement simple rate limiting based on MBAL tier."""
now = asyncio.get_event_loop().time()
# Check daily quota if using free tier
if not self.api_key and self.request_count >= 100:
# Free tier: 100 queries/day
# Simple approach: wait until reset (24 hours from first query)
if now < self.reset_time:
sleep_time = self.reset_time - now
logger.info(f"Waiting for MBAL daily quota reset: {sleep_time:.1f}s")
await asyncio.sleep(sleep_time)
# Ensure minimum delay between requests
time_since_last = now - self.last_request_time
min_delay = 1.0 # At least 1s between requests
if time_since_last < min_delay:
await asyncio.sleep(min_delay - time_since_last)
# Update counters
self.last_request_time = now
if now >= self.reset_time:
self.request_count = 0
self.reset_time = now + 86400 # Reset in 24 hours
self.request_count += 1
async def health_check(self) -> bool:
"""
Check if MBAL service is accessible.
Returns:
True if MBAL API responds successfully
"""
try:
response = await self.client.get(f"{self.base_url}/health")
return response.status_code == 200
except:
return False
# Global instance
_mbalsy_instance: Optional[MBALService] = None
def get_mbalsy_service() -> MBALService:
"""Get the global MBAL service instance."""
global _mbalsy_instance
if _mbalsy_instance is None:
import os
api_key = os.getenv("MBAL_API_KEY", None)
_mbalsy_instance = MBALService(api_key=api_key)
return _mbalsy_instance

View file

@ -0,0 +1,168 @@
"""MetaSleuth (BlockSec AML) source — live API at aml.blocksec.com.
Endpoint: POST https://aml.blocksec.com/address-label/api/v3/labels
Headers: API-KEY: <key>
Body: {"chain_id": 1, "address": "0x..."}
Verified working: 2026-06-22 (Binance 0x3d12... returned main_entity=Binance,
entity categories EXCHANGE + OTC DESK).
Limit: 10 calls/day per API key on the free tier. So we cache aggressively.
"""
from __future__ import annotations
import asyncio
import logging
import os
from datetime import UTC, datetime
import httpx
from app.domain.labels.models import Label, LabelSource
log = logging.getLogger(__name__)
METASLEUTH_BASE = "https://aml.blocksec.com/address-label/api/v3"
# Free tier: 10 calls/day per key on risk-level, similar on labels
DEFAULT_TIMEOUT = 8.0
# Chain name -> MetaSleuth chain_id
_CHAIN_MAP = {
"ethereum": 1,
"bitcoin": -1,
"tron": -2,
"solana": -3,
"monero": -4,
"sui": 5,
"optimism": 10,
"cronos": 25,
"bsc": 56,
"base": 8453,
"arbitrum": 42161,
"polygon": 137,
}
async def query_metasleuth(address: str, chain: str = "ethereum") -> list[Label]:
"""Query MetaSleuth for an address's entity info.
Returns:
- 1 entity label with the main entity name (e.g. "Binance")
- Optional category labels (EXCHANGE, OTC DESK, etc.)
- Attributes (DEPOSIT ADDRESS, etc.)
"""
api_key = os.getenv("METASLEUTH_LABELS_API_KEY", "")
if not api_key:
# Try gopass via the standard pattern
try:
import subprocess
result = subprocess.run(
["gopass", "show", "-o", "rmi/infra/metasleuth_labels_api_key"],
capture_output=True, text=True, timeout=2,
)
if result.returncode == 0 and result.stdout.strip():
api_key = result.stdout.strip()
except Exception:
pass
if not api_key:
log.debug("metasleuth_no_api_key configured, skipping")
return []
chain_id = _CHAIN_MAP.get(chain.lower())
if chain_id is None:
log.debug("metasleuth_unsupported_chain chain=%s", chain)
return []
def _call_api() -> dict:
"""Sync HTTP call (run in thread)."""
try:
r = httpx.post(
f"{METASLEUTH_BASE}/labels",
headers={"API-KEY": api_key, "Content-Type": "application/json"},
json={"chain_id": chain_id, "address": address},
timeout=DEFAULT_TIMEOUT,
)
r.raise_for_status()
data = r.json()
if data.get("code") == 200000:
return data.get("data") or {}
log.warning("metasleuth_api_err code=%s msg=%s", data.get("code"), data.get("message"))
return {}
except httpx.HTTPError as e:
log.warning("metasleuth_api_http_err: %s", e)
return {}
except Exception as e:
log.warning("metasleuth_api_err: %s", e)
return {}
data = await asyncio.to_thread(_call_api)
if not data:
return []
labels: list[Label] = []
main_entity = data.get("main_entity")
if main_entity:
labels.append(Label(
address=address.lower(),
chain=chain,
label_type="entity",
label_value=main_entity,
source=LabelSource.METASLEUTH,
confidence=0.95,
verified_at=datetime.now(UTC),
entity=main_entity,
entity_category=_first_category(data),
attributes=_extract_attributes(data),
))
# Also emit labels for each category
categories = data.get("main_entity_info", {}).get("categories", [])
if not isinstance(categories, list):
categories = []
for cat in categories:
if not isinstance(cat, dict):
continue
cat_name = cat.get("name")
if cat_name and cat_name != main_entity:
labels.append(Label(
address=address.lower(),
chain=chain,
label_type="category",
label_value=cat_name,
source=LabelSource.METASLEUTH,
confidence=0.9,
verified_at=datetime.now(UTC),
entity=main_entity,
entity_category=cat_name,
attributes={"code": cat.get("code")},
))
return labels
def _first_category(data: dict) -> str | None:
"""Extract first category name from MetaSleuth response."""
cats = data.get("main_entity_info", {}).get("categories", [])
if cats and isinstance(cats[0], dict):
return cats[0].get("name")
return None
def _extract_attributes(data: dict) -> dict:
"""Extract attributes from MetaSleuth response."""
attrs = data.get("main_entity_info", {}).get("attributes", [])
out = {}
for a in attrs:
if isinstance(a, dict):
name = a.get("name", "")
if name:
out[name] = a.get("code")
# Add description (website/twitter) if present
desc = data.get("main_entity_info", {}).get("description", {})
if isinstance(desc, dict):
for k, v in desc.items():
if v:
out[k] = v
return out

View file

@ -0,0 +1,22 @@
"""Open Labels Initiative (OLI) source — placeholder.
TODO: wire up when OLI endpoint is confirmed. GitHub repo:
https://github.com/openlabelsinitiative/OLI
For now returns []. Stub.
"""
from __future__ import annotations
import logging
from app.domain.labels.models import Label
log = logging.getLogger(__name__)
async def query_open_labels(address: str, chain: str = "ethereum") -> list[Label]:
"""Query OLI for an address's labels.
Returns [] until OLI endpoint is confirmed and we have API access.
"""
return []

View file

@ -0,0 +1,74 @@
"""Solana wallet labels CSV source — 103K SOL labels via DuckDB."""
from __future__ import annotations
import asyncio
import logging
from datetime import UTC, datetime
from app.domain.labels.models import Label, LabelSource
log = logging.getLogger(__name__)
CSV_PATH = "/home/dev/rmi/backend/data/wallet-labels-clean/wallet_labels_solana.csv"
async def query_solana_labels_csv(address: str, chain: str = "solana") -> list[Label]:
"""Query the Solana wallet labels CSV via DuckDB.
Schema (likely): address,source,name,entity_type,threat_group,risk_level
Only matches if chain == "solana" returns [] for EVM chains.
"""
if chain != "solana":
return []
def _query() -> list[Label]:
from app.core.duckdb_analytics import DuckDBAnalytics
try:
d = DuckDBAnalytics()
try:
rows = d.query_parquet(
CSV_PATH,
"""
SELECT *
FROM data
WHERE LOWER(address) = ?
LIMIT 20
""",
[address.lower()],
)
labels = []
for row in rows:
# Field names may differ — try common names
label_value = (
row.get("name")
or row.get("label")
or row.get("entity_type")
or row.get("source")
)
if not label_value:
continue
labels.append(
Label(
address=row.get("address", address).lower(),
chain="solana",
label_type="entity",
label_value=label_value,
source=LabelSource.SOLANA_LABELS_CSV,
confidence=0.8,
verified_at=datetime.now(UTC),
attributes={
k: v for k, v in row.items()
if k not in ("address", "name", "label") and v
},
)
)
return labels
finally:
d.close()
except Exception as e:
log.warning("solana_labels_csv_query_fail address=%s err=%s", address[:10], e)
raise
return await asyncio.to_thread(_query)