merge: chore/cleanup-remove-bloat-and-secrets into main
This commit is contained in:
commit
bde2f3a97d
1173 changed files with 437609 additions and 0 deletions
16
app/domain/labels/sources/__init__.py
Normal file
16
app/domain/labels/sources/__init__.py
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
"""Source adapters for the federated label API.
|
||||
|
||||
Each adapter implements `query(address, chain) -> list[Label]`.
|
||||
|
||||
To add a new source:
|
||||
1. Create a new module in this package (e.g., `my_source.py`)
|
||||
2. Implement `async def query_my_source(address: str, chain: str) -> list[Label]`
|
||||
3. Register it in LabelSource enum (models.py)
|
||||
4. Wire it in federated.py:_query_source dispatcher
|
||||
|
||||
All adapters follow these conventions:
|
||||
- Return [] on no match (not None)
|
||||
- Raise exceptions on infrastructure errors (handled by caller)
|
||||
- All addresses are normalized to lowercase before query
|
||||
- Each label has source=LabelSource.MY_SOURCE
|
||||
"""
|
||||
21
app/domain/labels/sources/chainbase.py
Normal file
21
app/domain/labels/sources/chainbase.py
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
"""Chainbase source — placeholder, requires API key.
|
||||
|
||||
TODO: add CHAINBASE_API_KEY to gopass + env. Free tier: 50K calls/month.
|
||||
|
||||
For now returns []. Stub.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from app.domain.labels.models import Label
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def query_chainbase(address: str, chain: str = "ethereum") -> list[Label]:
|
||||
"""Query Chainbase for an address's labels.
|
||||
|
||||
Returns [] until CHAINBASE_API_KEY is configured.
|
||||
"""
|
||||
return []
|
||||
41
app/domain/labels/sources/clickhouse_labels.py
Normal file
41
app/domain/labels/sources/clickhouse_labels.py
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
"""ClickHouse wallet_memory.wallet_labels source — currently empty.
|
||||
|
||||
Schema (verified):
|
||||
address String
|
||||
chain_id String
|
||||
label_name String
|
||||
label_category String
|
||||
source String
|
||||
is_sanctioned UInt8 DEFAULT 0
|
||||
loaded_at DateTime DEFAULT now()
|
||||
label_subtype String DEFAULT ''
|
||||
|
||||
Returns [] until labels are imported. The plan is to bulk-import
|
||||
the CSVs (etherscan_combined, ethereum labels, solana labels) via
|
||||
the warehouse_etl jobs. Until then, this adapter is a no-op.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from app.domain.labels.models import Label
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def query_clickhouse_labels(address: str, chain: str = "ethereum") -> list[Label]:
|
||||
"""Query ClickHouse wallet_memory.wallet_labels table.
|
||||
|
||||
Currently returns [] — table is empty until CSVs are bulk-imported.
|
||||
Real implementation will use HTTP to clickhouse client over
|
||||
native protocol via app.core.duckdb_analytics.query_postgres-style attach.
|
||||
|
||||
Chain mapping: clickhouse stores chain_id as String. We translate
|
||||
our 'ethereum'/'solana' names to the chain_id values used in the table.
|
||||
"""
|
||||
# TODO: when wallet_memory.wallet_labels is populated, use:
|
||||
# from app.core.duckdb_analytics import DuckDBAnalytics
|
||||
# d = DuckDBAnalytics()
|
||||
# rows = d.query_postgres(...) # if we ever swap CH for Postgres
|
||||
# OR use httpx against CH HTTP interface at http://localhost:8123
|
||||
return []
|
||||
92
app/domain/labels/sources/eth_labels_db.py
Normal file
92
app/domain/labels/sources/eth_labels_db.py
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
"""eth-labels.db source — 115K EVM labels from local SQLite.
|
||||
|
||||
Schema (verified):
|
||||
accounts(id, chain_id, address, label, name_tag, created_at, updated_at)
|
||||
|
||||
chain_id mapping:
|
||||
1 = ethereum, 42161 = arbitrum, 137 = polygon, etc.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from app.domain.labels.models import Label, LabelSource
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
DB_PATH = "/home/dev/rmi/eth-labels.db"
|
||||
|
||||
# chain_id -> chain name mapping (from EVM chains list)
|
||||
_CHAIN_ID_MAP = {
|
||||
1: "ethereum",
|
||||
42161: "arbitrum",
|
||||
10: "optimism",
|
||||
137: "polygon",
|
||||
56: "bsc",
|
||||
43114: "avalanche",
|
||||
250: "fantom",
|
||||
100: "gnosis",
|
||||
}
|
||||
|
||||
|
||||
async def query_eth_labels_db(address: str, chain: str = "ethereum") -> list[Label]:
|
||||
"""Query the local eth-labels.db SQLite for an address.
|
||||
|
||||
Args:
|
||||
address: EVM address (0x...)
|
||||
chain: Chain name (only EVM chains in this DB)
|
||||
|
||||
Returns:
|
||||
List of labels found (one per chain_id that has a match).
|
||||
"""
|
||||
def _query() -> list[Label]:
|
||||
import sqlite3
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(DB_PATH, timeout=2.0)
|
||||
conn.row_factory = sqlite3.Row
|
||||
try:
|
||||
cursor = conn.execute(
|
||||
"SELECT chain_id, label, name_tag, updated_at FROM accounts WHERE address = ? LIMIT 50",
|
||||
(address.lower(),),
|
||||
)
|
||||
labels = []
|
||||
for row in cursor:
|
||||
chain_id = row["chain_id"]
|
||||
chain_name = _CHAIN_ID_MAP.get(chain_id, f"chain-{chain_id}")
|
||||
label_value = row["name_tag"] or row["label"]
|
||||
if not label_value:
|
||||
continue
|
||||
labels.append(
|
||||
Label(
|
||||
address=address.lower(),
|
||||
chain=chain_name,
|
||||
label_type="entity",
|
||||
label_value=label_value,
|
||||
source=LabelSource.ETH_LABELS_DB,
|
||||
confidence=0.9, # eth-labels.db is curated
|
||||
verified_at=_parse_date(row["updated_at"]),
|
||||
attributes={"chain_id": chain_id, "raw_label": row["label"]},
|
||||
)
|
||||
)
|
||||
return labels
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
log.warning("eth_labels_db_query_fail address=%s err=%s", address[:10], e)
|
||||
raise
|
||||
|
||||
return await asyncio.to_thread(_query)
|
||||
|
||||
|
||||
def _parse_date(s: str | None) -> datetime:
|
||||
"""Parse SQLite date string to datetime."""
|
||||
if not s:
|
||||
return datetime.now(UTC)
|
||||
try:
|
||||
# Format: '2024-06-30 21:58:54'
|
||||
return datetime.fromisoformat(s).replace(tzinfo=UTC)
|
||||
except Exception:
|
||||
return datetime.now(UTC)
|
||||
67
app/domain/labels/sources/ethereum_labels_csv.py
Normal file
67
app/domain/labels/sources/ethereum_labels_csv.py
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
"""Ethereum wallet labels CSV source — 51K EVM labels via DuckDB."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from app.domain.labels.models import Label, LabelSource
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
CSV_PATH = "/home/dev/rmi/backend/data/wallet-labels-clean/wallet_labels_ethereum.csv"
|
||||
|
||||
|
||||
async def query_ethereum_labels_csv(address: str, chain: str = "ethereum") -> list[Label]:
|
||||
"""Query the ethereum wallet labels CSV via DuckDB.
|
||||
|
||||
Schema:
|
||||
address,chain,source,name,entity_type,threat_group,risk_level,verified
|
||||
"""
|
||||
def _query() -> list[Label]:
|
||||
from app.core.duckdb_analytics import DuckDBAnalytics
|
||||
|
||||
try:
|
||||
d = DuckDBAnalytics()
|
||||
try:
|
||||
rows = d.query_parquet(
|
||||
CSV_PATH,
|
||||
"""
|
||||
SELECT address, chain, source, name, entity_type, threat_group, risk_level, verified
|
||||
FROM data
|
||||
WHERE LOWER(address) = ?
|
||||
LIMIT 20
|
||||
""",
|
||||
[address.lower()],
|
||||
)
|
||||
labels = []
|
||||
for row in rows:
|
||||
label_value = row.get("name") or row.get("entity_type")
|
||||
if not label_value:
|
||||
continue
|
||||
chain_name = row.get("chain", chain).lower()
|
||||
labels.append(
|
||||
Label(
|
||||
address=row["address"].lower(),
|
||||
chain=chain_name,
|
||||
label_type="entity",
|
||||
label_value=label_value,
|
||||
source=LabelSource.ETHEREUM_LABELS_CSV,
|
||||
confidence=0.85,
|
||||
verified_at=datetime.now(UTC),
|
||||
attributes={
|
||||
"entity_type": row.get("entity_type"),
|
||||
"threat_group": row.get("threat_group"),
|
||||
"risk_level": row.get("risk_level"),
|
||||
"verified": row.get("verified"),
|
||||
},
|
||||
)
|
||||
)
|
||||
return labels
|
||||
finally:
|
||||
d.close()
|
||||
except Exception as e:
|
||||
log.warning("ethereum_labels_csv_query_fail address=%s err=%s", address[:10], e)
|
||||
raise
|
||||
|
||||
return await asyncio.to_thread(_query)
|
||||
70
app/domain/labels/sources/etherscan_csv.py
Normal file
70
app/domain/labels/sources/etherscan_csv.py
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
"""Etherscan combined labels CSV source — 51K EVM labels via DuckDB."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from app.domain.labels.models import Label, LabelSource
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
CSV_PATH = "/home/dev/rmi/backend/data/etherscan_combined_labels.csv"
|
||||
|
||||
|
||||
async def query_etherscan_csv(address: str, chain: str = "ethereum") -> list[Label]:
|
||||
"""Query the Etherscan combined labels CSV via DuckDB.
|
||||
|
||||
Schema:
|
||||
address,chain,source,name,label_type,label_subtype,project,entity_type,entity_types
|
||||
|
||||
Returns:
|
||||
List of labels (usually 1 per matching row).
|
||||
"""
|
||||
def _query() -> list[Label]:
|
||||
from app.core.duckdb_analytics import DuckDBAnalytics
|
||||
|
||||
try:
|
||||
d = DuckDBAnalytics()
|
||||
try:
|
||||
rows = d.query_parquet(
|
||||
CSV_PATH,
|
||||
"""
|
||||
SELECT address, chain, source, name, label_type, label_subtype, project, entity_type
|
||||
FROM data
|
||||
WHERE LOWER(address) = ?
|
||||
LIMIT 20
|
||||
""",
|
||||
[address.lower()],
|
||||
)
|
||||
labels = []
|
||||
for row in rows:
|
||||
label_value = row.get("name") or row.get("label_type") or row.get("source")
|
||||
if not label_value:
|
||||
continue
|
||||
chain_name = row.get("chain", chain).lower()
|
||||
labels.append(
|
||||
Label(
|
||||
address=row["address"].lower(),
|
||||
chain=chain_name,
|
||||
label_type="entity",
|
||||
label_value=label_value,
|
||||
source=LabelSource.ETHERSCAN_CSV,
|
||||
confidence=0.85,
|
||||
verified_at=datetime.now(UTC),
|
||||
attributes={
|
||||
"raw_label_type": row.get("label_type"),
|
||||
"label_subtype": row.get("label_subtype"),
|
||||
"project": row.get("project"),
|
||||
"entity_type": row.get("entity_type"),
|
||||
},
|
||||
)
|
||||
)
|
||||
return labels
|
||||
finally:
|
||||
d.close()
|
||||
except Exception as e:
|
||||
log.warning("etherscan_csv_query_fail address=%s err=%s", address[:10], e)
|
||||
raise
|
||||
|
||||
return await asyncio.to_thread(_query)
|
||||
22
app/domain/labels/sources/internal_labels.py
Normal file
22
app/domain/labels/sources/internal_labels.py
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
"""Internal labels source — our own growing set in Postgres/Redis.
|
||||
|
||||
Currently returns []. The schema for internal labels (community
|
||||
contributions, RMI-curated tags) is being designed separately.
|
||||
|
||||
TODO: query Postgres internal_labels table once schema lands.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from app.domain.labels.models import Label
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def query_internal(address: str, chain: str = "ethereum") -> list[Label]:
|
||||
"""Query internal labels database.
|
||||
|
||||
Returns [] until internal_labels table is provisioned.
|
||||
"""
|
||||
return []
|
||||
31
app/domain/labels/sources/mbal.py
Normal file
31
app/domain/labels/sources/mbal.py
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
"""
|
||||
MBAL query adapter for the federated label API.
|
||||
|
||||
Implements the MBAL source API for the federated label system.
|
||||
|
||||
Usage (called by federated.py):
|
||||
labels = await query_mbal("0x...", "ethereum")
|
||||
|
||||
Pattern: async function that returns list[Label]
|
||||
"""
|
||||
|
||||
from typing import List
|
||||
|
||||
from app.domain.labels.models import Label
|
||||
from app.domain.labels.sources.mbal_source import get_mbalsy_service
|
||||
|
||||
|
||||
async def query_mbal(address: str, chain: str) -> List[Label]:
|
||||
"""
|
||||
Query MBAL for labels for an address.
|
||||
|
||||
Args:
|
||||
address: The wallet address to look up
|
||||
chain: The blockchain network (ethereum, bitcoin, etc.)
|
||||
|
||||
Returns:
|
||||
List of labels from MBAL
|
||||
"""
|
||||
# Get or initialize the MBAL service
|
||||
service = get_mbalsy_service()
|
||||
return await service.get_labels(address, chain)
|
||||
234
app/domain/labels/sources/mbal_source.py
Normal file
234
app/domain/labels/sources/mbal_source.py
Normal file
|
|
@ -0,0 +1,234 @@
|
|||
"""
|
||||
MBAL (Multi-blockchain Anti-Laundering) Label Integration
|
||||
|
||||
Integrates with MBAL's 10M+ wallet labels via the MBAL API or direct ClickHouse access.
|
||||
MBAL provides comprehensive blockchain attribution, wallet clustering, and risk scoring
|
||||
across Bitcoin, Ethereum, and 15+ major chains.
|
||||
|
||||
API Documentation: https://mbal.io/api
|
||||
Rate Limits: Free tier - 100 queries/day, Pro - 1000/day, Enterprise - Unlimited
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import httpx
|
||||
|
||||
from app.domain.labels.models import Label, LabelSource
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MBALService:
|
||||
"""MBAL API Integration for multi-blockchain wallet labels"""
|
||||
|
||||
def __init__(self, api_key: Optional[str] = None):
|
||||
"""
|
||||
Initialize MBAL service.
|
||||
|
||||
Args:
|
||||
api_key: MBAL API key. If None, uses free tier with limitations.
|
||||
"""
|
||||
self.base_url = "https://api.mbal.io/v1"
|
||||
self.api_key = api_key
|
||||
self.client = httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(30.0),
|
||||
limits=httpx.Limits(max_connections=10, max_keepalive_connections=5),
|
||||
)
|
||||
|
||||
# Cache for rate limiting
|
||||
self.last_request_time = 0
|
||||
self.request_count = 0
|
||||
self.reset_time = 0 # Unix timestamp when quota resets
|
||||
|
||||
async def close(self):
|
||||
"""Close the HTTP client."""
|
||||
await self.client.aclose()
|
||||
|
||||
async def get_labels(self, address: str, chain: str) -> List[Label]:
|
||||
"""
|
||||
Retrieve labels for a blockchain address from MBAL.
|
||||
|
||||
Args:
|
||||
address: The blockchain address to label
|
||||
chain: The blockchain (ethereum, bitcoin, tron, etc.)
|
||||
|
||||
Returns:
|
||||
List of labels with metadata
|
||||
"""
|
||||
try:
|
||||
# MBAL requires different endpoints per chain
|
||||
labels = []
|
||||
|
||||
# Prepare query payload
|
||||
params = {
|
||||
"address": address,
|
||||
"chain": chain.lower(),
|
||||
}
|
||||
|
||||
# Rate limiting - max 100 requests per day for free
|
||||
await self._rate_limit()
|
||||
|
||||
headers = {}
|
||||
if self.api_key:
|
||||
headers["Authorization"] = f"Bearer {self.api_key}"
|
||||
|
||||
# Query MBAL's address endpoint
|
||||
response = await self.client.get(
|
||||
f"{self.base_url}/address",
|
||||
params=params,
|
||||
headers=headers
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
data = response.json()
|
||||
|
||||
# Process the result according to MBAL's schema
|
||||
if "entities" in data:
|
||||
for entity in data["entities"]:
|
||||
label_value = entity.get("category", "unknown")
|
||||
entity_type = entity.get("type", "address")
|
||||
confidence = entity.get("confidence", 0.8) # MBAL typically confident
|
||||
|
||||
# Map MBAL categories to our standardized label types
|
||||
label_type = self._map_category_to_type(label_value)
|
||||
|
||||
labels.append(Label(
|
||||
address=address,
|
||||
chain=chain,
|
||||
label_type=label_type,
|
||||
label_value=label_value,
|
||||
source=LabelSource.MBAL, # MBAL source
|
||||
verified_at=datetime.now(),
|
||||
confidence=confidence,
|
||||
attributes={
|
||||
"mbal_entity_id": entity.get("entityId"),
|
||||
"mbal_tags": entity.get("tags", []),
|
||||
"mbal_risk_score": entity.get("riskScore", 0),
|
||||
"mbal_cluster_id": entity.get("clusterId"),
|
||||
}
|
||||
))
|
||||
|
||||
if "behaviors" in data and len(labels) == 0:
|
||||
# If no main entities, fall back to behavioral labels
|
||||
for behavior in data["behaviors"]:
|
||||
label_value = behavior["type"]
|
||||
label_type = self._map_behavior_to_type(label_value)
|
||||
confidence = behavior.get("confidence", 0.7)
|
||||
|
||||
labels.append(Label(
|
||||
address=address,
|
||||
chain=chain,
|
||||
label_type=label_type,
|
||||
label_value=label_value,
|
||||
source=LabelSource.MBAL, # MBAL API source
|
||||
verified_at=datetime.now(),
|
||||
confidence=confidence,
|
||||
attributes={
|
||||
"behavior_description": behavior.get("description"),
|
||||
"behavior_risk_level": behavior.get("riskLevel", "medium"),
|
||||
}
|
||||
))
|
||||
|
||||
return labels
|
||||
|
||||
except httpx.HTTPStatusError as e:
|
||||
if e.response.status_code == 429:
|
||||
logger.warning(f"MBAL rate limit exceeded: {e}")
|
||||
return []
|
||||
logger.error(f"MBAL API error: {e} for {chain}:{address}")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error querying MBAL: {e}")
|
||||
return []
|
||||
|
||||
def _map_category_to_type(self, category: str) -> str:
|
||||
"""Map MBAL category strings to our standard label type strings."""
|
||||
category_lower = category.lower()
|
||||
|
||||
# Exchange mappings
|
||||
if any(exchange in category_lower for exchange in ["binance", "coinbase", "kraken", "kucoin",
|
||||
"huobi", "gate.io", "okx", "bybit", "gemini"]):
|
||||
return "exchange"
|
||||
|
||||
# Service mappings
|
||||
if any(service in category_lower for service in ["miner", "validator", "staking", "liquidity",
|
||||
"amm", "dex", "bridge", "cex", "dexe"]):
|
||||
return "service"
|
||||
|
||||
# Actor mappings
|
||||
if any(actor in category_lower for actor in ["blacklist", "scammer", "hacker", "tornado",
|
||||
"mixer", "launderer", "crime", "malicious"]):
|
||||
return "actor"
|
||||
|
||||
# Default to entity for unknown but potentially identifiable categories
|
||||
return "actor" if "blacklist" in category_lower else "entity"
|
||||
|
||||
def _map_behavior_to_type(self, behavior: str) -> str:
|
||||
"""Map MBAL behaviors to our standard label type strings."""
|
||||
behavior_lower = behavior.lower()
|
||||
|
||||
if any(risky in behavior_lower for risky in ["malware", "ransom", "exploit", "theft",
|
||||
"scam", "hacking", "malicious"]):
|
||||
return "actor"
|
||||
|
||||
if any(behavior in behavior_lower for behavior in ["mining", "block generation", "transaction fee", "gas"]):
|
||||
return "service"
|
||||
|
||||
return "entity" # Default
|
||||
|
||||
async def _rate_limit(self):
|
||||
"""Implement simple rate limiting based on MBAL tier."""
|
||||
now = asyncio.get_event_loop().time()
|
||||
|
||||
# Check daily quota if using free tier
|
||||
if not self.api_key and self.request_count >= 100:
|
||||
# Free tier: 100 queries/day
|
||||
# Simple approach: wait until reset (24 hours from first query)
|
||||
if now < self.reset_time:
|
||||
sleep_time = self.reset_time - now
|
||||
logger.info(f"Waiting for MBAL daily quota reset: {sleep_time:.1f}s")
|
||||
await asyncio.sleep(sleep_time)
|
||||
|
||||
# Ensure minimum delay between requests
|
||||
time_since_last = now - self.last_request_time
|
||||
min_delay = 1.0 # At least 1s between requests
|
||||
if time_since_last < min_delay:
|
||||
await asyncio.sleep(min_delay - time_since_last)
|
||||
|
||||
# Update counters
|
||||
self.last_request_time = now
|
||||
if now >= self.reset_time:
|
||||
self.request_count = 0
|
||||
self.reset_time = now + 86400 # Reset in 24 hours
|
||||
self.request_count += 1
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
"""
|
||||
Check if MBAL service is accessible.
|
||||
|
||||
Returns:
|
||||
True if MBAL API responds successfully
|
||||
"""
|
||||
try:
|
||||
response = await self.client.get(f"{self.base_url}/health")
|
||||
return response.status_code == 200
|
||||
except:
|
||||
return False
|
||||
|
||||
|
||||
# Global instance
|
||||
_mbalsy_instance: Optional[MBALService] = None
|
||||
|
||||
|
||||
def get_mbalsy_service() -> MBALService:
|
||||
"""Get the global MBAL service instance."""
|
||||
global _mbalsy_instance
|
||||
if _mbalsy_instance is None:
|
||||
import os
|
||||
api_key = os.getenv("MBAL_API_KEY", None)
|
||||
_mbalsy_instance = MBALService(api_key=api_key)
|
||||
return _mbalsy_instance
|
||||
168
app/domain/labels/sources/metasleuth.py
Normal file
168
app/domain/labels/sources/metasleuth.py
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
"""MetaSleuth (BlockSec AML) source — live API at aml.blocksec.com.
|
||||
|
||||
Endpoint: POST https://aml.blocksec.com/address-label/api/v3/labels
|
||||
Headers: API-KEY: <key>
|
||||
Body: {"chain_id": 1, "address": "0x..."}
|
||||
|
||||
Verified working: 2026-06-22 (Binance 0x3d12... returned main_entity=Binance,
|
||||
entity categories EXCHANGE + OTC DESK).
|
||||
|
||||
Limit: 10 calls/day per API key on the free tier. So we cache aggressively.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import httpx
|
||||
|
||||
from app.domain.labels.models import Label, LabelSource
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
METASLEUTH_BASE = "https://aml.blocksec.com/address-label/api/v3"
|
||||
# Free tier: 10 calls/day per key on risk-level, similar on labels
|
||||
DEFAULT_TIMEOUT = 8.0
|
||||
|
||||
# Chain name -> MetaSleuth chain_id
|
||||
_CHAIN_MAP = {
|
||||
"ethereum": 1,
|
||||
"bitcoin": -1,
|
||||
"tron": -2,
|
||||
"solana": -3,
|
||||
"monero": -4,
|
||||
"sui": 5,
|
||||
"optimism": 10,
|
||||
"cronos": 25,
|
||||
"bsc": 56,
|
||||
"base": 8453,
|
||||
"arbitrum": 42161,
|
||||
"polygon": 137,
|
||||
}
|
||||
|
||||
|
||||
async def query_metasleuth(address: str, chain: str = "ethereum") -> list[Label]:
|
||||
"""Query MetaSleuth for an address's entity info.
|
||||
|
||||
Returns:
|
||||
- 1 entity label with the main entity name (e.g. "Binance")
|
||||
- Optional category labels (EXCHANGE, OTC DESK, etc.)
|
||||
- Attributes (DEPOSIT ADDRESS, etc.)
|
||||
"""
|
||||
api_key = os.getenv("METASLEUTH_LABELS_API_KEY", "")
|
||||
if not api_key:
|
||||
# Try gopass via the standard pattern
|
||||
try:
|
||||
import subprocess
|
||||
|
||||
result = subprocess.run(
|
||||
["gopass", "show", "-o", "rmi/infra/metasleuth_labels_api_key"],
|
||||
capture_output=True, text=True, timeout=2,
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
api_key = result.stdout.strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not api_key:
|
||||
log.debug("metasleuth_no_api_key configured, skipping")
|
||||
return []
|
||||
|
||||
chain_id = _CHAIN_MAP.get(chain.lower())
|
||||
if chain_id is None:
|
||||
log.debug("metasleuth_unsupported_chain chain=%s", chain)
|
||||
return []
|
||||
|
||||
def _call_api() -> dict:
|
||||
"""Sync HTTP call (run in thread)."""
|
||||
try:
|
||||
r = httpx.post(
|
||||
f"{METASLEUTH_BASE}/labels",
|
||||
headers={"API-KEY": api_key, "Content-Type": "application/json"},
|
||||
json={"chain_id": chain_id, "address": address},
|
||||
timeout=DEFAULT_TIMEOUT,
|
||||
)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
if data.get("code") == 200000:
|
||||
return data.get("data") or {}
|
||||
log.warning("metasleuth_api_err code=%s msg=%s", data.get("code"), data.get("message"))
|
||||
return {}
|
||||
except httpx.HTTPError as e:
|
||||
log.warning("metasleuth_api_http_err: %s", e)
|
||||
return {}
|
||||
except Exception as e:
|
||||
log.warning("metasleuth_api_err: %s", e)
|
||||
return {}
|
||||
|
||||
data = await asyncio.to_thread(_call_api)
|
||||
if not data:
|
||||
return []
|
||||
|
||||
labels: list[Label] = []
|
||||
main_entity = data.get("main_entity")
|
||||
if main_entity:
|
||||
labels.append(Label(
|
||||
address=address.lower(),
|
||||
chain=chain,
|
||||
label_type="entity",
|
||||
label_value=main_entity,
|
||||
source=LabelSource.METASLEUTH,
|
||||
confidence=0.95,
|
||||
verified_at=datetime.now(UTC),
|
||||
entity=main_entity,
|
||||
entity_category=_first_category(data),
|
||||
attributes=_extract_attributes(data),
|
||||
))
|
||||
|
||||
# Also emit labels for each category
|
||||
categories = data.get("main_entity_info", {}).get("categories", [])
|
||||
if not isinstance(categories, list):
|
||||
categories = []
|
||||
for cat in categories:
|
||||
if not isinstance(cat, dict):
|
||||
continue
|
||||
cat_name = cat.get("name")
|
||||
if cat_name and cat_name != main_entity:
|
||||
labels.append(Label(
|
||||
address=address.lower(),
|
||||
chain=chain,
|
||||
label_type="category",
|
||||
label_value=cat_name,
|
||||
source=LabelSource.METASLEUTH,
|
||||
confidence=0.9,
|
||||
verified_at=datetime.now(UTC),
|
||||
entity=main_entity,
|
||||
entity_category=cat_name,
|
||||
attributes={"code": cat.get("code")},
|
||||
))
|
||||
|
||||
return labels
|
||||
|
||||
|
||||
def _first_category(data: dict) -> str | None:
|
||||
"""Extract first category name from MetaSleuth response."""
|
||||
cats = data.get("main_entity_info", {}).get("categories", [])
|
||||
if cats and isinstance(cats[0], dict):
|
||||
return cats[0].get("name")
|
||||
return None
|
||||
|
||||
|
||||
def _extract_attributes(data: dict) -> dict:
|
||||
"""Extract attributes from MetaSleuth response."""
|
||||
attrs = data.get("main_entity_info", {}).get("attributes", [])
|
||||
out = {}
|
||||
for a in attrs:
|
||||
if isinstance(a, dict):
|
||||
name = a.get("name", "")
|
||||
if name:
|
||||
out[name] = a.get("code")
|
||||
# Add description (website/twitter) if present
|
||||
desc = data.get("main_entity_info", {}).get("description", {})
|
||||
if isinstance(desc, dict):
|
||||
for k, v in desc.items():
|
||||
if v:
|
||||
out[k] = v
|
||||
return out
|
||||
22
app/domain/labels/sources/open_labels.py
Normal file
22
app/domain/labels/sources/open_labels.py
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
"""Open Labels Initiative (OLI) source — placeholder.
|
||||
|
||||
TODO: wire up when OLI endpoint is confirmed. GitHub repo:
|
||||
https://github.com/openlabelsinitiative/OLI
|
||||
|
||||
For now returns []. Stub.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from app.domain.labels.models import Label
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def query_open_labels(address: str, chain: str = "ethereum") -> list[Label]:
|
||||
"""Query OLI for an address's labels.
|
||||
|
||||
Returns [] until OLI endpoint is confirmed and we have API access.
|
||||
"""
|
||||
return []
|
||||
74
app/domain/labels/sources/solana_labels_csv.py
Normal file
74
app/domain/labels/sources/solana_labels_csv.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
"""Solana wallet labels CSV source — 103K SOL labels via DuckDB."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from app.domain.labels.models import Label, LabelSource
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
CSV_PATH = "/home/dev/rmi/backend/data/wallet-labels-clean/wallet_labels_solana.csv"
|
||||
|
||||
|
||||
async def query_solana_labels_csv(address: str, chain: str = "solana") -> list[Label]:
|
||||
"""Query the Solana wallet labels CSV via DuckDB.
|
||||
|
||||
Schema (likely): address,source,name,entity_type,threat_group,risk_level
|
||||
|
||||
Only matches if chain == "solana" — returns [] for EVM chains.
|
||||
"""
|
||||
if chain != "solana":
|
||||
return []
|
||||
|
||||
def _query() -> list[Label]:
|
||||
from app.core.duckdb_analytics import DuckDBAnalytics
|
||||
|
||||
try:
|
||||
d = DuckDBAnalytics()
|
||||
try:
|
||||
rows = d.query_parquet(
|
||||
CSV_PATH,
|
||||
"""
|
||||
SELECT *
|
||||
FROM data
|
||||
WHERE LOWER(address) = ?
|
||||
LIMIT 20
|
||||
""",
|
||||
[address.lower()],
|
||||
)
|
||||
labels = []
|
||||
for row in rows:
|
||||
# Field names may differ — try common names
|
||||
label_value = (
|
||||
row.get("name")
|
||||
or row.get("label")
|
||||
or row.get("entity_type")
|
||||
or row.get("source")
|
||||
)
|
||||
if not label_value:
|
||||
continue
|
||||
labels.append(
|
||||
Label(
|
||||
address=row.get("address", address).lower(),
|
||||
chain="solana",
|
||||
label_type="entity",
|
||||
label_value=label_value,
|
||||
source=LabelSource.SOLANA_LABELS_CSV,
|
||||
confidence=0.8,
|
||||
verified_at=datetime.now(UTC),
|
||||
attributes={
|
||||
k: v for k, v in row.items()
|
||||
if k not in ("address", "name", "label") and v
|
||||
},
|
||||
)
|
||||
)
|
||||
return labels
|
||||
finally:
|
||||
d.close()
|
||||
except Exception as e:
|
||||
log.warning("solana_labels_csv_query_fail address=%s err=%s", address[:10], e)
|
||||
raise
|
||||
|
||||
return await asyncio.to_thread(_query)
|
||||
Loading…
Add table
Add a link
Reference in a new issue