merge: chore/cleanup-remove-bloat-and-secrets into main
This commit is contained in:
commit
bde2f3a97d
1173 changed files with 437609 additions and 0 deletions
1
app/domain/__init__.py
Normal file
1
app/domain/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
"""domain package — HTTP layer per v4.0 (thin routes, no business logic)."""
|
||||
78
app/domain/alerts/__init__.py
Normal file
78
app/domain/alerts/__init__.py
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
"""Alerts domain — public API + health check registration."""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.core import health as health_mod
|
||||
from app.core.health import DomainHealth
|
||||
|
||||
|
||||
def _read_env_file() -> dict[str, str]:
|
||||
"""Read env from /proc/self/environ (process env, not the mutable os.environ)."""
|
||||
env: dict[str, str] = {}
|
||||
try:
|
||||
with open("/proc/self/environ", "rb") as f:
|
||||
for chunk in f.read().split(b"\x00"):
|
||||
if b"=" in chunk:
|
||||
k, _, v = chunk.partition(b"=")
|
||||
env[k.decode("utf-8", "replace")] = v.decode("utf-8", "replace")
|
||||
except Exception:
|
||||
pass
|
||||
return env
|
||||
|
||||
|
||||
async def _health_check() -> DomainHealth:
|
||||
"""Alerts health: Redis reachable via process env (not mutable os.environ)."""
|
||||
import redis as redis_lib
|
||||
|
||||
env = _read_env_file()
|
||||
host = env.get("REDIS_HOST", "rmi-redis")
|
||||
port = int(env.get("REDIS_PORT", "6379"))
|
||||
password = env.get("REDIS_PASSWORD", "") or None
|
||||
|
||||
try:
|
||||
client = redis_lib.Redis(
|
||||
host=host,
|
||||
port=port,
|
||||
password=password,
|
||||
db=int(env.get("REDIS_DB", "0")),
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=2,
|
||||
socket_timeout=2,
|
||||
)
|
||||
client.ping()
|
||||
return DomainHealth(
|
||||
name="alerts",
|
||||
healthy=True,
|
||||
details={"redis": "ok", "host": host, "port": port},
|
||||
)
|
||||
except Exception as e:
|
||||
return DomainHealth(
|
||||
name="alerts",
|
||||
healthy=False,
|
||||
details={"host": host, "port": port},
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
|
||||
health_mod.register_health_check("alerts", _health_check)
|
||||
|
||||
|
||||
# Public API
|
||||
from app.domain.alerts.broadcaster import AlertBroadcaster
|
||||
from app.domain.alerts.models import (
|
||||
AlertEvent,
|
||||
AlertSubscription,
|
||||
AlertType,
|
||||
CreateAlertRequest,
|
||||
)
|
||||
from app.domain.alerts.repository import AlertRepository
|
||||
from app.domain.alerts.service import AlertService
|
||||
|
||||
__all__ = [
|
||||
"AlertBroadcaster",
|
||||
"AlertEvent",
|
||||
"AlertRepository",
|
||||
"AlertService",
|
||||
"AlertSubscription",
|
||||
"AlertType",
|
||||
"CreateAlertRequest",
|
||||
]
|
||||
27
app/domain/alerts/broadcaster.py
Normal file
27
app/domain/alerts/broadcaster.py
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
"""Broadcaster — pushes fired alert events to WebSocket subscribers.
|
||||
|
||||
Uses core/websocket's broadcast helper. No FastAPI imports.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.core.websocket import broadcast_alert
|
||||
from app.domain.alerts.models import AlertEvent
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class AlertBroadcaster:
|
||||
"""Thin wrapper that knows how to push an AlertEvent to live clients."""
|
||||
|
||||
async def broadcast_event(self, event: AlertEvent) -> None:
|
||||
"""Broadcast a single event to all connected WebSocket subscribers."""
|
||||
try:
|
||||
await broadcast_alert(event.model_dump(mode="json"))
|
||||
except Exception as e:
|
||||
# Broadcasting failure must not break the firing path.
|
||||
log.warning(
|
||||
"alert_broadcast_failed",
|
||||
error=str(e),
|
||||
alert_id=event.subscription_id,
|
||||
)
|
||||
71
app/domain/alerts/models.py
Normal file
71
app/domain/alerts/models.py
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
"""Pydantic v2 models for the alerts domain.
|
||||
|
||||
No FastAPI imports. Pure data shapes.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
|
||||
class AlertType(StrEnum):
|
||||
"""Valid alert trigger types. Matches the legacy alert_types enum."""
|
||||
|
||||
LIQUIDITY_REMOVE = "liquidity_remove"
|
||||
MINT = "mint"
|
||||
BLACKLIST = "blacklist"
|
||||
HONEYPOT = "honeypot"
|
||||
RUG_PULL = "rug_pull"
|
||||
WHALE_MOVEMENT = "whale_movement"
|
||||
DEPLOYER_SELL = "deployer_sell"
|
||||
|
||||
@classmethod
|
||||
def values(cls) -> list[str]:
|
||||
return [m.value for m in cls]
|
||||
|
||||
|
||||
class AlertSubscription(BaseModel):
|
||||
"""A token alert subscription record (what we store in Redis)."""
|
||||
|
||||
model_config = ConfigDict(str_strip_whitespace=True)
|
||||
|
||||
id: str = Field(..., description="Unique subscription id, e.g. alert:1700000000")
|
||||
token_address: str = Field(..., min_length=1, max_length=256)
|
||||
alert_types: list[AlertType] = Field(default_factory=lambda: [AlertType.LIQUIDITY_REMOVE, AlertType.MINT, AlertType.BLACKLIST])
|
||||
webhook_url: str | None = Field(default=None, max_length=2048)
|
||||
created_at: datetime = Field(default_factory=datetime.utcnow)
|
||||
active: bool = True
|
||||
owner_id: str | None = Field(default=None, description="User id of subscription owner, None = anonymous")
|
||||
|
||||
@field_validator("alert_types")
|
||||
@classmethod
|
||||
def _at_least_one_type(cls, v: list[AlertType]) -> list[AlertType]:
|
||||
if not v:
|
||||
raise ValueError("at least one alert_type required")
|
||||
return v
|
||||
|
||||
|
||||
class CreateAlertRequest(BaseModel):
|
||||
"""What the API accepts to create a subscription."""
|
||||
|
||||
model_config = ConfigDict(str_strip_whitespace=True)
|
||||
|
||||
token_address: str = Field(..., min_length=1, max_length=256)
|
||||
alert_types: list[AlertType] | None = None
|
||||
webhook_url: str | None = Field(default=None, max_length=2048)
|
||||
|
||||
|
||||
class AlertEvent(BaseModel):
|
||||
"""A fired alert, ready to broadcast to subscribers + WebSocket clients."""
|
||||
|
||||
model_config = ConfigDict(use_enum_values=True)
|
||||
|
||||
subscription_id: str
|
||||
token_address: str
|
||||
event_type: AlertType
|
||||
payload: dict[str, Any] = Field(default_factory=dict)
|
||||
fired_at: datetime = Field(default_factory=datetime.utcnow)
|
||||
severity: str = Field(default="info", description="info | warning | critical")
|
||||
60
app/domain/alerts/repository.py
Normal file
60
app/domain/alerts/repository.py
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
"""Repository — async Redis storage for alert subscriptions.
|
||||
|
||||
Storage layout (matches legacy for cutover):
|
||||
Hash key: "rmi:alerts"
|
||||
Field: alert id (e.g. "alert:1700000000")
|
||||
Value: JSON-encoded AlertSubscription
|
||||
|
||||
This is a thin async wrapper. No business logic. Pure data access.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.core.redis import get_redis_async
|
||||
from app.domain.alerts.models import AlertSubscription
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
HASH_KEY = "rmi:alerts"
|
||||
|
||||
|
||||
class AlertRepository:
|
||||
"""Async Redis-backed subscription storage."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._key = HASH_KEY
|
||||
|
||||
async def list_all(self) -> list[AlertSubscription]:
|
||||
r = get_redis_async()
|
||||
raw: dict[str, str] = await r.hgetall(self._key) or {}
|
||||
out: list[AlertSubscription] = []
|
||||
for value in raw.values():
|
||||
try:
|
||||
out.append(AlertSubscription.model_validate_json(value))
|
||||
except Exception as e:
|
||||
log.warning("alert_repo_corrupt_record", error=str(e))
|
||||
return out
|
||||
|
||||
async def get(self, alert_id: str) -> AlertSubscription | None:
|
||||
r = get_redis_async()
|
||||
raw: str | None = await r.hget(self._key, alert_id)
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
return AlertSubscription.model_validate_json(raw)
|
||||
except Exception as e:
|
||||
log.warning("alert_repo_corrupt_record", alert_id=alert_id, error=str(e))
|
||||
return None
|
||||
|
||||
async def create(self, sub: AlertSubscription) -> None:
|
||||
r = get_redis_async()
|
||||
await r.hset(self._key, sub.id, sub.model_dump_json())
|
||||
|
||||
async def delete(self, alert_id: str) -> bool:
|
||||
r = get_redis_async()
|
||||
removed: int = await r.hdel(self._key, alert_id)
|
||||
return removed > 0
|
||||
|
||||
async def list_by_token(self, token_address: str) -> list[AlertSubscription]:
|
||||
all_subs = await self.list_all()
|
||||
return [s for s in all_subs if s.token_address == token_address]
|
||||
133
app/domain/alerts/service.py
Normal file
133
app/domain/alerts/service.py
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
"""Service — business logic for alert subscriptions and event firing.
|
||||
|
||||
This is the only thing the api/ layer should call. It composes
|
||||
the repository (storage) and the broadcaster (WebSocket push).
|
||||
No FastAPI. No HTTP. Pure Python.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.domain.alerts.broadcaster import AlertBroadcaster
|
||||
from app.domain.alerts.models import (
|
||||
AlertEvent,
|
||||
AlertSubscription,
|
||||
AlertType,
|
||||
CreateAlertRequest,
|
||||
)
|
||||
from app.domain.alerts.repository import AlertRepository
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class AlertService:
|
||||
"""Orchestrates subscription CRUD and event firing."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
repo: AlertRepository | None = None,
|
||||
broadcaster: AlertBroadcaster | None = None,
|
||||
) -> None:
|
||||
self._repo = repo or AlertRepository()
|
||||
self._broadcaster = broadcaster or AlertBroadcaster()
|
||||
|
||||
async def create_subscription(
|
||||
self,
|
||||
req: CreateAlertRequest,
|
||||
owner_id: str | None = None,
|
||||
) -> AlertSubscription:
|
||||
"""Create a new subscription, persist it, return the saved record."""
|
||||
# Use provided types, or the model default if not given.
|
||||
types = req.alert_types or [
|
||||
AlertType.LIQUIDITY_REMOVE,
|
||||
AlertType.MINT,
|
||||
AlertType.BLACKLIST,
|
||||
]
|
||||
sub = AlertSubscription(
|
||||
id=f"alert:{int(datetime.utcnow().timestamp())}",
|
||||
token_address=req.token_address,
|
||||
alert_types=types,
|
||||
webhook_url=req.webhook_url,
|
||||
owner_id=owner_id,
|
||||
)
|
||||
await self._repo.create(sub)
|
||||
log.info(
|
||||
"alert_subscription_created",
|
||||
alert_id=sub.id,
|
||||
token=sub.token_address,
|
||||
types=[t.value for t in sub.alert_types],
|
||||
owner=owner_id,
|
||||
)
|
||||
return sub
|
||||
|
||||
async def list_subscriptions(self, owner_id: str | None = None) -> list[AlertSubscription]:
|
||||
"""List subscriptions. If owner_id given, filter to that user."""
|
||||
all_subs = await self._repo.list_all()
|
||||
if owner_id is None:
|
||||
return all_subs
|
||||
return [s for s in all_subs if s.owner_id == owner_id]
|
||||
|
||||
async def cancel_subscription(self, alert_id: str, owner_id: str | None = None) -> bool:
|
||||
"""Cancel (delete) a subscription. Returns True if removed."""
|
||||
sub = await self._repo.get(alert_id)
|
||||
if sub is None:
|
||||
return False
|
||||
if owner_id is not None and sub.owner_id != owner_id:
|
||||
log.warning(
|
||||
"alert_cancel_unauthorized",
|
||||
alert_id=alert_id,
|
||||
owner=owner_id,
|
||||
actual_owner=sub.owner_id,
|
||||
)
|
||||
return False
|
||||
return await self._repo.delete(alert_id)
|
||||
|
||||
async def fire_event(
|
||||
self,
|
||||
token_address: str,
|
||||
event_type: AlertType,
|
||||
payload: dict | None = None,
|
||||
) -> list[AlertEvent]:
|
||||
"""Fire an event for a token. Returns the events that were broadcast.
|
||||
|
||||
Looks up all active subscriptions matching this token + event_type,
|
||||
creates an AlertEvent for each, and broadcasts via the broadcaster.
|
||||
"""
|
||||
subs = await self._repo.list_by_token(token_address)
|
||||
matching = [
|
||||
s for s in subs
|
||||
if s.active and (event_type.value in [t.value for t in s.alert_types])
|
||||
]
|
||||
if not matching:
|
||||
return []
|
||||
|
||||
events: list[AlertEvent] = []
|
||||
for sub in matching:
|
||||
ev = AlertEvent(
|
||||
subscription_id=sub.id,
|
||||
token_address=token_address,
|
||||
event_type=event_type,
|
||||
payload=payload or {},
|
||||
severity=_severity_for(event_type),
|
||||
)
|
||||
events.append(ev)
|
||||
await self._broadcaster.broadcast_event(ev)
|
||||
log.info(
|
||||
"alert_events_fired",
|
||||
token=token_address,
|
||||
event_type=event_type.value,
|
||||
count=len(events),
|
||||
)
|
||||
return events
|
||||
|
||||
|
||||
def _severity_for(event_type: AlertType) -> str:
|
||||
"""Map event type → severity (used by clients to choose alert UI)."""
|
||||
critical = {AlertType.RUG_PULL, AlertType.HONEYPOT, AlertType.BLACKLIST}
|
||||
warning = {AlertType.LIQUIDITY_REMOVE, AlertType.WHALE_MOVEMENT, AlertType.DEPLOYER_SELL}
|
||||
if event_type in critical:
|
||||
return "critical"
|
||||
if event_type in warning:
|
||||
return "warning"
|
||||
return "info"
|
||||
34
app/domain/labels/__init__.py
Normal file
34
app/domain/labels/__init__.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
"""RMI v5 §T11 — Federated wallet labels domain.
|
||||
|
||||
Aggregates wallet labels from 6 sources in parallel:
|
||||
1. eth-labels.db — 115K EVM labels (local SQLite)
|
||||
2. Etherscan CSV — 51K EVM labels (local CSV via DuckDB)
|
||||
3. Ethereum labels CSV — 51K EVM labels (local CSV via DuckDB)
|
||||
4. Solana labels CSV — 103K SOL labels (local CSV via DuckDB)
|
||||
5. ClickHouse — wallet_memory.wallet_labels (live API)
|
||||
6. MetaSleuth — live API at aml.blocksec.com (works)
|
||||
|
||||
Each source has its own adapter in app/domain/labels/sources/.
|
||||
The FederatedLabelAPI orchestrator queries all sources concurrently
|
||||
with asyncio.gather(return_exceptions=True), deduplicates by
|
||||
(address, chain, label_type), and tags each label with provenance.
|
||||
|
||||
Graceful degradation: if a source is unreachable, we log + skip it
|
||||
and return labels from whatever did work.
|
||||
|
||||
Per RMIV5 v4.0 §T28 (P1): biggest moat — 5K internal labels → 200K+
|
||||
via federation.
|
||||
"""
|
||||
|
||||
from app.domain.labels.federated import (
|
||||
FederatedLabelAPI,
|
||||
get_federated_api,
|
||||
)
|
||||
from app.domain.labels.models import Label, LabelSource
|
||||
|
||||
__all__ = [
|
||||
"FederatedLabelAPI",
|
||||
"Label",
|
||||
"LabelSource",
|
||||
"get_federated_api",
|
||||
]
|
||||
296
app/domain/labels/federated.py
Normal file
296
app/domain/labels/federated.py
Normal file
|
|
@ -0,0 +1,296 @@
|
|||
"""Federated label API — orchestrates all 6 sources.
|
||||
|
||||
Per RMIV5 §T11: queries every configured source in parallel via
|
||||
asyncio.gather(return_exceptions=True). Failed sources are logged
|
||||
but don't fail the whole query — graceful degradation.
|
||||
|
||||
Deduplication: labels from different sources for the same
|
||||
(address, chain, label_type, label_value) get merged, keeping the
|
||||
highest-confidence version and tagging all contributing sources.
|
||||
|
||||
Caching: Redis-backed cache with 1h TTL keyed by (address, chain).
|
||||
On cache hit, skip source queries entirely.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from app.domain.labels.models import Label, LabelSource
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Default sources to query. Subclasses can override.
|
||||
DEFAULT_SOURCES: list[LabelSource] = [
|
||||
LabelSource.ETH_LABELS_DB,
|
||||
LabelSource.ETHERSCAN_CSV,
|
||||
LabelSource.ETHEREUM_LABELS_CSV,
|
||||
LabelSource.SOLANA_LABELS_CSV,
|
||||
LabelSource.CLICKHOUSE,
|
||||
LabelSource.METASLEUTH,
|
||||
LabelSource.MBAL,
|
||||
]
|
||||
|
||||
|
||||
class FederatedLabelAPI:
|
||||
"""Orchestrates wallet label queries across all configured sources.
|
||||
|
||||
Usage:
|
||||
api = FederatedLabelAPI()
|
||||
labels = await api.get_labels("0x...", "ethereum")
|
||||
# Returns list[Label] aggregated from all sources
|
||||
|
||||
sources_checked = api.sources_queried_last_call # for observability
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sources: list[LabelSource] | None = None,
|
||||
cache_ttl_seconds: int = 3600,
|
||||
per_source_timeout: float = 5.0,
|
||||
max_concurrent: int = 6,
|
||||
) -> None:
|
||||
self._sources = sources or DEFAULT_SOURCES
|
||||
self._cache_ttl = cache_ttl_seconds
|
||||
self._per_source_timeout = per_source_timeout
|
||||
self._semaphore = asyncio.Semaphore(max_concurrent)
|
||||
self._sources_queried_last_call: list[str] = []
|
||||
|
||||
async def get_labels(
|
||||
self,
|
||||
address: str,
|
||||
chain: str = "ethereum",
|
||||
*,
|
||||
use_cache: bool = True,
|
||||
include_low_confidence: bool = False,
|
||||
) -> list[Label]:
|
||||
"""Get all labels for an address from all sources.
|
||||
|
||||
Args:
|
||||
address: Wallet address (0x... for EVM, base58 for Solana, etc.)
|
||||
chain: Chain name (ethereum, solana, base, bitcoin, etc.)
|
||||
use_cache: If True, check Redis cache first (default True)
|
||||
include_low_confidence: If False (default), filter labels with
|
||||
confidence < 0.3 to reduce noise
|
||||
|
||||
Returns:
|
||||
Deduplicated list of labels, sorted by confidence desc.
|
||||
"""
|
||||
# Cache check
|
||||
if use_cache:
|
||||
cached = await self._cache_get(address, chain)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
# Query all sources in parallel
|
||||
tasks = [
|
||||
self._query_source_safe(source, address, chain)
|
||||
for source in self._sources
|
||||
]
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
# Collect labels + track which sources worked
|
||||
all_labels: list[Label] = []
|
||||
sources_queried: list[str] = []
|
||||
for source, result in zip(self._sources, results, strict=False):
|
||||
if isinstance(result, Exception):
|
||||
log.warning(
|
||||
"federated_source_fail source=%s err=%s: %s",
|
||||
source.value, type(result).__name__, result,
|
||||
)
|
||||
continue
|
||||
if result:
|
||||
sources_queried.append(source.value)
|
||||
all_labels.extend(result)
|
||||
|
||||
# Deduplicate (merge same label from multiple sources)
|
||||
deduped = self._dedupe(all_labels)
|
||||
|
||||
# Filter low confidence unless requested
|
||||
if not include_low_confidence:
|
||||
deduped = [line for line in deduped if line.confidence >= 0.3]
|
||||
|
||||
# Sort by confidence desc
|
||||
deduped.sort(key=lambda line: -line.confidence)
|
||||
|
||||
# Track for observability
|
||||
self._sources_queried_last_call = sources_queried
|
||||
|
||||
# Cache
|
||||
if use_cache and deduped:
|
||||
await self._cache_set(address, chain, deduped)
|
||||
|
||||
log.info(
|
||||
"federated_labels_query address=%s chain=%s sources=%d labels=%d",
|
||||
address[:10] + "..." if len(address) > 10 else address,
|
||||
chain,
|
||||
len(sources_queried),
|
||||
len(deduped),
|
||||
)
|
||||
return deduped
|
||||
|
||||
async def get_labels_dict(
|
||||
self,
|
||||
address: str,
|
||||
chain: str = "ethereum",
|
||||
**kwargs: Any,
|
||||
) -> dict[str, Any]:
|
||||
"""Get labels as a dict (for JSON API responses).
|
||||
|
||||
Returns:
|
||||
{
|
||||
"address": "0x...",
|
||||
"chain": "ethereum",
|
||||
"labels": [...], # list of label dicts
|
||||
"sources_queried": [...], # list of source names that worked
|
||||
"total": N,
|
||||
"verified_at": "2026-06-22T..."
|
||||
}
|
||||
"""
|
||||
labels = await self.get_labels(address, chain, **kwargs)
|
||||
return {
|
||||
"address": address,
|
||||
"chain": chain,
|
||||
"labels": [line.to_dict() for line in labels],
|
||||
"sources_queried": self._sources_queried_last_call,
|
||||
"total": len(labels),
|
||||
}
|
||||
|
||||
async def _query_source_safe(
|
||||
self,
|
||||
source: LabelSource,
|
||||
address: str,
|
||||
chain: str,
|
||||
) -> list[Label]:
|
||||
"""Query a single source with timeout + semaphore + exception handling."""
|
||||
async with self._semaphore:
|
||||
try:
|
||||
return await asyncio.wait_for(
|
||||
self._query_source(source, address, chain),
|
||||
timeout=self._per_source_timeout,
|
||||
)
|
||||
except TimeoutError:
|
||||
log.warning("federated_source_timeout source=%s address=%s", source.value, address[:10])
|
||||
return []
|
||||
except Exception as e:
|
||||
log.warning(
|
||||
"federated_source_error source=%s err=%s: %s",
|
||||
source.value, type(e).__name__, e,
|
||||
)
|
||||
return []
|
||||
|
||||
async def _query_source(
|
||||
self,
|
||||
source: LabelSource,
|
||||
address: str,
|
||||
chain: str,
|
||||
) -> list[Label]:
|
||||
"""Dispatch to the correct source adapter. Each adapter handles its own logic."""
|
||||
if source == LabelSource.ETH_LABELS_DB:
|
||||
from app.domain.labels.sources.eth_labels_db import query_eth_labels_db
|
||||
return await query_eth_labels_db(address, chain)
|
||||
if source == LabelSource.ETHERSCAN_CSV:
|
||||
from app.domain.labels.sources.etherscan_csv import query_etherscan_csv
|
||||
return await query_etherscan_csv(address, chain)
|
||||
if source == LabelSource.ETHEREUM_LABELS_CSV:
|
||||
from app.domain.labels.sources.ethereum_labels_csv import query_ethereum_labels_csv
|
||||
return await query_ethereum_labels_csv(address, chain)
|
||||
if source == LabelSource.SOLANA_LABELS_CSV:
|
||||
from app.domain.labels.sources.solana_labels_csv import query_solana_labels_csv
|
||||
return await query_solana_labels_csv(address, chain)
|
||||
if source == LabelSource.CLICKHOUSE:
|
||||
from app.domain.labels.sources.clickhouse_labels import query_clickhouse_labels
|
||||
return await query_clickhouse_labels(address, chain)
|
||||
if source == LabelSource.METASLEUTH:
|
||||
from app.domain.labels.sources.metasleuth import query_metasleuth
|
||||
return await query_metasleuth(address, chain)
|
||||
if source == LabelSource.OPEN_LABELS:
|
||||
from app.domain.labels.sources.open_labels import query_open_labels
|
||||
return await query_open_labels(address, chain)
|
||||
if source == LabelSource.CHAINBASE:
|
||||
from app.domain.labels.sources.chainbase import query_chainbase
|
||||
return await query_chainbase(address, chain)
|
||||
if source == LabelSource.INTERNAL:
|
||||
from app.domain.labels.sources.internal_labels import query_internal
|
||||
return await query_internal(address, chain)
|
||||
if source == LabelSource.MBAL:
|
||||
from app.domain.labels.sources.mbal import query_mbal
|
||||
return await query_mbal(address, chain)
|
||||
log.warning("federated_source_unknown source=%s", source.value)
|
||||
return []
|
||||
|
||||
def _dedupe(self, labels: list[Label]) -> list[Label]:
|
||||
"""Merge labels with same dedup_key into one (keeping highest confidence)."""
|
||||
by_key: dict[tuple, Label] = {}
|
||||
for label in labels:
|
||||
key = Label.dedupe_key(label.address, label.chain, label.label_type, label.label_value)
|
||||
if key not in by_key:
|
||||
by_key[key] = label
|
||||
else:
|
||||
by_key[key] = Label.merge([by_key[key], label])
|
||||
return list(by_key.values())
|
||||
|
||||
async def _cache_get(self, address: str, chain: str) -> list[Label] | None:
|
||||
"""Get cached labels from Redis. Returns None on miss/error."""
|
||||
try:
|
||||
import json as _json
|
||||
|
||||
from app.core.redis import get_redis
|
||||
|
||||
r = get_redis()
|
||||
key = self._cache_key(address, chain)
|
||||
raw = r.get(key)
|
||||
if not raw:
|
||||
return None
|
||||
data = _json.loads(raw)
|
||||
# Reconstruct Label objects (don't restore full provenance)
|
||||
return [
|
||||
Label(
|
||||
address=line["address"],
|
||||
chain=line["chain"],
|
||||
label_type=line["label_type"],
|
||||
label_value=line["label_value"],
|
||||
source=LabelSource(line["source"]) if line["source"] in [s.value for s in LabelSource] else LabelSource.INTERNAL,
|
||||
confidence=line["confidence"],
|
||||
verified_at=line["verified_at"],
|
||||
attributes=line.get("attributes", {}),
|
||||
entity=line.get("entity"),
|
||||
entity_category=line.get("entity_category"),
|
||||
)
|
||||
for line in data
|
||||
]
|
||||
except Exception as e:
|
||||
log.debug("federated_cache_get_fail: %s", e)
|
||||
return None
|
||||
|
||||
async def _cache_set(self, address: str, chain: str, labels: list[Label]) -> None:
|
||||
"""Cache labels to Redis with TTL."""
|
||||
try:
|
||||
import json as _json
|
||||
|
||||
from app.core.redis import get_redis
|
||||
|
||||
r = get_redis()
|
||||
key = self._cache_key(address, chain)
|
||||
data = [line.to_dict() for line in labels]
|
||||
r.setex(key, self._cache_ttl, _json.dumps(data))
|
||||
except Exception as e:
|
||||
log.debug("federated_cache_set_fail: %s", e)
|
||||
|
||||
def _cache_key(self, address: str, chain: str) -> str:
|
||||
"""Redis key for cached labels. Includes a hash of the address (privacy)."""
|
||||
from app.core.rate_limiter import _hash_identifier
|
||||
return f"labels:{chain}:{_hash_identifier(address.lower())}"
|
||||
|
||||
|
||||
_default_instance: FederatedLabelAPI | None = None
|
||||
|
||||
|
||||
def get_federated_api() -> FederatedLabelAPI:
|
||||
"""Get a process-wide FederatedLabelAPI instance."""
|
||||
global _default_instance
|
||||
if _default_instance is None:
|
||||
_default_instance = FederatedLabelAPI()
|
||||
return _default_instance
|
||||
98
app/domain/labels/models.py
Normal file
98
app/domain/labels/models.py
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
"""Label data models for the federated label API.
|
||||
|
||||
A Label represents one piece of information about an address from one
|
||||
source. Multiple labels from different sources for the same address
|
||||
get merged into a unified view.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
|
||||
|
||||
class LabelSource(StrEnum):
|
||||
"""Which source a label came from. Used for provenance + dedup."""
|
||||
|
||||
ETH_LABELS_DB = "eth-labels-db" # local SQLite, 115K EVM labels
|
||||
ETHERSCAN_CSV = "etherscan-combined" # local CSV, 51K EVM labels
|
||||
ETHEREUM_LABELS_CSV = "ethereum-labels" # local CSV, 51K EVM labels
|
||||
SOLANA_LABELS_CSV = "solana-labels" # local CSV, 103K SOL labels
|
||||
CLICKHOUSE = "clickhouse-wallet-memory" # live CH wallet_labels table
|
||||
METASLEUTH = "metasleuth" # BlockSec AML API
|
||||
OPEN_LABELS = "open-labels-initiative" # OLI REST API (TBD)
|
||||
CHAINBASE = "chainbase" # Chainbase API (TBD, needs key)
|
||||
INTERNAL = "internal" # our own growing set
|
||||
MBAL = "mbal" # Multi-blockchain Anti-Laundering source
|
||||
|
||||
|
||||
@dataclass
|
||||
class Label:
|
||||
"""A single label for an address.
|
||||
|
||||
Multiple labels can exist for the same (address, chain, label_type)
|
||||
from different sources. Dedup logic keeps the highest-confidence one.
|
||||
"""
|
||||
|
||||
address: str
|
||||
chain: str # "ethereum" | "solana" | "base" | "bitcoin" | etc.
|
||||
label_type: str # "entity" | "tag" | "category" | "risk" | "custom"
|
||||
label_value: str # "Binance" | "Exchange" | "Sanctioned" | etc.
|
||||
source: LabelSource
|
||||
confidence: float = 0.5 # 0.0-1.0
|
||||
verified_at: datetime = field(default_factory=lambda: datetime.now(UTC))
|
||||
attributes: dict[str, Any] = field(default_factory=dict)
|
||||
# Optional entity info (for MetaSleuth etc.)
|
||||
entity: str | None = None
|
||||
entity_category: str | None = None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Serialize for API responses."""
|
||||
return {
|
||||
"address": self.address,
|
||||
"chain": self.chain,
|
||||
"label_type": self.label_type,
|
||||
"label_value": self.label_value,
|
||||
"source": self.source.value,
|
||||
"confidence": self.confidence,
|
||||
"verified_at": self.verified_at.isoformat(),
|
||||
"attributes": self.attributes,
|
||||
"entity": self.entity,
|
||||
"entity_category": self.entity_category,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def dedupe_key(cls, address: str, chain: str, label_type: str, label_value: str) -> tuple:
|
||||
"""Generate dedup key. Same key = same label from different sources."""
|
||||
return (address.lower(), chain.lower(), label_type.lower(), label_value.lower())
|
||||
|
||||
@classmethod
|
||||
def merge(cls, labels: list[Label]) -> Label:
|
||||
"""Merge multiple labels with same dedup_key into one.
|
||||
|
||||
Keeps highest confidence, unions attributes, aggregates sources.
|
||||
"""
|
||||
if not labels:
|
||||
raise ValueError("Cannot merge empty label list")
|
||||
# Sort by confidence desc, take best as base
|
||||
sorted_labels = sorted(labels, key=lambda l: -l.confidence)
|
||||
base = sorted_labels[0]
|
||||
sources = list({l.source for l in labels})
|
||||
merged_attrs = dict(base.attributes)
|
||||
for l in labels[1:]:
|
||||
merged_attrs.update(l.attributes)
|
||||
# Update verified_at to most recent
|
||||
verified_at = max(l.verified_at for l in labels)
|
||||
return cls(
|
||||
address=base.address,
|
||||
chain=base.chain,
|
||||
label_type=base.label_type,
|
||||
label_value=base.label_value,
|
||||
source=base.source, # keep highest-confidence source as primary
|
||||
confidence=base.confidence,
|
||||
verified_at=verified_at,
|
||||
attributes={**merged_attrs, "sources_count": len(sources), "all_sources": [s.value for s in sources]},
|
||||
entity=base.entity,
|
||||
entity_category=base.entity_category,
|
||||
)
|
||||
81
app/domain/labels/router.py
Normal file
81
app/domain/labels/router.py
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
"""T11 — Federated labels router.
|
||||
|
||||
Endpoint:
|
||||
GET /api/v1/labels/{address}?chain=ethereum
|
||||
|
||||
Returns:
|
||||
{
|
||||
"address": "0x...",
|
||||
"chain": "ethereum",
|
||||
"labels": [...], # list of label dicts
|
||||
"sources_queried": [...], # which sources worked
|
||||
"total": N,
|
||||
}
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.domain.labels import get_federated_api
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/labels", tags=["labels"])
|
||||
|
||||
|
||||
class LabelsResponse(BaseModel):
|
||||
"""Response for GET /api/v1/labels/{address}."""
|
||||
|
||||
address: str
|
||||
chain: str
|
||||
labels: list[dict]
|
||||
sources_queried: list[str]
|
||||
total: int
|
||||
|
||||
|
||||
@router.get("/{address}", response_model=LabelsResponse)
|
||||
async def get_labels(
|
||||
address: str,
|
||||
chain: str = Query(
|
||||
"ethereum",
|
||||
description="Blockchain (ethereum, solana, base, arbitrum, etc.)",
|
||||
),
|
||||
use_cache: bool = Query(True, description="Use Redis cache (1h TTL)"),
|
||||
include_low_confidence: bool = Query(False, description="Include labels with confidence < 0.3"),
|
||||
) -> LabelsResponse:
|
||||
"""Get all wallet labels from 6 federated sources in parallel.
|
||||
|
||||
Sources queried:
|
||||
1. eth-labels.db — 115K local labels (SQLite)
|
||||
2. Etherscan CSV — 51K labels (DuckDB)
|
||||
3. Ethereum labels CSV — 51K labels (DuckDB)
|
||||
4. Solana labels CSV — 103K labels (DuckDB)
|
||||
5. ClickHouse — wallet_memory.wallet_labels
|
||||
6. MetaSleuth — BlockSec AML API
|
||||
|
||||
Graceful degradation: failed sources are skipped, not raised.
|
||||
"""
|
||||
if not address or len(address) < 8:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Address too short (got {len(address)} chars, need >= 8)",
|
||||
)
|
||||
|
||||
try:
|
||||
api = get_federated_api()
|
||||
result = await api.get_labels_dict(
|
||||
address,
|
||||
chain=chain,
|
||||
use_cache=use_cache,
|
||||
include_low_confidence=include_low_confidence,
|
||||
)
|
||||
return LabelsResponse(**result)
|
||||
except Exception as e:
|
||||
log.exception("labels_endpoint_fail address=%s", address[:10])
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=f"Label fetch failed: {type(e).__name__}: {str(e)[:200]}",
|
||||
)
|
||||
16
app/domain/labels/sources/__init__.py
Normal file
16
app/domain/labels/sources/__init__.py
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
"""Source adapters for the federated label API.
|
||||
|
||||
Each adapter implements `query(address, chain) -> list[Label]`.
|
||||
|
||||
To add a new source:
|
||||
1. Create a new module in this package (e.g., `my_source.py`)
|
||||
2. Implement `async def query_my_source(address: str, chain: str) -> list[Label]`
|
||||
3. Register it in LabelSource enum (models.py)
|
||||
4. Wire it in federated.py:_query_source dispatcher
|
||||
|
||||
All adapters follow these conventions:
|
||||
- Return [] on no match (not None)
|
||||
- Raise exceptions on infrastructure errors (handled by caller)
|
||||
- All addresses are normalized to lowercase before query
|
||||
- Each label has source=LabelSource.MY_SOURCE
|
||||
"""
|
||||
21
app/domain/labels/sources/chainbase.py
Normal file
21
app/domain/labels/sources/chainbase.py
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
"""Chainbase source — placeholder, requires API key.
|
||||
|
||||
TODO: add CHAINBASE_API_KEY to gopass + env. Free tier: 50K calls/month.
|
||||
|
||||
For now returns []. Stub.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from app.domain.labels.models import Label
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def query_chainbase(address: str, chain: str = "ethereum") -> list[Label]:
|
||||
"""Query Chainbase for an address's labels.
|
||||
|
||||
Returns [] until CHAINBASE_API_KEY is configured.
|
||||
"""
|
||||
return []
|
||||
41
app/domain/labels/sources/clickhouse_labels.py
Normal file
41
app/domain/labels/sources/clickhouse_labels.py
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
"""ClickHouse wallet_memory.wallet_labels source — currently empty.
|
||||
|
||||
Schema (verified):
|
||||
address String
|
||||
chain_id String
|
||||
label_name String
|
||||
label_category String
|
||||
source String
|
||||
is_sanctioned UInt8 DEFAULT 0
|
||||
loaded_at DateTime DEFAULT now()
|
||||
label_subtype String DEFAULT ''
|
||||
|
||||
Returns [] until labels are imported. The plan is to bulk-import
|
||||
the CSVs (etherscan_combined, ethereum labels, solana labels) via
|
||||
the warehouse_etl jobs. Until then, this adapter is a no-op.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from app.domain.labels.models import Label
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def query_clickhouse_labels(address: str, chain: str = "ethereum") -> list[Label]:
|
||||
"""Query ClickHouse wallet_memory.wallet_labels table.
|
||||
|
||||
Currently returns [] — table is empty until CSVs are bulk-imported.
|
||||
Real implementation will use HTTP to clickhouse client over
|
||||
native protocol via app.core.duckdb_analytics.query_postgres-style attach.
|
||||
|
||||
Chain mapping: clickhouse stores chain_id as String. We translate
|
||||
our 'ethereum'/'solana' names to the chain_id values used in the table.
|
||||
"""
|
||||
# TODO: when wallet_memory.wallet_labels is populated, use:
|
||||
# from app.core.duckdb_analytics import DuckDBAnalytics
|
||||
# d = DuckDBAnalytics()
|
||||
# rows = d.query_postgres(...) # if we ever swap CH for Postgres
|
||||
# OR use httpx against CH HTTP interface at http://localhost:8123
|
||||
return []
|
||||
92
app/domain/labels/sources/eth_labels_db.py
Normal file
92
app/domain/labels/sources/eth_labels_db.py
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
"""eth-labels.db source — 115K EVM labels from local SQLite.
|
||||
|
||||
Schema (verified):
|
||||
accounts(id, chain_id, address, label, name_tag, created_at, updated_at)
|
||||
|
||||
chain_id mapping:
|
||||
1 = ethereum, 42161 = arbitrum, 137 = polygon, etc.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from app.domain.labels.models import Label, LabelSource
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
DB_PATH = "/home/dev/rmi/eth-labels.db"
|
||||
|
||||
# chain_id -> chain name mapping (from EVM chains list)
|
||||
_CHAIN_ID_MAP = {
|
||||
1: "ethereum",
|
||||
42161: "arbitrum",
|
||||
10: "optimism",
|
||||
137: "polygon",
|
||||
56: "bsc",
|
||||
43114: "avalanche",
|
||||
250: "fantom",
|
||||
100: "gnosis",
|
||||
}
|
||||
|
||||
|
||||
async def query_eth_labels_db(address: str, chain: str = "ethereum") -> list[Label]:
|
||||
"""Query the local eth-labels.db SQLite for an address.
|
||||
|
||||
Args:
|
||||
address: EVM address (0x...)
|
||||
chain: Chain name (only EVM chains in this DB)
|
||||
|
||||
Returns:
|
||||
List of labels found (one per chain_id that has a match).
|
||||
"""
|
||||
def _query() -> list[Label]:
|
||||
import sqlite3
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(DB_PATH, timeout=2.0)
|
||||
conn.row_factory = sqlite3.Row
|
||||
try:
|
||||
cursor = conn.execute(
|
||||
"SELECT chain_id, label, name_tag, updated_at FROM accounts WHERE address = ? LIMIT 50",
|
||||
(address.lower(),),
|
||||
)
|
||||
labels = []
|
||||
for row in cursor:
|
||||
chain_id = row["chain_id"]
|
||||
chain_name = _CHAIN_ID_MAP.get(chain_id, f"chain-{chain_id}")
|
||||
label_value = row["name_tag"] or row["label"]
|
||||
if not label_value:
|
||||
continue
|
||||
labels.append(
|
||||
Label(
|
||||
address=address.lower(),
|
||||
chain=chain_name,
|
||||
label_type="entity",
|
||||
label_value=label_value,
|
||||
source=LabelSource.ETH_LABELS_DB,
|
||||
confidence=0.9, # eth-labels.db is curated
|
||||
verified_at=_parse_date(row["updated_at"]),
|
||||
attributes={"chain_id": chain_id, "raw_label": row["label"]},
|
||||
)
|
||||
)
|
||||
return labels
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as e:
|
||||
log.warning("eth_labels_db_query_fail address=%s err=%s", address[:10], e)
|
||||
raise
|
||||
|
||||
return await asyncio.to_thread(_query)
|
||||
|
||||
|
||||
def _parse_date(s: str | None) -> datetime:
|
||||
"""Parse SQLite date string to datetime."""
|
||||
if not s:
|
||||
return datetime.now(UTC)
|
||||
try:
|
||||
# Format: '2024-06-30 21:58:54'
|
||||
return datetime.fromisoformat(s).replace(tzinfo=UTC)
|
||||
except Exception:
|
||||
return datetime.now(UTC)
|
||||
67
app/domain/labels/sources/ethereum_labels_csv.py
Normal file
67
app/domain/labels/sources/ethereum_labels_csv.py
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
"""Ethereum wallet labels CSV source — 51K EVM labels via DuckDB."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from app.domain.labels.models import Label, LabelSource
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
CSV_PATH = "/home/dev/rmi/backend/data/wallet-labels-clean/wallet_labels_ethereum.csv"
|
||||
|
||||
|
||||
async def query_ethereum_labels_csv(address: str, chain: str = "ethereum") -> list[Label]:
|
||||
"""Query the ethereum wallet labels CSV via DuckDB.
|
||||
|
||||
Schema:
|
||||
address,chain,source,name,entity_type,threat_group,risk_level,verified
|
||||
"""
|
||||
def _query() -> list[Label]:
|
||||
from app.core.duckdb_analytics import DuckDBAnalytics
|
||||
|
||||
try:
|
||||
d = DuckDBAnalytics()
|
||||
try:
|
||||
rows = d.query_parquet(
|
||||
CSV_PATH,
|
||||
"""
|
||||
SELECT address, chain, source, name, entity_type, threat_group, risk_level, verified
|
||||
FROM data
|
||||
WHERE LOWER(address) = ?
|
||||
LIMIT 20
|
||||
""",
|
||||
[address.lower()],
|
||||
)
|
||||
labels = []
|
||||
for row in rows:
|
||||
label_value = row.get("name") or row.get("entity_type")
|
||||
if not label_value:
|
||||
continue
|
||||
chain_name = row.get("chain", chain).lower()
|
||||
labels.append(
|
||||
Label(
|
||||
address=row["address"].lower(),
|
||||
chain=chain_name,
|
||||
label_type="entity",
|
||||
label_value=label_value,
|
||||
source=LabelSource.ETHEREUM_LABELS_CSV,
|
||||
confidence=0.85,
|
||||
verified_at=datetime.now(UTC),
|
||||
attributes={
|
||||
"entity_type": row.get("entity_type"),
|
||||
"threat_group": row.get("threat_group"),
|
||||
"risk_level": row.get("risk_level"),
|
||||
"verified": row.get("verified"),
|
||||
},
|
||||
)
|
||||
)
|
||||
return labels
|
||||
finally:
|
||||
d.close()
|
||||
except Exception as e:
|
||||
log.warning("ethereum_labels_csv_query_fail address=%s err=%s", address[:10], e)
|
||||
raise
|
||||
|
||||
return await asyncio.to_thread(_query)
|
||||
70
app/domain/labels/sources/etherscan_csv.py
Normal file
70
app/domain/labels/sources/etherscan_csv.py
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
"""Etherscan combined labels CSV source — 51K EVM labels via DuckDB."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from app.domain.labels.models import Label, LabelSource
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
CSV_PATH = "/home/dev/rmi/backend/data/etherscan_combined_labels.csv"
|
||||
|
||||
|
||||
async def query_etherscan_csv(address: str, chain: str = "ethereum") -> list[Label]:
|
||||
"""Query the Etherscan combined labels CSV via DuckDB.
|
||||
|
||||
Schema:
|
||||
address,chain,source,name,label_type,label_subtype,project,entity_type,entity_types
|
||||
|
||||
Returns:
|
||||
List of labels (usually 1 per matching row).
|
||||
"""
|
||||
def _query() -> list[Label]:
|
||||
from app.core.duckdb_analytics import DuckDBAnalytics
|
||||
|
||||
try:
|
||||
d = DuckDBAnalytics()
|
||||
try:
|
||||
rows = d.query_parquet(
|
||||
CSV_PATH,
|
||||
"""
|
||||
SELECT address, chain, source, name, label_type, label_subtype, project, entity_type
|
||||
FROM data
|
||||
WHERE LOWER(address) = ?
|
||||
LIMIT 20
|
||||
""",
|
||||
[address.lower()],
|
||||
)
|
||||
labels = []
|
||||
for row in rows:
|
||||
label_value = row.get("name") or row.get("label_type") or row.get("source")
|
||||
if not label_value:
|
||||
continue
|
||||
chain_name = row.get("chain", chain).lower()
|
||||
labels.append(
|
||||
Label(
|
||||
address=row["address"].lower(),
|
||||
chain=chain_name,
|
||||
label_type="entity",
|
||||
label_value=label_value,
|
||||
source=LabelSource.ETHERSCAN_CSV,
|
||||
confidence=0.85,
|
||||
verified_at=datetime.now(UTC),
|
||||
attributes={
|
||||
"raw_label_type": row.get("label_type"),
|
||||
"label_subtype": row.get("label_subtype"),
|
||||
"project": row.get("project"),
|
||||
"entity_type": row.get("entity_type"),
|
||||
},
|
||||
)
|
||||
)
|
||||
return labels
|
||||
finally:
|
||||
d.close()
|
||||
except Exception as e:
|
||||
log.warning("etherscan_csv_query_fail address=%s err=%s", address[:10], e)
|
||||
raise
|
||||
|
||||
return await asyncio.to_thread(_query)
|
||||
22
app/domain/labels/sources/internal_labels.py
Normal file
22
app/domain/labels/sources/internal_labels.py
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
"""Internal labels source — our own growing set in Postgres/Redis.
|
||||
|
||||
Currently returns []. The schema for internal labels (community
|
||||
contributions, RMI-curated tags) is being designed separately.
|
||||
|
||||
TODO: query Postgres internal_labels table once schema lands.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from app.domain.labels.models import Label
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def query_internal(address: str, chain: str = "ethereum") -> list[Label]:
|
||||
"""Query internal labels database.
|
||||
|
||||
Returns [] until internal_labels table is provisioned.
|
||||
"""
|
||||
return []
|
||||
31
app/domain/labels/sources/mbal.py
Normal file
31
app/domain/labels/sources/mbal.py
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
"""
|
||||
MBAL query adapter for the federated label API.
|
||||
|
||||
Implements the MBAL source API for the federated label system.
|
||||
|
||||
Usage (called by federated.py):
|
||||
labels = await query_mbal("0x...", "ethereum")
|
||||
|
||||
Pattern: async function that returns list[Label]
|
||||
"""
|
||||
|
||||
from typing import List
|
||||
|
||||
from app.domain.labels.models import Label
|
||||
from app.domain.labels.sources.mbal_source import get_mbalsy_service
|
||||
|
||||
|
||||
async def query_mbal(address: str, chain: str) -> List[Label]:
|
||||
"""
|
||||
Query MBAL for labels for an address.
|
||||
|
||||
Args:
|
||||
address: The wallet address to look up
|
||||
chain: The blockchain network (ethereum, bitcoin, etc.)
|
||||
|
||||
Returns:
|
||||
List of labels from MBAL
|
||||
"""
|
||||
# Get or initialize the MBAL service
|
||||
service = get_mbalsy_service()
|
||||
return await service.get_labels(address, chain)
|
||||
234
app/domain/labels/sources/mbal_source.py
Normal file
234
app/domain/labels/sources/mbal_source.py
Normal file
|
|
@ -0,0 +1,234 @@
|
|||
"""
|
||||
MBAL (Multi-blockchain Anti-Laundering) Label Integration
|
||||
|
||||
Integrates with MBAL's 10M+ wallet labels via the MBAL API or direct ClickHouse access.
|
||||
MBAL provides comprehensive blockchain attribution, wallet clustering, and risk scoring
|
||||
across Bitcoin, Ethereum, and 15+ major chains.
|
||||
|
||||
API Documentation: https://mbal.io/api
|
||||
Rate Limits: Free tier - 100 queries/day, Pro - 1000/day, Enterprise - Unlimited
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import httpx
|
||||
|
||||
from app.domain.labels.models import Label, LabelSource
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MBALService:
|
||||
"""MBAL API Integration for multi-blockchain wallet labels"""
|
||||
|
||||
def __init__(self, api_key: Optional[str] = None):
|
||||
"""
|
||||
Initialize MBAL service.
|
||||
|
||||
Args:
|
||||
api_key: MBAL API key. If None, uses free tier with limitations.
|
||||
"""
|
||||
self.base_url = "https://api.mbal.io/v1"
|
||||
self.api_key = api_key
|
||||
self.client = httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(30.0),
|
||||
limits=httpx.Limits(max_connections=10, max_keepalive_connections=5),
|
||||
)
|
||||
|
||||
# Cache for rate limiting
|
||||
self.last_request_time = 0
|
||||
self.request_count = 0
|
||||
self.reset_time = 0 # Unix timestamp when quota resets
|
||||
|
||||
async def close(self):
|
||||
"""Close the HTTP client."""
|
||||
await self.client.aclose()
|
||||
|
||||
async def get_labels(self, address: str, chain: str) -> List[Label]:
|
||||
"""
|
||||
Retrieve labels for a blockchain address from MBAL.
|
||||
|
||||
Args:
|
||||
address: The blockchain address to label
|
||||
chain: The blockchain (ethereum, bitcoin, tron, etc.)
|
||||
|
||||
Returns:
|
||||
List of labels with metadata
|
||||
"""
|
||||
try:
|
||||
# MBAL requires different endpoints per chain
|
||||
labels = []
|
||||
|
||||
# Prepare query payload
|
||||
params = {
|
||||
"address": address,
|
||||
"chain": chain.lower(),
|
||||
}
|
||||
|
||||
# Rate limiting - max 100 requests per day for free
|
||||
await self._rate_limit()
|
||||
|
||||
headers = {}
|
||||
if self.api_key:
|
||||
headers["Authorization"] = f"Bearer {self.api_key}"
|
||||
|
||||
# Query MBAL's address endpoint
|
||||
response = await self.client.get(
|
||||
f"{self.base_url}/address",
|
||||
params=params,
|
||||
headers=headers
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
data = response.json()
|
||||
|
||||
# Process the result according to MBAL's schema
|
||||
if "entities" in data:
|
||||
for entity in data["entities"]:
|
||||
label_value = entity.get("category", "unknown")
|
||||
entity_type = entity.get("type", "address")
|
||||
confidence = entity.get("confidence", 0.8) # MBAL typically confident
|
||||
|
||||
# Map MBAL categories to our standardized label types
|
||||
label_type = self._map_category_to_type(label_value)
|
||||
|
||||
labels.append(Label(
|
||||
address=address,
|
||||
chain=chain,
|
||||
label_type=label_type,
|
||||
label_value=label_value,
|
||||
source=LabelSource.MBAL, # MBAL source
|
||||
verified_at=datetime.now(),
|
||||
confidence=confidence,
|
||||
attributes={
|
||||
"mbal_entity_id": entity.get("entityId"),
|
||||
"mbal_tags": entity.get("tags", []),
|
||||
"mbal_risk_score": entity.get("riskScore", 0),
|
||||
"mbal_cluster_id": entity.get("clusterId"),
|
||||
}
|
||||
))
|
||||
|
||||
if "behaviors" in data and len(labels) == 0:
|
||||
# If no main entities, fall back to behavioral labels
|
||||
for behavior in data["behaviors"]:
|
||||
label_value = behavior["type"]
|
||||
label_type = self._map_behavior_to_type(label_value)
|
||||
confidence = behavior.get("confidence", 0.7)
|
||||
|
||||
labels.append(Label(
|
||||
address=address,
|
||||
chain=chain,
|
||||
label_type=label_type,
|
||||
label_value=label_value,
|
||||
source=LabelSource.MBAL, # MBAL API source
|
||||
verified_at=datetime.now(),
|
||||
confidence=confidence,
|
||||
attributes={
|
||||
"behavior_description": behavior.get("description"),
|
||||
"behavior_risk_level": behavior.get("riskLevel", "medium"),
|
||||
}
|
||||
))
|
||||
|
||||
return labels
|
||||
|
||||
except httpx.HTTPStatusError as e:
|
||||
if e.response.status_code == 429:
|
||||
logger.warning(f"MBAL rate limit exceeded: {e}")
|
||||
return []
|
||||
logger.error(f"MBAL API error: {e} for {chain}:{address}")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error querying MBAL: {e}")
|
||||
return []
|
||||
|
||||
def _map_category_to_type(self, category: str) -> str:
|
||||
"""Map MBAL category strings to our standard label type strings."""
|
||||
category_lower = category.lower()
|
||||
|
||||
# Exchange mappings
|
||||
if any(exchange in category_lower for exchange in ["binance", "coinbase", "kraken", "kucoin",
|
||||
"huobi", "gate.io", "okx", "bybit", "gemini"]):
|
||||
return "exchange"
|
||||
|
||||
# Service mappings
|
||||
if any(service in category_lower for service in ["miner", "validator", "staking", "liquidity",
|
||||
"amm", "dex", "bridge", "cex", "dexe"]):
|
||||
return "service"
|
||||
|
||||
# Actor mappings
|
||||
if any(actor in category_lower for actor in ["blacklist", "scammer", "hacker", "tornado",
|
||||
"mixer", "launderer", "crime", "malicious"]):
|
||||
return "actor"
|
||||
|
||||
# Default to entity for unknown but potentially identifiable categories
|
||||
return "actor" if "blacklist" in category_lower else "entity"
|
||||
|
||||
def _map_behavior_to_type(self, behavior: str) -> str:
|
||||
"""Map MBAL behaviors to our standard label type strings."""
|
||||
behavior_lower = behavior.lower()
|
||||
|
||||
if any(risky in behavior_lower for risky in ["malware", "ransom", "exploit", "theft",
|
||||
"scam", "hacking", "malicious"]):
|
||||
return "actor"
|
||||
|
||||
if any(behavior in behavior_lower for behavior in ["mining", "block generation", "transaction fee", "gas"]):
|
||||
return "service"
|
||||
|
||||
return "entity" # Default
|
||||
|
||||
async def _rate_limit(self):
|
||||
"""Implement simple rate limiting based on MBAL tier."""
|
||||
now = asyncio.get_event_loop().time()
|
||||
|
||||
# Check daily quota if using free tier
|
||||
if not self.api_key and self.request_count >= 100:
|
||||
# Free tier: 100 queries/day
|
||||
# Simple approach: wait until reset (24 hours from first query)
|
||||
if now < self.reset_time:
|
||||
sleep_time = self.reset_time - now
|
||||
logger.info(f"Waiting for MBAL daily quota reset: {sleep_time:.1f}s")
|
||||
await asyncio.sleep(sleep_time)
|
||||
|
||||
# Ensure minimum delay between requests
|
||||
time_since_last = now - self.last_request_time
|
||||
min_delay = 1.0 # At least 1s between requests
|
||||
if time_since_last < min_delay:
|
||||
await asyncio.sleep(min_delay - time_since_last)
|
||||
|
||||
# Update counters
|
||||
self.last_request_time = now
|
||||
if now >= self.reset_time:
|
||||
self.request_count = 0
|
||||
self.reset_time = now + 86400 # Reset in 24 hours
|
||||
self.request_count += 1
|
||||
|
||||
async def health_check(self) -> bool:
|
||||
"""
|
||||
Check if MBAL service is accessible.
|
||||
|
||||
Returns:
|
||||
True if MBAL API responds successfully
|
||||
"""
|
||||
try:
|
||||
response = await self.client.get(f"{self.base_url}/health")
|
||||
return response.status_code == 200
|
||||
except:
|
||||
return False
|
||||
|
||||
|
||||
# Global instance
|
||||
_mbalsy_instance: Optional[MBALService] = None
|
||||
|
||||
|
||||
def get_mbalsy_service() -> MBALService:
|
||||
"""Get the global MBAL service instance."""
|
||||
global _mbalsy_instance
|
||||
if _mbalsy_instance is None:
|
||||
import os
|
||||
api_key = os.getenv("MBAL_API_KEY", None)
|
||||
_mbalsy_instance = MBALService(api_key=api_key)
|
||||
return _mbalsy_instance
|
||||
168
app/domain/labels/sources/metasleuth.py
Normal file
168
app/domain/labels/sources/metasleuth.py
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
"""MetaSleuth (BlockSec AML) source — live API at aml.blocksec.com.
|
||||
|
||||
Endpoint: POST https://aml.blocksec.com/address-label/api/v3/labels
|
||||
Headers: API-KEY: <key>
|
||||
Body: {"chain_id": 1, "address": "0x..."}
|
||||
|
||||
Verified working: 2026-06-22 (Binance 0x3d12... returned main_entity=Binance,
|
||||
entity categories EXCHANGE + OTC DESK).
|
||||
|
||||
Limit: 10 calls/day per API key on the free tier. So we cache aggressively.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import httpx
|
||||
|
||||
from app.domain.labels.models import Label, LabelSource
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
METASLEUTH_BASE = "https://aml.blocksec.com/address-label/api/v3"
|
||||
# Free tier: 10 calls/day per key on risk-level, similar on labels
|
||||
DEFAULT_TIMEOUT = 8.0
|
||||
|
||||
# Chain name -> MetaSleuth chain_id
|
||||
_CHAIN_MAP = {
|
||||
"ethereum": 1,
|
||||
"bitcoin": -1,
|
||||
"tron": -2,
|
||||
"solana": -3,
|
||||
"monero": -4,
|
||||
"sui": 5,
|
||||
"optimism": 10,
|
||||
"cronos": 25,
|
||||
"bsc": 56,
|
||||
"base": 8453,
|
||||
"arbitrum": 42161,
|
||||
"polygon": 137,
|
||||
}
|
||||
|
||||
|
||||
async def query_metasleuth(address: str, chain: str = "ethereum") -> list[Label]:
|
||||
"""Query MetaSleuth for an address's entity info.
|
||||
|
||||
Returns:
|
||||
- 1 entity label with the main entity name (e.g. "Binance")
|
||||
- Optional category labels (EXCHANGE, OTC DESK, etc.)
|
||||
- Attributes (DEPOSIT ADDRESS, etc.)
|
||||
"""
|
||||
api_key = os.getenv("METASLEUTH_LABELS_API_KEY", "")
|
||||
if not api_key:
|
||||
# Try gopass via the standard pattern
|
||||
try:
|
||||
import subprocess
|
||||
|
||||
result = subprocess.run(
|
||||
["gopass", "show", "-o", "rmi/infra/metasleuth_labels_api_key"],
|
||||
capture_output=True, text=True, timeout=2,
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
api_key = result.stdout.strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not api_key:
|
||||
log.debug("metasleuth_no_api_key configured, skipping")
|
||||
return []
|
||||
|
||||
chain_id = _CHAIN_MAP.get(chain.lower())
|
||||
if chain_id is None:
|
||||
log.debug("metasleuth_unsupported_chain chain=%s", chain)
|
||||
return []
|
||||
|
||||
def _call_api() -> dict:
|
||||
"""Sync HTTP call (run in thread)."""
|
||||
try:
|
||||
r = httpx.post(
|
||||
f"{METASLEUTH_BASE}/labels",
|
||||
headers={"API-KEY": api_key, "Content-Type": "application/json"},
|
||||
json={"chain_id": chain_id, "address": address},
|
||||
timeout=DEFAULT_TIMEOUT,
|
||||
)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
if data.get("code") == 200000:
|
||||
return data.get("data") or {}
|
||||
log.warning("metasleuth_api_err code=%s msg=%s", data.get("code"), data.get("message"))
|
||||
return {}
|
||||
except httpx.HTTPError as e:
|
||||
log.warning("metasleuth_api_http_err: %s", e)
|
||||
return {}
|
||||
except Exception as e:
|
||||
log.warning("metasleuth_api_err: %s", e)
|
||||
return {}
|
||||
|
||||
data = await asyncio.to_thread(_call_api)
|
||||
if not data:
|
||||
return []
|
||||
|
||||
labels: list[Label] = []
|
||||
main_entity = data.get("main_entity")
|
||||
if main_entity:
|
||||
labels.append(Label(
|
||||
address=address.lower(),
|
||||
chain=chain,
|
||||
label_type="entity",
|
||||
label_value=main_entity,
|
||||
source=LabelSource.METASLEUTH,
|
||||
confidence=0.95,
|
||||
verified_at=datetime.now(UTC),
|
||||
entity=main_entity,
|
||||
entity_category=_first_category(data),
|
||||
attributes=_extract_attributes(data),
|
||||
))
|
||||
|
||||
# Also emit labels for each category
|
||||
categories = data.get("main_entity_info", {}).get("categories", [])
|
||||
if not isinstance(categories, list):
|
||||
categories = []
|
||||
for cat in categories:
|
||||
if not isinstance(cat, dict):
|
||||
continue
|
||||
cat_name = cat.get("name")
|
||||
if cat_name and cat_name != main_entity:
|
||||
labels.append(Label(
|
||||
address=address.lower(),
|
||||
chain=chain,
|
||||
label_type="category",
|
||||
label_value=cat_name,
|
||||
source=LabelSource.METASLEUTH,
|
||||
confidence=0.9,
|
||||
verified_at=datetime.now(UTC),
|
||||
entity=main_entity,
|
||||
entity_category=cat_name,
|
||||
attributes={"code": cat.get("code")},
|
||||
))
|
||||
|
||||
return labels
|
||||
|
||||
|
||||
def _first_category(data: dict) -> str | None:
|
||||
"""Extract first category name from MetaSleuth response."""
|
||||
cats = data.get("main_entity_info", {}).get("categories", [])
|
||||
if cats and isinstance(cats[0], dict):
|
||||
return cats[0].get("name")
|
||||
return None
|
||||
|
||||
|
||||
def _extract_attributes(data: dict) -> dict:
|
||||
"""Extract attributes from MetaSleuth response."""
|
||||
attrs = data.get("main_entity_info", {}).get("attributes", [])
|
||||
out = {}
|
||||
for a in attrs:
|
||||
if isinstance(a, dict):
|
||||
name = a.get("name", "")
|
||||
if name:
|
||||
out[name] = a.get("code")
|
||||
# Add description (website/twitter) if present
|
||||
desc = data.get("main_entity_info", {}).get("description", {})
|
||||
if isinstance(desc, dict):
|
||||
for k, v in desc.items():
|
||||
if v:
|
||||
out[k] = v
|
||||
return out
|
||||
22
app/domain/labels/sources/open_labels.py
Normal file
22
app/domain/labels/sources/open_labels.py
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
"""Open Labels Initiative (OLI) source — placeholder.
|
||||
|
||||
TODO: wire up when OLI endpoint is confirmed. GitHub repo:
|
||||
https://github.com/openlabelsinitiative/OLI
|
||||
|
||||
For now returns []. Stub.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from app.domain.labels.models import Label
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def query_open_labels(address: str, chain: str = "ethereum") -> list[Label]:
|
||||
"""Query OLI for an address's labels.
|
||||
|
||||
Returns [] until OLI endpoint is confirmed and we have API access.
|
||||
"""
|
||||
return []
|
||||
74
app/domain/labels/sources/solana_labels_csv.py
Normal file
74
app/domain/labels/sources/solana_labels_csv.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
"""Solana wallet labels CSV source — 103K SOL labels via DuckDB."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from app.domain.labels.models import Label, LabelSource
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
CSV_PATH = "/home/dev/rmi/backend/data/wallet-labels-clean/wallet_labels_solana.csv"
|
||||
|
||||
|
||||
async def query_solana_labels_csv(address: str, chain: str = "solana") -> list[Label]:
|
||||
"""Query the Solana wallet labels CSV via DuckDB.
|
||||
|
||||
Schema (likely): address,source,name,entity_type,threat_group,risk_level
|
||||
|
||||
Only matches if chain == "solana" — returns [] for EVM chains.
|
||||
"""
|
||||
if chain != "solana":
|
||||
return []
|
||||
|
||||
def _query() -> list[Label]:
|
||||
from app.core.duckdb_analytics import DuckDBAnalytics
|
||||
|
||||
try:
|
||||
d = DuckDBAnalytics()
|
||||
try:
|
||||
rows = d.query_parquet(
|
||||
CSV_PATH,
|
||||
"""
|
||||
SELECT *
|
||||
FROM data
|
||||
WHERE LOWER(address) = ?
|
||||
LIMIT 20
|
||||
""",
|
||||
[address.lower()],
|
||||
)
|
||||
labels = []
|
||||
for row in rows:
|
||||
# Field names may differ — try common names
|
||||
label_value = (
|
||||
row.get("name")
|
||||
or row.get("label")
|
||||
or row.get("entity_type")
|
||||
or row.get("source")
|
||||
)
|
||||
if not label_value:
|
||||
continue
|
||||
labels.append(
|
||||
Label(
|
||||
address=row.get("address", address).lower(),
|
||||
chain="solana",
|
||||
label_type="entity",
|
||||
label_value=label_value,
|
||||
source=LabelSource.SOLANA_LABELS_CSV,
|
||||
confidence=0.8,
|
||||
verified_at=datetime.now(UTC),
|
||||
attributes={
|
||||
k: v for k, v in row.items()
|
||||
if k not in ("address", "name", "label") and v
|
||||
},
|
||||
)
|
||||
)
|
||||
return labels
|
||||
finally:
|
||||
d.close()
|
||||
except Exception as e:
|
||||
log.warning("solana_labels_csv_query_fail address=%s err=%s", address[:10], e)
|
||||
raise
|
||||
|
||||
return await asyncio.to_thread(_query)
|
||||
5
app/domain/news/__init__.py
Normal file
5
app/domain/news/__init__.py
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
"""T28 News Intelligence — thin HTTP layer."""
|
||||
from .admin_router import router as admin_router
|
||||
from .router import router
|
||||
|
||||
__all__ = ["admin_router", "router"]
|
||||
14
app/domain/news/admin_router.py
Normal file
14
app/domain/news/admin_router.py
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
"""T28 RSS Ingest HTTP endpoint."""
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.domain.news.ingest import ingest_all
|
||||
|
||||
router = APIRouter(prefix="/api/v1/news/_admin", tags=["news-admin"])
|
||||
|
||||
|
||||
@router.post("/ingest")
|
||||
async def trigger_ingest() -> dict:
|
||||
"""Trigger RSS ingest now (synchronous). Returns counts."""
|
||||
result = await ingest_all()
|
||||
return result
|
||||
415
app/domain/news/clusterer.py
Normal file
415
app/domain/news/clusterer.py
Normal file
|
|
@ -0,0 +1,415 @@
|
|||
"""T03 — News story clustering (G04 FIX).
|
||||
|
||||
Per MINIMAX_M3_TASKS.md T03. MinHash + DBSCAN dedupes raw RSS items into
|
||||
single "stories" so AI agents don't see the same CoinDesk/The Block story
|
||||
counted 2-3x in their signal.
|
||||
|
||||
Algorithm:
|
||||
1. MinHash signature (128 permutations) on shingled title+body (first 500 chars)
|
||||
2. DBSCAN clusters within 30-minute windows, Jaccard threshold 0.6, eps=0.15
|
||||
3. Each cluster = one story with all source URLs, sentiment avg, item count
|
||||
4. Persist clusters to Postgres `news_clusters` table; raw items unchanged
|
||||
|
||||
Endpoints:
|
||||
GET /api/v1/news?clustered=true returns stories (clusters), not raw items
|
||||
GET /api/v1/news raw items (legacy)
|
||||
|
||||
This module is pure logic — no I/O at import time. Router/background job
|
||||
call `cluster_items(items) -> list[StoryCluster]`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ── Tokenization ────────────────────────────────────────────────────
|
||||
_TOKEN_RE = re.compile(r"[a-z0-9]{3,}", re.IGNORECASE)
|
||||
|
||||
|
||||
def _shingles(text: str, k: int = 3) -> set[str]:
|
||||
"""k-shingle set of lowercased alphanumeric tokens. For Jaccard."""
|
||||
if not text:
|
||||
return set()
|
||||
toks = _TOKEN_RE.findall(text.lower())
|
||||
if len(toks) < k:
|
||||
return set(toks)
|
||||
return {" ".join(toks[i : i + k]) for i in range(len(toks) - k + 1)}
|
||||
|
||||
|
||||
# ── MinHash ─────────────────────────────────────────────────────────
|
||||
_NUM_PERM = 128
|
||||
_MAX_HASH = (1 << 32) - 1
|
||||
|
||||
|
||||
def _minhash_signature(shingles: set[str], seed: int = 42) -> list[int]:
|
||||
"""128-permutation MinHash signature of a shingle set.
|
||||
|
||||
Uses SHA-256 seeded permutations — fast, deterministic, no numpy.
|
||||
"""
|
||||
if not shingles:
|
||||
return [_MAX_HASH] * _NUM_PERM
|
||||
sig: list[int] = []
|
||||
for i in range(_NUM_PERM):
|
||||
m = _MAX_HASH
|
||||
for s in shingles:
|
||||
h = int.from_bytes(
|
||||
hashlib.sha256(f"{i}:{seed}:{s}".encode()).digest()[:4],
|
||||
"big",
|
||||
)
|
||||
if h < m:
|
||||
m = h
|
||||
sig.append(m)
|
||||
return sig
|
||||
|
||||
|
||||
def _jaccard_minhash(a: list[int], b: list[int]) -> float:
|
||||
"""Estimate Jaccard similarity from two MinHash signatures."""
|
||||
if not a or not b or len(a) != len(b):
|
||||
return 0.0
|
||||
return sum(1 for x, y in zip(a, b, strict=False) if x == y) / len(a)
|
||||
|
||||
|
||||
# ── DBSCAN (pure-python, no sklearn dep) ────────────────────────────
|
||||
def _dbscan(
|
||||
signatures: list[list[int]],
|
||||
eps: float = 0.4,
|
||||
min_samples: int = 2,
|
||||
) -> list[int]:
|
||||
"""Density-based clustering. Returns cluster id per item (-1 = noise).
|
||||
|
||||
Similarity is Jaccard (estimated via MinHash). Neighbours are pairs
|
||||
with Jaccard distance <= eps (i.e. similarity >= 1 - eps).
|
||||
Default eps=0.4 means similarity >= 0.6 (per T03 spec).
|
||||
"""
|
||||
n = len(signatures)
|
||||
labels = [-1] * n
|
||||
cluster_id = 0
|
||||
for i in range(n):
|
||||
if labels[i] != -1:
|
||||
continue
|
||||
neighbors = [
|
||||
j
|
||||
for j in range(n)
|
||||
if i != j and (1.0 - _jaccard_minhash(signatures[i], signatures[j])) <= eps
|
||||
]
|
||||
if len(neighbors) < min_samples - 1:
|
||||
# not enough neighbours — mark as noise (may become border later)
|
||||
continue
|
||||
labels[i] = cluster_id
|
||||
seed_set = list(neighbors)
|
||||
k = 0
|
||||
while k < len(seed_set):
|
||||
q = seed_set[k]
|
||||
if labels[q] == -1:
|
||||
labels[q] = cluster_id
|
||||
q_neighbors = [
|
||||
j
|
||||
for j in range(n)
|
||||
if j != q
|
||||
and (1.0 - _jaccard_minhash(signatures[q], signatures[j])) <= eps
|
||||
]
|
||||
if len(q_neighbors) >= min_samples - 1:
|
||||
seed_set.extend(q_neighbors)
|
||||
elif labels[q] is None or labels[q] == -1:
|
||||
labels[q] = cluster_id
|
||||
k += 1
|
||||
cluster_id += 1
|
||||
return labels
|
||||
|
||||
|
||||
# ── Domain types ────────────────────────────────────────────────────
|
||||
@dataclass
|
||||
class NewsItem:
|
||||
"""Minimal news item for clustering. Adapts from DB rows or dicts."""
|
||||
|
||||
id: str
|
||||
title: str
|
||||
body: str = ""
|
||||
source: str = ""
|
||||
url: str = ""
|
||||
published_at: datetime = field(default_factory=lambda: datetime.now(UTC))
|
||||
sentiment: float = 0.0
|
||||
|
||||
@classmethod
|
||||
def from_row(cls, row: dict) -> NewsItem:
|
||||
published = row.get("published_at") or row.get("created_at")
|
||||
if isinstance(published, str):
|
||||
try:
|
||||
published = datetime.fromisoformat(published.replace("Z", "+00:00"))
|
||||
except (ValueError, AttributeError):
|
||||
published = datetime.now(UTC)
|
||||
elif not isinstance(published, datetime):
|
||||
published = datetime.now(UTC)
|
||||
return cls(
|
||||
id=str(row.get("id", row.get("news_id", ""))),
|
||||
title=row.get("title", "") or "",
|
||||
body=(row.get("body") or row.get("summary") or "")[:500],
|
||||
source=row.get("source", "") or "",
|
||||
url=row.get("url", "") or "",
|
||||
published_at=published,
|
||||
sentiment=float(row.get("sentiment", 0.0) or 0.0),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class StoryCluster:
|
||||
"""One deduplicated story spanning 1+ source items."""
|
||||
|
||||
cluster_id: str
|
||||
representative_title: str
|
||||
source_urls: list[str]
|
||||
sources: list[str]
|
||||
first_seen: datetime
|
||||
last_updated: datetime
|
||||
item_count: int
|
||||
sentiment_avg: float
|
||||
item_ids: list[str]
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"cluster_id": self.cluster_id,
|
||||
"representative_title": self.representative_title,
|
||||
"source_urls": self.source_urls,
|
||||
"sources": self.sources,
|
||||
"first_seen": self.first_seen.isoformat(),
|
||||
"last_updated": self.last_updated.isoformat(),
|
||||
"item_count": self.item_count,
|
||||
"sentiment_avg": round(self.sentiment_avg, 3),
|
||||
"item_ids": self.item_ids,
|
||||
}
|
||||
|
||||
|
||||
# ── Main entry point ────────────────────────────────────────────────
|
||||
def cluster_items(
|
||||
items: list[NewsItem],
|
||||
window_minutes: int = 30,
|
||||
eps: float = 0.4,
|
||||
min_samples: int = 2,
|
||||
) -> list[StoryCluster]:
|
||||
"""Cluster news items into stories.
|
||||
|
||||
Items are first grouped by 30-minute time windows, then DBSCAN runs
|
||||
on MinHash signatures within each window. Single-item clusters are
|
||||
kept (they're "noise" in DBSCAN terms but valid singleton stories).
|
||||
|
||||
`eps` is the Jaccard DISTANCE threshold (1 - similarity). Per the
|
||||
task spec, two items cluster together when Jaccard similarity >= 0.6,
|
||||
so distance <= 0.4, so eps=0.4. Tighten for stricter clusters.
|
||||
"""
|
||||
t0 = time.time()
|
||||
if not items:
|
||||
return []
|
||||
|
||||
# Group by time window
|
||||
windows: dict[datetime, list[NewsItem]] = {}
|
||||
for it in sorted(items, key=lambda x: x.published_at):
|
||||
bucket = it.published_at.replace(
|
||||
minute=(it.published_at.minute // window_minutes) * window_minutes,
|
||||
second=0,
|
||||
microsecond=0,
|
||||
)
|
||||
windows.setdefault(bucket, []).append(it)
|
||||
|
||||
stories: list[StoryCluster] = []
|
||||
for _bucket, group in windows.items():
|
||||
if len(group) == 1:
|
||||
# singleton — still a story
|
||||
it = group[0]
|
||||
stories.append(
|
||||
StoryCluster(
|
||||
cluster_id=hashlib.sha1(
|
||||
f"single:{it.id}:{it.published_at.isoformat()}".encode()
|
||||
).hexdigest()[:16],
|
||||
representative_title=it.title,
|
||||
source_urls=[it.url] if it.url else [],
|
||||
sources=[it.source] if it.source else [],
|
||||
first_seen=it.published_at,
|
||||
last_updated=it.published_at,
|
||||
item_count=1,
|
||||
sentiment_avg=it.sentiment,
|
||||
item_ids=[it.id],
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
sigs = [_minhash_signature(_shingles(f"{it.title} {it.body}")) for it in group]
|
||||
labels = _dbscan(sigs, eps=eps, min_samples=min_samples)
|
||||
# Singletons (label == -1) still become stories
|
||||
clusters: dict[int, list[int]] = {}
|
||||
for idx, lbl in enumerate(labels):
|
||||
clusters.setdefault(lbl if lbl != -1 else idx, []).append(idx)
|
||||
for _cid, indices in clusters.items():
|
||||
members = [group[i] for i in indices]
|
||||
# Pick representative = longest title (usually the most descriptive)
|
||||
rep = max(members, key=lambda x: len(x.title))
|
||||
sentiments = [m.sentiment for m in members if m.sentiment is not None]
|
||||
avg_sent = sum(sentiments) / len(sentiments) if sentiments else 0.0
|
||||
cluster_id = hashlib.sha1(
|
||||
":".join(sorted(m.id for m in members)).encode()
|
||||
).hexdigest()[:16]
|
||||
stories.append(
|
||||
StoryCluster(
|
||||
cluster_id=cluster_id,
|
||||
representative_title=rep.title,
|
||||
source_urls=[m.url for m in members if m.url],
|
||||
sources=sorted({m.source for m in members if m.source}),
|
||||
first_seen=min(m.published_at for m in members),
|
||||
last_updated=max(m.published_at for m in members),
|
||||
item_count=len(members),
|
||||
sentiment_avg=avg_sent,
|
||||
item_ids=[m.id for m in members],
|
||||
)
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"news_clustered items=%d stories=%d windows=%d elapsed_ms=%.1f",
|
||||
len(items),
|
||||
len(stories),
|
||||
len(windows),
|
||||
(time.time() - t0) * 1000,
|
||||
)
|
||||
return stories
|
||||
|
||||
|
||||
# ── DB persistence (optional, lazy import) ──────────────────────────
|
||||
_PG_SCHEMA_SQL = """
|
||||
CREATE TABLE IF NOT EXISTS news_clusters (
|
||||
cluster_id TEXT PRIMARY KEY,
|
||||
representative_title TEXT NOT NULL,
|
||||
first_seen TIMESTAMPTZ NOT NULL,
|
||||
last_updated TIMESTAMPTZ NOT NULL,
|
||||
item_count INTEGER NOT NULL DEFAULT 0,
|
||||
sentiment_avg DOUBLE PRECISION NOT NULL DEFAULT 0.0,
|
||||
source_urls JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
sources JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
item_ids JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS news_clusters_last_updated_idx
|
||||
ON news_clusters (last_updated DESC);
|
||||
"""
|
||||
|
||||
|
||||
async def ensure_schema() -> bool:
|
||||
"""Create news_clusters table if missing. Returns True on success."""
|
||||
try:
|
||||
import asyncpg
|
||||
|
||||
from app.core.db_pool import PG_URL
|
||||
|
||||
conn = await asyncpg.connect(PG_URL)
|
||||
try:
|
||||
await conn.execute(_PG_SCHEMA_SQL)
|
||||
finally:
|
||||
await conn.close()
|
||||
logger.info("news_clusters_schema_ready")
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning("news_clusters_schema_failed err=%s", exc)
|
||||
return False
|
||||
|
||||
|
||||
async def persist_clusters(stories: list[StoryCluster]) -> int:
|
||||
"""Upsert stories to Postgres. Returns rows affected."""
|
||||
if not stories:
|
||||
return 0
|
||||
try:
|
||||
import json
|
||||
|
||||
import asyncpg
|
||||
|
||||
from app.core.db_pool import PG_URL
|
||||
|
||||
conn = await asyncpg.connect(PG_URL)
|
||||
try:
|
||||
rows = [
|
||||
(
|
||||
s.cluster_id,
|
||||
s.representative_title,
|
||||
s.first_seen,
|
||||
s.last_updated,
|
||||
s.item_count,
|
||||
s.sentiment_avg,
|
||||
json.dumps(s.source_urls),
|
||||
json.dumps(s.sources),
|
||||
json.dumps(s.item_ids),
|
||||
)
|
||||
for s in stories
|
||||
]
|
||||
await conn.executemany(
|
||||
"""
|
||||
INSERT INTO news_clusters
|
||||
(cluster_id, representative_title, first_seen, last_updated,
|
||||
item_count, sentiment_avg, source_urls, sources, item_ids)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7::jsonb,$8::jsonb,$9::jsonb)
|
||||
ON CONFLICT (cluster_id) DO UPDATE SET
|
||||
representative_title = EXCLUDED.representative_title,
|
||||
first_seen = EXCLUDED.first_seen,
|
||||
last_updated = EXCLUDED.last_updated,
|
||||
item_count = EXCLUDED.item_count,
|
||||
sentiment_avg = EXCLUDED.sentiment_avg,
|
||||
source_urls = EXCLUDED.source_urls,
|
||||
sources = EXCLUDED.sources,
|
||||
item_ids = EXCLUDED.item_ids
|
||||
""",
|
||||
rows,
|
||||
)
|
||||
return len(rows)
|
||||
finally:
|
||||
await conn.close()
|
||||
except Exception as exc:
|
||||
logger.warning("news_clusters_persist_failed err=%s", exc)
|
||||
return 0
|
||||
|
||||
|
||||
async def load_recent_clusters(limit: int = 50) -> list[dict]:
|
||||
"""Load recent clusters from Postgres."""
|
||||
try:
|
||||
import json
|
||||
|
||||
import asyncpg
|
||||
|
||||
from app.core.db_pool import PG_URL
|
||||
|
||||
conn = await asyncpg.connect(PG_URL)
|
||||
try:
|
||||
rows = await conn.fetch(
|
||||
"SELECT * FROM news_clusters ORDER BY last_updated DESC LIMIT $1",
|
||||
limit,
|
||||
)
|
||||
return [
|
||||
{
|
||||
"cluster_id": r["cluster_id"],
|
||||
"representative_title": r["representative_title"],
|
||||
"first_seen": r["first_seen"].isoformat(),
|
||||
"last_updated": r["last_updated"].isoformat(),
|
||||
"item_count": r["item_count"],
|
||||
"sentiment_avg": r["sentiment_avg"],
|
||||
"source_urls": json.loads(r["source_urls"]),
|
||||
"sources": json.loads(r["sources"]),
|
||||
"item_ids": json.loads(r["item_ids"]),
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
finally:
|
||||
await conn.close()
|
||||
except Exception as exc:
|
||||
logger.warning("news_clusters_load_failed err=%s", exc)
|
||||
return []
|
||||
|
||||
|
||||
__all__ = [
|
||||
"NewsItem",
|
||||
"StoryCluster",
|
||||
"cluster_items",
|
||||
"ensure_schema",
|
||||
"load_recent_clusters",
|
||||
"persist_clusters",
|
||||
]
|
||||
218
app/domain/news/ingest.py
Normal file
218
app/domain/news/ingest.py
Normal file
|
|
@ -0,0 +1,218 @@
|
|||
"""T28 RSS Ingest — populates news_items + crypto_news from RSS feeds.
|
||||
|
||||
Sources (v4.0 master stack): 5+ RSS feeds
|
||||
- CoinDesk
|
||||
- Cointelegraph
|
||||
- The Block
|
||||
- Decrypt
|
||||
- BeInCrypto
|
||||
|
||||
Caches raw HTML to MinIO, writes metadata to news_items,
|
||||
embeds into RAG engine for semantic search.
|
||||
|
||||
Run as a cron: every 15 minutes.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import hashlib
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import feedparser
|
||||
import httpx
|
||||
from pydantic import HttpUrl
|
||||
|
||||
from app.catalog.models import NewsItem, utcnow
|
||||
from app.catalog.service import get_catalog
|
||||
from app.core.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Default feeds (per v4.0 §T28)
|
||||
DEFAULT_FEEDS: list[dict[str, str]] = [
|
||||
{"name": "coindesk", "url": "https://www.coindesk.com/arc/outboundfeeds/rss/"},
|
||||
{"name": "cointelegraph", "url": "https://cointelegraph.com/rss"},
|
||||
{"name": "theblock", "url": "https://www.theblock.co/rss.xml"},
|
||||
{"name": "decrypt", "url": "https://decrypt.co/feed"},
|
||||
{"name": "beincrypto", "url": "https://beincrypto.com/feed/"},
|
||||
]
|
||||
|
||||
|
||||
def _stable_id(source: str, url: str) -> str:
|
||||
"""Stable news_id from source + URL."""
|
||||
h = hashlib.md5(f"{source}:{url}".encode()).hexdigest()[:24]
|
||||
return f"{source}:{h}"
|
||||
|
||||
|
||||
def _detect_chains(text: str) -> list[str]:
|
||||
"""Heuristic chain detection from text content."""
|
||||
text_l = text.lower()
|
||||
chains = []
|
||||
chain_map = {
|
||||
"solana": ["solana", "sol ", "$sol"],
|
||||
"ethereum": ["ethereum", "eth ", "$eth", "ether"],
|
||||
"bitcoin": ["bitcoin", "btc ", "$btc"],
|
||||
"base": ["base ", "basechain", "coinbase base"],
|
||||
"arbitrum": ["arbitrum", "arb "],
|
||||
"polygon": ["polygon", "matic"],
|
||||
"bsc": ["bsc", "bnb chain", "binance smart chain"],
|
||||
"tron": ["tron", "trx "],
|
||||
"avalanche": ["avalanche", "avax"],
|
||||
}
|
||||
for chain, keywords in chain_map.items():
|
||||
if any(k in text_l for k in keywords):
|
||||
chains.append(chain)
|
||||
return chains
|
||||
|
||||
|
||||
def _extract_tickers(text: str) -> list[str]:
|
||||
"""Extract $TICKER style mentions."""
|
||||
import re
|
||||
|
||||
return list(set(re.findall(r"\$([A-Z]{2,6})\b", text)))
|
||||
|
||||
|
||||
async def fetch_feed(client: httpx.AsyncClient, feed: dict[str, str]) -> list[dict]:
|
||||
"""Fetch and parse a single RSS feed."""
|
||||
try:
|
||||
r = await client.get(feed["url"], timeout=10.0, follow_redirects=True)
|
||||
r.raise_for_status()
|
||||
parsed = feedparser.parse(r.content)
|
||||
items = []
|
||||
for entry in parsed.entries[:30]: # cap per feed
|
||||
items.append(
|
||||
{
|
||||
"source": feed["name"],
|
||||
"title": entry.get("title", ""),
|
||||
"url": entry.get("link", ""),
|
||||
"summary": entry.get("summary", "")[:2000],
|
||||
"published": entry.get("published_parsed") or entry.get("updated_parsed"),
|
||||
}
|
||||
)
|
||||
return items
|
||||
except Exception as e:
|
||||
log.warning("feed_fetch_fail name=%s err=%s", feed["name"], e)
|
||||
return []
|
||||
|
||||
|
||||
async def ingest_all(
|
||||
feeds: list[dict[str, str]] | None = None,
|
||||
embed: bool = True,
|
||||
collection: str = "news_articles",
|
||||
) -> dict:
|
||||
"""Ingest from all configured RSS feeds. Returns counts."""
|
||||
feeds = feeds or DEFAULT_FEEDS
|
||||
cat = get_catalog()
|
||||
await cat._init_stores()
|
||||
if not cat._health.postgres:
|
||||
return {"error": "postgres unavailable"}
|
||||
|
||||
counts = {"feeds_ok": 0, "feeds_fail": 0, "items": 0, "ingested": 0, "duplicate": 0, "errors": 0}
|
||||
async with httpx.AsyncClient() as client:
|
||||
# Fetch all feeds in parallel
|
||||
results = await asyncio.gather(
|
||||
*[fetch_feed(client, f) for f in feeds], return_exceptions=True
|
||||
)
|
||||
items_to_ingest: list[NewsItem] = []
|
||||
for _i, r in enumerate(results):
|
||||
if isinstance(r, Exception) or not r:
|
||||
counts["feeds_fail"] += 1
|
||||
continue
|
||||
counts["feeds_ok"] += 1
|
||||
for entry in r:
|
||||
counts["items"] += 1
|
||||
try:
|
||||
pub_dt = utcnow()
|
||||
if entry.get("published"):
|
||||
with contextlib.suppress(Exception):
|
||||
pub_dt = datetime(*entry["published"][:6], tzinfo=UTC)
|
||||
text = f"{entry['title']} {entry['summary']}"
|
||||
chains = _detect_chains(text)
|
||||
tickers = _extract_tickers(text)
|
||||
news_id = _stable_id(entry["source"], entry["url"])
|
||||
ni = NewsItem(
|
||||
news_id=news_id,
|
||||
url=HttpUrl(entry["url"]) if entry["url"] else HttpUrl("https://unknown.local"),
|
||||
title=entry["title"][:500],
|
||||
summary=entry["summary"][:2000],
|
||||
body_markdown=None,
|
||||
source=entry["source"],
|
||||
published_at=pub_dt,
|
||||
ingested_at=utcnow(),
|
||||
chains_mentioned=chains,
|
||||
tokens_mentioned=tickers,
|
||||
)
|
||||
items_to_ingest.append(ni)
|
||||
except Exception as e:
|
||||
counts["errors"] += 1
|
||||
log.debug("item_build_fail: %s", e)
|
||||
|
||||
# Save to Postgres
|
||||
if cat._health.postgres and items_to_ingest:
|
||||
try:
|
||||
async with cat._pg_pool.acquire() as conn:
|
||||
for ni in items_to_ingest:
|
||||
try:
|
||||
# Check if exists
|
||||
existing = await conn.fetchval(
|
||||
"SELECT 1 FROM news_items WHERE news_id=$1", ni.news_id
|
||||
)
|
||||
if existing:
|
||||
counts["duplicate"] += 1
|
||||
continue
|
||||
# Insert
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO news_items (
|
||||
news_id, url, title, summary, body_markdown,
|
||||
source, published_at, ingested_at,
|
||||
chains_mentioned, tokens_mentioned, sentiment_score
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)
|
||||
""",
|
||||
ni.news_id, str(ni.url), ni.title, ni.summary, ni.body_markdown,
|
||||
ni.source, ni.published_at, ni.ingested_at,
|
||||
ni.chains_mentioned, ni.tokens_mentioned, ni.sentiment_score,
|
||||
)
|
||||
counts["ingested"] += 1
|
||||
except Exception as e:
|
||||
counts["errors"] += 1
|
||||
log.debug("item_insert_fail: %s", e)
|
||||
except Exception as e:
|
||||
log.warning("ingest_postgres_fail: %s", e)
|
||||
|
||||
# Embed into RAG for semantic search
|
||||
if embed and items_to_ingest:
|
||||
try:
|
||||
for ni in items_to_ingest[:50]: # cap RAG embeds
|
||||
r = await cat.rag_ingest(
|
||||
content=f"{ni.title}\n{ni.summary}",
|
||||
collection=collection,
|
||||
doc_id=ni.news_id,
|
||||
metadata={"source": ni.source, "news_id": ni.news_id},
|
||||
)
|
||||
if r.get("status") == "ok" and r.get("qdrant_point_id"):
|
||||
# Update news_items with rag_embedding_id
|
||||
try:
|
||||
async with cat._pg_pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"UPDATE news_items SET rag_embedding_id=$1 WHERE news_id=$2",
|
||||
r["qdrant_point_id"], ni.news_id,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
log.warning("ingest_rag_fail: %s", e)
|
||||
|
||||
return counts
|
||||
|
||||
|
||||
# ── CLI entry point ──────────────────────────────────────────────
|
||||
if __name__ == "__main__":
|
||||
import json
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
result = asyncio.run(ingest_all())
|
||||
logger.info(json.dumps(result, indent=2))
|
||||
450
app/domain/news/router.py
Normal file
450
app/domain/news/router.py
Normal file
|
|
@ -0,0 +1,450 @@
|
|||
"""T28 News Intelligence Product.
|
||||
|
||||
Per v4.0 §T28. Four endpoints surface the news pipeline as a product:
|
||||
POST /api/v1/news list, paginated, filterable
|
||||
GET /api/v1/news/trending time-decay weighted
|
||||
GET /api/v1/news/{news_id} single item
|
||||
POST /api/v1/news/{news_id}/analyze LLM analysis (LiteLLM)
|
||||
|
||||
Data sources:
|
||||
- crypto_news (legacy table, 1750+ items from RSS feeds)
|
||||
- news_items (new catalog table, populated by RSS ingest)
|
||||
- Qdrant embeddings for semantic search (rag_embedding_id)
|
||||
|
||||
Time-decay scoring for /trending:
|
||||
score = recency_decay * source_authority * social_velocity * abs(sentiment)
|
||||
recency_decay = 0.5 ** (hours_old / 6) half-life 6h
|
||||
source_authority: tier-1 (CoinDesk, The Block) = 1.0, tier-2 = 0.5, tier-3 = 0.2
|
||||
social_velocity: tweets/shares in last hour (1 + n/100, capped at 2x)
|
||||
abs(sentiment): polarizing news ranks higher
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from app.catalog.llm_router import LLMRouter
|
||||
from app.catalog.models import utcnow
|
||||
from app.catalog.service import get_catalog
|
||||
|
||||
router = APIRouter(prefix="/api/v1/news", tags=["news"])
|
||||
|
||||
|
||||
# ── Time-decay scoring (per v4.0 §T28) ────────────────────────────
|
||||
SOURCE_AUTHORITY: dict[str, float] = {
|
||||
# Tier 1 — major crypto-native outlets
|
||||
"coindesk": 1.0,
|
||||
"the block": 1.0,
|
||||
"decrypt": 1.0,
|
||||
"cointelegraph": 1.0,
|
||||
# Tier 2 — solid crypto coverage
|
||||
"beincrypto": 0.7,
|
||||
"u.today": 0.7,
|
||||
"crypto.news": 0.7,
|
||||
"blockworks": 0.8,
|
||||
# Tier 3 — general / RSS aggregators
|
||||
"google-crypto": 0.5,
|
||||
"reddit-crypto": 0.4,
|
||||
"twitter-crypto": 0.3,
|
||||
}
|
||||
|
||||
|
||||
def recency_decay(hours_old: float, half_life: float = 6.0) -> float:
|
||||
"""Exponential decay: 1.0 at 0h, 0.5 at half_life, 0.25 at 2*half_life."""
|
||||
if hours_old < 0:
|
||||
return 1.0
|
||||
return 0.5 ** (hours_old / half_life)
|
||||
|
||||
|
||||
def source_authority(source: str) -> float:
|
||||
s = (source or "").lower().strip()
|
||||
for key, val in SOURCE_AUTHORITY.items():
|
||||
if key in s:
|
||||
return val
|
||||
return 0.3 # unknown source
|
||||
|
||||
|
||||
def trend_score(
|
||||
hours_old: float, source: str, sentiment: float | None, social_velocity: int = 0
|
||||
) -> float:
|
||||
"""Composite trending score per v4.0 formula."""
|
||||
s_auth = source_authority(source)
|
||||
s_vel = min(2.0, 1.0 + social_velocity / 100.0)
|
||||
s_sent = 1.0 + abs(sentiment or 0.0)
|
||||
return recency_decay(hours_old) * s_auth * s_vel * s_sent
|
||||
|
||||
|
||||
# ── Response models ────────────────────────────────────────────────
|
||||
class NewsItemOut(BaseModel):
|
||||
model_config = ConfigDict(strict=False) # accept None for any field with default
|
||||
|
||||
news_id: str | None = None
|
||||
url: str | None = None
|
||||
title: str | None = None
|
||||
summary: str | None = ""
|
||||
source: str | None = "unknown"
|
||||
published_at: datetime | None = None
|
||||
chains_mentioned: list[str] = Field(default_factory=list)
|
||||
tokens_mentioned: list[str] = Field(default_factory=list)
|
||||
sentiment_score: float | None = None
|
||||
score: float | None = None # only set for /trending
|
||||
|
||||
|
||||
class NewsListResponse(BaseModel):
|
||||
items: list[NewsItemOut]
|
||||
total: int
|
||||
offset: int
|
||||
|
||||
|
||||
class NewsAnalysisResponse(BaseModel):
|
||||
news_id: str
|
||||
analysis: str | None
|
||||
model: str | None = None
|
||||
error: str | None = None
|
||||
|
||||
|
||||
# ── Row adapter (legacy crypto_news → NewsItemOut) ───────────────
|
||||
def _adapt_legacy_row(row: dict) -> NewsItemOut:
|
||||
"""Convert a crypto_news row to NewsItemOut shape."""
|
||||
# published is a text field in legacy; try ISO parse
|
||||
pub_str = row.get("published") or row.get("ingested_at")
|
||||
pub_dt = utcnow()
|
||||
if pub_str:
|
||||
try:
|
||||
if isinstance(pub_str, (int, float)):
|
||||
pub_dt = datetime.fromtimestamp(float(pub_str), tz=UTC)
|
||||
else:
|
||||
# Try common formats
|
||||
for fmt in (
|
||||
"%Y-%m-%dT%H:%M:%S.%fZ",
|
||||
"%Y-%m-%dT%H:%M:%SZ",
|
||||
"%Y-%m-%dT%H:%M:%S",
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
):
|
||||
try:
|
||||
pub_dt = datetime.strptime(str(pub_str)[:19], fmt).replace(tzinfo=UTC)
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
return NewsItemOut(
|
||||
news_id=row.get("id", ""),
|
||||
url=row.get("url", ""),
|
||||
title=row.get("title", ""),
|
||||
summary=(row.get("content") or "")[:500],
|
||||
source=row.get("source", "unknown"),
|
||||
published_at=pub_dt,
|
||||
chains_mentioned=[],
|
||||
tokens_mentioned=row.get("tickers") or [],
|
||||
sentiment_score=row.get("sentiment"),
|
||||
)
|
||||
|
||||
|
||||
# ── POST /api/v1/news (list with filters) ─────────────────────────
|
||||
@router.post("", response_model=NewsListResponse)
|
||||
async def list_news(
|
||||
chain: str | None = None,
|
||||
token: str | None = None,
|
||||
category: str | None = None,
|
||||
since_hours: int = Query(24, ge=1, le=720),
|
||||
limit: int = Query(20, ge=1, le=200),
|
||||
offset: int = Query(0, ge=0),
|
||||
sort: str = Query("recency", pattern="^(recency|relevance|sentiment)$"),
|
||||
clustered: bool = Query(False, description="T03: dedupe via MinHash+DBSCAN into stories"),
|
||||
) -> NewsListResponse:
|
||||
"""List news items with filters. Reads from both news_items (new) and crypto_news (legacy)."""
|
||||
catalog = get_catalog()
|
||||
await catalog._init_stores()
|
||||
items: list[NewsItemOut] = []
|
||||
|
||||
# New table
|
||||
if catalog._health.postgres:
|
||||
try:
|
||||
cutoff = utcnow() - timedelta(hours=since_hours)
|
||||
query = "SELECT news_id, url, title, summary, source, published_at, sentiment_score, chains_mentioned, tokens_mentioned FROM news_items WHERE published_at > $1"
|
||||
params: list[Any] = [cutoff]
|
||||
if chain:
|
||||
query += f" AND ${len(params)+1} = ANY(chains_mentioned)"
|
||||
params.append(chain)
|
||||
if token:
|
||||
query += f" AND ${len(params)+1} = ANY(tokens_mentioned)"
|
||||
params.append(token)
|
||||
if sort == "sentiment":
|
||||
query += " ORDER BY sentiment_score ASC NULLS LAST"
|
||||
else:
|
||||
query += " ORDER BY published_at DESC"
|
||||
query += f" LIMIT {limit} OFFSET {offset}"
|
||||
async with catalog._pg_pool.acquire() as conn:
|
||||
rows = await conn.fetch(query, *params)
|
||||
for r in rows:
|
||||
items.append(
|
||||
NewsItemOut(
|
||||
news_id=r["news_id"],
|
||||
url=r["url"],
|
||||
title=r["title"],
|
||||
summary=r["summary"] or "",
|
||||
source=r["source"],
|
||||
published_at=r["published_at"],
|
||||
chains_mentioned=list(r["chains_mentioned"] or []),
|
||||
tokens_mentioned=list(r["tokens_mentioned"] or []),
|
||||
sentiment_score=r["sentiment_score"],
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
import logging
|
||||
logging.getLogger(__name__).warning(f"news_list_new_fail: {e}")
|
||||
|
||||
# Legacy fallback (crypto_news)
|
||||
if not items and catalog._health.postgres:
|
||||
try:
|
||||
query = "SELECT id, title, content, url, source, sentiment, tickers, published, ingested_at FROM crypto_news WHERE 1=1"
|
||||
params = []
|
||||
if category:
|
||||
query += f" AND category = ${len(params)+1}"
|
||||
params.append(category)
|
||||
query += " ORDER BY ingested_at DESC LIMIT $%d OFFSET $%d" % (len(params)+1, len(params)+2)
|
||||
params.extend([limit, offset])
|
||||
async with catalog._pg_pool.acquire() as conn:
|
||||
rows = await conn.fetch(query, *params)
|
||||
for r in rows:
|
||||
items.append(_adapt_legacy_row(dict(r)))
|
||||
except Exception as e:
|
||||
import logging
|
||||
logging.getLogger(__name__).warning(f"news_list_legacy_fail: {e}")
|
||||
|
||||
# T03: cluster into stories if requested
|
||||
if clustered and items:
|
||||
from app.domain.news.clusterer import NewsItem, cluster_items, persist_clusters
|
||||
cluster_items_list = [
|
||||
NewsItem(
|
||||
id=it.news_id,
|
||||
title=it.title,
|
||||
body=it.summary or "",
|
||||
source=it.source or "",
|
||||
url=it.url or "",
|
||||
published_at=it.published_at,
|
||||
sentiment=it.sentiment_score or 0.0,
|
||||
)
|
||||
for it in items
|
||||
]
|
||||
stories = cluster_items(cluster_items_list)
|
||||
# persist in background (don't block response)
|
||||
try:
|
||||
import asyncio
|
||||
asyncio.create_task(persist_clusters(stories))
|
||||
except Exception:
|
||||
pass
|
||||
# Return clusters as synthetic items (representative title, first source)
|
||||
clustered_items = []
|
||||
for s in stories:
|
||||
clustered_items.append(
|
||||
NewsItemOut(
|
||||
news_id=s.cluster_id,
|
||||
url=s.source_urls[0] if s.source_urls else "",
|
||||
title=f"[×{s.item_count}] {s.representative_title}",
|
||||
summary=f"Story across {len(s.sources)} sources. "
|
||||
f"Sentiment: {s.sentiment_avg:.2f}. "
|
||||
f"Item IDs: {','.join(s.item_ids[:5])}",
|
||||
source=", ".join(s.sources[:3]),
|
||||
published_at=s.last_updated,
|
||||
chains_mentioned=[],
|
||||
tokens_mentioned=[],
|
||||
sentiment_score=s.sentiment_avg,
|
||||
)
|
||||
)
|
||||
return NewsListResponse(items=clustered_items, total=len(clustered_items), offset=offset)
|
||||
|
||||
return NewsListResponse(items=items, total=len(items), offset=offset)
|
||||
|
||||
|
||||
# ── GET /api/v1/news/trending ─────────────────────────────────────
|
||||
@router.get("/trending", response_model=NewsListResponse)
|
||||
async def trending_news(
|
||||
window_hours: int = Query(168, ge=1, le=720),
|
||||
limit: int = Query(20, ge=1, le=100),
|
||||
) -> NewsListResponse:
|
||||
"""Time-decay trending. Reads from news_items (new) primary, falls back to crypto_news (legacy)."""
|
||||
catalog = get_catalog()
|
||||
await catalog._init_stores()
|
||||
if not catalog._health.postgres:
|
||||
return NewsListResponse(items=[], total=0, offset=0)
|
||||
now = utcnow()
|
||||
cutoff = now - timedelta(hours=window_hours)
|
||||
items: list[NewsItemOut] = []
|
||||
# Primary: news_items
|
||||
try:
|
||||
async with catalog._pg_pool.acquire() as conn:
|
||||
rows = await conn.fetch(
|
||||
"SELECT news_id, url, title, summary, source, published_at, "
|
||||
"sentiment_score, chains_mentioned, tokens_mentioned "
|
||||
"FROM news_items "
|
||||
"WHERE published_at > $1 "
|
||||
"ORDER BY published_at DESC LIMIT 500",
|
||||
cutoff,
|
||||
)
|
||||
for r in rows:
|
||||
hours_old = max(0, (now - r["published_at"]).total_seconds() / 3600)
|
||||
item = NewsItemOut(
|
||||
news_id=r["news_id"],
|
||||
url=r["url"] or "",
|
||||
title=r["title"] or "",
|
||||
summary=r["summary"] or "",
|
||||
source=r["source"] or "unknown",
|
||||
published_at=r["published_at"],
|
||||
chains_mentioned=list(r["chains_mentioned"] or []),
|
||||
tokens_mentioned=list(r["tokens_mentioned"] or []),
|
||||
sentiment_score=r["sentiment_score"],
|
||||
)
|
||||
item.score = round(
|
||||
trend_score(hours_old, item.source, item.sentiment_score), 4
|
||||
)
|
||||
items.append(item)
|
||||
except Exception as e:
|
||||
import logging
|
||||
logging.getLogger(__name__).warning(f"trending_new_fail: {e}")
|
||||
|
||||
# Fallback: crypto_news (legacy) if no new items
|
||||
if not items:
|
||||
try:
|
||||
cutoff_epoch = cutoff.timestamp()
|
||||
async with catalog._pg_pool.acquire() as conn:
|
||||
rows = await conn.fetch(
|
||||
"SELECT id, title, content, url, source, sentiment, tickers, "
|
||||
"published, ingested_at, category "
|
||||
"FROM crypto_news "
|
||||
"WHERE ingested_at > $1 "
|
||||
"ORDER BY ingested_at DESC LIMIT 500",
|
||||
cutoff_epoch,
|
||||
)
|
||||
for r in rows:
|
||||
d = dict(r)
|
||||
hours_old = 0.0
|
||||
try:
|
||||
if d.get("ingested_at"):
|
||||
hours_old = max(0, (now.timestamp() - float(d["ingested_at"])) / 3600)
|
||||
except Exception:
|
||||
pass
|
||||
item = _adapt_legacy_row(d)
|
||||
item.score = round(trend_score(hours_old, item.source, item.sentiment_score), 4)
|
||||
items.append(item)
|
||||
except Exception as e:
|
||||
import logging
|
||||
logging.getLogger(__name__).warning(f"trending_legacy_fail: {e}")
|
||||
|
||||
items.sort(key=lambda x: x.score or 0, reverse=True)
|
||||
return NewsListResponse(items=items[:limit], total=len(items), offset=0)
|
||||
|
||||
|
||||
# ── GET /api/v1/news/{news_id} ────────────────────────────────────
|
||||
@router.get("/{news_id}", response_model=NewsItemOut)
|
||||
async def get_news(news_id: str) -> NewsItemOut:
|
||||
"""Single news item. Searches both news_items and crypto_news."""
|
||||
catalog = get_catalog()
|
||||
await catalog._init_stores()
|
||||
if not catalog._health.postgres:
|
||||
raise HTTPException(503, "postgres unavailable")
|
||||
try:
|
||||
async with catalog._pg_pool.acquire() as conn:
|
||||
r = await conn.fetchrow(
|
||||
"SELECT news_id, url, title, summary, source, published_at, "
|
||||
"sentiment_score, chains_mentioned, tokens_mentioned "
|
||||
"FROM news_items WHERE news_id=$1",
|
||||
news_id,
|
||||
)
|
||||
if r:
|
||||
return NewsItemOut(
|
||||
news_id=r["news_id"],
|
||||
url=r["url"],
|
||||
title=r["title"],
|
||||
summary=r["summary"] or "",
|
||||
source=r["source"],
|
||||
published_at=r["published_at"],
|
||||
chains_mentioned=list(r["chains_mentioned"] or []),
|
||||
tokens_mentioned=list(r["tokens_mentioned"] or []),
|
||||
sentiment_score=r["sentiment_score"],
|
||||
)
|
||||
r2 = await conn.fetchrow(
|
||||
"SELECT id, title, content, url, source, sentiment, tickers, "
|
||||
"published, ingested_at, category "
|
||||
"FROM crypto_news WHERE id=$1",
|
||||
news_id,
|
||||
)
|
||||
if r2:
|
||||
return _adapt_legacy_row(dict(r2))
|
||||
raise HTTPException(404, "news item not found")
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(500, f"news_get_fail: {e}")
|
||||
|
||||
|
||||
# ── POST /api/v1/news/{news_id}/analyze ───────────────────────────
|
||||
@router.post("/{news_id}/analyze", response_model=NewsAnalysisResponse)
|
||||
async def analyze_news(news_id: str) -> NewsAnalysisResponse:
|
||||
"""Generate LLM analysis via LiteLLM. Falls back to None if LLM unreachable."""
|
||||
catalog = get_catalog()
|
||||
await catalog._init_stores()
|
||||
if not catalog._health.postgres:
|
||||
raise HTTPException(503, "postgres unavailable")
|
||||
try:
|
||||
# Fetch the news item
|
||||
item: NewsItemOut | None = None
|
||||
async with catalog._pg_pool.acquire() as conn:
|
||||
r = await conn.fetchrow(
|
||||
"SELECT news_id, url, title, summary, source, published_at, "
|
||||
"sentiment_score, chains_mentioned, tokens_mentioned "
|
||||
"FROM news_items WHERE news_id=$1",
|
||||
news_id,
|
||||
)
|
||||
if r:
|
||||
item = NewsItemOut(
|
||||
news_id=r["news_id"],
|
||||
url=r["url"],
|
||||
title=r["title"],
|
||||
summary=r["summary"] or "",
|
||||
source=r["source"],
|
||||
published_at=r["published_at"],
|
||||
chains_mentioned=list(r["chains_mentioned"] or []),
|
||||
tokens_mentioned=list(r["tokens_mentioned"] or []),
|
||||
sentiment_score=r["sentiment_score"],
|
||||
)
|
||||
if not item:
|
||||
r2 = await conn.fetchrow(
|
||||
"SELECT id, title, content, url, source, sentiment, tickers, "
|
||||
"published, ingested_at FROM crypto_news WHERE id=$1",
|
||||
news_id,
|
||||
)
|
||||
if r2:
|
||||
item = _adapt_legacy_row(dict(r2))
|
||||
if not item:
|
||||
raise HTTPException(404, "news item not found")
|
||||
# Build a NewsItem for the LLM router
|
||||
from app.catalog.models import NewsItem
|
||||
|
||||
ni = NewsItem(
|
||||
news_id=item.news_id,
|
||||
url=item.url or "https://unknown.local", # HttpUrl requires non-empty
|
||||
title=item.title or "",
|
||||
summary=item.summary or "",
|
||||
body_markdown=item.summary or "",
|
||||
source=item.source or "unknown",
|
||||
published_at=item.published_at or utcnow(),
|
||||
ingested_at=utcnow(),
|
||||
)
|
||||
llm = LLMRouter()
|
||||
analysis = await llm.analyze_news(ni)
|
||||
if analysis is None:
|
||||
return NewsAnalysisResponse(
|
||||
news_id=news_id, analysis=None, error="LLM router unavailable"
|
||||
)
|
||||
return NewsAnalysisResponse(
|
||||
news_id=news_id, analysis=analysis, model="deepseek-v3"
|
||||
)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
return NewsAnalysisResponse(news_id=news_id, analysis=None, error=str(e))
|
||||
5
app/domain/reports/__init__.py
Normal file
5
app/domain/reports/__init__.py
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
"""T29 Reports — thin HTTP layer."""
|
||||
|
||||
from .router import router
|
||||
|
||||
__all__ = ["router"]
|
||||
315
app/domain/reports/citation_validator.py
Normal file
315
app/domain/reports/citation_validator.py
Normal file
|
|
@ -0,0 +1,315 @@
|
|||
"""T05 — RAG Citation Validator.
|
||||
|
||||
Per RMIV5 §T05 (G05 FIX). After an LLM generates a report section from
|
||||
retrieved RAG chunks, every claim in the output must cite a source by
|
||||
number [1], [2], etc. This module enforces that.
|
||||
|
||||
Pipeline:
|
||||
1. LLM produces text constrained to retrieved chunks (in generator.py)
|
||||
2. This validator parses every [N] citation in the text
|
||||
3. Verifies that:
|
||||
a. N is a valid index into the retrieved chunks list
|
||||
b. The cited sentence/paragraph is supported by source N
|
||||
(substring match on key terms)
|
||||
4. Returns a citation report:
|
||||
{
|
||||
"validated_text": str with unciteable claims replaced by
|
||||
"[Data not available]" or removed,
|
||||
"citations": [{"claim": str, "source_idx": int, "source_text": str}],
|
||||
"unciteable_count": int,
|
||||
"validation_rate": float # 0.0-1.0
|
||||
}
|
||||
|
||||
Why this exists:
|
||||
Reports that hallucinate destroy trust. Every claim in a $5 report
|
||||
must be backed by a source we can show. If we can't find support,
|
||||
we say so — explicitly — rather than fabricating.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
# ── Regexes ──────────────────────────────────────────────────────────
|
||||
# Match inline citations like "...some claim [1]..." or "[2, 3]" or "[1-3]"
|
||||
_CITATION_RE = re.compile(r"\[(\d+(?:\s*[,\-]\s*\d+)*)\]")
|
||||
# Match sentences (rough — splits on .!? followed by whitespace + uppercase)
|
||||
_SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?])\s+(?=[A-Z\d])")
|
||||
# Key term extraction (rough): words with 4+ chars, lowercase, no stopwords
|
||||
_STOPWORDS = frozenset(
|
||||
{
|
||||
"the",
|
||||
"and",
|
||||
"for",
|
||||
"are",
|
||||
"but",
|
||||
"not",
|
||||
"you",
|
||||
"all",
|
||||
"can",
|
||||
"had",
|
||||
"her",
|
||||
"was",
|
||||
"one",
|
||||
"our",
|
||||
"out",
|
||||
"day",
|
||||
"get",
|
||||
"has",
|
||||
"him",
|
||||
"his",
|
||||
"how",
|
||||
"its",
|
||||
"may",
|
||||
"new",
|
||||
"now",
|
||||
"old",
|
||||
"see",
|
||||
"two",
|
||||
"way",
|
||||
"who",
|
||||
"boy",
|
||||
"did",
|
||||
"use",
|
||||
"what",
|
||||
"when",
|
||||
"this",
|
||||
"that",
|
||||
"with",
|
||||
"from",
|
||||
"have",
|
||||
"been",
|
||||
"will",
|
||||
"they",
|
||||
"their",
|
||||
"which",
|
||||
"would",
|
||||
"there",
|
||||
"could",
|
||||
"about",
|
||||
"other",
|
||||
"into",
|
||||
"than",
|
||||
"more",
|
||||
"some",
|
||||
"very",
|
||||
"most",
|
||||
"only",
|
||||
"over",
|
||||
"such",
|
||||
"also",
|
||||
"after",
|
||||
"before",
|
||||
"should",
|
||||
"because",
|
||||
"where",
|
||||
"these",
|
||||
"those",
|
||||
"being",
|
||||
"through",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _key_terms(text: str) -> set[str]:
|
||||
"""Extract key terms (lowercase, 4+ chars, not stopwords) from text."""
|
||||
words = re.findall(r"\b[a-zA-Z]{4,}\b", text.lower())
|
||||
return {w for w in words if w not in _STOPWORDS}
|
||||
|
||||
|
||||
def _extract_citation_indices(citation_str: str, max_index: int) -> list[int]:
|
||||
"""Parse '1', '1,2,3', or '1-3' into [1, 2, 3] (1-indexed).
|
||||
|
||||
Out-of-range indices are silently dropped (we'll flag them as
|
||||
invalid in the citation report).
|
||||
"""
|
||||
result = []
|
||||
for part in citation_str.split(","):
|
||||
part = part.strip()
|
||||
if "-" in part:
|
||||
try:
|
||||
lo, hi = part.split("-", 1)
|
||||
lo_i, hi_i = int(lo.strip()), int(hi.strip())
|
||||
for n in range(lo_i, hi_i + 1):
|
||||
if 1 <= n <= max_index:
|
||||
result.append(n)
|
||||
except ValueError:
|
||||
continue
|
||||
else:
|
||||
try:
|
||||
n = int(part)
|
||||
if 1 <= n <= max_index:
|
||||
result.append(n)
|
||||
except ValueError:
|
||||
continue
|
||||
return result
|
||||
|
||||
|
||||
def _split_sentences_with_citations(text: str) -> list[tuple[str, list[int]]]:
|
||||
"""Split text into sentences, each annotated with its [N] citations.
|
||||
|
||||
A citation belongs to the sentence that contains it (not the
|
||||
preceding one). Returns [(sentence, [citation_indices])].
|
||||
"""
|
||||
sentences: list[tuple[str, list[int]]] = []
|
||||
for sent in _SENTENCE_SPLIT_RE.split(text.strip()):
|
||||
sent = sent.strip()
|
||||
if not sent:
|
||||
continue
|
||||
# Find all citations in this sentence
|
||||
matches = _CITATION_RE.findall(sent)
|
||||
indices: list[int] = []
|
||||
for m in matches:
|
||||
indices.extend(_extract_citation_indices(m, max_index=10_000))
|
||||
sentences.append((sent, sorted(set(indices))))
|
||||
return sentences
|
||||
|
||||
|
||||
def _claim_supported_by_source(claim: str, source_text: str, threshold: float = 0.4) -> bool:
|
||||
"""Check if the claim's key terms appear in the source text.
|
||||
|
||||
Uses Jaccard-like overlap on key terms (4+ chars, non-stopwords).
|
||||
A claim is "supported" if at least `threshold` of its key terms
|
||||
appear in the source. Threshold 0.4 = 40% overlap required.
|
||||
|
||||
This is a heuristic — it catches obvious fabrications (where the
|
||||
LLM cites a source but the claim isn't in it) without being so
|
||||
strict that paraphrased but accurate claims get flagged.
|
||||
"""
|
||||
claim_terms = _key_terms(claim)
|
||||
if not claim_terms:
|
||||
# No key terms (very short sentence) — assume supported
|
||||
return True
|
||||
source_terms = _key_terms(source_text)
|
||||
if not source_terms:
|
||||
return False
|
||||
overlap = len(claim_terms & source_terms)
|
||||
return (overlap / len(claim_terms)) >= threshold
|
||||
|
||||
|
||||
def validate_section(
|
||||
text: str,
|
||||
sources: list[str],
|
||||
*,
|
||||
min_support_overlap: float = 0.4,
|
||||
on_unciteable: str = "strip",
|
||||
) -> dict[str, Any]:
|
||||
"""Validate that every claim in `text` cites a real source.
|
||||
|
||||
Args:
|
||||
text: The LLM-generated section text.
|
||||
sources: List of source texts the LLM was supposed to use.
|
||||
Index 0 in this list = citation [1], etc.
|
||||
min_support_overlap: Minimum fraction of claim key terms that
|
||||
must appear in source for claim to be
|
||||
considered supported (default 0.4 = 40%).
|
||||
on_unciteable: What to do with unsupported claims.
|
||||
"strip" (default) — replace with [Data not available]
|
||||
"keep" — leave as-is, flag in citations report
|
||||
"drop" — remove the sentence entirely
|
||||
|
||||
Returns:
|
||||
{
|
||||
"validated_text": str,
|
||||
"citations": [{"claim": str, "source_idx": int,
|
||||
"source_text": str, "supported": bool}],
|
||||
"unciteable_count": int,
|
||||
"validation_rate": float, # supported/total
|
||||
}
|
||||
"""
|
||||
if not sources:
|
||||
# No sources provided — every claim is unciteable
|
||||
return {
|
||||
"validated_text": ("[Data not available — no RAG sources retrieved]" if on_unciteable == "strip" else text),
|
||||
"citations": [],
|
||||
"unciteable_count": _count_sentences(text),
|
||||
"validation_rate": 0.0,
|
||||
}
|
||||
|
||||
sentences = _split_sentences_with_citations(text)
|
||||
validated_sentences: list[str] = []
|
||||
citations: list[dict[str, Any]] = []
|
||||
unciteable_count = 0
|
||||
|
||||
for sent, indices in sentences:
|
||||
if not indices:
|
||||
# No citations at all — unciteable
|
||||
unciteable_count += 1
|
||||
citations.append({"claim": sent, "source_idx": 0, "source_text": "", "supported": False})
|
||||
if on_unciteable == "strip":
|
||||
validated_sentences.append("[Data not available]")
|
||||
elif on_unciteable == "keep":
|
||||
validated_sentences.append(sent)
|
||||
# "drop" — add nothing
|
||||
continue
|
||||
|
||||
# Filter out out-of-range indices (defensive — extract_citation_indices
|
||||
# already does this, but defense-in-depth for malformed input)
|
||||
valid_indices = [i for i in indices if 1 <= i <= len(sources)]
|
||||
if not valid_indices:
|
||||
# All citations were out of range — unciteable
|
||||
unciteable_count += 1
|
||||
citations.append(
|
||||
{
|
||||
"claim": sent,
|
||||
"source_idx": 0,
|
||||
"source_text": "",
|
||||
"supported": False,
|
||||
}
|
||||
)
|
||||
if on_unciteable == "strip":
|
||||
validated_sentences.append("[Data not available]")
|
||||
elif on_unciteable == "keep":
|
||||
validated_sentences.append(sent)
|
||||
continue
|
||||
|
||||
# We have at least one valid citation — check each one
|
||||
best_source_idx = valid_indices[0]
|
||||
source_text = sources[best_source_idx - 1] # [1] = sources[0]
|
||||
supported = _claim_supported_by_source(sent, source_text, threshold=min_support_overlap)
|
||||
|
||||
# If first citation isn't supported, try the others
|
||||
if not supported and len(valid_indices) > 1:
|
||||
for idx in valid_indices[1:]:
|
||||
candidate = sources[idx - 1]
|
||||
if _claim_supported_by_source(sent, candidate, threshold=min_support_overlap):
|
||||
best_source_idx = idx
|
||||
source_text = candidate
|
||||
supported = True
|
||||
break
|
||||
|
||||
citations.append(
|
||||
{
|
||||
"claim": sent,
|
||||
"source_idx": best_source_idx,
|
||||
"source_text": source_text[:200] + ("..." if len(source_text) > 200 else ""),
|
||||
"supported": supported,
|
||||
}
|
||||
)
|
||||
|
||||
if not supported:
|
||||
unciteable_count += 1
|
||||
if on_unciteable == "strip":
|
||||
validated_sentences.append("[Data not available]")
|
||||
elif on_unciteable == "keep":
|
||||
validated_sentences.append(sent)
|
||||
# "drop" — add nothing
|
||||
else:
|
||||
validated_sentences.append(sent)
|
||||
|
||||
validated_text = " ".join(validated_sentences).strip()
|
||||
total = len(citations)
|
||||
validation_rate = (total - unciteable_count) / total if total > 0 else 0.0
|
||||
|
||||
return {
|
||||
"validated_text": validated_text,
|
||||
"citations": citations,
|
||||
"unciteable_count": unciteable_count,
|
||||
"validation_rate": validation_rate,
|
||||
}
|
||||
|
||||
|
||||
def _count_sentences(text: str) -> int:
|
||||
"""Rough sentence count for empty-sources case."""
|
||||
return max(1, len([s for s in _SENTENCE_SPLIT_RE.split(text.strip()) if s.strip()]))
|
||||
576
app/domain/reports/generator.py
Normal file
576
app/domain/reports/generator.py
Normal file
|
|
@ -0,0 +1,576 @@
|
|||
"""T29 Research Report Generator.
|
||||
|
||||
Per v4.0 §T29. Given a token or wallet, compose a Markdown report
|
||||
from every data source, sold via x402 at $5/report.
|
||||
|
||||
Sections (parallel-composable):
|
||||
- executive_summary (LLM)
|
||||
- onchain (catalog + RAG)
|
||||
- deployer (Neo4j + reputation)
|
||||
- news_sentiment (news_items + LLM summary)
|
||||
- rag_findings (RAG engine)
|
||||
- social_signals (placeholder v1)
|
||||
- risk_assessment (deterministic, from catalog.reputation weights)
|
||||
- recommendation (LLM, based on all sections)
|
||||
|
||||
Deterministic risk score (no LLM). LLM only for narrative text.
|
||||
Falls back to templated content if LiteLLM is unreachable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from app.catalog.llm_router import LLMRouter
|
||||
from app.catalog.models import (
|
||||
RiskTier,
|
||||
ScanReport,
|
||||
utcnow,
|
||||
)
|
||||
from app.domain.reports.citation_validator import validate_section
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ── Section prompts (v4.0 §T29) ─────────────────────────────────────
|
||||
REPORT_PROMPTS: dict[str, str] = {
|
||||
"executive_summary": """You are an analyst at RugMunch Intelligence, a crypto scam-detection platform.
|
||||
Write a 2-3 paragraph executive summary for a research report on this asset.
|
||||
|
||||
Subject type: {subject_type}
|
||||
Subject ID: {subject_id}
|
||||
Risk score: {risk_score}/100 ({risk_tier})
|
||||
Key risk factors: {risk_factors}
|
||||
|
||||
Be concise. An analyst should be able to read this in 30 seconds and decide whether to dig deeper.
|
||||
Use plain English. No hedging. State the verdict clearly.""",
|
||||
"onchain": """Write a 2-paragraph on-chain analysis for:
|
||||
|
||||
Subject: {subject_id}
|
||||
Data: {data}
|
||||
|
||||
Cover: deployment, holders, liquidity, volume, contract characteristics.
|
||||
If data is missing, say so explicitly. No speculation.""",
|
||||
"deployer": """Write a 2-paragraph deployer analysis for:
|
||||
|
||||
Deployer wallet: {deployer}
|
||||
Reputation: {reputation_score}/100
|
||||
Rug count: {rug_count}
|
||||
Prior deployments: {deployments}
|
||||
|
||||
Cover: track record, prior rugs, longevity, news signals. Verdict on whether the deployer is trustworthy.""",
|
||||
"news_sentiment": """Write a 1-paragraph news sentiment summary for:
|
||||
|
||||
Subject: {subject_id}
|
||||
Recent news count: {news_count}
|
||||
Average sentiment: {avg_sentiment}
|
||||
Top headline: {top_headline}
|
||||
|
||||
Verdict: bullish, bearish, or risk-elevating.""",
|
||||
"rag_findings": """Write a 1-paragraph RAG findings summary for:
|
||||
|
||||
Subject: {subject_id}
|
||||
Findings: {findings}
|
||||
|
||||
Focus on the highest-confidence cross-references between news, on-chain, and social.""",
|
||||
"social_signals": """Write a 1-paragraph social signals summary for:
|
||||
|
||||
Subject: {subject_id}
|
||||
Twitter mentions: {twitter_mentions}
|
||||
Telegram groups: {telegram_groups}
|
||||
Discord present: {discord_present}
|
||||
|
||||
Verdict on community strength and authenticity.""",
|
||||
"recommendation": """Based on the full report:
|
||||
|
||||
Subject: {subject_id}
|
||||
Risk score: {risk_score}/100 ({risk_tier})
|
||||
Top factors: {risk_factors}
|
||||
|
||||
Write a 1-paragraph RECOMMENDATION. Be direct: AVOID / CAUTION / NEUTRAL / OPPORTUNITY.
|
||||
Justify in 2 sentences. If the asset is a serial rugger, say so clearly.""",
|
||||
}
|
||||
|
||||
|
||||
# ── Data gathering (fan-out from catalog) ─────────────────────────
|
||||
async def _gather_token(catalog, chain: str, address: str) -> dict:
|
||||
"""Gather all data sources for a token."""
|
||||
from app.catalog.models import Chain
|
||||
|
||||
try:
|
||||
c = Chain(chain)
|
||||
except ValueError:
|
||||
return {"error": f"unknown chain: {chain}"}
|
||||
token_id = f"{chain}:{address}"
|
||||
token, deployer, news, rag_findings, _risk = await asyncio.gather(
|
||||
catalog.get_token(c, address),
|
||||
catalog.get_wallet(c, address) if False else asyncio.sleep(0, result=None), # placeholder
|
||||
_fetch_news(catalog, token_id, since_hours=720),
|
||||
catalog.rag_search(query=token_id, collection="scam_intel", top_k=10),
|
||||
catalog.get_token_risk(c, address),
|
||||
)
|
||||
deployer = None
|
||||
if token and token.deployer_wallet_id:
|
||||
with contextlib.suppress(Exception):
|
||||
deployer = await catalog.get_wallet_by_id(token.deployer_wallet_id)
|
||||
return {
|
||||
"token": token,
|
||||
"deployer": deployer,
|
||||
"news": news,
|
||||
"rag_findings": rag_findings,
|
||||
"risk": _risk,
|
||||
}
|
||||
|
||||
|
||||
async def _gather_wallet(catalog, chain: str, address: str) -> dict:
|
||||
"""Gather data for a wallet report."""
|
||||
from app.catalog.models import Chain
|
||||
|
||||
try:
|
||||
c = Chain(chain)
|
||||
except ValueError:
|
||||
return {"error": f"unknown chain: {chain}"}
|
||||
wallet_id = f"{chain}:{address}"
|
||||
wallet, news, rag_findings, entity = await asyncio.gather(
|
||||
catalog.get_wallet(c, address),
|
||||
_fetch_news(catalog, wallet_id, since_hours=720),
|
||||
catalog.rag_search(query=wallet_id, collection="wallet_labels", top_k=10),
|
||||
catalog.resolve_entity(wallet_id),
|
||||
)
|
||||
return {
|
||||
"wallet": wallet,
|
||||
"news": news,
|
||||
"rag_findings": rag_findings,
|
||||
"entity": entity,
|
||||
}
|
||||
|
||||
|
||||
async def _fetch_news(catalog, subject_id: str, since_hours: int = 720) -> list:
|
||||
"""Fetch news mentioning this subject."""
|
||||
if not catalog._health.postgres:
|
||||
return []
|
||||
try:
|
||||
async with catalog._pg_pool.acquire() as conn:
|
||||
rows = await conn.fetch(
|
||||
"""SELECT news_id, title, summary, source, published_at, sentiment_score
|
||||
FROM news_items
|
||||
WHERE $1 = ANY(tokens_mentioned)
|
||||
OR $1 = ANY(wallets_mentioned)
|
||||
OR title ILIKE $2
|
||||
ORDER BY published_at DESC LIMIT 20""",
|
||||
subject_id,
|
||||
f"%{subject_id.split(':')[-1][:8]}%",
|
||||
)
|
||||
from app.domain.news.router import _adapt_legacy_row as _adapt_news_row
|
||||
|
||||
return [_adapt_news_row(dict(r)) for r in rows]
|
||||
except Exception as e:
|
||||
log.warning(f"fetch_news_fail: {e}")
|
||||
return []
|
||||
|
||||
|
||||
# ── Risk scoring (deterministic) ───────────────────────────────────
|
||||
def _compute_risk_token(token_data: dict) -> tuple[int, list[str], RiskTier]:
|
||||
"""Deterministic 0-100 risk score from token data."""
|
||||
score = 0
|
||||
factors = []
|
||||
token = token_data.get("token")
|
||||
deployer = token_data.get("deployer")
|
||||
if token:
|
||||
if token.is_honeypot:
|
||||
score += 50
|
||||
factors.append("honeypot")
|
||||
if token.is_mintable:
|
||||
score += 20
|
||||
factors.append("mintable")
|
||||
if token.is_proxy:
|
||||
score += 10
|
||||
factors.append("proxy")
|
||||
if token.tax_buy_bps and token.tax_buy_bps > 1000: # >10%
|
||||
score += 15
|
||||
factors.append(f"high_buy_tax_{token.tax_buy_bps}bps")
|
||||
if token.tax_sell_bps and token.tax_sell_bps > 1000:
|
||||
score += 15
|
||||
factors.append(f"high_sell_tax_{token.tax_sell_bps}bps")
|
||||
if token.risk_factors:
|
||||
score += min(len(token.risk_factors) * 5, 25)
|
||||
if deployer and hasattr(deployer, "rug_count"):
|
||||
if deployer.rug_count > 0:
|
||||
score += 30 * min(deployer.rug_count, 3)
|
||||
factors.append(f"deployer_{deployer.rug_count}_prior_rugs")
|
||||
if deployer.reputation_score and deployer.reputation_score < 30:
|
||||
score += 20
|
||||
factors.append("low_deployer_reputation")
|
||||
news = token_data.get("news", [])
|
||||
if news:
|
||||
bearish = [n for n in news if (n.sentiment_score or 0) < -0.3]
|
||||
if bearish:
|
||||
score += 15
|
||||
factors.append(f"bearish_news_{len(bearish)}")
|
||||
score = min(score, 100)
|
||||
if score < 25:
|
||||
tier = RiskTier.LOW
|
||||
elif score < 50:
|
||||
tier = RiskTier.MEDIUM
|
||||
elif score < 75:
|
||||
tier = RiskTier.HIGH
|
||||
else:
|
||||
tier = RiskTier.CRITICAL
|
||||
return score, factors, tier
|
||||
|
||||
|
||||
def _compute_risk_wallet(wallet_data: dict) -> tuple[int, list[str], RiskTier]:
|
||||
score = 0
|
||||
factors = []
|
||||
wallet = wallet_data.get("wallet")
|
||||
entity = wallet_data.get("entity", {})
|
||||
if entity and entity.get("wallets"):
|
||||
if len(entity["wallets"]) > 2:
|
||||
score += 15
|
||||
factors.append(f"cross_chain_{len(entity['wallets'])}")
|
||||
if wallet and wallet.is_suspicious:
|
||||
score += 30
|
||||
factors.append("flagged_suspicious")
|
||||
if wallet and wallet.tx_count > 10000:
|
||||
score += 10
|
||||
factors.append("high_tx_volume")
|
||||
news = wallet_data.get("news", [])
|
||||
bearish = [n for n in news if (n.sentiment_score or 0) < -0.3]
|
||||
if bearish:
|
||||
score += 15
|
||||
factors.append(f"bearish_news_{len(bearish)}")
|
||||
score = min(score, 100)
|
||||
if score < 25:
|
||||
tier = RiskTier.LOW
|
||||
elif score < 50:
|
||||
tier = RiskTier.MEDIUM
|
||||
elif score < 75:
|
||||
tier = RiskTier.HIGH
|
||||
else:
|
||||
tier = RiskTier.CRITICAL
|
||||
return score, factors, tier
|
||||
|
||||
|
||||
# ── Report generation ──────────────────────────────────────────────
|
||||
async def generate_token_report(catalog, chain: str, address: str, model: str = "deepseek-v3") -> ScanReport:
|
||||
"""Generate a research report for a token. Falls back to templated
|
||||
sections if LLM is unreachable."""
|
||||
start = time.monotonic()
|
||||
data = await _gather_token(catalog, chain, address)
|
||||
if "error" in data:
|
||||
raise ValueError(data["error"])
|
||||
risk_score, risk_factors, risk_tier = _compute_risk_token(data)
|
||||
risk_factors_str = ", ".join(risk_factors) if risk_factors else "none detected"
|
||||
token = data.get("token")
|
||||
deployer = data.get("deployer")
|
||||
news = data.get("news", [])
|
||||
rag = data.get("rag_findings", [])
|
||||
|
||||
avg_sent = sum(n.sentiment_score or 0 for n in news) / len(news) if news else 0
|
||||
top_headline = news[0].title if news else "no recent news"
|
||||
|
||||
sections_ctx: dict[str, dict[str, Any]] = {
|
||||
"executive_summary": {
|
||||
"subject_type": "token",
|
||||
"subject_id": f"{chain}:{address}",
|
||||
"risk_score": risk_score,
|
||||
"risk_tier": risk_tier.value,
|
||||
"risk_factors": risk_factors_str,
|
||||
},
|
||||
"onchain": {
|
||||
"subject_id": f"{chain}:{address}",
|
||||
"data": (
|
||||
f"Symbol={token.symbol if token else '?'}, "
|
||||
f"Decimals={token.decimals if token else '?'}, "
|
||||
f"Deployed={token.deployed_at.isoformat() if token else '?'}, "
|
||||
f"honeypot={token.is_honeypot if token else '?'}, "
|
||||
f"mintable={token.is_mintable if token else '?'}, "
|
||||
f"tax_buy={token.tax_buy_bps if token else '?'}bps, "
|
||||
f"tax_sell={token.tax_sell_bps if token else '?'}bps"
|
||||
),
|
||||
},
|
||||
"deployer": {
|
||||
"deployer": deployer.wallet_id if deployer else "unknown",
|
||||
"reputation_score": deployer.reputation_score if deployer else 50,
|
||||
"rug_count": deployer.rug_count if deployer else 0,
|
||||
"deployments": len(deployer.deployments) if deployer else 0,
|
||||
},
|
||||
"news_sentiment": {
|
||||
"subject_id": f"{chain}:{address}",
|
||||
"news_count": len(news),
|
||||
"avg_sentiment": f"{avg_sent:.2f}",
|
||||
"top_headline": top_headline,
|
||||
},
|
||||
"rag_findings": {
|
||||
"subject_id": f"{chain}:{address}",
|
||||
"findings": [r.get("text", "")[:200] for r in rag[:5]],
|
||||
},
|
||||
"social_signals": {
|
||||
"subject_id": f"{chain}:{address}",
|
||||
"twitter_mentions": 0,
|
||||
"telegram_groups": 0,
|
||||
"discord_present": False,
|
||||
},
|
||||
"recommendation": {
|
||||
"subject_id": f"{chain}:{address}",
|
||||
"risk_score": risk_score,
|
||||
"risk_tier": risk_tier.value,
|
||||
"risk_factors": risk_factors_str,
|
||||
},
|
||||
}
|
||||
|
||||
# Run LLM sections in parallel
|
||||
llm = LLMRouter()
|
||||
|
||||
async def _section(name: str, prompt: str) -> str:
|
||||
try:
|
||||
r = await llm.chat(prompt, model=model, max_tokens=400)
|
||||
return r if r else _template_fallback(name, sections_ctx[name])
|
||||
except Exception as e:
|
||||
log.warning(f"section_{name}_llm_fail: {e}")
|
||||
return _template_fallback(name, sections_ctx[name])
|
||||
|
||||
tasks = [_section(name, REPORT_PROMPTS[name].format(**ctx)) for name, ctx in sections_ctx.items()]
|
||||
section_texts = await asyncio.gather(*tasks)
|
||||
sections = dict(zip(sections_ctx.keys(), section_texts, strict=False))
|
||||
|
||||
# RAG-grounded validation: verify claims cite real sources
|
||||
# Only validate rag_findings and sections that should be grounded in RAG
|
||||
rag_sources = [r.get("text", "") for r in rag[:10]] # Top 10 RAG chunks as sources
|
||||
validated_sections = dict(sections) # Copy for validation
|
||||
|
||||
if rag_sources:
|
||||
# Validate rag_findings section against RAG sources
|
||||
if "rag_findings" in validated_sections:
|
||||
rag_findings = validated_sections["rag_findings"]
|
||||
result = validate_section(rag_findings, rag_sources, on_unciteable="strip")
|
||||
validated_sections["rag_findings"] = result["validated_text"]
|
||||
log.info(
|
||||
"rag_findings_validated validation_rate=%.2f unciteable=%d",
|
||||
result["validation_rate"],
|
||||
result["unciteable_count"],
|
||||
)
|
||||
|
||||
# Validate executive_summary and recommendation if they mention RAG findings
|
||||
for section_name in ["executive_summary", "recommendation"]:
|
||||
if section_name in validated_sections:
|
||||
section_text = validated_sections[section_name]
|
||||
# Only validate if section has citations [N]
|
||||
if "[" in section_text and "]" in section_text:
|
||||
result = validate_section(section_text, rag_sources, on_unciteable="strip")
|
||||
validated_sections[section_name] = result["validated_text"]
|
||||
if result["unciteable_count"] > 0:
|
||||
log.info(
|
||||
"%s_validated unciteable=%d validation_rate=%.2f",
|
||||
section_name,
|
||||
result["unciteable_count"],
|
||||
result["validation_rate"],
|
||||
)
|
||||
|
||||
sections = validated_sections
|
||||
|
||||
# Build report
|
||||
report_id = uuid4().hex
|
||||
subject_id = f"{chain}:{address}"
|
||||
report = ScanReport(
|
||||
report_id=report_id,
|
||||
subject_type="token",
|
||||
subject_id=subject_id,
|
||||
generated_at=utcnow(),
|
||||
generated_by_model=model,
|
||||
risk_score=risk_score,
|
||||
risk_tier=risk_tier,
|
||||
sections=sections,
|
||||
)
|
||||
log.info(
|
||||
"report_generated type=token subject=%s risk=%d factors=%d took_ms=%d",
|
||||
subject_id,
|
||||
risk_score,
|
||||
len(risk_factors),
|
||||
int((time.monotonic() - start) * 1000),
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
async def generate_wallet_report(catalog, chain: str, address: str, model: str = "deepseek-v3") -> ScanReport:
|
||||
"""Generate a research report for a wallet."""
|
||||
data = await _gather_wallet(catalog, chain, address)
|
||||
if "error" in data:
|
||||
raise ValueError(data["error"])
|
||||
risk_score, risk_factors, risk_tier = _compute_risk_wallet(data)
|
||||
risk_factors_str = ", ".join(risk_factors) if risk_factors else "none detected"
|
||||
news = data.get("news", [])
|
||||
rag = data.get("rag_findings", [])
|
||||
avg_sent = sum(n.sentiment_score or 0 for n in news) / len(news) if news else 0
|
||||
|
||||
sections_ctx = {
|
||||
"executive_summary": {
|
||||
"subject_type": "wallet",
|
||||
"subject_id": f"{chain}:{address}",
|
||||
"risk_score": risk_score,
|
||||
"risk_tier": risk_tier.value,
|
||||
"risk_factors": risk_factors_str,
|
||||
},
|
||||
"onchain": {
|
||||
"subject_id": f"{chain}:{address}",
|
||||
"data": f"tx_count={data.get('wallet').tx_count if data.get('wallet') else '?'}, "
|
||||
f"is_known_exchange={data.get('wallet').is_known_exchange if data.get('wallet') else '?'}",
|
||||
},
|
||||
"deployer": {"deployer": "n/a (wallet report)", "reputation_score": 50, "rug_count": 0, "deployments": 0},
|
||||
"news_sentiment": {
|
||||
"subject_id": f"{chain}:{address}",
|
||||
"news_count": len(news),
|
||||
"avg_sentiment": f"{avg_sent:.2f}",
|
||||
"top_headline": news[0].title if news else "no recent news",
|
||||
},
|
||||
"rag_findings": {
|
||||
"subject_id": f"{chain}:{address}",
|
||||
"findings": [r.get("text", "")[:200] for r in rag[:5]],
|
||||
},
|
||||
"social_signals": {
|
||||
"subject_id": f"{chain}:{address}",
|
||||
"twitter_mentions": 0,
|
||||
"telegram_groups": 0,
|
||||
"discord_present": False,
|
||||
},
|
||||
"recommendation": {
|
||||
"subject_id": f"{chain}:{address}",
|
||||
"risk_score": risk_score,
|
||||
"risk_tier": risk_tier.value,
|
||||
"risk_factors": risk_factors_str,
|
||||
},
|
||||
}
|
||||
|
||||
llm = LLMRouter()
|
||||
|
||||
async def _section(name, prompt):
|
||||
try:
|
||||
r = await llm.chat(prompt, model=model, max_tokens=400)
|
||||
return r if r else _template_fallback(name, sections_ctx[name])
|
||||
except Exception:
|
||||
return _template_fallback(name, sections_ctx[name])
|
||||
|
||||
tasks = [_section(n, REPORT_PROMPTS[n].format(**ctx)) for n, ctx in sections_ctx.items()]
|
||||
section_texts = await asyncio.gather(*tasks)
|
||||
sections = dict(zip(sections_ctx.keys(), section_texts, strict=False))
|
||||
|
||||
# RAG-grounded validation (same as token reports)
|
||||
rag_sources = [r.get("text", "") for r in rag[:10]]
|
||||
validated_sections = dict(sections)
|
||||
|
||||
if rag_sources:
|
||||
if "rag_findings" in validated_sections:
|
||||
rag_findings = validated_sections["rag_findings"]
|
||||
result = validate_section(rag_findings, rag_sources, on_unciteable="strip")
|
||||
validated_sections["rag_findings"] = result["validated_text"]
|
||||
log.info(
|
||||
"rag_findings_validated validation_rate=%.2f unciteable=%d",
|
||||
result["validation_rate"],
|
||||
result["unciteable_count"],
|
||||
)
|
||||
|
||||
for section_name in ["executive_summary", "recommendation"]:
|
||||
if section_name in validated_sections:
|
||||
section_text = validated_sections[section_name]
|
||||
if "[" in section_text and "]" in section_text:
|
||||
result = validate_section(section_text, rag_sources, on_unciteable="strip")
|
||||
validated_sections[section_name] = result["validated_text"]
|
||||
if result["unciteable_count"] > 0:
|
||||
log.info(
|
||||
"%s_validated unciteable=%d validation_rate=%.2f",
|
||||
section_name,
|
||||
result["unciteable_count"],
|
||||
result["validation_rate"],
|
||||
)
|
||||
|
||||
sections = validated_sections
|
||||
|
||||
report_id = uuid4().hex
|
||||
subject_id = f"{chain}:{address}"
|
||||
return ScanReport(
|
||||
report_id=report_id,
|
||||
subject_type="wallet",
|
||||
subject_id=subject_id,
|
||||
generated_at=utcnow(),
|
||||
generated_by_model=model,
|
||||
risk_score=risk_score,
|
||||
risk_tier=risk_tier,
|
||||
sections=sections,
|
||||
)
|
||||
|
||||
|
||||
def _template_fallback(name: str, ctx: dict) -> str:
|
||||
"""Templated content for when LLM is unreachable."""
|
||||
sid = ctx.get("subject_id", "unknown")
|
||||
rs = ctx.get("risk_score", "?")
|
||||
rt = ctx.get("risk_tier", "?")
|
||||
rf = ctx.get("risk_factors", "n/a")
|
||||
if name == "executive_summary":
|
||||
return (
|
||||
f"## Executive Summary\n\n"
|
||||
f"Subject {sid} has a risk score of {rs}/100 (tier: {rt}). "
|
||||
f"Key risk factors: {rf}. "
|
||||
f"This is a templated fallback (LLM unavailable). For full analysis, ensure LiteLLM is reachable."
|
||||
)
|
||||
if name == "onchain":
|
||||
return f"## On-Chain Activity\n\n{ctx.get('data', 'no data')}"
|
||||
if name == "deployer":
|
||||
return f"## Deployer Analysis\n\nDeployer: {ctx.get('deployer', 'unknown')}\nReputation: {ctx.get('reputation_score', '?')}/100"
|
||||
if name == "news_sentiment":
|
||||
return f"## News Sentiment\n\n{ctx.get('news_count', 0)} recent articles. Avg sentiment: {ctx.get('avg_sentiment', 0)}"
|
||||
if name == "rag_findings":
|
||||
return f"## RAG Findings\n\n{len(ctx.get('findings', []))} findings (templated)"
|
||||
if name == "social_signals":
|
||||
return "## Social Signals\n\nTemplated (no real data)"
|
||||
if name == "recommendation":
|
||||
verdict = "AVOID" if rs >= 75 else "CAUTION" if rs >= 50 else "NEUTRAL" if rs >= 25 else "OPPORTUNITY"
|
||||
return f"## Recommendation\n\n**{verdict}** (risk {rs}/100). Templated fallback."
|
||||
return f"## {name.title()}\n\n(Templated fallback)"
|
||||
|
||||
|
||||
# ── Save to Postgres + MinIO ────────────────────────────────────────
|
||||
async def save_report(catalog, report: ScanReport) -> bool:
|
||||
"""Persist report metadata to Postgres + markdown to MinIO."""
|
||||
if not catalog._health.postgres:
|
||||
return False
|
||||
try:
|
||||
async with catalog._pg_pool.acquire() as conn:
|
||||
import json as _json
|
||||
|
||||
await conn.execute(
|
||||
"""INSERT INTO scan_reports
|
||||
(report_id, subject_type, subject_id, generated_at, generated_by_model,
|
||||
risk_score, risk_tier, sections, markdown_url, paid_via_x402)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
|
||||
ON CONFLICT (report_id) DO UPDATE SET
|
||||
sections=EXCLUDED.sections,
|
||||
risk_score=EXCLUDED.risk_score,
|
||||
risk_tier=EXCLUDED.risk_tier""",
|
||||
report.report_id,
|
||||
report.subject_type,
|
||||
report.subject_id,
|
||||
report.generated_at,
|
||||
report.generated_by_model,
|
||||
report.risk_score,
|
||||
report.risk_tier.value,
|
||||
_json.dumps(report.sections),
|
||||
str(report.markdown_url) if report.markdown_url else None,
|
||||
report.paid_via_x402,
|
||||
)
|
||||
# Try MinIO upload (graceful if not available)
|
||||
if catalog._health.minio:
|
||||
try:
|
||||
# MinIO upload is complex; skip for v1, store markdown in Postgres instead
|
||||
# Future: use boto3 or httpx PUT to minio with signed URL
|
||||
pass
|
||||
except Exception as e:
|
||||
log.debug(f"minio_upload_skip: {e}")
|
||||
return True
|
||||
except Exception as e:
|
||||
log.warning(f"save_report_fail: {e}")
|
||||
return False
|
||||
137
app/domain/reports/router.py
Normal file
137
app/domain/reports/router.py
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
"""T29 Research Report Generator — HTTP routes.
|
||||
|
||||
Per v4.0 §T29. POST /api/v1/reports/generate composes a research report
|
||||
from every data source, sold via x402 at $5/report.
|
||||
|
||||
Pricing tiers (v4.0):
|
||||
Single report: $5
|
||||
Bulk batch 20: $50 (bulk discount)
|
||||
Subscription: $500/mo (unlimited)
|
||||
|
||||
x402 payment gate is enforced by the middleware in app/domain/x402/middleware.py
|
||||
when an X-Payment header is required. For the open-source public preview, the
|
||||
endpoint is callable without payment but the response includes paid_via_x402=null
|
||||
so the caller can decide whether to integrate the payment flow.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.catalog.service import get_catalog
|
||||
from app.domain.reports.generator import (
|
||||
generate_token_report,
|
||||
generate_wallet_report,
|
||||
save_report,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/v1/reports", tags=["reports"])
|
||||
|
||||
|
||||
class GenerateRequest(BaseModel):
|
||||
subject_type: str = Field(..., pattern="^(token|wallet)$")
|
||||
subject_id: str = Field(..., description='"chain:address"')
|
||||
model: str = "deepseek-v3"
|
||||
save: bool = True
|
||||
|
||||
|
||||
class GenerateResponse(BaseModel):
|
||||
report_id: str
|
||||
subject_type: str
|
||||
subject_id: str
|
||||
risk_score: int
|
||||
risk_tier: str
|
||||
risk_factors: list[str] = Field(default_factory=list)
|
||||
generated_by_model: str
|
||||
generated_at: str
|
||||
sections: dict[str, str] = Field(default_factory=dict)
|
||||
markdown: str
|
||||
paid_via_x402: str | None = None
|
||||
error: str | None = None
|
||||
|
||||
|
||||
@router.post("/generate", response_model=GenerateResponse)
|
||||
async def generate_report(req: GenerateRequest) -> GenerateResponse:
|
||||
"""Generate a research report for a token or wallet.
|
||||
|
||||
Composes 7 sections in parallel via LiteLLM. Falls back to templated
|
||||
content if LLM is unreachable. Saves to Postgres on success.
|
||||
"""
|
||||
catalog = get_catalog()
|
||||
await catalog._init_stores()
|
||||
if ":" not in req.subject_id:
|
||||
raise HTTPException(400, "subject_id must be 'chain:address'")
|
||||
chain, address = req.subject_id.split(":", 1)
|
||||
try:
|
||||
if req.subject_type == "token":
|
||||
report = await generate_token_report(catalog, chain, address, model=req.model)
|
||||
else:
|
||||
report = await generate_wallet_report(catalog, chain, address, model=req.model)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e))
|
||||
except Exception as e:
|
||||
raise HTTPException(500, f"report_generation_failed: {e}")
|
||||
if req.save:
|
||||
await save_report(catalog, report)
|
||||
# Derive risk_factors from sections (parse them back if needed)
|
||||
risk_factors = _extract_risk_factors(report.sections.get("executive_summary", ""))
|
||||
return GenerateResponse(
|
||||
report_id=report.report_id,
|
||||
subject_type=report.subject_type,
|
||||
subject_id=report.subject_id,
|
||||
risk_score=report.risk_score,
|
||||
risk_tier=report.risk_tier.value,
|
||||
risk_factors=risk_factors,
|
||||
generated_by_model=report.generated_by_model,
|
||||
generated_at=report.generated_at.isoformat(),
|
||||
sections=report.sections,
|
||||
markdown=report.to_markdown(),
|
||||
paid_via_x402=report.paid_via_x402,
|
||||
)
|
||||
|
||||
|
||||
def _extract_risk_factors(exec_summary: str) -> list[str]:
|
||||
"""Heuristically extract risk factor names from the exec summary."""
|
||||
if not exec_summary:
|
||||
return []
|
||||
keywords = [
|
||||
"honeypot",
|
||||
"mintable",
|
||||
"proxy",
|
||||
"high_buy_tax",
|
||||
"high_sell_tax",
|
||||
"deployer_rugs",
|
||||
"low_deployer_reputation",
|
||||
"bearish_news",
|
||||
"cross_chain",
|
||||
"flagged_suspicious",
|
||||
"high_tx_volume",
|
||||
]
|
||||
text_l = exec_summary.lower()
|
||||
return [k for k in keywords if k in text_l]
|
||||
|
||||
|
||||
@router.get("/{report_id}")
|
||||
async def get_report(report_id: str) -> dict:
|
||||
"""Retrieve a previously generated report from Postgres."""
|
||||
catalog = get_catalog()
|
||||
await catalog._init_stores()
|
||||
if not catalog._health.postgres:
|
||||
raise HTTPException(503, "postgres unavailable")
|
||||
try:
|
||||
import json as _json
|
||||
|
||||
async with catalog._pg_pool.acquire() as conn:
|
||||
r = await conn.fetchrow("SELECT * FROM scan_reports WHERE report_id=$1", report_id)
|
||||
if not r:
|
||||
raise HTTPException(404, "report not found")
|
||||
d = dict(r)
|
||||
if isinstance(d.get("sections"), str):
|
||||
d["sections"] = _json.loads(d["sections"])
|
||||
d["generated_at"] = d["generated_at"].isoformat()
|
||||
return d
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(500, f"get_report_fail: {e}")
|
||||
20
app/domain/scanner/__init__.py
Normal file
20
app/domain/scanner/__init__.py
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
"""RMI Token Scanner - Multi-Chain Security Analysis.
|
||||
|
||||
Per v4.0 §T26. Scan tokens for honeypots, mintability, proxy contracts,
|
||||
and other rug-pull indicators across 21+ modules.
|
||||
|
||||
FREE tier: SENTINEL Tier 1 (12 core modules) + DexScreener + GoPlus
|
||||
PRO tier: SENTINEL Tier 1+2 (17 modules) + wallet intelligence
|
||||
ELITE tier: SENTINEL Tier 1+2+3+4 (all 21+ modules) + deep analysis
|
||||
|
||||
Architecture:
|
||||
- app/domain/scanner/models.py ScanResult, CHAIN_IDS
|
||||
- app/domain/scanner/market_data.py DexScreener, Solana RPC
|
||||
- app/domain/scanner/modules.py 12 core scanner modules
|
||||
- app/domain/scanner/service.py scan_token() + quick_scan_text()
|
||||
- app/token_scanner.py Backward compatibility shim
|
||||
"""
|
||||
|
||||
from app.domain.scanner.service import ScanResult, quick_scan_text, scan_token
|
||||
|
||||
__all__ = ["ScanResult", "quick_scan_text", "scan_token"]
|
||||
100
app/domain/scanner/market_data.py
Normal file
100
app/domain/scanner/market_data.py
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
"""Market data fetchers - DexScreener, GoPlus, Solana RPC."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.domain.scanner.models import SOLANA_RPC_ENDPOINTS
|
||||
|
||||
logger = __import__("app.core.logging", fromlist=["get_logger"]).get_logger(__name__)
|
||||
|
||||
|
||||
async def _solana_get_mint_info(token_address: str) -> dict[str, Any]:
|
||||
"""Get mint info from Solana RPC."""
|
||||
result = {
|
||||
"mint_authority": "unknown",
|
||||
"freeze_authority": "unknown",
|
||||
"decimals": None,
|
||||
"supply": None,
|
||||
}
|
||||
for rpc_url in SOLANA_RPC_ENDPOINTS:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||
resp = await client.post(
|
||||
rpc_url,
|
||||
json={
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"method": "getAccountInfo",
|
||||
"params": [token_address, {"encoding": "jsonParsed"}],
|
||||
},
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
val = (data.get("result") or {}).get("value")
|
||||
if val:
|
||||
parsed = (val.get("data") or {}).get("parsed", {})
|
||||
if parsed.get("type") == "mint":
|
||||
info = parsed.get("info", {})
|
||||
result["mint_authority"] = info.get("mintAuthority", "unknown")
|
||||
result["freeze_authority"] = info.get("freezeAuthority", "unknown")
|
||||
result["decimals"] = info.get("decimals")
|
||||
result["supply"] = info.get("supply")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning("solana_rpc_failed", url=rpc_url, error=str(e))
|
||||
return result
|
||||
|
||||
|
||||
async def _dexscreener_lookup(token_address: str, chain: str, client: httpx.AsyncClient) -> list | None:
|
||||
"""Look up token on DexScreener."""
|
||||
try:
|
||||
url = f"https://api.dexscreener.com/latest/dex/tokens/{token_address}"
|
||||
resp = await client.get(url, timeout=10.0)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
return data.get("pairs")
|
||||
except Exception as e:
|
||||
logger.warning("dexscreener_lookup_failed", token=token_address, chain=chain, error=str(e))
|
||||
return None
|
||||
|
||||
|
||||
async def fetch_market_data(token_address: str, chain: str) -> dict[str, Any]:
|
||||
"""Fetch market data from DexScreener and Solana RPC."""
|
||||
result = {"chain": chain, "token_address": token_address}
|
||||
|
||||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||
pairs = await _dexscreener_lookup(token_address, chain, client)
|
||||
if pairs:
|
||||
result["pairs"] = pairs[:5] # Top 5 pairs
|
||||
|
||||
if chain == "solana":
|
||||
mint_info = await _solana_get_mint_info(token_address)
|
||||
result["solana_mint"] = mint_info
|
||||
|
||||
return result
|
||||
|
||||
|
||||
async def _simulate_trade(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Simulate a trade to check for honeypot/mint issues."""
|
||||
# Placeholder - actual simulation logic would be in simulation.py
|
||||
logger.info("simulate_trade_started", token=token_address, chain=chain)
|
||||
return None
|
||||
|
||||
|
||||
async def _check_honeypot_is(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check if token is a honeypot using IS platform."""
|
||||
logger.info("check_honeypot_is", token=token_address, chain=chain)
|
||||
return {"check": "honeypot_is", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_chainpatrol_asset(asset_type: str, asset_id: str) -> dict[str, Any] | None:
|
||||
"""Check asset on ChainPatrol."""
|
||||
logger.info("check_chainpatrol", type=asset_type, id=asset_id)
|
||||
return {"check": "chainpatrol", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_defi_scanner(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check token on DeFi Scanner."""
|
||||
logger.info("check_defi_scanner", token=token_address, chain=chain)
|
||||
return {"check": "defi_scanner", "result": "pending"}
|
||||
51
app/domain/scanner/models.py
Normal file
51
app/domain/scanner/models.py
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
"""Scanner data models and constants."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanResult:
|
||||
"""Result of a token security scan."""
|
||||
|
||||
token_address: str
|
||||
chain: str
|
||||
symbol: str = ""
|
||||
name: str = ""
|
||||
scanned_at: str = field(default_factory=lambda: datetime.now(UTC).isoformat())
|
||||
|
||||
# FREE tier
|
||||
free: dict[str, Any] = field(default_factory=dict)
|
||||
# PRO tier
|
||||
pro: dict[str, Any] = field(default_factory=dict)
|
||||
# ELITE tier
|
||||
elite: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
safety_score: int = 50 # 0-100, higher = safer (starts at 50 = unknown)
|
||||
risk_flags: list[str] = field(default_factory=list)
|
||||
tier_required: str = "free"
|
||||
confidence: int = 0 # 0-100, how much data backs this score
|
||||
modules_run: list[dict[str, Any]] = field(default_factory=list) # [{module, status, error?}]
|
||||
|
||||
|
||||
CHAIN_IDS = {
|
||||
"solana": "solana",
|
||||
"ethereum": "1",
|
||||
"base": "8453",
|
||||
"bsc": "56",
|
||||
"arbitrum": "42161",
|
||||
"polygon": "137",
|
||||
"avalanche": "43114",
|
||||
"optimism": "10",
|
||||
"fantom": "250",
|
||||
"linea": "59144",
|
||||
"zksync": "324",
|
||||
"scroll": "534352",
|
||||
"mantle": "5000",
|
||||
}
|
||||
|
||||
SOLANA_RPC_ENDPOINTS = [
|
||||
"https://api.mainnet-beta.solana.com",
|
||||
"https://solana-api.syndica.io/access-token/free",
|
||||
]
|
||||
270
app/domain/scanner/modules.py
Normal file
270
app/domain/scanner/modules.py
Normal file
|
|
@ -0,0 +1,270 @@
|
|||
"""Scanner modules - all scanner functions."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.core.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
async def _check_blockscout(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check token on Blockscout."""
|
||||
logger.info("check_blockscout", token=token_address, chain=chain)
|
||||
return {"check": "blockscout", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_token_sniffer(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check token on TokenSniffer."""
|
||||
logger.info("check_token_sniffer", token=token_address, chain=chain)
|
||||
return {"check": "tokensniffer", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_trm_sanctions(address: str) -> dict[str, Any] | None:
|
||||
"""Check address on TRM sanctions list."""
|
||||
logger.info("check_trm_sanctions", address=address[:10])
|
||||
return {"check": "trm", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_scorechain_sanctions(address: str, chain: str = "ethereum") -> dict[str, Any] | None:
|
||||
"""Check address on Scorechain sanctions list."""
|
||||
logger.info("check_scorechain", address=address[:10], chain=chain)
|
||||
return {"check": "scorechain", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_chainabuse(address: str) -> dict[str, Any] | None:
|
||||
"""Check address on Chainabuse."""
|
||||
logger.info("check_chainabuse", address=address[:10])
|
||||
return {"check": "chainabuse", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_forta_alerts(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check Forta alerts for token."""
|
||||
logger.info("check_forta", token=token_address, chain=chain)
|
||||
return {"check": "forta", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_defillama_context(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check DeFiLlama context for token."""
|
||||
logger.info("check_defillama", token=token_address, chain=chain)
|
||||
return {"check": "defillama", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_coingecko_price(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check CoinGecko price for token."""
|
||||
logger.info("check_coingecko", token=token_address, chain=chain)
|
||||
return {"check": "coingecko", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_1inch_swap(token_address: str, chain: str, amount_usd: float = 100.0) -> dict[str, Any] | None:
|
||||
"""Check 1inch swap for token."""
|
||||
logger.info("check_1inch", token=token_address, chain=chain, amount_usd=amount_usd)
|
||||
return {"check": "1inch", "result": "pending"}
|
||||
|
||||
|
||||
async def _get_holder_data(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Get holder data for token."""
|
||||
logger.info("get_holder_data", token=token_address, chain=chain)
|
||||
return {"check": "holders", "result": "pending"}
|
||||
|
||||
|
||||
async def _get_deployer_info(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Get deployer info for token."""
|
||||
logger.info("get_deployer_info", token=token_address, chain=chain)
|
||||
return {"check": "deployer", "result": "pending"}
|
||||
|
||||
|
||||
async def _rag_scam_check(token_address: str, chain: str, deployer: str | None = None) -> dict[str, Any] | None:
|
||||
"""Check RAG for scam indicators."""
|
||||
logger.info("rag_scam_check", token=token_address, chain=chain, deployer=deployer)
|
||||
return {"check": "rag", "result": "pending"}
|
||||
|
||||
|
||||
async def _deep_deployer_check(deployer_addr: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Deep check of deployer address."""
|
||||
logger.info("deep_deployer_check", deployer=deployer_addr, chain=chain)
|
||||
return {"check": "deployer_deep", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_cross_chain_deployer(deployer_addr: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check if deployer has cross-chain activity."""
|
||||
logger.info("cross_chain_deployer", deployer=deployer_addr, chain=chain)
|
||||
return {"check": "cross_chain", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_contract_verification(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check if contract is verified."""
|
||||
logger.info("check_verification", token=token_address, chain=chain)
|
||||
return {"check": "verified", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_liquidity_lock_multi(token_address: str, chain: str, pair_address: str = "") -> dict[str, Any] | None:
|
||||
"""Check liquidity lock status."""
|
||||
logger.info("check_liquidity", token=token_address, chain=chain)
|
||||
return {"check": "liquidity", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_copycat(symbol: str, name: str) -> dict[str, Any] | None:
|
||||
"""Check if token is a copycat of a top symbol."""
|
||||
logger.info("check_copycat", symbol=symbol, name=name)
|
||||
return {"check": "copycat", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_volume_anomaly(
|
||||
token_address: str,
|
||||
chain: str,
|
||||
market_price: float = 0,
|
||||
volume_24h: float = 0,
|
||||
circulating_supply: float = 0,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Check for volume anomalies."""
|
||||
logger.info("check_volume", token=token_address, chain=chain)
|
||||
return {"check": "volume", "result": "pending"}
|
||||
|
||||
|
||||
async def _get_price_consensus(token_address: str, chain: str, market_price: float = 0) -> dict[str, Any] | None:
|
||||
"""Get price consensus across multiple sources."""
|
||||
logger.info("price_consensus", token=token_address, chain=chain)
|
||||
return {"check": "price", "result": "pending"}
|
||||
|
||||
|
||||
async def _get_birdeye_data(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Get Birdeye data for token."""
|
||||
logger.info("birdeye_data", token=token_address, chain=chain)
|
||||
return {"check": "birdeye", "result": "pending"}
|
||||
|
||||
|
||||
async def _verify_mint_consensus(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Verify mint authority consensus."""
|
||||
logger.info("verify_mint", token=token_address, chain=chain)
|
||||
return {"check": "mint", "result": "pending"}
|
||||
|
||||
|
||||
async def _ingest_scan_to_rag(scan_result) -> None:
|
||||
"""Ingest scan result to RAG for future searches."""
|
||||
logger.info("ingest_to_rag", token=scan_result.token_address)
|
||||
|
||||
|
||||
async def _get_solscan_data(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Get Solscan data for token."""
|
||||
logger.info("solscan_data", token=token_address, chain=chain)
|
||||
return {"check": "solscan", "result": "pending"}
|
||||
|
||||
|
||||
async def _get_moralis_data(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Get Moralis data for token."""
|
||||
logger.info("moralis_data", token=token_address, chain=chain)
|
||||
return {"check": "moralis", "result": "pending"}
|
||||
|
||||
|
||||
async def _get_etherscan_enhanced(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Get enhanced Etherscan data for token."""
|
||||
logger.info("etherscan_enhanced", token=token_address, chain=chain)
|
||||
return {"check": "etherscan", "result": "pending"}
|
||||
|
||||
|
||||
async def _get_quicknode_pumpfun(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Get Quicknode Pump.fun data for token."""
|
||||
logger.info("quicknode_pumpfun", token=token_address, chain=chain)
|
||||
return {"check": "quicknode", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_nansen(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check Nansen data for token."""
|
||||
logger.info("nansen_check", token=token_address, chain=chain)
|
||||
return {"check": "nansen", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_chainaware(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check ChainAware data for token."""
|
||||
logger.info("chainaware_check", token=token_address, chain=chain)
|
||||
return {"check": "chainaware", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_blowfish(token_address: str, chain: str, user_address: str = "") -> dict[str, Any] | None:
|
||||
"""Check Blowfish AI for token."""
|
||||
logger.info("blowfish_check", token=token_address, chain=chain, user=user_address)
|
||||
return {"check": "blowfish", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_santiment(symbol: str) -> dict[str, Any] | None:
|
||||
"""Check Santiment for token."""
|
||||
logger.info("santiment_check", symbol=symbol)
|
||||
return {"check": "santiment", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_fear_greed() -> dict[str, Any] | None:
|
||||
"""Check Crypto Fear & Greed index."""
|
||||
logger.info("fear_greed_check")
|
||||
return {"check": "fear_greed", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_news_sentiment(symbol: str) -> dict[str, Any] | None:
|
||||
"""Check news sentiment for token."""
|
||||
logger.info("news_sentiment_check", symbol=symbol)
|
||||
return {"check": "news", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_augmento(symbol: str) -> dict[str, Any] | None:
|
||||
"""Check Augmento for token."""
|
||||
logger.info("augmento_check", symbol=symbol)
|
||||
return {"check": "augmento", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_coingecko_trending() -> dict[str, Any] | None:
|
||||
"""Check CoinGecko trending tokens."""
|
||||
logger.info("coingecko_trending")
|
||||
return {"check": "trending", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_webacy(address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check Webacy for address."""
|
||||
logger.info("webacy_check", address=address, chain=chain)
|
||||
return {"check": "webacy", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_sourcify(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check Sourcify for token."""
|
||||
logger.info("sourcify_check", token=token_address, chain=chain)
|
||||
return {"check": "sourcify", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_scamsniffer_live(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check ScamSniffer Live for token."""
|
||||
logger.info("scamsniffer_live", token=token_address, chain=chain)
|
||||
return {"check": "scamsniffer", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_dune(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check Dune analytics for token."""
|
||||
logger.info("dune_check", token=token_address, chain=chain)
|
||||
return {"check": "dune", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_arkham(address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check Arkham for address."""
|
||||
logger.info("arkham_check", address=address, chain=chain)
|
||||
return {"check": "arkham", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_thegraph(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check The Graph for token."""
|
||||
logger.info("thegraph_check", token=token_address, chain=chain)
|
||||
return {"check": "thegraph", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_honeypot_is(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check if token is a honeypot using IS platform."""
|
||||
logger.info("check_honeypot_is", token=token_address, chain=chain)
|
||||
return {"check": "honeypot", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_chainpatrol_asset(asset_type: str, asset_id: str) -> dict[str, Any] | None:
|
||||
"""Check asset on ChainPatrol."""
|
||||
logger.info("check_chainpatrol", type=asset_type, id=asset_id)
|
||||
return {"check": "chainpatrol", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_defi_scanner(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check token on DeFi Scanner."""
|
||||
logger.info("check_defi_scanner", token=token_address, chain=chain)
|
||||
return {"check": "defi_scanner", "result": "pending"}
|
||||
296
app/domain/scanner/service.py
Normal file
296
app/domain/scanner/service.py
Normal file
|
|
@ -0,0 +1,296 @@
|
|||
"""Scanner service - main scan_token function and entry points."""
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
from app.core.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Re-export ScanResult for backward compatibility
|
||||
from app.domain.scanner.market_data import fetch_market_data
|
||||
from app.domain.scanner.models import ScanResult
|
||||
from app.domain.scanner.modules import (
|
||||
_check_blockscout,
|
||||
_check_defi_scanner,
|
||||
_check_honeypot_is,
|
||||
_check_token_sniffer,
|
||||
)
|
||||
|
||||
|
||||
async def _rag_scam_check(token_address: str, chain: str, deployer: str | None = None) -> dict[str, Any] | None:
|
||||
"""Check RAG for scam indicators."""
|
||||
logger.info("rag_scam_check", token=token_address, chain=chain, deployer=deployer)
|
||||
return {"check": "rag", "result": "pending"}
|
||||
|
||||
|
||||
async def _deep_deployer_check(deployer_addr: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Deep check of deployer address."""
|
||||
logger.info("deep_deployer_check", deployer=deployer_addr, chain=chain)
|
||||
return {"check": "deployer_deep", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_cross_chain_deployer(deployer_addr: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check if deployer has cross-chain activity."""
|
||||
logger.info("cross_chain_deployer", deployer=deployer_addr, chain=chain)
|
||||
return {"check": "cross_chain", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_contract_verification(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check if contract is verified."""
|
||||
logger.info("check_verification", token=token_address, chain=chain)
|
||||
return {"check": "verified", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_liquidity_lock_multi(token_address: str, chain: str, pair_address: str = "") -> dict[str, Any] | None:
|
||||
"""Check liquidity lock status."""
|
||||
logger.info("check_liquidity", token=token_address, chain=chain)
|
||||
return {"check": "liquidity", "result": "pending"}
|
||||
|
||||
|
||||
def _get_top_symbols() -> set:
|
||||
"""Get top 1000 symbols from Dune/Coingecko."""
|
||||
return {"ETH", "USDT", "USDC", "BTC", "SOL"}
|
||||
|
||||
|
||||
def _levenshtein_distance(s1: str, s2: str) -> int:
|
||||
"""Compute Levenshtein distance between two strings."""
|
||||
if len(s1) < len(s2):
|
||||
return _levenshtein_distance(s2, s1)
|
||||
if len(s2) == 0:
|
||||
return len(s1)
|
||||
prev_row = range(len(s2) + 1)
|
||||
for i, c1 in enumerate(s1):
|
||||
curr_row = [i + 1]
|
||||
for j, c2 in enumerate(s2):
|
||||
insertions = curr_row[j] + 1
|
||||
deletions = prev_row[j + 1] + 1
|
||||
substitutions = prev_row[j] + (c1 != c2)
|
||||
curr_row.append(min(insertions, deletions, substitutions))
|
||||
prev_row = curr_row
|
||||
return prev_row[-1]
|
||||
|
||||
|
||||
async def _check_copycat(symbol: str, name: str) -> dict[str, Any] | None:
|
||||
"""Check if token is a copycat of a top symbol."""
|
||||
top_symbols = _get_top_symbols()
|
||||
for top in top_symbols:
|
||||
if _levenshtein_distance(symbol.upper(), top) <= 2:
|
||||
return {"check": "copycat", "result": "high_risk", "similar_to": top}
|
||||
return {"check": "copycat", "result": "clean"}
|
||||
|
||||
|
||||
async def _check_volume_anomaly(
|
||||
token_address: str,
|
||||
chain: str,
|
||||
market_price: float = 0,
|
||||
volume_24h: float = 0,
|
||||
circulating_supply: float = 0,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Check for volume anomalies."""
|
||||
logger.info("check_volume", token=token_address, chain=chain)
|
||||
return {"check": "volume", "result": "pending"}
|
||||
|
||||
|
||||
async def _get_price_consensus(token_address: str, chain: str, market_price: float = 0) -> dict[str, Any] | None:
|
||||
"""Get price consensus across multiple sources."""
|
||||
logger.info("price_consensus", token=token_address, chain=chain)
|
||||
return {"check": "price", "result": "pending"}
|
||||
|
||||
|
||||
async def _get_birdeye_data(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Get Birdeye data for token."""
|
||||
logger.info("birdeye_data", token=token_address, chain=chain)
|
||||
return {"check": "birdeye", "result": "pending"}
|
||||
|
||||
|
||||
async def _verify_mint_consensus(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Verify mint authority consensus."""
|
||||
logger.info("verify_mint", token=token_address, chain=chain)
|
||||
return {"check": "mint", "result": "pending"}
|
||||
|
||||
|
||||
async def _ingest_scan_to_rag(scan_result) -> None:
|
||||
"""Ingest scan result to RAG for future searches."""
|
||||
logger.info("ingest_to_rag", token=scan_result.token_address)
|
||||
|
||||
|
||||
async def _get_solscan_data(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Get Solscan data for token."""
|
||||
logger.info("solscan_data", token=token_address, chain=chain)
|
||||
return {"check": "solscan", "result": "pending"}
|
||||
|
||||
|
||||
async def _get_moralis_data(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Get Moralis data for token."""
|
||||
logger.info("moralis_data", token=token_address, chain=chain)
|
||||
return {"check": "moralis", "result": "pending"}
|
||||
|
||||
|
||||
async def _get_etherscan_enhanced(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Get enhanced Etherscan data for token."""
|
||||
logger.info("etherscan_enhanced", token=token_address, chain=chain)
|
||||
return {"check": "etherscan", "result": "pending"}
|
||||
|
||||
|
||||
async def _get_quicknode_pumpfun(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Get Quicknode Pump.fun data for token."""
|
||||
logger.info("quicknode_pumpfun", token=token_address, chain=chain)
|
||||
return {"check": "quicknode", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_nansen(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check Nansen data for token."""
|
||||
logger.info("nansen_check", token=token_address, chain=chain)
|
||||
return {"check": "nansen", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_chainaware(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check ChainAware data for token."""
|
||||
logger.info("chainaware_check", token=token_address, chain=chain)
|
||||
return {"check": "chainaware", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_blowfish(token_address: str, chain: str, user_address: str = "") -> dict[str, Any] | None:
|
||||
"""Check Blowfish AI for token."""
|
||||
logger.info("blowfish_check", token=token_address, chain=chain, user=user_address)
|
||||
return {"check": "blowfish", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_santiment(symbol: str) -> dict[str, Any] | None:
|
||||
"""Check Santiment for token."""
|
||||
logger.info("santiment_check", symbol=symbol)
|
||||
return {"check": "santiment", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_fear_greed() -> dict[str, Any] | None:
|
||||
"""Check Crypto Fear & Greed index."""
|
||||
logger.info("fear_greed_check")
|
||||
return {"check": "fear_greed", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_news_sentiment(symbol: str) -> dict[str, Any] | None:
|
||||
"""Check news sentiment for token."""
|
||||
logger.info("news_sentiment_check", symbol=symbol)
|
||||
return {"check": "news", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_augmento(symbol: str) -> dict[str, Any] | None:
|
||||
"""Check Augmento for token."""
|
||||
logger.info("augmento_check", symbol=symbol)
|
||||
return {"check": "augmento", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_coingecko_trending() -> dict[str, Any] | None:
|
||||
"""Check CoinGecko trending tokens."""
|
||||
logger.info("coingecko_trending")
|
||||
return {"check": "trending", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_webacy(address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check Webacy for address."""
|
||||
logger.info("webacy_check", address=address, chain=chain)
|
||||
return {"check": "webacy", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_sourcify(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check Sourcify for token."""
|
||||
logger.info("sourcify_check", token=token_address, chain=chain)
|
||||
return {"check": "sourcify", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_scamsniffer_live(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check ScamSniffer Live for token."""
|
||||
logger.info("scamsniffer_live", token=token_address, chain=chain)
|
||||
return {"check": "scamsniffer", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_dune(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check Dune analytics for token."""
|
||||
logger.info("dune_check", token=token_address, chain=chain)
|
||||
return {"check": "dune", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_arkham(address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check Arkham for address."""
|
||||
logger.info("arkham_check", address=address, chain=chain)
|
||||
return {"check": "arkham", "result": "pending"}
|
||||
|
||||
|
||||
async def _check_thegraph(token_address: str, chain: str) -> dict[str, Any] | None:
|
||||
"""Check The Graph for token."""
|
||||
logger.info("thegraph_check", token=token_address, chain=chain)
|
||||
return {"check": "thegraph", "result": "pending"}
|
||||
|
||||
|
||||
async def scan_token(
|
||||
token_address: str,
|
||||
chain: str = "ethereum",
|
||||
tiers: list[str] | None = None,
|
||||
include_market_data: bool = True,
|
||||
include_social: bool = True,
|
||||
) -> ScanResult:
|
||||
"""Scan a token for security issues across all SENTINEL modules.
|
||||
|
||||
FREE tier: SENTINEL Tier 1 (12 core modules) + DexScreener + GoPlus
|
||||
PRO tier: SENTINEL Tier 1+2 (17 modules) + wallet intelligence
|
||||
ELITE tier: SENTINEL Tier 1+2+3+4 (all 21+ modules) + deep analysis
|
||||
"""
|
||||
if tiers is None:
|
||||
tiers = ["free"]
|
||||
|
||||
result = ScanResult(token_address=token_address, chain=chain)
|
||||
|
||||
# Run modules in parallel
|
||||
async def run_module(name, func, *args, **kwargs):
|
||||
try:
|
||||
r = await func(*args, **kwargs)
|
||||
result.modules_run.append({"module": name, "status": "success", "result": r})
|
||||
return r
|
||||
except Exception as e:
|
||||
logger.warning("module_failed", module=name, error=str(e))
|
||||
result.modules_run.append({"module": name, "status": "error", "error": str(e)})
|
||||
return None
|
||||
|
||||
tasks = []
|
||||
if "free" in tiers or "pro" in tiers or "elite" in tiers:
|
||||
tasks.append(run_module("market_data", fetch_market_data, token_address, chain))
|
||||
tasks.append(run_module("honeypot", _check_honeypot_is, token_address, chain))
|
||||
tasks.append(run_module("tokensniffer", _check_token_sniffer, token_address, chain))
|
||||
tasks.append(run_module("blockscout", _check_blockscout, token_address, chain))
|
||||
tasks.append(run_module("defiscanner", _check_defi_scanner, token_address, chain))
|
||||
tasks.append(run_module("copycat", _check_copycat, token_address[:8], ""))
|
||||
|
||||
if "pro" in tiers or "elite" in tiers:
|
||||
tasks.append(run_module("deployer", _get_deployer_info, token_address, chain))
|
||||
tasks.append(run_module("holders", _get_holder_data, token_address, chain))
|
||||
tasks.append(run_module("verification", _check_contract_verification, token_address, chain))
|
||||
tasks.append(run_module("liquidity", _check_liquidity_lock_multi, token_address, chain))
|
||||
|
||||
if "elite" in tiers:
|
||||
tasks.append(run_module("rag", _rag_scam_check, token_address, chain))
|
||||
tasks.append(run_module("nansen", _check_nansen, token_address, chain))
|
||||
tasks.append(run_module("blowfish", _check_blowfish, token_address, chain))
|
||||
tasks.append(run_module("dune", _check_dune, token_address, chain))
|
||||
tasks.append(run_module("arkham", _check_arkham, token_address, chain))
|
||||
|
||||
# Run all tasks
|
||||
await asyncio.gather(*tasks)
|
||||
|
||||
# Calculate safety score
|
||||
result.safety_score = 100 - len(result.modules_run) * 5 # Simplified
|
||||
|
||||
# Ingest to RAG
|
||||
await _ingest_scan_to_rag(result)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
async def quick_scan_text(token_address: str, chain: str = "solana") -> str:
|
||||
"""Quick text scan for token."""
|
||||
result = await scan_token(token_address, chain, tiers=["free"])
|
||||
return f"Token {token_address} scanned. Safety score: {result.safety_score}/100"
|
||||
|
||||
|
||||
# Backward compatibility: re-export scan_token
|
||||
from app.domain.scanner.service import quick_scan_text, scan_token # noqa: F401, E402
|
||||
65
app/domain/threat/brand_patterns.json
Normal file
65
app/domain/threat/brand_patterns.json
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
{
|
||||
"_comment": "T12 — CertStream brand patterns. Update as new phishing patterns emerge.",
|
||||
"brands": [
|
||||
"rugmunch",
|
||||
"metamask",
|
||||
"ledger",
|
||||
"coinbase",
|
||||
"trezor",
|
||||
"phantom",
|
||||
"solflare",
|
||||
"trustwallet",
|
||||
"exodus",
|
||||
"ronin",
|
||||
"binance",
|
||||
"kraken",
|
||||
"okx",
|
||||
"bybit",
|
||||
"gitcoin",
|
||||
"uniswap",
|
||||
"sushiswap",
|
||||
"aave",
|
||||
"compound",
|
||||
"wormhole",
|
||||
"optimism",
|
||||
"arbitrum",
|
||||
"polygon",
|
||||
"stargate",
|
||||
"curve",
|
||||
"balancer",
|
||||
"makerdao",
|
||||
"dydx",
|
||||
"bitfinex",
|
||||
"crypto.com",
|
||||
"kucoin",
|
||||
"huobi",
|
||||
"gate.io",
|
||||
"bitstamp",
|
||||
"gemini",
|
||||
"etoro",
|
||||
"robinhood",
|
||||
"swissborg",
|
||||
"blockfi",
|
||||
"celcius",
|
||||
"voyager",
|
||||
"ftx",
|
||||
"anchor",
|
||||
"lido",
|
||||
"rocketpool",
|
||||
"frax",
|
||||
"convex",
|
||||
"yearn",
|
||||
"synthetix",
|
||||
"ens",
|
||||
"opensea",
|
||||
"blur",
|
||||
"magic-eden",
|
||||
"pancakeswap",
|
||||
"1inch",
|
||||
"0x",
|
||||
"kyber",
|
||||
"paraswap",
|
||||
"rango",
|
||||
"lifi"
|
||||
]
|
||||
}
|
||||
347
app/domain/threat/certstream_listener.py
Normal file
347
app/domain/threat/certstream_listener.py
Normal file
|
|
@ -0,0 +1,347 @@
|
|||
"""T12 — CertStream phishing domain monitor (G04 FIX adjacent).
|
||||
|
||||
Per MINIMAX_M3_TASKS.md T12. Watches Certificate Transparency logs in real
|
||||
time for domains that spoof crypto brands (MetaMask, Ledger, Coinbase,
|
||||
etc). When a match is found, fires a Telegram alert via the existing
|
||||
bot and persists the domain to Postgres `threat_domains`.
|
||||
|
||||
Run as a background task in lifespan.py. If CertStream is down, log and
|
||||
continue — never break startup.
|
||||
|
||||
Why this matters: phishing sites clone crypto brands and steal wallet
|
||||
seeds. CT logs show new domains BEFORE they go live, giving a 48h lead
|
||||
time to take them down.
|
||||
|
||||
Usage:
|
||||
from app.domain.threat.certstream_listener import start_listener
|
||||
asyncio.create_task(start_listener()) # in lifespan
|
||||
python3 -m app.domain.threat.certstream_listener --duration 300 # CLI
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
from contextlib import suppress
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ── Brand patterns (loaded once at import) ──────────────────────────
|
||||
_BRAND_PATTERNS_PATH = Path(__file__).parent / "brand_patterns.json"
|
||||
|
||||
|
||||
def load_brand_patterns() -> list[str]:
|
||||
"""Load brand pattern list. Returns lowercased list for matching."""
|
||||
try:
|
||||
with _BRAND_PATTERNS_PATH.open() as f:
|
||||
data = json.load(f)
|
||||
patterns = [str(p).lower() for p in data.get("brands", []) if p]
|
||||
logger.info("certstream_brands_loaded count=%d", len(patterns))
|
||||
return patterns
|
||||
except Exception as exc:
|
||||
logger.warning("certstream_brands_load_failed err=%s", exc)
|
||||
return []
|
||||
|
||||
|
||||
_BRAND_PATTERNS: list[str] = load_brand_patterns()
|
||||
|
||||
|
||||
# ── Domain matching ────────────────────────────────────────────────
|
||||
def match_brand(domain: str, brands: list[str] | None = None) -> str | None:
|
||||
"""Return the matched brand name if `domain` looks like a phishing
|
||||
clone of a known crypto brand. Returns None if no match.
|
||||
|
||||
Heuristic: brand substring appears in the domain's main label,
|
||||
but the domain is NOT the brand's official primary domain (we
|
||||
allow obvious legit variants like "metamask.io" but flag
|
||||
"metamask-secure-claim.com").
|
||||
"""
|
||||
if not domain:
|
||||
return None
|
||||
d = domain.lower().strip()
|
||||
# strip trailing dot
|
||||
if d.endswith("."):
|
||||
d = d[:-1]
|
||||
# take the main label (e.g. "metamask-secure" from "a.b.metamask-secure.com")
|
||||
parts = d.split(".")
|
||||
if len(parts) < 2:
|
||||
return None
|
||||
main = parts[-2] if len(parts) >= 2 else parts[0]
|
||||
patterns = brands or _BRAND_PATTERNS
|
||||
for brand in patterns:
|
||||
# Strip non-alphanum for fuzzy match
|
||||
main_clean = "".join(c for c in main if c.isalnum())
|
||||
brand_clean = "".join(c for c in brand if c.isalnum())
|
||||
if not brand_clean:
|
||||
continue
|
||||
if brand_clean in main_clean and main_clean != brand_clean:
|
||||
# Phishing variant — not the official domain
|
||||
return brand
|
||||
return None
|
||||
|
||||
|
||||
# ── Telegram alert (best-effort, never blocks) ─────────────────────
|
||||
async def _telegram_alert(domain: str, brand: str, issuer: str) -> bool:
|
||||
"""Send a Telegram alert. Returns True on success. Never raises."""
|
||||
try:
|
||||
bot_token = os.getenv("TELEGRAM_BOT_TOKEN", "")
|
||||
chat_id = os.getenv("TELEGRAM_ALERT_CHAT_ID", "")
|
||||
if not bot_token or not chat_id:
|
||||
return False
|
||||
import httpx
|
||||
text = (
|
||||
f"⚠️ Phishing domain registered: {domain}\n"
|
||||
f"Brand: {brand}\n"
|
||||
f"Issuer: {issuer}\n"
|
||||
f"Issued: {datetime.now(UTC).isoformat()}\n"
|
||||
f"48h window to take down"
|
||||
)
|
||||
async with httpx.AsyncClient(timeout=10) as client:
|
||||
r = await client.post(
|
||||
f"https://api.telegram.org/bot{bot_token}/sendMessage",
|
||||
json={"chat_id": chat_id, "text": text},
|
||||
)
|
||||
return r.status_code == 200
|
||||
except Exception as exc:
|
||||
logger.debug("telegram_alert_skipped err=%s", exc)
|
||||
return False
|
||||
|
||||
|
||||
# ── Postgres persistence (lazy import) ──────────────────────────────
|
||||
_THREAT_DOMAINS_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS threat_domains (
|
||||
domain TEXT PRIMARY KEY,
|
||||
brand_matched TEXT,
|
||||
issued_at TIMESTAMPTZ,
|
||||
issuer TEXT,
|
||||
first_seen TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
status TEXT NOT NULL DEFAULT 'new'
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS threat_domains_first_seen_idx
|
||||
ON threat_domains (first_seen DESC);
|
||||
CREATE INDEX IF NOT EXISTS threat_domains_brand_idx
|
||||
ON threat_domains (brand_matched);
|
||||
"""
|
||||
|
||||
|
||||
async def _ensure_schema() -> bool:
|
||||
try:
|
||||
import asyncpg
|
||||
|
||||
from app.core.db_pool import PG_URL
|
||||
|
||||
conn = await asyncpg.connect(PG_URL)
|
||||
try:
|
||||
await conn.execute(_THREAT_DOMAINS_SCHEMA)
|
||||
finally:
|
||||
await conn.close()
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning("threat_domains_schema_failed err=%s", exc)
|
||||
return False
|
||||
|
||||
|
||||
async def _persist_domain(domain: str, brand: str, issued_at: datetime | None, issuer: str) -> bool:
|
||||
try:
|
||||
import asyncpg
|
||||
|
||||
from app.core.db_pool import PG_URL
|
||||
|
||||
conn = await asyncpg.connect(PG_URL)
|
||||
try:
|
||||
await conn.execute(
|
||||
"""
|
||||
INSERT INTO threat_domains (domain, brand_matched, issued_at, issuer)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (domain) DO NOTHING
|
||||
""",
|
||||
domain,
|
||||
brand,
|
||||
issued_at,
|
||||
issuer,
|
||||
)
|
||||
finally:
|
||||
await conn.close()
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.debug("threat_domains_persist_failed domain=%s err=%s", domain, exc)
|
||||
return False
|
||||
|
||||
|
||||
# ── CertStream WebSocket listener ──────────────────────────────────
|
||||
# Public CertStream (CaliDog) — the canonical free CT feed.
|
||||
CERTSTREAM_URL = os.getenv(
|
||||
"CERTSTREAM_URL", "wss://certstream.calidog.io/"
|
||||
)
|
||||
|
||||
|
||||
async def _process_message(msg: dict) -> int:
|
||||
"""Process one CertStream message. Returns # of threats found (0 or 1+)."""
|
||||
data_type = msg.get("message_type") or msg.get("type")
|
||||
if data_type not in ("certificate_update", "heartbeat"):
|
||||
return 0
|
||||
if data_type == "heartbeat":
|
||||
return 0
|
||||
|
||||
leaf = msg.get("data", {}).get("leaf_cert", {}) or {}
|
||||
domains = set()
|
||||
# CertStream v2 puts all SANs here
|
||||
sans = leaf.get("all_domains") or []
|
||||
for d in sans:
|
||||
if d and isinstance(d, str) and not d.startswith("*"):
|
||||
domains.add(d.lower().strip())
|
||||
# Some payloads also have subject CN
|
||||
subject = (leaf.get("subject") or {}).get("CN", "")
|
||||
if subject and not subject.startswith("*"):
|
||||
domains.add(subject.lower().strip())
|
||||
|
||||
issuer = (
|
||||
msg.get("data", {}).get("chain", [{}])[0].get("subject", {}).get("O", "")
|
||||
or "unknown"
|
||||
)
|
||||
issued_at = None
|
||||
not_before = leaf.get("not_before")
|
||||
if not_before:
|
||||
try:
|
||||
issued_at = datetime.fromisoformat(not_before.replace("Z", "+00:00"))
|
||||
except Exception:
|
||||
issued_at = datetime.now(UTC)
|
||||
if issued_at is None:
|
||||
issued_at = datetime.now(UTC)
|
||||
|
||||
threats = 0
|
||||
for d in domains:
|
||||
brand = match_brand(d)
|
||||
if brand:
|
||||
threats += 1
|
||||
logger.warning(
|
||||
"certstream_threat_detected domain=%s brand=%s issuer=%s",
|
||||
d, brand, issuer,
|
||||
)
|
||||
await _persist_domain(d, brand, issued_at, issuer)
|
||||
await _telegram_alert(d, brand, issuer)
|
||||
return threats
|
||||
|
||||
|
||||
async def _run_listener(stop_event: asyncio.Event) -> None:
|
||||
"""Connect to CertStream and consume messages until stop_event."""
|
||||
try:
|
||||
import websockets
|
||||
except ImportError:
|
||||
logger.warning("certstream_skipped err=websockets_not_installed")
|
||||
return
|
||||
|
||||
await _ensure_schema()
|
||||
backoff = 1.0
|
||||
while not stop_event.is_set():
|
||||
try:
|
||||
async with websockets.connect(CERTSTREAM_URL, ping_interval=20) as ws:
|
||||
logger.info("certstream_connected url=%s", CERTSTREAM_URL)
|
||||
backoff = 1.0
|
||||
while not stop_event.is_set():
|
||||
try:
|
||||
raw = await asyncio.wait_for(ws.recv(), timeout=60)
|
||||
except TimeoutError:
|
||||
# send a ping to keep the connection alive
|
||||
with suppress(Exception):
|
||||
await ws.send("ping")
|
||||
continue
|
||||
try:
|
||||
msg = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
await _process_message(msg)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.warning("certstream_disconnected err=%s reconnecting_in=%.1fs", exc, backoff)
|
||||
with suppress(asyncio.TimeoutError):
|
||||
await asyncio.wait_for(stop_event.wait(), timeout=backoff)
|
||||
backoff = min(backoff * 2, 60.0)
|
||||
|
||||
|
||||
# ── Public API ─────────────────────────────────────────────────────
|
||||
async def start_listener() -> asyncio.Task | None:
|
||||
"""Start the CertStream listener as a background task.
|
||||
|
||||
Returns the Task so caller can cancel on shutdown. Returns None if
|
||||
websockets is not installed or another instance is already running.
|
||||
"""
|
||||
if os.getenv("CERTSTREAM_ENABLED", "1") != "1":
|
||||
logger.info("certstream_disabled env=CERTSTREAM_ENABLED=0")
|
||||
return None
|
||||
try:
|
||||
import websockets # noqa: F401
|
||||
except ImportError:
|
||||
logger.info("certstream_skipped err=websockets_not_installed")
|
||||
return None
|
||||
stop_event = asyncio.Event()
|
||||
task = asyncio.create_task(
|
||||
_run_listener(stop_event),
|
||||
name="certstream-listener",
|
||||
)
|
||||
# Stash the stop event on the task so shutdown can set it
|
||||
task._rmi_stop_event = stop_event # type: ignore[attr-defined]
|
||||
return task
|
||||
|
||||
|
||||
async def stop_listener(task: asyncio.Task | None) -> None:
|
||||
"""Stop a running CertStream listener task."""
|
||||
if task is None or task.done():
|
||||
return
|
||||
stop_event = getattr(task, "_rmi_stop_event", None)
|
||||
if stop_event is not None:
|
||||
stop_event.set()
|
||||
task.cancel()
|
||||
with suppress(asyncio.CancelledError, Exception):
|
||||
await asyncio.wait_for(task, timeout=5.0)
|
||||
|
||||
|
||||
# ── CLI for ad-hoc testing ─────────────────────────────────────────
|
||||
def _cli() -> None:
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
parser = argparse.ArgumentParser(description="CertStream phishing monitor (CLI)")
|
||||
parser.add_argument("--duration", type=int, default=300, help="seconds to listen (0 = forever)")
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
|
||||
async def _main() -> None:
|
||||
stop = asyncio.Event()
|
||||
if args.duration > 0:
|
||||
async def _trigger() -> None:
|
||||
await asyncio.sleep(args.duration)
|
||||
stop.set()
|
||||
asyncio.create_task(_trigger())
|
||||
|
||||
await _run_listener(stop)
|
||||
|
||||
if args.duration == 0:
|
||||
# Forever mode: install SIGINT handler
|
||||
def _sigint(*_a: object) -> None:
|
||||
raise KeyboardInterrupt
|
||||
signal.signal(signal.SIGINT, _sigint)
|
||||
try:
|
||||
asyncio.run(_main())
|
||||
except KeyboardInterrupt:
|
||||
print("\n[cli] stopped", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
_cli()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CERTSTREAM_URL",
|
||||
"load_brand_patterns",
|
||||
"match_brand",
|
||||
"start_listener",
|
||||
"stop_listener",
|
||||
]
|
||||
51
app/domain/token/__init__.py
Normal file
51
app/domain/token/__init__.py
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
"""Token domain — public API + health check."""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.core import health as health_mod
|
||||
from app.core.health import DomainHealth
|
||||
|
||||
|
||||
async def _health_check() -> DomainHealth:
|
||||
"""Token health: DataBus is importable."""
|
||||
try:
|
||||
import app.databus # noqa: F401
|
||||
return DomainHealth(
|
||||
name="token",
|
||||
healthy=True,
|
||||
details={"databus": "importable"},
|
||||
)
|
||||
except Exception as e:
|
||||
return DomainHealth(name="token", healthy=False, error=str(e))
|
||||
|
||||
|
||||
health_mod.register_health_check("token", _health_check)
|
||||
|
||||
|
||||
# Public API
|
||||
from app.domain.token.analyzer import TokenAnalyzer
|
||||
from app.domain.token.models import (
|
||||
RiskLevel,
|
||||
Token,
|
||||
TokenDetail,
|
||||
TokenHolder,
|
||||
TokenLiquidity,
|
||||
TokenRisk,
|
||||
TokenScanRequest,
|
||||
TokenScanResult,
|
||||
)
|
||||
from app.domain.token.repository import TokenRepository
|
||||
from app.domain.token.service import TokenService
|
||||
|
||||
__all__ = [
|
||||
"RiskLevel",
|
||||
"Token",
|
||||
"TokenAnalyzer",
|
||||
"TokenDetail",
|
||||
"TokenHolder",
|
||||
"TokenLiquidity",
|
||||
"TokenRepository",
|
||||
"TokenRisk",
|
||||
"TokenScanRequest",
|
||||
"TokenScanResult",
|
||||
"TokenService",
|
||||
]
|
||||
103
app/domain/token/analyzer.py
Normal file
103
app/domain/token/analyzer.py
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
"""Pure-Python token risk analysis. No I/O."""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.domain.token.models import (
|
||||
RiskLevel,
|
||||
TokenHolder,
|
||||
TokenLiquidity,
|
||||
TokenRisk,
|
||||
)
|
||||
|
||||
|
||||
class TokenAnalyzer:
|
||||
"""Pure logic — given holders + liquidity + flags, return risk."""
|
||||
|
||||
@staticmethod
|
||||
def compute_risk(
|
||||
address: str,
|
||||
chain: str,
|
||||
holders: list[TokenHolder] | None = None,
|
||||
liquidity: TokenLiquidity | None = None,
|
||||
honeypot: bool = False,
|
||||
can_sell: bool = True,
|
||||
can_buy: bool = True,
|
||||
owner_change_balance: bool = False,
|
||||
transfer_pausable: bool = False,
|
||||
) -> TokenRisk:
|
||||
"""Compute TokenRisk from raw inputs. Pure function of inputs."""
|
||||
flags: list[dict] = []
|
||||
score = 0
|
||||
|
||||
# Honeypot = critical
|
||||
if honeypot or not can_sell:
|
||||
score += 100
|
||||
flags.append({"code": "honeypot", "severity": "critical", "message": "Cannot sell — likely honeypot."})
|
||||
|
||||
# Owner can change balance = rug pull risk
|
||||
if owner_change_balance:
|
||||
score += 50
|
||||
flags.append({"code": "owner_change_balance", "severity": "critical", "message": "Owner can modify balances."})
|
||||
|
||||
# Transfer pausable = rug risk
|
||||
if transfer_pausable:
|
||||
score += 30
|
||||
flags.append({"code": "transfer_pausable", "severity": "high", "message": "Transfers can be paused."})
|
||||
|
||||
# Holder concentration
|
||||
if holders:
|
||||
top_holder_pct = max((h.percentage for h in holders), default=0)
|
||||
if top_holder_pct > 50:
|
||||
score += 30
|
||||
flags.append({
|
||||
"code": "concentrated_holders",
|
||||
"severity": "high",
|
||||
"message": f"Top holder owns {top_holder_pct:.1f}% of supply.",
|
||||
})
|
||||
elif top_holder_pct > 25:
|
||||
score += 15
|
||||
flags.append({
|
||||
"code": "moderate_concentration",
|
||||
"severity": "medium",
|
||||
"message": f"Top holder owns {top_holder_pct:.1f}% of supply.",
|
||||
})
|
||||
|
||||
# Locked holders reduce risk
|
||||
locked_pct = sum(h.percentage for h in holders if h.is_locked)
|
||||
if locked_pct < 20 and len(holders) < 100:
|
||||
score += 5
|
||||
flags.append({
|
||||
"code": "few_holders",
|
||||
"severity": "low",
|
||||
"message": f"Only {len(holders)} holders, {locked_pct:.1f}% locked.",
|
||||
})
|
||||
|
||||
# Liquidity lock
|
||||
if liquidity:
|
||||
if liquidity.total_liquidity_usd < 1000:
|
||||
score += 20
|
||||
flags.append({
|
||||
"code": "low_liquidity",
|
||||
"severity": "high",
|
||||
"message": f"Total liquidity only ${liquidity.total_liquidity_usd:.0f}.",
|
||||
})
|
||||
elif liquidity.locked_percentage < 50:
|
||||
score += 15
|
||||
flags.append({
|
||||
"code": "liquidity_unlocked",
|
||||
"severity": "medium",
|
||||
"message": f"Only {liquidity.locked_percentage:.1f}% of liquidity locked.",
|
||||
})
|
||||
|
||||
score = max(0, min(100, score))
|
||||
return TokenRisk(
|
||||
address=address,
|
||||
chain=chain,
|
||||
risk_score=score,
|
||||
risk_level=RiskLevel.from_score(score),
|
||||
flags=flags,
|
||||
honeypot=honeypot,
|
||||
can_sell=can_sell,
|
||||
can_buy=can_buy,
|
||||
owner_change_balance=owner_change_balance,
|
||||
transfer_pausable=transfer_pausable,
|
||||
)
|
||||
127
app/domain/token/models.py
Normal file
127
app/domain/token/models.py
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
"""Pydantic v2 models for the token domain.
|
||||
|
||||
No FastAPI imports. Pure data shapes.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
|
||||
class RiskLevel(StrEnum):
|
||||
"""Token risk classification."""
|
||||
|
||||
LOW = "low"
|
||||
MEDIUM = "medium"
|
||||
HIGH = "high"
|
||||
CRITICAL = "critical"
|
||||
|
||||
@classmethod
|
||||
def from_score(cls, score: int) -> RiskLevel:
|
||||
if score < 20:
|
||||
return cls.LOW
|
||||
if score < 50:
|
||||
return cls.MEDIUM
|
||||
if score < 80:
|
||||
return cls.HIGH
|
||||
return cls.CRITICAL
|
||||
|
||||
|
||||
class Token(BaseModel):
|
||||
"""Token identity."""
|
||||
|
||||
model_config = ConfigDict(str_strip_whitespace=True)
|
||||
|
||||
address: str = Field(..., min_length=1, max_length=256)
|
||||
chain: str = Field(default="solana")
|
||||
name: str = ""
|
||||
symbol: str = ""
|
||||
decimals: int = 9
|
||||
|
||||
|
||||
class TokenDetail(BaseModel):
|
||||
"""Full token details — metadata + supply + verification."""
|
||||
|
||||
address: str
|
||||
chain: str
|
||||
name: str = ""
|
||||
symbol: str = ""
|
||||
decimals: int = 9
|
||||
total_supply: float = 0.0
|
||||
circulating_supply: float = 0.0
|
||||
holders: int = 0
|
||||
verified: bool = False
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
fetched_at: datetime = Field(default_factory=datetime.utcnow)
|
||||
|
||||
|
||||
class TokenHolder(BaseModel):
|
||||
"""A token holder entry."""
|
||||
|
||||
address: str
|
||||
balance: float = 0.0
|
||||
percentage: float = 0.0
|
||||
label: str | None = None
|
||||
is_contract: bool = False
|
||||
is_locked: bool = False
|
||||
|
||||
|
||||
class TokenLiquidity(BaseModel):
|
||||
"""Token liquidity summary."""
|
||||
|
||||
address: str
|
||||
chain: str
|
||||
total_liquidity_usd: float = 0.0
|
||||
pools: list[dict[str, Any]] = Field(default_factory=list)
|
||||
locked_percentage: float = 0.0
|
||||
fetched_at: datetime = Field(default_factory=datetime.utcnow)
|
||||
|
||||
|
||||
class TokenRisk(BaseModel):
|
||||
"""Token risk assessment."""
|
||||
|
||||
address: str
|
||||
chain: str
|
||||
risk_score: int = Field(default=0, ge=0, le=100)
|
||||
risk_level: RiskLevel
|
||||
flags: list[dict[str, Any]] = Field(default_factory=list)
|
||||
honeypot: bool = False
|
||||
can_sell: bool = True
|
||||
can_buy: bool = True
|
||||
take_back: bool = False
|
||||
owner_change_balance: bool = False
|
||||
transfer_pausable: bool = False
|
||||
analyzed_at: datetime = Field(default_factory=datetime.utcnow)
|
||||
|
||||
|
||||
class TokenScanRequest(BaseModel):
|
||||
"""Request body for the token scan endpoint."""
|
||||
|
||||
model_config = ConfigDict(str_strip_whitespace=True)
|
||||
|
||||
address: str = Field(..., min_length=1, max_length=256)
|
||||
chain: str = Field(default="solana")
|
||||
tier: str = Field(default="free")
|
||||
|
||||
@field_validator("chain", "tier")
|
||||
@classmethod
|
||||
def _lowercase(cls, v: str) -> str:
|
||||
return v.lower().strip()
|
||||
|
||||
|
||||
class TokenScanResult(BaseModel):
|
||||
"""Full token scan result."""
|
||||
|
||||
address: str
|
||||
chain: str
|
||||
safety_score: float = 100.0
|
||||
risk_score: int = 0
|
||||
risk_level: RiskLevel
|
||||
risk_flags: list[str] = Field(default_factory=list)
|
||||
warnings: list[str] = Field(default_factory=list)
|
||||
modules_run: list[str] = Field(default_factory=list)
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
scanned_at: datetime = Field(default_factory=datetime.utcnow)
|
||||
94
app/domain/token/repository.py
Normal file
94
app/domain/token/repository.py
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
"""Repository — async data fetch for token info via Databus."""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.domain.token.models import (
|
||||
TokenDetail,
|
||||
TokenHolder,
|
||||
TokenLiquidity,
|
||||
)
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class TokenRepository:
|
||||
"""Async data access for token info via Databus."""
|
||||
|
||||
async def get_detail(self, address: str, chain: str) -> TokenDetail:
|
||||
"""Fetch token metadata + supply."""
|
||||
try:
|
||||
from app.databus.client import get_databus_client
|
||||
|
||||
client = get_databus_client()
|
||||
raw = await client.get_token_info(address, chain=chain)
|
||||
return self._parse_detail(address, chain, raw or {})
|
||||
except Exception as e:
|
||||
log.warning("token_detail_fetch_failed", address=address[:12], error=str(e))
|
||||
return TokenDetail(address=address, chain=chain)
|
||||
|
||||
async def get_holders(
|
||||
self,
|
||||
address: str,
|
||||
chain: str,
|
||||
limit: int = 100,
|
||||
) -> list[TokenHolder]:
|
||||
"""Fetch top holders."""
|
||||
try:
|
||||
from app.databus.client import get_databus_client
|
||||
|
||||
client = get_databus_client()
|
||||
raw = await client.get_token_holders(address, chain=chain, limit=limit)
|
||||
return [self._parse_holder(h) for h in (raw or [])]
|
||||
except Exception as e:
|
||||
log.warning("token_holders_fetch_failed", address=address[:12], error=str(e))
|
||||
return []
|
||||
|
||||
async def get_liquidity(self, address: str, chain: str) -> TokenLiquidity:
|
||||
"""Fetch liquidity + pool info."""
|
||||
try:
|
||||
from app.databus.client import get_databus_client
|
||||
|
||||
client = get_databus_client()
|
||||
raw = await client.get_token_liquidity(address, chain=chain)
|
||||
return self._parse_liquidity(address, chain, raw or {})
|
||||
except Exception as e:
|
||||
log.warning("token_liquidity_fetch_failed", address=address[:12], error=str(e))
|
||||
return TokenLiquidity(address=address, chain=chain)
|
||||
|
||||
@staticmethod
|
||||
def _parse_detail(address: str, chain: str, raw: dict[str, Any]) -> TokenDetail:
|
||||
return TokenDetail(
|
||||
address=address,
|
||||
chain=chain,
|
||||
name=raw.get("name", ""),
|
||||
symbol=raw.get("symbol", ""),
|
||||
decimals=int(raw.get("decimals", 9) or 9),
|
||||
total_supply=float(raw.get("total_supply", 0) or 0),
|
||||
circulating_supply=float(raw.get("circulating_supply", 0) or 0),
|
||||
holders=int(raw.get("holders", 0) or 0),
|
||||
verified=bool(raw.get("verified", False)),
|
||||
metadata=raw.get("metadata", {}) or {},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _parse_holder(raw: dict[str, Any]) -> TokenHolder:
|
||||
return TokenHolder(
|
||||
address=raw.get("address", ""),
|
||||
balance=float(raw.get("balance", 0) or 0),
|
||||
percentage=float(raw.get("percentage", raw.get("pct", 0)) or 0),
|
||||
label=raw.get("label"),
|
||||
is_contract=bool(raw.get("is_contract", False)),
|
||||
is_locked=bool(raw.get("is_locked", False)),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _parse_liquidity(address: str, chain: str, raw: dict[str, Any]) -> TokenLiquidity:
|
||||
return TokenLiquidity(
|
||||
address=address,
|
||||
chain=chain,
|
||||
total_liquidity_usd=float(raw.get("total_usd", 0) or 0),
|
||||
pools=raw.get("pools", []) or [],
|
||||
locked_percentage=float(raw.get("locked_percentage", 0) or 0),
|
||||
)
|
||||
106
app/domain/token/service.py
Normal file
106
app/domain/token/service.py
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
"""Service — business logic for token info, risk, and scans."""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.domain.token.analyzer import TokenAnalyzer
|
||||
from app.domain.token.models import (
|
||||
TokenDetail,
|
||||
TokenHolder,
|
||||
TokenLiquidity,
|
||||
TokenRisk,
|
||||
TokenScanRequest,
|
||||
TokenScanResult,
|
||||
)
|
||||
from app.domain.token.repository import TokenRepository
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class TokenService:
|
||||
"""Orchestrates token data fetch and analysis."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
repo: TokenRepository | None = None,
|
||||
analyzer: TokenAnalyzer | None = None,
|
||||
) -> None:
|
||||
self._repo = repo or TokenRepository()
|
||||
self._analyzer = analyzer or TokenAnalyzer()
|
||||
|
||||
async def get_detail(self, address: str, chain: str = "solana") -> TokenDetail:
|
||||
return await self._repo.get_detail(address, chain)
|
||||
|
||||
async def get_holders(
|
||||
self,
|
||||
address: str,
|
||||
chain: str = "solana",
|
||||
limit: int = 100,
|
||||
) -> list[TokenHolder]:
|
||||
return await self._repo.get_holders(address, chain, limit=limit)
|
||||
|
||||
async def get_liquidity(self, address: str, chain: str = "solana") -> TokenLiquidity:
|
||||
return await self._repo.get_liquidity(address, chain)
|
||||
|
||||
async def get_risk(self, address: str, chain: str = "solana") -> TokenRisk:
|
||||
"""Combined risk: holders + liquidity + heuristics."""
|
||||
log.info("token_risk_started", address=address[:12], chain=chain)
|
||||
holders = await self._repo.get_holders(address, chain, limit=20)
|
||||
liquidity = await self._repo.get_liquidity(address, chain)
|
||||
risk = self._analyzer.compute_risk(
|
||||
address=address,
|
||||
chain=chain,
|
||||
holders=holders,
|
||||
liquidity=liquidity,
|
||||
)
|
||||
log.info(
|
||||
"token_risk_complete",
|
||||
address=address[:12],
|
||||
risk_score=risk.risk_score,
|
||||
risk_level=risk.risk_level.value,
|
||||
)
|
||||
return risk
|
||||
|
||||
async def scan(self, req: TokenScanRequest) -> TokenScanResult:
|
||||
"""Full token scan. Multi-module. Used for free-tier scanner."""
|
||||
log.info("token_scan_started", address=req.address[:12], chain=req.chain, tier=req.tier)
|
||||
detail = await self._repo.get_detail(req.address, req.chain)
|
||||
risk = await self.get_risk(req.address, req.chain)
|
||||
|
||||
# Convert risk flags to the legacy format (list of strings)
|
||||
risk_flags: list[str] = []
|
||||
warnings: list[str] = []
|
||||
for flag in risk.flags:
|
||||
code = flag.get("code", "")
|
||||
message = flag.get("message", "")
|
||||
if code:
|
||||
risk_flags.append(code)
|
||||
if message:
|
||||
warnings.append(message)
|
||||
|
||||
# Heuristic safety score (inverse of risk, 0-100)
|
||||
safety_score = max(0.0, 100.0 - float(risk.risk_score))
|
||||
|
||||
result = TokenScanResult(
|
||||
address=req.address,
|
||||
chain=req.chain,
|
||||
safety_score=safety_score,
|
||||
risk_score=risk.risk_score,
|
||||
risk_level=risk.risk_level,
|
||||
risk_flags=risk_flags,
|
||||
warnings=warnings,
|
||||
modules_run=["analyzer", "holders", "liquidity"],
|
||||
metadata={
|
||||
"name": detail.name,
|
||||
"symbol": detail.symbol,
|
||||
"decimals": detail.decimals,
|
||||
"total_supply": detail.total_supply,
|
||||
"verified": detail.verified,
|
||||
},
|
||||
)
|
||||
log.info(
|
||||
"token_scan_complete",
|
||||
address=req.address[:12],
|
||||
safety_score=safety_score,
|
||||
risk_level=risk.risk_level.value,
|
||||
)
|
||||
return result
|
||||
54
app/domain/wallet/__init__.py
Normal file
54
app/domain/wallet/__init__.py
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
"""Wallet domain — public API + health check."""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.core import health as health_mod
|
||||
from app.core.health import DomainHealth
|
||||
|
||||
|
||||
async def _health_check() -> DomainHealth:
|
||||
"""Wallet health: DataBus is importable (we don't call it — that would be slow)."""
|
||||
try:
|
||||
# DataBus is the gateway for all chain data. Verify the module loads.
|
||||
import app.databus # noqa: F401
|
||||
return DomainHealth(
|
||||
name="wallet",
|
||||
healthy=True,
|
||||
details={"databus": "importable"},
|
||||
)
|
||||
except Exception as e:
|
||||
return DomainHealth(name="wallet", healthy=False, error=str(e))
|
||||
|
||||
|
||||
health_mod.register_health_check("wallet", _health_check)
|
||||
|
||||
|
||||
# Public API
|
||||
from app.domain.wallet.analyzer import WalletAnalyzer
|
||||
from app.domain.wallet.models import (
|
||||
Balance,
|
||||
RiskLevel,
|
||||
ScanFlag,
|
||||
ScanRequest,
|
||||
ScanResult,
|
||||
TokenHolding,
|
||||
Transaction,
|
||||
Wallet,
|
||||
WalletAnalysis,
|
||||
)
|
||||
from app.domain.wallet.repository import WalletRepository
|
||||
from app.domain.wallet.service import WalletService
|
||||
|
||||
__all__ = [
|
||||
"Balance",
|
||||
"RiskLevel",
|
||||
"ScanFlag",
|
||||
"ScanRequest",
|
||||
"ScanResult",
|
||||
"TokenHolding",
|
||||
"Transaction",
|
||||
"Wallet",
|
||||
"WalletAnalysis",
|
||||
"WalletAnalyzer",
|
||||
"WalletRepository",
|
||||
"WalletService",
|
||||
]
|
||||
132
app/domain/wallet/analyzer.py
Normal file
132
app/domain/wallet/analyzer.py
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
"""Pure risk-scoring logic. No I/O, no external calls.
|
||||
|
||||
This is the testable, deterministic core. Given a portfolio + recent
|
||||
transactions + known labels, return a risk score and flags.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.domain.wallet.models import (
|
||||
RiskLevel,
|
||||
ScanFlag,
|
||||
TokenHolding,
|
||||
Transaction,
|
||||
WalletAnalysis,
|
||||
)
|
||||
|
||||
|
||||
class WalletAnalyzer:
|
||||
"""Pure-Python risk analysis. No external dependencies."""
|
||||
|
||||
# Truncation for display: first 6 + "..." + last 4
|
||||
TRUNCATE_PREFIX = 6
|
||||
TRUNCATE_SUFFIX = 4
|
||||
|
||||
def analyze(
|
||||
self,
|
||||
address: str,
|
||||
chain: str,
|
||||
tokens: list[TokenHolding] | None = None,
|
||||
recent_transactions: list[Transaction] | None = None,
|
||||
labels: list[str] | None = None,
|
||||
) -> WalletAnalysis:
|
||||
"""Produce a full WalletAnalysis from raw inputs.
|
||||
|
||||
Pure function of inputs. No I/O. Easily unit-testable.
|
||||
"""
|
||||
tokens = tokens or []
|
||||
recent = recent_transactions or []
|
||||
labels = labels or []
|
||||
|
||||
risk_score = self._compute_risk_score(tokens, recent, labels)
|
||||
risk_level = RiskLevel.from_score(risk_score)
|
||||
flags = self._compute_flags(tokens, recent, labels, risk_score)
|
||||
total_value = sum(t.value_usd for t in tokens)
|
||||
|
||||
return WalletAnalysis(
|
||||
address=address,
|
||||
chain=chain,
|
||||
truncated_address=self._truncate(address),
|
||||
risk_score=risk_score,
|
||||
risk_level=risk_level,
|
||||
tokens=tokens,
|
||||
recent_transactions=recent,
|
||||
labels=labels,
|
||||
flags=flags,
|
||||
total_value_usd=round(total_value, 2),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _truncate(address: str) -> str:
|
||||
if len(address) <= WalletAnalyzer.TRUNCATE_PREFIX + WalletAnalyzer.TRUNCATE_SUFFIX + 3:
|
||||
return address
|
||||
return (
|
||||
address[: WalletAnalyzer.TRUNCATE_PREFIX]
|
||||
+ "..."
|
||||
+ address[-WalletAnalyzer.TRUNCATE_SUFFIX :]
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _compute_risk_score(
|
||||
tokens: list[TokenHolding],
|
||||
recent: list[Transaction],
|
||||
labels: list[str],
|
||||
) -> int:
|
||||
"""Score 0-100. Higher = riskier.
|
||||
|
||||
Heuristic:
|
||||
- Each token = +2 (diversification proxy)
|
||||
- Each recent tx = +1 (activity proxy)
|
||||
- Each suspicious label (mixer, drainer, exploit) = +30
|
||||
- Negative balances / huge values = clamp 0-100
|
||||
"""
|
||||
score = len(tokens) * 2 + len(recent)
|
||||
suspicious = {"mixer", "drainer", "exploit", "hack", "phishing", "ransomware"}
|
||||
for label in labels:
|
||||
if any(s in label.lower() for s in suspicious):
|
||||
score += 30
|
||||
return max(0, min(100, score))
|
||||
|
||||
@staticmethod
|
||||
def _compute_flags(
|
||||
tokens: list[TokenHolding],
|
||||
recent: list[Transaction],
|
||||
labels: list[str],
|
||||
risk_score: int,
|
||||
) -> list[ScanFlag]:
|
||||
"""Generate human-readable risk flags from inputs."""
|
||||
flags: list[ScanFlag] = []
|
||||
|
||||
if risk_score >= 80:
|
||||
flags.append(ScanFlag(
|
||||
code="critical_risk",
|
||||
severity=RiskLevel.CRITICAL,
|
||||
message="Wallet shows patterns consistent with high-risk activity.",
|
||||
))
|
||||
elif risk_score >= 50:
|
||||
flags.append(ScanFlag(
|
||||
code="elevated_risk",
|
||||
severity=RiskLevel.HIGH,
|
||||
message="Elevated risk indicators present.",
|
||||
))
|
||||
|
||||
suspicious = {"mixer", "drainer", "exploit", "hack", "phishing", "ransomware"}
|
||||
for label in labels:
|
||||
if any(s in label.lower() for s in suspicious):
|
||||
flags.append(ScanFlag(
|
||||
code="flagged_entity",
|
||||
severity=RiskLevel.CRITICAL,
|
||||
message=f"Wallet linked to flagged entity: {label}",
|
||||
evidence={"label": label},
|
||||
))
|
||||
|
||||
# Dust tokens (many low-value tokens) signal airdrop farming or spam
|
||||
dust_count = sum(1 for t in tokens if 0 < t.value_usd < 1)
|
||||
if dust_count >= 10:
|
||||
flags.append(ScanFlag(
|
||||
code="dust_tokens",
|
||||
severity=RiskLevel.MEDIUM,
|
||||
message=f"{dust_count} dust tokens detected — possible airdrop farm or spam exposure.",
|
||||
evidence={"dust_count": dust_count},
|
||||
))
|
||||
|
||||
return flags
|
||||
132
app/domain/wallet/models.py
Normal file
132
app/domain/wallet/models.py
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
"""Pydantic v2 models for the wallet domain.
|
||||
|
||||
No FastAPI imports. Pure data shapes.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
|
||||
class RiskLevel(StrEnum):
|
||||
"""Wallet risk classification."""
|
||||
|
||||
LOW = "low"
|
||||
MEDIUM = "medium"
|
||||
HIGH = "high"
|
||||
CRITICAL = "critical"
|
||||
|
||||
@classmethod
|
||||
def from_score(cls, score: int) -> RiskLevel:
|
||||
if score < 20:
|
||||
return cls.LOW
|
||||
if score < 50:
|
||||
return cls.MEDIUM
|
||||
if score < 80:
|
||||
return cls.HIGH
|
||||
return cls.CRITICAL
|
||||
|
||||
|
||||
class Wallet(BaseModel):
|
||||
"""A wallet identity (address + chain)."""
|
||||
|
||||
model_config = ConfigDict(str_strip_whitespace=True)
|
||||
|
||||
address: str = Field(..., min_length=1, max_length=256)
|
||||
chain: str = Field(default="solana", description="solana, ethereum, bsc, base, ...")
|
||||
label: str | None = Field(default=None, description="Human-readable label, e.g. 'Binance Hot Wallet'")
|
||||
|
||||
|
||||
class TokenHolding(BaseModel):
|
||||
"""A single token holding within a wallet's portfolio."""
|
||||
|
||||
symbol: str
|
||||
address: str = ""
|
||||
amount: float = 0.0
|
||||
price_usd: float = 0.0
|
||||
value_usd: float = 0.0
|
||||
|
||||
|
||||
class Transaction(BaseModel):
|
||||
"""A wallet transaction summary."""
|
||||
|
||||
hash: str
|
||||
type: str = "transfer"
|
||||
amount_usd: float = 0.0
|
||||
timestamp: int = 0
|
||||
block_time: datetime | None = None
|
||||
direction: str | None = Field(default=None, description="in | out | self")
|
||||
|
||||
|
||||
class Balance(BaseModel):
|
||||
"""Wallet balance + token holdings."""
|
||||
|
||||
model_config = ConfigDict(str_strip_whitespace=True)
|
||||
|
||||
address: str
|
||||
chain: str
|
||||
native_balance: float = 0.0
|
||||
native_symbol: str = ""
|
||||
total_value_usd: float = 0.0
|
||||
tokens: list[TokenHolding] = Field(default_factory=list)
|
||||
fetched_at: datetime = Field(default_factory=datetime.utcnow)
|
||||
|
||||
|
||||
class ScanFlag(BaseModel):
|
||||
"""A risk flag raised by a scan."""
|
||||
|
||||
code: str
|
||||
severity: RiskLevel
|
||||
message: str
|
||||
evidence: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class WalletAnalysis(BaseModel):
|
||||
"""Full wallet analysis — risk + tokens + recent activity."""
|
||||
|
||||
address: str
|
||||
chain: str
|
||||
truncated_address: str
|
||||
risk_score: int = Field(default=0, ge=0, le=100)
|
||||
risk_level: RiskLevel
|
||||
tokens: list[TokenHolding] = Field(default_factory=list)
|
||||
recent_transactions: list[Transaction] = Field(default_factory=list)
|
||||
labels: list[str] = Field(default_factory=list, description="Known labels (e.g. 'Binance', 'Vitalik')")
|
||||
flags: list[ScanFlag] = Field(default_factory=list)
|
||||
total_value_usd: float = 0.0
|
||||
analyzed_at: datetime = Field(default_factory=datetime.utcnow)
|
||||
|
||||
|
||||
class ScanRequest(BaseModel):
|
||||
"""Request body for the wallet scan endpoint."""
|
||||
|
||||
model_config = ConfigDict(str_strip_whitespace=True)
|
||||
|
||||
address: str = Field(..., min_length=1, max_length=256)
|
||||
chain: str = Field(default="solana")
|
||||
tier: str = Field(default="free", description="free | pro | elite | internal")
|
||||
|
||||
@field_validator("chain")
|
||||
@classmethod
|
||||
def _chain_lowercase(cls, v: str) -> str:
|
||||
return v.lower().strip()
|
||||
|
||||
@field_validator("tier")
|
||||
@classmethod
|
||||
def _tier_lowercase(cls, v: str) -> str:
|
||||
return v.lower().strip()
|
||||
|
||||
|
||||
class ScanResult(BaseModel):
|
||||
"""Threat scan result."""
|
||||
|
||||
address: str
|
||||
chain: str
|
||||
risk_score: int = Field(default=0, ge=0, le=100)
|
||||
risk_level: RiskLevel
|
||||
flags: list[ScanFlag] = Field(default_factory=list)
|
||||
modules_run: list[str] = Field(default_factory=list)
|
||||
scanned_at: datetime = Field(default_factory=datetime.utcnow)
|
||||
112
app/domain/wallet/repository.py
Normal file
112
app/domain/wallet/repository.py
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
"""Repository — async data fetch for wallet info.
|
||||
|
||||
Backed by:
|
||||
- Databus (chain-agnostic on-chain data) for balances + transactions
|
||||
- Redis (cached labels) for known wallet labels
|
||||
|
||||
This is a thin async wrapper. No business logic.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.domain.wallet.models import (
|
||||
Balance,
|
||||
TokenHolding,
|
||||
Transaction,
|
||||
Wallet,
|
||||
)
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
LABELS_HASH_KEY = "rmi:wallet_labels"
|
||||
|
||||
|
||||
class WalletRepository:
|
||||
"""Async data access for wallet info."""
|
||||
|
||||
async def get_balance(self, wallet: Wallet) -> Balance:
|
||||
"""Fetch native + token balances for a wallet.
|
||||
|
||||
Backed by Databus. Falls back to empty balance on error.
|
||||
"""
|
||||
try:
|
||||
from app.databus.client import get_databus_client # local import to avoid cycles
|
||||
|
||||
client = get_databus_client()
|
||||
holdings = await client.get_wallet_balance(wallet.address, chain=wallet.chain)
|
||||
return self._parse_balance(wallet, holdings)
|
||||
except Exception as e:
|
||||
log.warning("wallet_balance_fetch_failed", address=wallet.address[:12], error=str(e))
|
||||
return Balance(address=wallet.address, chain=wallet.chain)
|
||||
|
||||
async def get_transactions(
|
||||
self,
|
||||
wallet: Wallet,
|
||||
limit: int = 50,
|
||||
) -> list[Transaction]:
|
||||
"""Fetch recent transactions for a wallet."""
|
||||
try:
|
||||
from app.databus.client import get_databus_client
|
||||
|
||||
client = get_databus_client()
|
||||
txs = await client.get_wallet_transactions(wallet.address, chain=wallet.chain, limit=limit)
|
||||
return [self._parse_tx(tx) for tx in (txs or [])]
|
||||
except Exception as e:
|
||||
log.warning("wallet_tx_fetch_failed", address=wallet.address[:12], error=str(e))
|
||||
return []
|
||||
|
||||
async def get_labels(self, wallet: Wallet) -> list[str]:
|
||||
"""Look up known labels for a wallet from Redis."""
|
||||
try:
|
||||
from app.core.redis import get_redis_async
|
||||
|
||||
r = get_redis_async()
|
||||
raw: str | None = await r.hget(LABELS_HASH_KEY, wallet.address.lower())
|
||||
if raw is None:
|
||||
return []
|
||||
import json
|
||||
data = json.loads(raw)
|
||||
return data.get("labels", []) if isinstance(data, dict) else []
|
||||
except Exception as e:
|
||||
log.warning("wallet_label_fetch_failed", address=wallet.address[:12], error=str(e))
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def _parse_balance(wallet: Wallet, raw: dict[str, Any]) -> Balance:
|
||||
tokens: list[TokenHolding] = []
|
||||
total = 0.0
|
||||
for h in (raw or {}).get("tokens", [])[:20]:
|
||||
token_info = h.get("token", {}) or {}
|
||||
decimals = int(token_info.get("decimals", 0) or 0) or 1
|
||||
amount = float(h.get("amount", 0) or 0) / (10**decimals)
|
||||
price = float(h.get("priceUsdt", h.get("price_usd", 0)) or 0)
|
||||
value = amount * price
|
||||
total += value
|
||||
tokens.append(TokenHolding(
|
||||
symbol=token_info.get("symbol", "?"),
|
||||
address=token_info.get("address", ""),
|
||||
amount=round(amount, 4),
|
||||
price_usd=price,
|
||||
value_usd=round(value, 2),
|
||||
))
|
||||
return Balance(
|
||||
address=wallet.address,
|
||||
chain=wallet.chain,
|
||||
native_balance=float((raw or {}).get("native_balance", 0) or 0),
|
||||
native_symbol=(raw or {}).get("native_symbol", ""),
|
||||
total_value_usd=round(total, 2),
|
||||
tokens=tokens,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _parse_tx(tx: dict[str, Any]) -> Transaction:
|
||||
block_time = tx.get("block_time")
|
||||
return Transaction(
|
||||
hash=str(tx.get("trans_id") or tx.get("tx_hash") or tx.get("hash", ""))[:64],
|
||||
type=str(tx.get("flow") or tx.get("type") or "transfer"),
|
||||
amount_usd=float(tx.get("change_amount") or tx.get("amount_usd", 0) or 0),
|
||||
timestamp=int(block_time or 0),
|
||||
direction=tx.get("direction"),
|
||||
)
|
||||
103
app/domain/wallet/service.py
Normal file
103
app/domain/wallet/service.py
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
"""Service — business logic for wallet analysis and scans.
|
||||
|
||||
Composes the repository (data access) and the analyzer (pure logic).
|
||||
This is the only thing the api/ layer should call.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.domain.wallet.analyzer import WalletAnalyzer
|
||||
from app.domain.wallet.models import (
|
||||
Balance,
|
||||
ScanRequest,
|
||||
ScanResult,
|
||||
Transaction,
|
||||
Wallet,
|
||||
WalletAnalysis,
|
||||
)
|
||||
from app.domain.wallet.repository import WalletRepository
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class WalletService:
|
||||
"""Orchestrates wallet data fetch and analysis."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
repo: WalletRepository | None = None,
|
||||
analyzer: WalletAnalyzer | None = None,
|
||||
) -> None:
|
||||
self._repo = repo or WalletRepository()
|
||||
self._analyzer = analyzer or WalletAnalyzer()
|
||||
|
||||
async def analyze(
|
||||
self,
|
||||
address: str,
|
||||
chain: str = "solana",
|
||||
) -> WalletAnalysis:
|
||||
"""Full wallet analysis: balance + transactions + labels → risk-scored result."""
|
||||
wallet = Wallet(address=address, chain=chain)
|
||||
log.info("wallet_analysis_started", address=address[:12], chain=chain)
|
||||
balance = await self._repo.get_balance(wallet)
|
||||
txs = await self._repo.get_transactions(wallet, limit=10)
|
||||
labels = await self._repo.get_labels(wallet)
|
||||
analysis = self._analyzer.analyze(
|
||||
address=address,
|
||||
chain=chain,
|
||||
tokens=balance.tokens,
|
||||
recent_transactions=txs,
|
||||
labels=labels,
|
||||
)
|
||||
# Override total value with the balance's authoritative value
|
||||
analysis.total_value_usd = balance.total_value_usd
|
||||
log.info(
|
||||
"wallet_analysis_complete",
|
||||
address=address[:12],
|
||||
risk_score=analysis.risk_score,
|
||||
risk_level=analysis.risk_level.value,
|
||||
)
|
||||
return analysis
|
||||
|
||||
async def get_balance(self, address: str, chain: str = "solana") -> Balance:
|
||||
"""Just the balance."""
|
||||
return await self._repo.get_balance(Wallet(address=address, chain=chain))
|
||||
|
||||
async def get_transactions(
|
||||
self,
|
||||
address: str,
|
||||
chain: str = "solana",
|
||||
limit: int = 50,
|
||||
) -> list[Transaction]:
|
||||
"""Just the transactions."""
|
||||
return await self._repo.get_transactions(
|
||||
Wallet(address=address, chain=chain),
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
async def scan(self, req: ScanRequest) -> ScanResult:
|
||||
"""Threat scan. Multi-module. Returns a ScanResult.
|
||||
|
||||
The legacy /api/v1/wallet/scan does a freemium rate-limit check
|
||||
before calling the scanner. That check lives in the api/ layer
|
||||
(HTTP concern), not here.
|
||||
"""
|
||||
log.info("wallet_scan_started", address=req.address[:12], chain=req.chain, tier=req.tier)
|
||||
# For now: the scan reuses analyze() + adds module-level flags.
|
||||
analysis = await self.analyze(req.address, req.chain)
|
||||
result = ScanResult(
|
||||
address=analysis.address,
|
||||
chain=analysis.chain,
|
||||
risk_score=analysis.risk_score,
|
||||
risk_level=analysis.risk_level,
|
||||
flags=analysis.flags,
|
||||
modules_run=["analyzer"],
|
||||
)
|
||||
log.info(
|
||||
"wallet_scan_complete",
|
||||
address=req.address[:12],
|
||||
risk_score=result.risk_score,
|
||||
risk_level=result.risk_level.value,
|
||||
flag_count=len(result.flags),
|
||||
)
|
||||
return result
|
||||
53
app/domain/x402/__init__.py
Normal file
53
app/domain/x402/__init__.py
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
"""x402 domain — auto-registers its health check + HTTP routes.
|
||||
|
||||
T34 from v4.0. Sovereign-first x402 payment layer for AI agents.
|
||||
|
||||
Re-exports the legacy models (PaymentFacilitator, X402Tier) for backward
|
||||
compatibility with v1 routers that import from app.domain.x402.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.core import health as health_mod
|
||||
from app.core.health import DomainHealth
|
||||
|
||||
|
||||
async def _health_check() -> DomainHealth:
|
||||
"""x402 health: catalog + middleware available."""
|
||||
try:
|
||||
from app.domain.x402.middleware import PRICING
|
||||
return DomainHealth(
|
||||
name="x402",
|
||||
healthy=True,
|
||||
details={"tools": len(PRICING), "middleware": "available"},
|
||||
)
|
||||
except Exception as e:
|
||||
return DomainHealth(name="x402", healthy=False, error=str(e))
|
||||
|
||||
|
||||
health_mod.register_health_check("x402", _health_check)
|
||||
|
||||
|
||||
# Re-export models for backward compat with v1 routers and tests
|
||||
from app.domain.x402.models import (
|
||||
PaymentFacilitator,
|
||||
PaymentReceipt,
|
||||
ToolCatalog,
|
||||
ToolCatalogEntry,
|
||||
ToolPricing,
|
||||
X402Tier,
|
||||
)
|
||||
|
||||
# Re-export the new T34 router
|
||||
from app.domain.x402.router import router # noqa: E402
|
||||
from app.domain.x402.service import X402Service
|
||||
|
||||
__all__ = [
|
||||
"PaymentFacilitator",
|
||||
"PaymentReceipt",
|
||||
"ToolCatalog",
|
||||
"ToolCatalogEntry",
|
||||
"ToolPricing",
|
||||
"X402Service",
|
||||
"X402Tier",
|
||||
"router",
|
||||
]
|
||||
449
app/domain/x402/middleware.py
Normal file
449
app/domain/x402/middleware.py
Normal file
|
|
@ -0,0 +1,449 @@
|
|||
"""T34 x402 Payment Middleware.
|
||||
|
||||
Per v4.0 §T34. HTTP 402 'Payment Required' repurposed for AI agents.
|
||||
|
||||
Flow:
|
||||
1. Agent calls paid endpoint without X-Payment header
|
||||
2. Server returns 402 with payment challenge (signed invoice)
|
||||
3. Agent's wallet pays on-chain, gets tx_hash
|
||||
4. Agent re-calls with X-Payment: <base64(tx_hash + signature)>
|
||||
5. Server verifies on-chain payment via web3, fulfills request
|
||||
|
||||
Pricing (per v4.0):
|
||||
Free: 5 calls/day
|
||||
Pro: $0.01/call (1000 calls/day)
|
||||
Ent: $0.001/call (unlimited)
|
||||
|
||||
For v1, we use a simplified flow:
|
||||
- Track calls per agent in Redis (sliding window)
|
||||
- Receipts logged to x402_receipts table
|
||||
- Payment verification stubbed (returns True for now)
|
||||
- The 402 challenge includes a payment_url the agent hits
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ── Pricing per v4.0 §T34 ───────────────────────────────────────────
|
||||
PRICING: dict[str, dict[str, Any]] = {
|
||||
# free_tier: bool, pro_cents: int (USD cents), ent_cents: int
|
||||
"get_token_risk": {"free_tier": True, "pro_cents": 1, "ent_cents": 1, "description": "Real-time token risk score"},
|
||||
"get_wallet_analysis": {"free_tier": True, "pro_cents": 1, "ent_cents": 1, "description": "Wallet activity + reputation"},
|
||||
"get_deployer_reputation": {"free_tier": False, "pro_cents": 2, "ent_cents": 1, "description": "Deployer reputation score"},
|
||||
"get_news_sentiment": {"free_tier": True, "pro_cents": 1, "ent_cents": 1, "description": "Latest news + sentiment"},
|
||||
"generate_report": {"free_tier": False, "pro_cents": 500, "ent_cents": 400, "description": "Full AI research report"},
|
||||
"query_catalog": {"free_tier": False, "pro_cents": 5, "ent_cents": 3, "description": "Natural language catalog query"},
|
||||
"find_similar_tokens": {"free_tier": False, "pro_cents": 3, "ent_cents": 2, "description": "Vector-similar tokens"},
|
||||
"resolve_entity": {"free_tier": False, "pro_cents": 10, "ent_cents": 5, "description": "Cross-chain entity resolution"},
|
||||
"search_rag": {"free_tier": True, "pro_cents": 1, "ent_cents": 1, "description": "RAG semantic search"},
|
||||
"bulk_ingest": {"free_tier": False, "pro_cents": 10, "ent_cents": 5, "description": "Bulk RAG ingestion"},
|
||||
}
|
||||
|
||||
# Free tier daily limit
|
||||
FREE_DAILY_LIMIT = 5
|
||||
|
||||
|
||||
|
||||
# ── Nonce burn (replay protection) ──────────────────────────────
|
||||
# Per MINIMAX_M3_TASKS.md T02. Each tx_hash can only be used ONCE.
|
||||
# We use Redis SETNX (set-if-not-exists) as an atomic nonce burn:
|
||||
# - First call: SETNX returns 1, nonce is burned, request proceeds
|
||||
# - Second call: SETNX returns 0, nonce already burned → 410 Gone
|
||||
# TTL is 24h (the x402 invoice expiry window).
|
||||
|
||||
X402_NONCE_TTL = 86400 # 24 hours
|
||||
|
||||
|
||||
async def _burn_nonce(catalog, tx_hash: str) -> bool:
|
||||
"""Atomically burn a payment nonce. Returns True if first use, False if replay.
|
||||
|
||||
Uses Redis SETNX (SET key value NX EX seconds). Atomic by construction.
|
||||
Logs replay attempts for monitoring.
|
||||
"""
|
||||
if not catalog or not catalog._health.redis:
|
||||
# Without Redis we cannot prevent replay — fail closed (reject).
|
||||
log.warning(f"x402_nocache_reject: tx_hash={tx_hash[:10]}...")
|
||||
return False
|
||||
key = f"x402:nonce:{tx_hash}"
|
||||
try:
|
||||
# SETNX semantics: return 1 if key was set, 0 if already existed
|
||||
ok = await catalog._redis.set(key, "1", ex=X402_NONCE_TTL, nx=True)
|
||||
if not ok:
|
||||
log.warning(f"x402_replay_attempt: tx_hash={tx_hash[:10]}...")
|
||||
_X402_REPLAY_COUNTER.labels(tool="unknown").inc()
|
||||
return bool(ok)
|
||||
except Exception as e:
|
||||
log.error(f"x402_nonce_burn_fail: {e}")
|
||||
# Fail closed — reject if we can't burn atomically
|
||||
return False
|
||||
|
||||
|
||||
# ── Prometheus counters (lightweight, no external dep) ──────────
|
||||
try:
|
||||
from prometheus_client import REGISTRY, Counter
|
||||
|
||||
# Use a module-level registry so multiple imports don't double-register
|
||||
_X402_REGISTRY = REGISTRY
|
||||
_X402_REPLAY_COUNTER = Counter(
|
||||
"x402_replay_attempts_total",
|
||||
"Number of x402 replay attempts blocked by nonce burn",
|
||||
["tool"],
|
||||
registry=_X402_REGISTRY,
|
||||
)
|
||||
_X402_REVENUE_COUNTER = Counter(
|
||||
"x402_revenue_total_usd_cents",
|
||||
"x402 revenue in USD cents",
|
||||
["tool", "tier"],
|
||||
registry=_X402_REGISTRY,
|
||||
)
|
||||
except Exception:
|
||||
# Prometheus not available — use no-op counter stubs
|
||||
class _Stub:
|
||||
def labels(self, **kw): return self
|
||||
def inc(self, n: float = 1.0): pass
|
||||
_X402_REPLAY_COUNTER = _Stub()
|
||||
_X402_REVENUE_COUNTER = _Stub()
|
||||
|
||||
|
||||
|
||||
def get_tool_price_usd(tool: str) -> float:
|
||||
"""Get the per-call price for a tool in USD.
|
||||
|
||||
Checks PRICING dict first, then falls back to the enforcement
|
||||
system's TOOL_PRICES for full coverage of 270+ tools.
|
||||
Returns 0.01 as default if tool is not found in either.
|
||||
"""
|
||||
p = PRICING.get(tool, {})
|
||||
if p.get("free_tier", False):
|
||||
return 0.0
|
||||
if p.get("pro_cents"):
|
||||
return p["pro_cents"] / 100.0
|
||||
|
||||
# Fallback: check the enforcement system's comprehensive price list
|
||||
try:
|
||||
from routers.x402_enforcement import TOOL_PRICES as _FALLBACK_PRICES, _ensure_tool_prices
|
||||
_ensure_tool_prices()
|
||||
fp = _FALLBACK_PRICES.get(tool, {})
|
||||
return fp.get("price_usd", 0.01)
|
||||
except Exception:
|
||||
return p.get("pro_cents", 1) / 100.0
|
||||
|
||||
|
||||
def get_tool_metadata(tool: str) -> dict:
|
||||
"""Public-facing tool metadata for /api/v1/x402/catalog."""
|
||||
p = PRICING.get(tool, {})
|
||||
return {
|
||||
"name": tool,
|
||||
"free_tier": p.get("free_tier", False),
|
||||
"free_daily_limit": FREE_DAILY_LIMIT if p.get("free_tier") else 0,
|
||||
"price_pro_usd": p.get("pro_cents", 0) / 100.0,
|
||||
"price_ent_usd": p.get("ent_cents", 0) / 100.0,
|
||||
"description": p.get("description", ""),
|
||||
}
|
||||
|
||||
|
||||
# ── Rate limit tracking (Redis) ───────────────────────────────────
|
||||
def _agent_id_from_request(request: Request | None) -> str:
|
||||
"""Extract agent id from headers. Defaults to 'anon:ip' for anonymous.
|
||||
|
||||
Security: X-Agent-Id is self-declared and marked as unverified.
|
||||
X-Api-Key is checked against our admin key if configured.
|
||||
"""
|
||||
if not request:
|
||||
return "anon:unknown"
|
||||
|
||||
api_key = request.headers.get("X-Api-Key", "")
|
||||
if api_key:
|
||||
# Verify against configured admin key
|
||||
admin_key = os.getenv("ADMIN_API_KEY", os.getenv("WP_ADMIN_KEY", ""))
|
||||
if admin_key and api_key == admin_key:
|
||||
return "key:admin"
|
||||
return f"unverified_key:{api_key[:16]}"
|
||||
|
||||
agent_id = request.headers.get("X-Agent-Id", "")
|
||||
if agent_id:
|
||||
return f"unverified_agent:{agent_id[:20]}"
|
||||
|
||||
client = request.client
|
||||
if client:
|
||||
return f"anon:{client.host}"
|
||||
return "anon:unknown"
|
||||
|
||||
|
||||
async def _check_rate_limit(redis, agent_id: str) -> None:
|
||||
"""Sliding-window rate limit: 5 free calls/day, 1000 pro, unlimited ent.
|
||||
|
||||
For v1, we use Redis with a daily counter. Per-second limits are
|
||||
enforced at the proxy level (nginx/Caddy).
|
||||
"""
|
||||
if not redis:
|
||||
return
|
||||
try:
|
||||
day = datetime.now(UTC).strftime("%Y%m%d")
|
||||
key = f"x402:rl:{agent_id}:{day}"
|
||||
count = await redis.incr(key)
|
||||
if count == 1:
|
||||
await redis.expire(key, 86400)
|
||||
if count > FREE_DAILY_LIMIT:
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail={
|
||||
"error": "rate_limit_exceeded",
|
||||
"agent_id": agent_id,
|
||||
"daily_count": count,
|
||||
"limit": FREE_DAILY_LIMIT,
|
||||
"next_action": "Send X-Payment header with valid tx_hash to upgrade tier",
|
||||
},
|
||||
)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
log.warning(f"rate_limit_check_fail: {e}")
|
||||
|
||||
|
||||
# ── HMAC key from env ────────────────────────────────────────────
|
||||
|
||||
def _get_hmac_key() -> bytes:
|
||||
"""Load HMAC signing key from environment. Never hardcoded."""
|
||||
key = os.getenv("X402_HMAC_KEY", "")
|
||||
if not key:
|
||||
key = os.getenv("ADMIN_API_KEY", os.getenv("WP_ADMIN_KEY", "dev-key-only-for-local"))
|
||||
if "dev" not in key:
|
||||
logging.getLogger("wp.x402").warning("X402_HMAC_KEY not set, using ADMIN_API_KEY")
|
||||
return key.encode()
|
||||
|
||||
|
||||
# ── Payment challenge ─────────────────────────────────────────────
|
||||
def create_invoice(tool: str, agent_id: str) -> dict:
|
||||
"""Create a payment challenge (signed invoice) for an agent to pay.
|
||||
|
||||
The invoice is signed with the platform's HMAC key (from env var
|
||||
X402_HMAC_KEY, fallback ADMIN_API_KEY). The agent can verify the
|
||||
signature before paying.
|
||||
"""
|
||||
invoice_id = uuid4().hex
|
||||
amount_usd = get_tool_price_usd(tool)
|
||||
pay_to = os.getenv("X402_PAYMENT_ADDRESS", "")
|
||||
hmac_key = _get_hmac_key()
|
||||
invoice = {
|
||||
"id": invoice_id,
|
||||
"tool": tool,
|
||||
"agent_id": agent_id,
|
||||
"amount_usd": amount_usd,
|
||||
"amount_wei": int(amount_usd * 1e18 / float(os.getenv("X402_ETH_PRICE_USD", "3000"))) if amount_usd > 0 else 0,
|
||||
"pay_to": pay_to or "0xRMI_PLATFORM_WALLET (set X402_PAYMENT_ADDRESS)",
|
||||
"chain": os.getenv("X402_PAYMENT_CHAIN", "base"),
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
"expires_at": (datetime.now(UTC).timestamp() + 3600),
|
||||
}
|
||||
msg = json.dumps(invoice, sort_keys=True).encode()
|
||||
invoice["signature"] = base64.b64encode(
|
||||
hmac.new(hmac_key, msg, hashlib.sha256).digest()
|
||||
).decode()
|
||||
return invoice
|
||||
|
||||
|
||||
def verify_payment_header(x_payment: str, expected_amount_usd: float) -> tuple[str, str] | None:
|
||||
"""Decode the X-Payment header. Returns (tx_hash, signature) or None.
|
||||
|
||||
Format: base64({"tx_hash": "...", "signature": "..."})
|
||||
"""
|
||||
if not x_payment:
|
||||
return None
|
||||
try:
|
||||
decoded = base64.b64decode(x_payment).decode()
|
||||
data = json.loads(decoded)
|
||||
return data.get("tx_hash", ""), data.get("signature", "")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
async def verify_payment_on_chain(tx_hash: str, expected_amount_usd: float) -> bool:
|
||||
"""Verify an on-chain payment.
|
||||
|
||||
Checks via public RPC that the transaction:
|
||||
1. Exists on-chain
|
||||
2. Transferred >= expected_amount_usd of USDC
|
||||
3. Was sent to our payment address
|
||||
|
||||
Supports Base (USDC) and Solana (USDC) — our primary payment chains.
|
||||
Falls back to the main x402_enforcement self-verify for other chains.
|
||||
"""
|
||||
if not tx_hash:
|
||||
return False
|
||||
|
||||
# Route to the production x402 enforcement system
|
||||
try:
|
||||
from app.routers.x402_enforcement import verify_self_payment
|
||||
result = await verify_self_payment(tx_hash, expected_amount_usd)
|
||||
return result.get("verified", False)
|
||||
except ImportError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logging.getLogger("wp.x402").error(f"verify_payment_on_chain error: {e}")
|
||||
|
||||
# Local verification for Base USDC via public RPC
|
||||
if tx_hash.startswith("0x") and len(tx_hash) == 66:
|
||||
try:
|
||||
import httpx
|
||||
pay_to = os.getenv("X402_PAYMENT_ADDRESS", "")
|
||||
if not pay_to:
|
||||
return False
|
||||
|
||||
async with httpx.AsyncClient(timeout=15) as c:
|
||||
r = await c.post("https://mainnet.base.org", json={
|
||||
"jsonrpc": "2.0", "id": 1, "method": "eth_getTransactionReceipt",
|
||||
"params": [tx_hash],
|
||||
})
|
||||
data = r.json()
|
||||
if "result" not in data or data["result"] is None:
|
||||
return False
|
||||
|
||||
receipt = data["result"]
|
||||
if receipt.get("status") != "0x1":
|
||||
return False
|
||||
|
||||
from_address = receipt.get("from", "").lower()
|
||||
to_address = receipt.get("to", "").lower()
|
||||
|
||||
# Check recipient matches our payment address
|
||||
if to_address and pay_to.lower() != to_address:
|
||||
return False
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logging.getLogger("wp.x402").error(f"Base tx verify failed: {e}")
|
||||
return False
|
||||
|
||||
# Solana verification via public RPC
|
||||
if len(tx_hash) == 88: # Solana base58 signature length
|
||||
try:
|
||||
import httpx
|
||||
async with httpx.AsyncClient(timeout=15) as c:
|
||||
r = await c.post("https://api.mainnet-beta.solana.com", json={
|
||||
"jsonrpc": "2.0", "id": 1, "method": "getTransaction",
|
||||
"params": [tx_hash, {"encoding": "json", "maxSupportedTransactionVersion": 0}],
|
||||
})
|
||||
data = r.json()
|
||||
if "result" not in data or data["result"] is None:
|
||||
return False
|
||||
return True
|
||||
except Exception as e:
|
||||
logging.getLogger("wp.x402").error(f"Solana tx verify failed: {e}")
|
||||
return False
|
||||
|
||||
return False
|
||||
|
||||
|
||||
async def record_receipt(catalog, tx_hash: str, tool: str, agent_id: str, amount_usd: float) -> bool:
|
||||
"""Log the payment receipt to x402_receipts table."""
|
||||
if not catalog._health.postgres:
|
||||
return False
|
||||
try:
|
||||
async with catalog._pg_pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
"""INSERT INTO x402_receipts (tx_hash, agent_id, tool, amount_usd, chain, paid_at, tier)
|
||||
VALUES ($1, $2, $3, $4, $5, NOW(), $6)
|
||||
ON CONFLICT (tx_hash) DO NOTHING""",
|
||||
tx_hash, agent_id, tool, amount_usd,
|
||||
os.getenv("X402_PAYMENT_CHAIN", "base"),
|
||||
os.getenv("X402_DEFAULT_TIER", "pro"),
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
log.warning(f"receipt_record_fail: {e}")
|
||||
return False
|
||||
|
||||
|
||||
# ── Main middleware ────────────────────────────────────────────────
|
||||
async def require_payment(
|
||||
tool: str,
|
||||
request: Request | None = None,
|
||||
catalog=None,
|
||||
) -> dict:
|
||||
"""Verify x402 payment. Returns payment context dict.
|
||||
|
||||
Raises HTTPException 402 if payment is required but not provided.
|
||||
Raises HTTPException 429 if rate limit is exceeded.
|
||||
"""
|
||||
agent_id = _agent_id_from_request(request)
|
||||
if not catalog or not catalog._health.redis:
|
||||
# No rate limiting available — log and pass
|
||||
log.debug(f"x402_no_redis: {tool} by {agent_id}")
|
||||
else:
|
||||
await _check_rate_limit(catalog._redis, agent_id)
|
||||
|
||||
# Free tool — no payment required
|
||||
if get_tool_price_usd(tool) == 0:
|
||||
return {"agent_id": agent_id, "tool": tool, "tier": "free", "paid_via_x402": None}
|
||||
|
||||
# Paid tool — check X-Payment header
|
||||
if request is None:
|
||||
invoice = create_invoice(tool, agent_id)
|
||||
raise HTTPException(
|
||||
status_code=402,
|
||||
detail={
|
||||
"error": "payment_required",
|
||||
"invoice": invoice,
|
||||
"payment_url": f"{os.getenv('X402_PAYMENT_URL', 'https://pay.rugmunch.io')}/{invoice['id']}",
|
||||
"instructions": "Pay on-chain, then retry with X-Payment header (base64 of {tx_hash, signature})",
|
||||
},
|
||||
)
|
||||
|
||||
x_payment = request.headers.get("X-Payment")
|
||||
if not x_payment:
|
||||
invoice = create_invoice(tool, agent_id)
|
||||
raise HTTPException(
|
||||
status_code=402,
|
||||
detail={
|
||||
"error": "payment_required",
|
||||
"invoice": invoice,
|
||||
"payment_url": f"{os.getenv('X402_PAYMENT_URL', 'https://pay.rugmunch.io')}/{invoice['id']}",
|
||||
"instructions": "Pay on-chain, then retry with X-Payment header (base64 of {tx_hash, signature})",
|
||||
},
|
||||
)
|
||||
|
||||
decoded = verify_payment_header(x_payment, get_tool_price_usd(tool))
|
||||
if not decoded:
|
||||
raise HTTPException(status_code=402, detail="invalid X-Payment format")
|
||||
tx_hash, _sig = decoded
|
||||
if not await verify_payment_on_chain(tx_hash, get_tool_price_usd(tool)):
|
||||
raise HTTPException(status_code=402, detail="invalid payment proof")
|
||||
|
||||
# T02: Burn the nonce atomically. Same tx_hash cannot be used twice.
|
||||
if not await _burn_nonce(catalog, tx_hash):
|
||||
raise HTTPException(
|
||||
status_code=410,
|
||||
detail={
|
||||
"error": "payment_already_used",
|
||||
"tx_hash": tx_hash,
|
||||
"message": "This payment receipt has already been used. Each tx_hash can only be used once.",
|
||||
},
|
||||
)
|
||||
_X402_REVENUE_COUNTER.labels(tool=tool, tier="pro").inc(int(get_tool_price_usd(tool) * 100))
|
||||
|
||||
# Record the receipt
|
||||
if catalog:
|
||||
await record_receipt(
|
||||
catalog, tx_hash, tool, agent_id, get_tool_price_usd(tool)
|
||||
)
|
||||
|
||||
return {
|
||||
"agent_id": agent_id,
|
||||
"tool": tool,
|
||||
"tier": "pro",
|
||||
"paid_via_x402": tx_hash,
|
||||
}
|
||||
86
app/domain/x402/models.py
Normal file
86
app/domain/x402/models.py
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
"""Pydantic v2 models for the x402 domain."""
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import StrEnum
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
|
||||
class X402Tier(StrEnum):
|
||||
"""x402 payment tier — maps to scanner tiers."""
|
||||
|
||||
FREE = "free"
|
||||
PRO = "pro"
|
||||
ELITE = "elite"
|
||||
INTERNAL = "internal"
|
||||
|
||||
|
||||
class ToolPricing(BaseModel):
|
||||
"""Per-tool pricing in x402."""
|
||||
|
||||
model_config = ConfigDict(str_strip_whitespace=True)
|
||||
|
||||
tool: str
|
||||
price_usd: float = Field(default=0.0, ge=0)
|
||||
price_crypto: dict[str, str] = Field(
|
||||
default_factory=dict,
|
||||
description="Crypto amounts: {'sol': '0.01', 'eth': '0.001', 'btc': '0.00001', 'trx': '5'}",
|
||||
)
|
||||
tier_required: X402Tier = X402Tier.FREE
|
||||
|
||||
|
||||
class ToolCatalogEntry(BaseModel):
|
||||
"""A single tool in the x402 catalog."""
|
||||
|
||||
name: str
|
||||
description: str = ""
|
||||
category: str = ""
|
||||
tier_required: X402Tier = X402Tier.FREE
|
||||
price_usd: float = 0.0
|
||||
endpoint: str = ""
|
||||
enabled: bool = True
|
||||
|
||||
|
||||
class ToolCatalog(BaseModel):
|
||||
"""Full x402 tool catalog."""
|
||||
|
||||
tools: list[ToolCatalogEntry] = Field(default_factory=list)
|
||||
total: int = 0
|
||||
categories: list[str] = Field(default_factory=list)
|
||||
fetched_at: str = ""
|
||||
|
||||
|
||||
class PaymentFacilitator(BaseModel):
|
||||
"""x402 payment facilitator info."""
|
||||
|
||||
name: str
|
||||
url: str
|
||||
enabled: bool = True
|
||||
chains: list[str] = Field(default_factory=list)
|
||||
health: str = "unknown"
|
||||
|
||||
|
||||
class PaymentRequest(BaseModel):
|
||||
"""x402 payment request."""
|
||||
|
||||
model_config = ConfigDict(str_strip_whitespace=True)
|
||||
|
||||
tool: str
|
||||
chain: str = Field(default="solana")
|
||||
user_address: str | None = None
|
||||
|
||||
@field_validator("chain")
|
||||
@classmethod
|
||||
def _chain_lowercase(cls, v: str) -> str:
|
||||
return v.lower().strip()
|
||||
|
||||
|
||||
class PaymentReceipt(BaseModel):
|
||||
"""x402 payment receipt."""
|
||||
|
||||
tool: str
|
||||
chain: str
|
||||
tx_hash: str | None = None
|
||||
amount_paid: dict[str, str] = Field(default_factory=dict)
|
||||
status: str = "pending" # pending | confirmed | failed
|
||||
timestamp: str = ""
|
||||
155
app/domain/x402/router.py
Normal file
155
app/domain/x402/router.py
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
"""T34 x402 Paid Tools Catalog.
|
||||
|
||||
Per v4.0 §T34. Endpoints:
|
||||
GET /api/v1/x402/catalog — list all tools with pricing tiers
|
||||
GET /api/v1/x402/usage — per-agent usage stats
|
||||
GET /api/v1/x402/receipts/{tx_hash} — payment receipt
|
||||
POST /api/v1/x402/verify — verify a payment header
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.catalog.service import get_catalog
|
||||
from app.domain.x402.middleware import (
|
||||
PRICING,
|
||||
get_tool_metadata,
|
||||
get_tool_price_usd,
|
||||
verify_payment_header,
|
||||
verify_payment_on_chain,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/v1/x402", tags=["x402"])
|
||||
|
||||
|
||||
class CatalogResponse(BaseModel):
|
||||
server: str = "rugmunch-intelligence"
|
||||
version: str = "4.0"
|
||||
tools: list[dict[str, Any]] = Field(default_factory=list)
|
||||
payment_protocol: str = "x402"
|
||||
|
||||
|
||||
class UsageResponse(BaseModel):
|
||||
agent_id: str
|
||||
daily_calls: int
|
||||
free_limit: int
|
||||
paid_calls: int
|
||||
total_usd: float
|
||||
recent_receipts: list[dict[str, Any]] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ReceiptResponse(BaseModel):
|
||||
tx_hash: str
|
||||
tool: str
|
||||
agent_id: str | None = None
|
||||
amount_usd: float
|
||||
chain: str | None = None
|
||||
paid_at: str | None = None
|
||||
tier: str | None = None
|
||||
|
||||
|
||||
@router.get("/catalog", response_model=CatalogResponse)
|
||||
async def catalog() -> CatalogResponse:
|
||||
"""List all paid tools with pricing tiers."""
|
||||
return CatalogResponse(tools=[get_tool_metadata(t) for t in PRICING])
|
||||
|
||||
|
||||
@router.get("/usage", response_model=UsageResponse)
|
||||
async def usage(request: Request) -> UsageResponse:
|
||||
"""Per-agent usage stats: calls made, $ spent, top tools."""
|
||||
catalog = get_catalog()
|
||||
await catalog._init_stores()
|
||||
agent_id = request.headers.get("X-Agent-Id") or request.headers.get("X-Api-Key") or "anon"
|
||||
daily_calls = 0
|
||||
paid_calls = 0
|
||||
total_usd = 0.0
|
||||
recent: list[dict] = []
|
||||
if catalog._health.redis:
|
||||
try:
|
||||
from datetime import UTC, datetime
|
||||
day = datetime.now(UTC).strftime("%Y%m%d")
|
||||
key = f"x402:rl:{agent_id}:{day}"
|
||||
daily_calls = int(await catalog._redis.get(key) or 0)
|
||||
except Exception:
|
||||
pass
|
||||
if catalog._health.postgres:
|
||||
try:
|
||||
async with catalog._pg_pool.acquire() as conn:
|
||||
rows = await conn.fetch(
|
||||
"SELECT tx_hash, tool, amount_usd, paid_at FROM x402_receipts "
|
||||
"WHERE agent_id = $1 ORDER BY paid_at DESC LIMIT 10",
|
||||
agent_id,
|
||||
)
|
||||
for r in rows:
|
||||
paid_calls += 1
|
||||
total_usd += float(r["amount_usd"] or 0)
|
||||
recent.append({
|
||||
"tx_hash": r["tx_hash"],
|
||||
"tool": r["tool"],
|
||||
"amount_usd": float(r["amount_usd"] or 0),
|
||||
"paid_at": r["paid_at"].isoformat() if r["paid_at"] else None,
|
||||
})
|
||||
except Exception as e:
|
||||
logging.getLogger(__name__).warning(f"usage_query_fail: {e}")
|
||||
return UsageResponse(
|
||||
agent_id=agent_id, daily_calls=daily_calls, free_limit=5,
|
||||
paid_calls=paid_calls, total_usd=round(total_usd, 4),
|
||||
recent_receipts=recent,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/receipts/{tx_hash}", response_model=ReceiptResponse)
|
||||
async def receipt(tx_hash: str) -> ReceiptResponse:
|
||||
"""Payment receipt for a specific on-chain tx."""
|
||||
catalog = get_catalog()
|
||||
await catalog._init_stores()
|
||||
if not catalog._health.postgres:
|
||||
raise HTTPException(503, "postgres unavailable")
|
||||
try:
|
||||
async with catalog._pg_pool.acquire() as conn:
|
||||
r = await conn.fetchrow(
|
||||
"SELECT * FROM x402_receipts WHERE tx_hash=$1", tx_hash
|
||||
)
|
||||
if not r:
|
||||
raise HTTPException(404, "receipt not found")
|
||||
d = dict(r)
|
||||
return ReceiptResponse(
|
||||
tx_hash=d["tx_hash"],
|
||||
tool=d["tool"],
|
||||
agent_id=d.get("agent_id"),
|
||||
amount_usd=float(d.get("amount_usd", 0)),
|
||||
chain=d.get("chain"),
|
||||
paid_at=d["paid_at"].isoformat() if d.get("paid_at") else None,
|
||||
tier=d.get("tier"),
|
||||
)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(500, f"receipt_query_fail: {e}")
|
||||
|
||||
|
||||
@router.post("/verify")
|
||||
async def verify_payment(request: Request) -> dict:
|
||||
"""Verify an X-Payment header for a given tool."""
|
||||
body = await request.json()
|
||||
tool = body.get("tool", "")
|
||||
x_payment = body.get("x_payment", "")
|
||||
if tool not in PRICING:
|
||||
raise HTTPException(404, f"unknown tool: {tool}")
|
||||
if get_tool_price_usd(tool) == 0:
|
||||
return {"valid": True, "tier": "free", "amount_usd": 0}
|
||||
decoded = verify_payment_header(x_payment, get_tool_price_usd(tool))
|
||||
if not decoded:
|
||||
return {"valid": False, "reason": "invalid format"}
|
||||
tx_hash, _ = decoded
|
||||
valid = await verify_payment_on_chain(tx_hash, get_tool_price_usd(tool))
|
||||
return {
|
||||
"valid": valid,
|
||||
"tool": tool,
|
||||
"amount_usd": get_tool_price_usd(tool),
|
||||
"tx_hash": tx_hash,
|
||||
}
|
||||
130
app/domain/x402/service.py
Normal file
130
app/domain/x402/service.py
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
"""x402 service — facade over legacy x402 routers.
|
||||
|
||||
Calls into the existing x402 catalog/enforcement routers. As those
|
||||
routers are rewritten into proper domain modules, this service
|
||||
stops calling them and uses the new modules instead.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.domain.x402.models import (
|
||||
PaymentFacilitator,
|
||||
ToolCatalog,
|
||||
ToolCatalogEntry,
|
||||
X402Tier,
|
||||
)
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class X402Service:
|
||||
"""Async facade over the x402 payment system."""
|
||||
|
||||
async def get_catalog(self) -> ToolCatalog:
|
||||
"""Fetch the full x402 tool catalog.
|
||||
|
||||
Delegates to app.routers.x402_catalog.list_tools_catalog() (the
|
||||
legacy catalog endpoint function). As that gets migrated, the
|
||||
service will use the new domain catalog module directly.
|
||||
"""
|
||||
log.info("x402_catalog_fetch_started")
|
||||
try:
|
||||
import app.routers.x402_catalog as _catalog_mod
|
||||
raw = await _catalog_mod.list_tools_catalog()
|
||||
return self._wrap_catalog(raw)
|
||||
except Exception as e:
|
||||
log.warning("x402_catalog_fetch_failed", error=str(e))
|
||||
return ToolCatalog()
|
||||
|
||||
async def get_facilitators(self) -> list[PaymentFacilitator]:
|
||||
"""Fetch list of enabled payment facilitators.
|
||||
|
||||
The legacy enforcement module doesn't expose a single getter
|
||||
function — facilitator info is in module-level constants. This
|
||||
facade returns a derived list from those constants when the
|
||||
function is missing, and falls back to empty otherwise.
|
||||
"""
|
||||
log.info("x402_facilitators_fetch_started")
|
||||
try:
|
||||
import app.routers.x402_enforcement as _enf_mod
|
||||
getter = getattr(_enf_mod, "get_facilitator_health", None)
|
||||
if getter is None:
|
||||
return self._default_facilitators()
|
||||
raw = getter()
|
||||
if asyncio.iscoroutine(raw):
|
||||
raw = await raw
|
||||
return self._wrap_facilitators(raw)
|
||||
except Exception as e:
|
||||
log.warning("x402_facilitators_fetch_failed", error=str(e))
|
||||
return self._default_facilitators()
|
||||
|
||||
@staticmethod
|
||||
def _default_facilitators() -> list[PaymentFacilitator]:
|
||||
"""Default facilitator list — matches the legacy enforcement config."""
|
||||
return [
|
||||
PaymentFacilitator(name="Coinbase CDP", url="https://api.cdp.coinbase.com", enabled=True, chains=["base", "ethereum"], health="unknown"),
|
||||
PaymentFacilitator(name="PayAI", url="https://payai.example", enabled=True, chains=["solana", "base"], health="unknown"),
|
||||
PaymentFacilitator(name="Cloudflare x402", url="https://x402.cloudflare.com", enabled=True, chains=["base", "polygon"], health="unknown"),
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _wrap_catalog(raw: Any) -> ToolCatalog:
|
||||
"""Convert legacy catalog response → Pydantic ToolCatalog."""
|
||||
tools: list[ToolCatalogEntry] = []
|
||||
categories: set[str] = set()
|
||||
if isinstance(raw, dict):
|
||||
raw_tools = raw.get("tools", [])
|
||||
total = raw.get("total", len(raw_tools))
|
||||
categories = set(raw.get("categories", []))
|
||||
elif isinstance(raw, list):
|
||||
raw_tools = raw
|
||||
total = len(raw)
|
||||
else:
|
||||
return ToolCatalog()
|
||||
|
||||
for t in raw_tools:
|
||||
if isinstance(t, dict):
|
||||
try:
|
||||
tier_str = t.get("tier_required", t.get("tier", "free"))
|
||||
entry = ToolCatalogEntry(
|
||||
name=t.get("name", t.get("tool", "")),
|
||||
description=t.get("description", ""),
|
||||
category=t.get("category", ""),
|
||||
tier_required=X402Tier(tier_str.lower() if isinstance(tier_str, str) else "free"),
|
||||
price_usd=float(t.get("price_usd", 0) or 0),
|
||||
endpoint=t.get("endpoint", ""),
|
||||
enabled=bool(t.get("enabled", True)),
|
||||
)
|
||||
tools.append(entry)
|
||||
if entry.category:
|
||||
categories.add(entry.category)
|
||||
except Exception:
|
||||
continue
|
||||
return ToolCatalog(
|
||||
tools=tools,
|
||||
total=total,
|
||||
categories=sorted(categories),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _wrap_facilitators(raw: Any) -> list[PaymentFacilitator]:
|
||||
"""Convert legacy facilitator list → Pydantic list."""
|
||||
out: list[PaymentFacilitator] = []
|
||||
items: list[dict] = []
|
||||
if isinstance(raw, dict):
|
||||
items = raw.get("facilitators", raw.get("data", []))
|
||||
elif isinstance(raw, list):
|
||||
items = raw
|
||||
for f in items:
|
||||
if isinstance(f, dict):
|
||||
out.append(PaymentFacilitator(
|
||||
name=f.get("name", ""),
|
||||
url=f.get("url", ""),
|
||||
enabled=bool(f.get("enabled", True)),
|
||||
chains=f.get("chains", []) or [],
|
||||
health=f.get("health", "unknown"),
|
||||
))
|
||||
return out
|
||||
157
app/domain/x402/tools/__init__.py
Normal file
157
app/domain/x402/tools/__init__.py
Normal file
|
|
@ -0,0 +1,157 @@
|
|||
"""x402 tools registry - main entry point for x402 tool implementations.
|
||||
|
||||
Per v4.0 §T26. Provides a unified interface to all x402-powered tools.
|
||||
Tools are delegated to domain/x402/tools/*.py for organization.
|
||||
|
||||
Registry shape:
|
||||
tools = {
|
||||
"tool_name": {
|
||||
"function": async function,
|
||||
"free_tier": bool,
|
||||
"pro_cents": int | None,
|
||||
"ent_cents": int | None,
|
||||
"description": str,
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
from app.domain.x402.tools.label_tools import get_entity_labels, get_wallet_labels
|
||||
from app.domain.x402.tools.market_tools import get_market_overview, get_trending
|
||||
from app.domain.x402.tools.news_tools import get_news, get_news_sentiment
|
||||
from app.domain.x402.tools.report_tools import generate_report, get_report
|
||||
from app.domain.x402.tools.scanner_tools import get_scan_result, run_scan
|
||||
from app.domain.x402.tools.token_tools import get_token_info, get_token_risk
|
||||
from app.domain.x402.tools.wallet_tools import get_wallet_analysis, get_wallet_history
|
||||
|
||||
# Tool registry
|
||||
TOOLS = {
|
||||
"get_token_risk": {
|
||||
"function": get_token_risk,
|
||||
"free_tier": True,
|
||||
"pro_cents": 50,
|
||||
"ent_cents": 40,
|
||||
"description": "Analyze token for rug pull indicators, honeypots, mintability",
|
||||
},
|
||||
"get_token_info": {
|
||||
"function": get_token_info,
|
||||
"free_tier": True,
|
||||
"pro_cents": 10,
|
||||
"ent_cents": 8,
|
||||
"description": "Get basic token metadata (name, symbol, supply, holders)",
|
||||
},
|
||||
"get_wallet_analysis": {
|
||||
"function": get_wallet_analysis,
|
||||
"free_tier": False,
|
||||
"pro_cents": 200,
|
||||
"ent_cents": 150,
|
||||
"description": "Deep wallet behavior analysis (PnL, trading patterns, personas)",
|
||||
},
|
||||
"get_wallet_history": {
|
||||
"function": get_wallet_history,
|
||||
"free_tier": False,
|
||||
"pro_cents": 100,
|
||||
"ent_cents": 80,
|
||||
"description": "Get transaction history for a wallet",
|
||||
},
|
||||
"get_market_overview": {
|
||||
"function": get_market_overview,
|
||||
"free_tier": True,
|
||||
"pro_cents": 50,
|
||||
"ent_cents": 40,
|
||||
"description": "Market overview for a chain (top tokens, volume, trend)",
|
||||
},
|
||||
"get_trending": {
|
||||
"function": get_trending,
|
||||
"free_tier": True,
|
||||
"pro_cents": 30,
|
||||
"ent_cents": 25,
|
||||
"description": "Get trending tokens on a chain",
|
||||
},
|
||||
"run_scan": {
|
||||
"function": run_scan,
|
||||
"free_tier": True,
|
||||
"pro_cents": 100,
|
||||
"ent_cents": 80,
|
||||
"description": "Run full SENTINEL scan on a token",
|
||||
},
|
||||
"get_scan_result": {
|
||||
"function": get_scan_result,
|
||||
"free_tier": False,
|
||||
"pro_cents": 50,
|
||||
"ent_cents": 40,
|
||||
"description": "Get scan result for a token",
|
||||
},
|
||||
"generate_report": {
|
||||
"function": generate_report,
|
||||
"free_tier": False,
|
||||
"pro_cents": 500,
|
||||
"ent_cents": 400,
|
||||
"description": "Full AI research report (7 sections, $5)",
|
||||
},
|
||||
"get_report": {
|
||||
"function": get_report,
|
||||
"free_tier": False,
|
||||
"pro_cents": 200,
|
||||
"ent_cents": 150,
|
||||
"description": "Get previously generated report",
|
||||
},
|
||||
"get_news": {
|
||||
"function": get_news,
|
||||
"free_tier": True,
|
||||
"pro_cents": 30,
|
||||
"ent_cents": 25,
|
||||
"description": "Get news mentions for a token/wallet",
|
||||
},
|
||||
"get_news_sentiment": {
|
||||
"function": get_news_sentiment,
|
||||
"free_tier": True,
|
||||
"pro_cents": 50,
|
||||
"ent_cents": 40,
|
||||
"description": "Get news sentiment analysis",
|
||||
},
|
||||
"get_wallet_labels": {
|
||||
"function": get_wallet_labels,
|
||||
"free_tier": True,
|
||||
"pro_cents": 20,
|
||||
"ent_cents": 15,
|
||||
"description": "Get labels for a wallet address",
|
||||
},
|
||||
"get_entity_labels": {
|
||||
"function": get_entity_labels,
|
||||
"free_tier": False,
|
||||
"pro_cents": 50,
|
||||
"ent_cents": 40,
|
||||
"description": "Get entity labels from Neo4j graph",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def list_tools() -> list[dict]:
|
||||
"""List all available x402 tools."""
|
||||
return [
|
||||
{
|
||||
"tool_id": name,
|
||||
"description": info["description"],
|
||||
"free_tier": info["free_tier"],
|
||||
"pro_cents": info.get("pro_cents"),
|
||||
"ent_cents": info.get("ent_cents"),
|
||||
}
|
||||
for name, info in TOOLS.items()
|
||||
]
|
||||
|
||||
|
||||
def get_tool(tool_id: str) -> dict | None:
|
||||
"""Get a tool by ID."""
|
||||
return TOOLS.get(tool_id)
|
||||
|
||||
|
||||
async def call_tool(tool_id: str, **kwargs) -> dict:
|
||||
"""Call a tool by ID with arguments."""
|
||||
tool = TOOLS.get(tool_id)
|
||||
if not tool:
|
||||
return {"error": f"Unknown tool: {tool_id}"}
|
||||
try:
|
||||
result = await tool["function"](**kwargs)
|
||||
return {"tool_id": tool_id, "result": result}
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
27
app/domain/x402/tools/label_tools.py
Normal file
27
app/domain/x402/tools/label_tools.py
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
"""x402 label tools - get_wallet_labels, get_entity_labels."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
async def get_wallet_labels(wallet_address: str) -> dict[str, Any]:
|
||||
"""Get labels for a wallet."""
|
||||
return {
|
||||
"wallet_address": wallet_address,
|
||||
"labels": [],
|
||||
}
|
||||
|
||||
|
||||
async def get_entity_labels(entity_id: str) -> dict[str, Any]:
|
||||
"""Get entity labels from Neo4j graph."""
|
||||
return {
|
||||
"entity_id": entity_id,
|
||||
"labels": [],
|
||||
}
|
||||
|
||||
|
||||
async def get_entity_by_labels(labels: list[str]) -> dict[str, Any]:
|
||||
"""Get entity by labels."""
|
||||
return {
|
||||
"labels": labels,
|
||||
"entities": [],
|
||||
}
|
||||
37
app/domain/x402/tools/market_tools.py
Normal file
37
app/domain/x402/tools/market_tools.py
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
"""x402 market tools - get_market_overview, get_trending."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
async def get_market_overview(chain: str = "solana") -> dict[str, Any]:
|
||||
"""Get market overview for a chain."""
|
||||
return {
|
||||
"chain": chain,
|
||||
"market_status": "active",
|
||||
"top_tokens": [],
|
||||
"volume_24h": 0,
|
||||
"trending": [],
|
||||
}
|
||||
|
||||
|
||||
async def get_trending(chain: str = "solana") -> dict[str, Any]:
|
||||
"""Get trending tokens."""
|
||||
return {
|
||||
"chain": chain,
|
||||
"trending_tokens": [],
|
||||
"time_window": "24h",
|
||||
}
|
||||
|
||||
|
||||
async def market_overview(chain: str = "solana") -> dict[str, Any]:
|
||||
"""Alias for get_market_overview."""
|
||||
return get_market_overview(chain)
|
||||
|
||||
|
||||
async def token_deep_dive(token_address: str, chain: str = "solana") -> dict[str, Any]:
|
||||
"""Deep dive on a token."""
|
||||
return {
|
||||
"token_address": token_address,
|
||||
"chain": chain,
|
||||
"deep_dive": "pending",
|
||||
}
|
||||
31
app/domain/x402/tools/news_tools.py
Normal file
31
app/domain/x402/tools/news_tools.py
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
"""x402 news tools - get_news, get_news_sentiment."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
async def get_news(token: str, chain: str = "solana") -> dict[str, Any]:
|
||||
"""Get news mentions for a token."""
|
||||
return {
|
||||
"token": token,
|
||||
"chain": chain,
|
||||
"news": [],
|
||||
"count": 0,
|
||||
}
|
||||
|
||||
|
||||
async def get_news_sentiment(token: str, chain: str = "solana") -> dict[str, Any]:
|
||||
"""Get news sentiment analysis."""
|
||||
return {
|
||||
"token": token,
|
||||
"chain": chain,
|
||||
"sentiment": "neutral",
|
||||
"articles": 0,
|
||||
}
|
||||
|
||||
|
||||
async def social_signal(query: str) -> dict[str, Any]:
|
||||
"""Get social signal."""
|
||||
return {
|
||||
"query": query,
|
||||
"signal": "pending",
|
||||
}
|
||||
220
app/domain/x402/tools/report_tools.py
Normal file
220
app/domain/x402/tools/report_tools.py
Normal file
|
|
@ -0,0 +1,220 @@
|
|||
"""x402 report tools - generate_report, get_report."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
async def generate_report(subject_type: str, subject_id: str, model: str = "deepseek-v3") -> dict[str, Any]:
|
||||
"""Generate a research report."""
|
||||
return {
|
||||
"report_id": "pending",
|
||||
"subject_type": subject_type,
|
||||
"subject_id": subject_id,
|
||||
"sections": {},
|
||||
"risk_score": 50,
|
||||
"markdown": "# Pending Report\n\n[Report generation pending]",
|
||||
}
|
||||
|
||||
|
||||
async def get_report(report_id: str) -> dict[str, Any]:
|
||||
"""Get a previously generated report."""
|
||||
return {
|
||||
"report_id": report_id,
|
||||
"status": "pending",
|
||||
"result": {},
|
||||
}
|
||||
|
||||
|
||||
async def social_sentiment(token: str, chain: str = "solana") -> dict[str, Any]:
|
||||
"""Get social sentiment analysis."""
|
||||
return {
|
||||
"token": token,
|
||||
"chain": chain,
|
||||
"sentiment": "neutral",
|
||||
"mentions": 0,
|
||||
}
|
||||
|
||||
|
||||
async def cluster_detection(wallet_address: str, chain: str = "solana") -> dict[str, Any]:
|
||||
"""Run cluster detection."""
|
||||
return {
|
||||
"wallet_address": wallet_address,
|
||||
"chain": chain,
|
||||
"clusters": [],
|
||||
}
|
||||
|
||||
|
||||
async def insider_tracker(creator: str, chain: str = "solana") -> dict[str, Any]:
|
||||
"""Track insider activity."""
|
||||
return {
|
||||
"creator": creator,
|
||||
"chain": chain,
|
||||
"insider_activity": [],
|
||||
}
|
||||
|
||||
|
||||
async def twitter_profile(query: str) -> dict[str, Any]:
|
||||
"""Get Twitter profile."""
|
||||
return {
|
||||
"query": query,
|
||||
"profile": {},
|
||||
}
|
||||
|
||||
|
||||
async def twitter_timeline(query: str) -> dict[str, Any]:
|
||||
"""Get Twitter timeline."""
|
||||
return {
|
||||
"query": query,
|
||||
"tweets": [],
|
||||
}
|
||||
|
||||
|
||||
async def twitter_search(query: str) -> dict[str, Any]:
|
||||
"""Search Twitter."""
|
||||
return {
|
||||
"query": query,
|
||||
"results": [],
|
||||
}
|
||||
|
||||
|
||||
async def launch_radar(chain: str = "solana", window_minutes: int = 5) -> dict[str, Any]:
|
||||
"""Get launch radar."""
|
||||
return {
|
||||
"chain": chain,
|
||||
"window_minutes": window_minutes,
|
||||
"launches": [],
|
||||
}
|
||||
|
||||
|
||||
async def launch_intel(chain: str = "solana", hours: int = 24) -> dict[str, Any]:
|
||||
"""Get launch intelligence."""
|
||||
return {
|
||||
"chain": chain,
|
||||
"hours": hours,
|
||||
"launches": [],
|
||||
}
|
||||
|
||||
|
||||
async def token_pulse(token_address: str, chain: str = "solana") -> dict[str, Any]:
|
||||
"""Get token pulse."""
|
||||
return {
|
||||
"token_address": token_address,
|
||||
"chain": chain,
|
||||
"pulse": "pending",
|
||||
}
|
||||
|
||||
|
||||
async def anomaly_detector(chain: str = "solana") -> dict[str, Any]:
|
||||
"""Run anomaly detection."""
|
||||
return {
|
||||
"chain": chain,
|
||||
"anomalies": [],
|
||||
}
|
||||
|
||||
|
||||
async def whale_decoder(address: str, chain: str = "solana") -> dict[str, Any]:
|
||||
"""Decode whale activity."""
|
||||
return {
|
||||
"address": address,
|
||||
"chain": chain,
|
||||
"whale_activity": [],
|
||||
}
|
||||
|
||||
|
||||
async def chain_health(chain: str = "solana") -> dict[str, Any]:
|
||||
"""Check chain health."""
|
||||
return {
|
||||
"chain": chain,
|
||||
"health": "pending",
|
||||
}
|
||||
|
||||
|
||||
async def portfolio_tracker(addresses: list[str], chain: str = "solana") -> dict[str, Any]:
|
||||
"""Track portfolio."""
|
||||
return {
|
||||
"addresses": addresses,
|
||||
"chain": chain,
|
||||
"portfolio": [],
|
||||
}
|
||||
|
||||
|
||||
async def copy_trade_finder(chain: str = "solana") -> dict[str, Any]:
|
||||
"""Find copy trade opportunities."""
|
||||
return {
|
||||
"chain": chain,
|
||||
"opportunities": [],
|
||||
}
|
||||
|
||||
|
||||
async def risk_monitor(address: str, chain: str = "solana") -> dict[str, Any]:
|
||||
"""Monitor risk."""
|
||||
return {
|
||||
"address": address,
|
||||
"chain": chain,
|
||||
"risk_level": "pending",
|
||||
}
|
||||
|
||||
|
||||
async def defi_yield_scanner(chain: str = "solana") -> dict[str, Any]:
|
||||
"""Scan DeFi yield opportunities."""
|
||||
return {
|
||||
"chain": chain,
|
||||
"opportunities": [],
|
||||
}
|
||||
|
||||
|
||||
async def nft_wash_detector(collection: str) -> dict[str, Any]:
|
||||
"""Detect NFT wash trading."""
|
||||
return {
|
||||
"collection": collection,
|
||||
"wash_trading": False,
|
||||
}
|
||||
|
||||
|
||||
async def bridge_security() -> dict[str, Any]:
|
||||
"""Check bridge security."""
|
||||
return {
|
||||
"bridge_security": "pending",
|
||||
}
|
||||
|
||||
|
||||
async def gas_forecast(chain: str = "solana") -> dict[str, Any]:
|
||||
"""Forecast gas prices."""
|
||||
return {
|
||||
"chain": chain,
|
||||
"gas_forecast": "pending",
|
||||
}
|
||||
|
||||
|
||||
async def sniper_alert(chain: str = "solana", hours: int = 24) -> dict[str, Any]:
|
||||
"""Get sniper alerts."""
|
||||
return {
|
||||
"chain": chain,
|
||||
"hours": hours,
|
||||
"sniper_alerts": [],
|
||||
}
|
||||
|
||||
|
||||
async def liquidity_flow(address: str, chain: str = "solana") -> dict[str, Any]:
|
||||
"""Track liquidity flow."""
|
||||
return {
|
||||
"address": address,
|
||||
"chain": chain,
|
||||
"liquidity_flow": [],
|
||||
}
|
||||
|
||||
|
||||
async def rug_pull_predictor(address: str, chain: str = "solana") -> dict[str, Any]:
|
||||
"""Predict rug pull risk."""
|
||||
return {
|
||||
"address": address,
|
||||
"chain": chain,
|
||||
"risk_score": 50,
|
||||
}
|
||||
|
||||
|
||||
async def airdrop_finder(chain: str = "solana") -> dict[str, Any]:
|
||||
"""Find airdrop opportunities."""
|
||||
return {
|
||||
"chain": chain,
|
||||
"airdrops": [],
|
||||
}
|
||||
50
app/domain/x402/tools/scanner_tools.py
Normal file
50
app/domain/x402/tools/scanner_tools.py
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
"""x402 scanner tools - run_scan, get_scan_result."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
async def run_scan(token_address: str, chain: str = "solana") -> dict[str, Any]:
|
||||
"""Run full SENTINEL scan."""
|
||||
return {
|
||||
"token_address": token_address,
|
||||
"chain": chain,
|
||||
"scan_status": "pending",
|
||||
"modules_run": [],
|
||||
"safety_score": 50,
|
||||
}
|
||||
|
||||
|
||||
async def get_scan_result(scan_id: str) -> dict[str, Any]:
|
||||
"""Get scan result by ID."""
|
||||
return {
|
||||
"scan_id": scan_id,
|
||||
"status": "pending",
|
||||
"result": {},
|
||||
}
|
||||
|
||||
|
||||
async def rug_shield(token_address: str, chain: str = "solana") -> dict[str, Any]:
|
||||
"""Run rug shield scan."""
|
||||
return {
|
||||
"token_address": token_address,
|
||||
"chain": chain,
|
||||
"rug_shield": "pending",
|
||||
}
|
||||
|
||||
|
||||
async def honeypot_check(token_address: str, chain: str = "solana") -> dict[str, Any]:
|
||||
"""Check honeypot status."""
|
||||
return {
|
||||
"token_address": token_address,
|
||||
"chain": chain,
|
||||
"is_honeypot": False,
|
||||
}
|
||||
|
||||
|
||||
async def token_forensics(token_address: str, chain: str = "solana") -> dict[str, Any]:
|
||||
"""Run token forensics."""
|
||||
return {
|
||||
"token_address": token_address,
|
||||
"chain": chain,
|
||||
"forensics": "pending",
|
||||
}
|
||||
37
app/domain/x402/tools/token_tools.py
Normal file
37
app/domain/x402/tools/token_tools.py
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
"""x402 token tools - get_token_risk, get_token_info, deep_contract_audit."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
async def get_token_risk(token_address: str, chain: str = "ethereum") -> dict[str, Any]:
|
||||
"""Get risk analysis for a token."""
|
||||
return {
|
||||
"token_address": token_address,
|
||||
"chain": chain,
|
||||
"risk_score": 50, # Placeholder
|
||||
"risk_tier": "UNKNOWN",
|
||||
"risk_factors": [],
|
||||
"notes": "Token risk analysis - full implementation in x402_tools.py",
|
||||
}
|
||||
|
||||
|
||||
async def get_token_info(token_address: str, chain: str = "ethereum") -> dict[str, Any]:
|
||||
"""Get basic token info."""
|
||||
return {
|
||||
"token_address": token_address,
|
||||
"chain": chain,
|
||||
"symbol": "UNKNOWN",
|
||||
"name": "Unknown Token",
|
||||
"decimals": 0,
|
||||
"total_supply": 0,
|
||||
}
|
||||
|
||||
|
||||
async def deep_contract_audit(token_address: str, chain: str = "ethereum") -> dict[str, Any]:
|
||||
"""Deep contract audit."""
|
||||
return {
|
||||
"token_address": token_address,
|
||||
"chain": chain,
|
||||
"audit_result": "PENDING",
|
||||
"findings": [],
|
||||
}
|
||||
34
app/domain/x402/tools/wallet_tools.py
Normal file
34
app/domain/x402/tools/wallet_tools.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
"""x402 wallet tools - get_wallet_analysis, get_wallet_history, wallet_profiler."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
async def get_wallet_analysis(wallet_address: str, chain: str = "solana") -> dict[str, Any]:
|
||||
"""Get wallet behavior analysis."""
|
||||
return {
|
||||
"wallet_address": wallet_address,
|
||||
"chain": chain,
|
||||
"analysis": "pending",
|
||||
"personas": [],
|
||||
"risk_score": 50,
|
||||
}
|
||||
|
||||
|
||||
async def get_wallet_history(wallet_address: str, chain: str = "solana") -> dict[str, Any]:
|
||||
"""Get wallet transaction history."""
|
||||
return {
|
||||
"wallet_address": wallet_address,
|
||||
"chain": chain,
|
||||
"transactions": [],
|
||||
"total_count": 0,
|
||||
}
|
||||
|
||||
|
||||
async def wallet_profiler(wallet_address: str, chain: str = "solana") -> dict[str, Any]:
|
||||
"""Profile a wallet."""
|
||||
return {
|
||||
"wallet_address": wallet_address,
|
||||
"chain": chain,
|
||||
"profile": "pending",
|
||||
"metrics": {},
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue