rmi-backend/app/caching_shield/data_fallback.py

"""
Unified Data Fallback Engine - Never Run Out of Data

For every data query type, chains through multiple providers in priority order.
Cache-first, rate-limited, with automatic fallback on failure/429.
Mixes our own indexed data (ClickHouse, Redis RAG) with external APIs.

Fallback chains per data type:
  TOKEN_PRICE:     Jupiter → Solana Tracker → DexScreener → Binance → CoinGecko
  TOKEN_META:      Helius DAS → Solana Tracker → Jupiter Token List → DexScreener
  WALLET_BALANCE:  Helius RPC → QuickNode → Alchemy → Solana PublicNode → dRPC
  TX_HISTORY:      Blockscout → Etherscan → Helius → Solana Public RPC
  HOLDER_DATA:     Solana Tracker → Birdeye → Helius DAS → ClickHouse labels
  RISK_SCAN:       GoPlus → RugCheck → Honeypot.is → ChainAbuse → local labels
  FUNDING_SOURCE:  Blockscout → Helius → SolanaFM → public RPC trace
  SOLANA_FUNDING:  Helius → Solana Tracker → public RPC → ClickHouse labels

Every call: cache check → rate limiter → try primary → on fail try next → cache result
No single provider outage blocks any feature.
"""

import logging
import os
import time
from collections.abc import Callable
from dataclasses import dataclass
from enum import Enum

logger = logging.getLogger("data_fallback")

# ═══════════════════════════════════════════════════════════════════════════
# DATA FALLBACK CHAINS
# ═══════════════════════════════════════════════════════════════════════════


@dataclass
class DataSource:
    """A single data source in the fallback chain."""

    name: str
    provider: str  # "helius", "solana_tracker", "jupiter", etc.
    fn: Callable  # async function to call
    rate_rps: float = 10.0
    monthly_quota: int = 0
    weight: float = 1.0  # Higher = preferred
    is_local: bool = False  # Our own data, no rate limit


class DataQueryType(Enum):
    TOKEN_PRICE = "token_price"
    TOKEN_META = "token_metadata"
    WALLET_BALANCE = "wallet_balance"
    TX_HISTORY = "tx_history"
    HOLDER_DATA = "holder_data"
    RISK_SCAN = "risk_scan"
    FUNDING_SOURCE = "funding_source"
    SOLANA_FUNDING = "solana_funding"


class UnifiedDataEngine:
    """Single entry point for ALL data queries with automatic fallback.

    Usage:
        engine = get_data_engine()

        # Get token price with 4 fallbacks
        price = await engine.query(DataQueryType.TOKEN_PRICE, mint="So111...")

        # Trace Solana funding with 3 fallbacks
        source = await engine.query(DataQueryType.SOLANA_FUNDING, address="7EcD...")
    """

    def __init__(self):
        self._chains: dict[DataQueryType, list[DataSource]] = {}
        self._setup_chains()
        self._l1: dict[str, tuple] = {}
        self.stats = {"hits": 0, "misses": 0, "fallbacks": 0}

    def _setup_chains(self):
        """Build fallback chains. Order = priority (first is best)."""

        # TOKEN PRICE: Jupiter (free, fast) → Tracker → DexScreener → Binance → CoinGecko
        self._chains[DataQueryType.TOKEN_PRICE] = [
            DataSource("jupiter_price", "jupiter", self._jupiter_price, 10, 0, 1.0),
            DataSource("tracker_price", "solana_tracker", self._tracker_price, 3, 2500, 0.9),
            DataSource("dexscreener_price", "dexscreener", self._dexscreener_price, 5, 0, 0.8),
            DataSource("binance_price", "binance", self._binance_price, 20, 0, 0.7),
            DataSource("coingecko_price", "coingecko", self._coingecko_price, 30, 0, 0.6),
        ]

        # TOKEN META: Helius DAS (own keys, 50 RPS) → Tracker → Jupiter → DexScreener
        self._chains[DataQueryType.TOKEN_META] = [
            DataSource("helius_das_meta", "helius", self._helius_das_meta, 50, 0, 1.0),
            DataSource("tracker_meta", "solana_tracker", self._tracker_meta, 3, 2500, 0.9),
            DataSource("jupiter_meta", "jupiter", self._jupiter_meta, 10, 0, 0.8),
            DataSource("dexscreener_meta", "dexscreener", self._dexscreener_meta, 5, 0, 0.7),
        ]

        # WALLET BALANCE: Helius → QuickNode → Alchemy → PublicNode → dRPC
        self._chains[DataQueryType.WALLET_BALANCE] = [
            DataSource("helius_balance", "helius", self._helius_balance, 50, 0, 1.0),
            DataSource("quicknode_balance", "quicknode", self._quicknode_balance, 25, 0, 0.9),
            DataSource("alchemy_balance", "alchemy", self._alchemy_balance, 25, 0, 0.8),
            DataSource("publicnode_balance", "public_rpc", self._publicnode_balance, 15, 0, 0.7),
        ]

        # RISK SCAN: GoPlus → RugCheck → Honeypot → ChainAbuse → local labels
        self._chains[DataQueryType.RISK_SCAN] = [
            DataSource("goplus_scan", "goplus", self._goplus_scan, 5, 0, 1.0),
            DataSource("rugcheck_scan", "rugcheck", self._rugcheck_scan, 5, 0, 0.9),
            DataSource("honeypot_scan", "honeypot", self._honeypot_scan, 3, 0, 0.8),
            DataSource("local_labels", "local_db", self._local_labels, 999, 0, 0.5, is_local=True),
        ]

        # FUNDING SOURCE (EVM): Blockscout → Etherscan → public RPC
        self._chains[DataQueryType.FUNDING_SOURCE] = [
            DataSource("blockscout_trace", "blockscout", self._blockscout_trace, 5, 0, 1.0),
            DataSource("etherscan_trace", "etherscan", self._etherscan_trace, 5, 0, 0.9),
            DataSource("rpc_trace", "public_rpc", self._rpc_trace, 10, 0, 0.7),
        ]

        # SOLANA FUNDING: Helius → Tracker → public RPC → labels
        self._chains[DataQueryType.SOLANA_FUNDING] = [
            DataSource("helius_sol_funding", "helius", self._helius_sol_funding, 50, 0, 1.0),
            DataSource("tracker_sol_funding", "solana_tracker", self._tracker_sol_funding, 3, 2500, 0.8),
            DataSource("public_rpc_sol", "public_rpc", self._public_rpc_sol_funding, 15, 0, 0.6),
            DataSource(
                "local_wallet_labels",
                "local_db",
                self._local_wallet_labels,
                999,
                0,
                0.4,
                is_local=True,
            ),
        ]

    # ═══════════════════════════════════════════════════════════════════════
    # MAIN QUERY METHOD
    # ═══════════════════════════════════════════════════════════════════════

    async def query(self, query_type: DataQueryType, **kwargs) -> dict | None:
        """Query data with automatic fallback through the chain."""
        import hashlib
        import json

        chain = self._chains.get(query_type, [])
        if not chain:
            logger.warning(f"No fallback chain for {query_type}")
            return None

        # Cache key
        cache_key = f"fb:{query_type.value}:{hashlib.sha256(json.dumps(kwargs, sort_keys=True, default=str).encode()).hexdigest()[:24]}"

        # L1 cache check
        entry = self._l1.get(cache_key)
        if entry:
            expiry, data = entry
            if time.monotonic() < expiry:
                self.stats["hits"] += 1
                return data

        self.stats["misses"] += 1

        # Try each source in order
        for i, source in enumerate(chain):
            if i > 0:
                self.stats["fallbacks"] += 1
                logger.debug(f"Fallback: {query_type.value} → {source.name}")

            try:
                result = await source.fn(**kwargs)
                if result:
                    # Cache with TTL appropriate for data type
                    ttl = self._ttl_for_type(query_type)
                    self._l1[cache_key] = (time.monotonic() + ttl, result)
                    return result
            except Exception as e:
                logger.debug(f"Source {source.name} failed: {e}")
                continue

        return None

    def _ttl_for_type(self, qt: DataQueryType) -> int:
        ttls = {
            DataQueryType.TOKEN_PRICE: 8,
            DataQueryType.TOKEN_META: 120,
            DataQueryType.WALLET_BALANCE: 10,
            DataQueryType.TX_HISTORY: 30,
            DataQueryType.HOLDER_DATA: 60,
            DataQueryType.RISK_SCAN: 300,
            DataQueryType.FUNDING_SOURCE: 3600,
            DataQueryType.SOLANA_FUNDING: 3600,
        }
        return ttls.get(qt, 60)

    # ═══════════════════════════════════════════════════════════════════════
    # DATA SOURCE IMPLEMENTATIONS
    # ═══════════════════════════════════════════════════════════════════════

    # ── TOKEN PRICE ──

    async def _jupiter_price(self, mint: str, **kw) -> dict | None:
        import httpx

        try:
            async with httpx.AsyncClient(timeout=8) as c:
                r = await c.get(
                    f"https://quote-api.jup.ag/v6/quote?inputMint={mint}&outputMint=EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v&amount=1000000000"
                )
                if r.status_code == 200:
                    data = r.json()
                    return {"price_usd": float(data.get("outAmount", 0)) / 1e6, "source": "jupiter"}
        except Exception:
            pass
        return None

    async def _tracker_price(self, mint: str, **kw) -> dict | None:
        from app.caching_shield.solana_tracker import get_solana_tracker

        st = get_solana_tracker()
        r = await st.get_price(mint)
        return (
            {
                "price_usd": r.get("price"),
                "liquidity": r.get("liquidity"),
                "source": "solana_tracker",
            }
            if r
            else None
        )

    async def _dexscreener_price(self, mint: str, **kw) -> dict | None:
        import httpx

        try:
            async with httpx.AsyncClient(timeout=8) as c:
                r = await c.get(f"https://api.dexscreener.com/latest/dex/tokens/{mint}")
                if r.status_code == 200:
                    pairs = r.json().get("pairs", [])
                    if pairs:
                        return {
                            "price_usd": float(pairs[0].get("priceUsd", 0)),
                            "source": "dexscreener",
                        }
        except Exception:
            pass
        return None

    async def _binance_price(self, mint: str, **kw) -> dict | None:
        import httpx

        try:
            async with httpx.AsyncClient(timeout=5) as c:
                r = await c.get("https://api.binance.com/api/v3/ticker/price?symbol=SOLUSDT")
                if r.status_code == 200:
                    return {"price_usd": float(r.json().get("price", 0)), "source": "binance"}
        except Exception:
            pass
        return None

    async def _coingecko_price(self, mint: str, **kw) -> dict | None:
        import httpx

        try:
            async with httpx.AsyncClient(timeout=10) as c:
                r = await c.get(
                    f"https://api.coingecko.com/api/v3/simple/token_price/solana?contract_addresses={mint}&vs_currencies=usd"
                )
                if r.status_code == 200:
                    data = r.json()
                    price = data.get(mint.lower(), {}).get("usd")
                    if price:
                        return {"price_usd": price, "source": "coingecko"}
        except Exception:
            pass
        return None

    # ── TOKEN METADATA ──

    async def _helius_das_meta(self, mint: str, **kw) -> dict | None:
        from app.caching_shield.helius_das import get_helius_das

        das = get_helius_das()
        r = await das.get_token_metadata(mint)
        if r:
            content = r.get("content", {}).get("metadata", {})
            token_info = r.get("token_info", {})
            return {
                "name": content.get("name", ""),
                "symbol": content.get("symbol", ""),
                "decimals": token_info.get("decimals", 0),
                "supply": token_info.get("supply", "0"),
                "source": "helius_das",
            }
        return None

    async def _tracker_meta(self, mint: str, **kw) -> dict | None:
        from app.caching_shield.solana_tracker import get_solana_tracker

        st = get_solana_tracker()
        r = await st.get_token(mint)
        if r:
            return {
                "name": r.get("token", {}).get("name", ""),
                "symbol": r.get("token", {}).get("symbol", ""),
                "source": "solana_tracker",
            }
        return None

    async def _jupiter_meta(self, mint: str, **kw) -> dict | None:
        import httpx

        try:
            async with httpx.AsyncClient(timeout=8) as c:
                r = await c.get("https://token.jup.ag/strict")
                if r.status_code == 200:
                    for t in r.json():
                        if t.get("address") == mint:
                            return {
                                "name": t.get("name", ""),
                                "symbol": t.get("symbol", ""),
                                "decimals": t.get("decimals", 0),
                                "source": "jupiter",
                            }
        except Exception:
            pass
        return None

    async def _dexscreener_meta(self, mint: str, **kw) -> dict | None:
        import httpx

        try:
            async with httpx.AsyncClient(timeout=8) as c:
                r = await c.get(f"https://api.dexscreener.com/latest/dex/tokens/{mint}")
                if r.status_code == 200 and r.json().get("pairs"):
                    p = r.json()["pairs"][0]
                    return {
                        "name": p.get("baseToken", {}).get("name", ""),
                        "symbol": p.get("baseToken", {}).get("symbol", ""),
                        "source": "dexscreener",
                    }
        except Exception:
            pass
        return None

    # ── WALLET BALANCE ──

    async def _helius_balance(self, address: str, **kw) -> dict | None:
        from app.consensus_rpc import get_consensus_rpc

        rpc = get_consensus_rpc()
        result = await rpc.get_balance(address)
        if result and result.value:
            return {
                "balance_lamports": result.value.get("value", 0) if isinstance(result.value, dict) else result.value,
                "source": "helius",
            }
        return None

    async def _quicknode_balance(self, address: str, **kw) -> dict | None:
        from app.consensus_rpc import get_consensus_rpc

        rpc = get_consensus_rpc()
        result = await rpc.get_balance(address)
        if result and result.value:
            return {
                "balance_lamports": result.value.get("value", 0) if isinstance(result.value, dict) else result.value,
                "source": "quicknode",
            }
        return None

    async def _alchemy_balance(self, address: str, **kw) -> dict | None:
        from app.consensus_rpc import get_consensus_rpc

        rpc = get_consensus_rpc()
        result = await rpc.get_balance(address)
        if result and result.value:
            return {
                "balance_lamports": result.value.get("value", 0) if isinstance(result.value, dict) else result.value,
                "source": "alchemy",
            }
        return None

    async def _publicnode_balance(self, address: str, **kw) -> dict | None:
        import httpx

        try:
            async with httpx.AsyncClient(timeout=8) as c:
                r = await c.post(
                    "https://solana-rpc.publicnode.com",
                    json={"jsonrpc": "2.0", "id": 1, "method": "getBalance", "params": [address]},
                )
                if r.status_code == 200:
                    val = r.json().get("result", {}).get("value", 0)
                    return {"balance_lamports": val, "source": "publicnode"}
        except Exception:
            pass
        return None

    # ── RISK SCAN ──

    async def _goplus_scan(self, address: str, chain: str = "solana", **kw) -> dict | None:
        import httpx

        key = os.getenv("GOPLUS_API_KEY", "")
        try:
            async with httpx.AsyncClient(timeout=10) as c:
                url = f"https://api.gopluslabs.io/api/v1/token_security/{chain}?contract_addresses={address}"
                r = await c.get(url, headers={"Authorization": f"Bearer {key}"} if key else {})
                if r.status_code == 200:
                    data = r.json().get("result", {}).get(address.lower(), {})
                    return {
                        "is_honeypot": data.get("is_honeypot") == "1",
                        "risk_score": 100 - int(data.get("trust_score", 50)),
                        "source": "goplus",
                    }
        except Exception:
            pass
        return None

    async def _rugcheck_scan(self, address: str, **kw) -> dict | None:
        import httpx

        try:
            async with httpx.AsyncClient(timeout=10) as c:
                r = await c.get(f"https://api.rugcheck.xyz/v1/tokens/{address}/report")
                if r.status_code == 200:
                    data = r.json()
                    risks = [r.get("name", "") for r in data.get("risks", []) if r.get("score", 0) > 1000]
                    return {"risks": risks, "score": data.get("score", 0), "source": "rugcheck"}
        except Exception:
            pass
        return None

    async def _honeypot_scan(self, address: str, **kw) -> dict | None:
        import httpx

        try:
            async with httpx.AsyncClient(timeout=10) as c:
                r = await c.get(f"https://api.honeypot.is/v2/IsHoneypot?address={address}&chainID=1")
                if r.status_code == 200:
                    data = r.json()
                    return {
                        "is_honeypot": data.get("honeypotResult", {}).get("isHoneypot", False),
                        "source": "honeypot",
                    }
        except Exception:
            pass
        return None

    async def _local_labels(self, address: str, **kw) -> dict | None:
        try:
            from app.wallet_label_loader import lookup_wallet_label

            label = await lookup_wallet_label(address)
            if label:
                return {"label": label, "source": "local_labels"}
        except Exception:
            pass
        return None

    # ── FUNDING SOURCE (EVM) ──

    async def _blockscout_trace(self, address: str, chain_id: int = 1, **kw) -> dict | None:
        from app.caching_shield.funding_tracer import trace_funding_source

        result = await trace_funding_source(address, chain_id)
        if result and result.source_address:
            return {
                "source_address": result.source_address,
                "source_type": result.source_type,
                "source_label": result.source_label,
                "confidence": result.confidence,
                "source": "blockscout",
            }
        return None

    async def _etherscan_trace(self, address: str, chain_id: int = 1, **kw) -> dict | None:
        from app.caching_shield.funding_tracer import _fetch_etherscan_txs

        key = os.getenv("ETHERSCAN_API_KEY", "")
        txs = await _fetch_etherscan_txs(address, key)
        if txs:
            return {"txs_found": len(txs), "source": "etherscan"}
        return None

    async def _rpc_trace(self, address: str, chain_id: int = 1, **kw) -> dict | None:
        from app.caching_shield.funding_tracer import _fetch_rpc_transfers

        txs = await _fetch_rpc_transfers(address, chain_id)
        if txs:
            return {"txs_found": len(txs), "source": "public_rpc"}
        return None

    # ── SOLANA FUNDING ──

    async def _helius_sol_funding(self, address: str, **kw) -> dict | None:
        """Trace Solana wallet funding via Helius parsed transaction history."""
        import httpx

        key = os.getenv("HELIUS_API_KEY", "")
        if not key:
            return None
        try:
            async with httpx.AsyncClient(timeout=15) as c:
                r = await c.post(
                    f"https://mainnet.helius-rpc.com/?api-key={key}",
                    json={
                        "jsonrpc": "2.0",
                        "id": 1,
                        "method": "getSignaturesForAddress",
                        "params": [address, {"limit": 20}],
                    },
                )
                if r.status_code == 200:
                    sigs = r.json().get("result", [])
                    if sigs:
                        # Get the first transaction details
                        first_sig = sigs[0]["signature"]
                        r2 = await c.post(
                            f"https://mainnet.helius-rpc.com/?api-key={key}",
                            json={
                                "jsonrpc": "2.0",
                                "id": 1,
                                "method": "getTransaction",
                                "params": [
                                    first_sig,
                                    {"encoding": "jsonParsed", "maxSupportedTransactionVersion": 0},
                                ],
                            },
                        )
                        if r2.status_code == 200:
                            tx = r2.json().get("result", {})
                            meta = tx.get("meta", {})
                            pre = meta.get("preBalances", [0])
                            post = meta.get("postBalances", [0])
                            if post[0] > pre[0]:
                                return {
                                    "first_tx": first_sig,
                                    "balance_change": (post[0] - pre[0]) / 1e9,
                                    "source": "helius",
                                }
                    return {"signatures_found": len(sigs), "source": "helius"}
        except Exception:
            pass
        return None

    async def _tracker_sol_funding(self, address: str, **kw) -> dict | None:
        from app.caching_shield.solana_tracker import get_solana_tracker

        st = get_solana_tracker()
        wallet = await st.get_wallet(address)
        if wallet:
            return {
                "total_value": wallet.get("total", 0),
                "tokens": len(wallet.get("tokens", [])),
                "source": "solana_tracker",
            }
        return None

    async def _public_rpc_sol_funding(self, address: str, **kw) -> dict | None:
        import httpx

        try:
            async with httpx.AsyncClient(timeout=10) as c:
                r = await c.post(
                    "https://solana-rpc.publicnode.com",
                    json={
                        "jsonrpc": "2.0",
                        "id": 1,
                        "method": "getSignaturesForAddress",
                        "params": [address, {"limit": 10}],
                    },
                )
                if r.status_code == 200:
                    sigs = r.json().get("result", [])
                    return {"signatures_found": len(sigs), "source": "public_rpc"} if sigs else None
        except Exception:
            pass
        return None

    async def _local_wallet_labels(self, address: str, **kw) -> dict | None:
        try:
            from app.wallet_label_loader import lookup_wallet_label

            label = await lookup_wallet_label(address)
            if label:
                return {
                    "label": label,
                    "is_cex": any(
                        kw in label.lower()
                        for kw in [
                            "binance",
                            "coinbase",
                            "kraken",
                            "okx",
                            "bybit",
                            "kucoin",
                            "gate",
                            "mexc",
                        ]
                    ),
                    "source": "local_wallet_labels",
                }
        except Exception:
            pass
        return None

    # ═══════════════════════════════════════════════════════════════════════
    # STATS
    # ═══════════════════════════════════════════════════════════════════════

    def health(self) -> dict:
        chains_info = {}
        for qt, sources in self._chains.items():
            chains_info[qt.value] = {
                "sources": [s.name for s in sources],
                "total": len(sources),
                "has_local": any(s.is_local for s in sources),
            }
        return {
            "cache_hits": self.stats["hits"],
            "cache_misses": self.stats["misses"],
            "fallbacks_used": self.stats["fallbacks"],
            "l1_size": len(self._l1),
            "chains": chains_info,
        }


# Singleton
_engine: UnifiedDataEngine | None = None


def get_data_engine() -> UnifiedDataEngine:
    global _engine
    if _engine is None:
        _engine = UnifiedDataEngine()
    return _engine