c762564d40
- Fix 71 invalid-syntax files (class-body newline-broken assignments) - Add from/None chain to 307 B904 raise-without-from sites - Add B008 ignore to ruff.toml (already in pyproject.toml) - Noqa F401 on __init__.py re-exports (137 sites) - Noqa E402 on deferred imports (63 sites) - Bulk-add stdlib/FastAPI/project imports for F821 (127 sites) - Replace ×→x, –→-, …→... in docstrings (4093 chars) - Manual refactor of 5 SIM103/SIM116 patterns Tests: 791 passed (66 deselected due to pre-existing Redis issues in test_rag.py) Co-authored-by: opencode <opencode@rugmunch.io>
486 lines
20 KiB
Python
486 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
TIER-1 SCAM INGESTION - Automated Knowledge Base Builder
|
|
=========================================================
|
|
Pulls scam/exploit data from public sources and indexes them.
|
|
|
|
Sources:
|
|
- REKT News DB (rekt.news) - largest crypto exploit database
|
|
- Chainabuse - community-reported scams
|
|
- GoPlus Security API - real-time token risk assessment
|
|
- QuickIntel API - honeypot/rug detection
|
|
- TokenSniffer - scam detection scores
|
|
- Web3IsGoingGreat - incident timeline
|
|
- SlowMist Hacked - investigation reports
|
|
- CertiK Alerts - audit findings
|
|
|
|
Architecture:
|
|
1. Periodically poll sources (configurable intervals)
|
|
2. Extract structured data (name, description, contract, txns, losses)
|
|
3. Embed using CryptoEmbedder
|
|
4. Store in pgvector + Redis (dual-write)
|
|
5. Trigger re-scan of related tokens/wallets
|
|
"""
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import logging
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
import feedparser
|
|
import httpx
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# ══════════════════════════════════════════════════════════════════════
|
|
# SOURCE CONNECTORS
|
|
# ══════════════════════════════════════════════════════════════════════
|
|
|
|
# RSS feeds from security sources - verified working May 2026
|
|
SECURITY_RSS_FEEDS = [
|
|
("https://www.web3isgoinggreat.com/feed", "W3IGG"),
|
|
("https://slowmist.medium.com/feed", "SlowMist"),
|
|
("https://peckshield.medium.com/feed", "PeckShield"),
|
|
("https://certik.medium.com/feed", "CertiK"),
|
|
("https://immunefi.medium.com/feed", "Immunefi"),
|
|
("https://blog.trailofbits.com/feed/", "TrailOfBits"),
|
|
("https://cryptosecurity.substack.com/feed", "CryptoSecurity"),
|
|
("https://cointelegraph.com/rss/tag/security", "CoinTelegraph-Security"),
|
|
("https://www.chainalysis.com/blog/feed/", "Chainalysis"),
|
|
]
|
|
|
|
# Scam-adjacent URLs for scraping (no API, no RSS - use HTML parsing)
|
|
SCRAPE_SOURCES = [
|
|
{
|
|
"name": "chainabuse",
|
|
"url": "https://chainabuse.com/reports",
|
|
"selector": "a[href*='/report/']",
|
|
},
|
|
{
|
|
"name": "rekt",
|
|
"url": "https://rekt.news/",
|
|
"selector": "article a",
|
|
},
|
|
]
|
|
|
|
|
|
class RSSFeedConnector:
|
|
"""Generic RSS/Atom feed connector for security feeds."""
|
|
|
|
@staticmethod
|
|
async def fetch_feed(url: str, source_name: str, limit: int = 20) -> list[dict]:
|
|
results = []
|
|
try:
|
|
async with httpx.AsyncClient(timeout=20, follow_redirects=True) as client:
|
|
resp = await client.get(url, headers={"User-Agent": "RMI-Scam-Indexer/1.0"})
|
|
if resp.status_code != 200:
|
|
logger.debug(f"RSS {source_name}: HTTP {resp.status_code}")
|
|
return results
|
|
|
|
feed = feedparser.parse(resp.text)
|
|
if not feed.entries:
|
|
logger.debug(f"RSS {source_name}: no entries found")
|
|
return results
|
|
|
|
for entry in feed.entries[:limit]:
|
|
title = entry.get("title", "")
|
|
link = entry.get("link", "")
|
|
desc = entry.get("summary", "") or entry.get("description", "") or title
|
|
|
|
# Parse date
|
|
pub_parsed = entry.get("published_parsed") or entry.get("updated_parsed")
|
|
if pub_parsed:
|
|
published = datetime(*pub_parsed[:6], tzinfo=UTC).isoformat()
|
|
else:
|
|
published = datetime.now(UTC).isoformat()
|
|
|
|
results.append(
|
|
{
|
|
"source": source_name.lower().replace(" ", "_"),
|
|
"type": "security_feed",
|
|
"name": title,
|
|
"description": desc[:500],
|
|
"date": published,
|
|
"url": link,
|
|
"category": "security_intel",
|
|
}
|
|
)
|
|
|
|
logger.info(f"RSS {source_name}: {len(results)} articles")
|
|
except Exception as e:
|
|
logger.warning(f"RSS {source_name} failed: {e}")
|
|
return results
|
|
|
|
|
|
class GoPlusConnector:
|
|
"""GoPlus Security API - real-time token risk detection.
|
|
Free tier: rate limited but no API key needed for basic checks."""
|
|
|
|
BASE = "https://api.gopluslabs.io/api/v1"
|
|
|
|
@staticmethod
|
|
async def check_token(chain_id: str, address: str) -> dict:
|
|
"""Check a single token for security risks."""
|
|
try:
|
|
async with httpx.AsyncClient(timeout=30) as client:
|
|
resp = await client.get(
|
|
f"{GoPlusConnector.BASE}/token_security/{chain_id}",
|
|
params={"contract_addresses": address},
|
|
)
|
|
if resp.status_code == 200:
|
|
data = resp.json()
|
|
result = data.get("result", {})
|
|
token_data = result.get(address.lower(), {})
|
|
return {
|
|
"source": "goplus",
|
|
"type": "token_scan",
|
|
"address": address,
|
|
"chain": chain_id,
|
|
"is_honeypot": token_data.get("is_honeypot") == "1",
|
|
"is_open_source": token_data.get("is_open_source") == "1",
|
|
"buy_tax": token_data.get("buy_tax", "0"),
|
|
"sell_tax": token_data.get("sell_tax", "0"),
|
|
"can_take_back_ownership": token_data.get("can_take_back_ownership") == "1",
|
|
"hidden_owner": token_data.get("hidden_owner") == "1",
|
|
"is_blacklisted": token_data.get("is_blacklisted") == "1",
|
|
"is_whitelisted": token_data.get("is_whitelisted") == "1",
|
|
"is_proxy": token_data.get("is_proxy") == "1",
|
|
"owner_address": token_data.get("owner_address", ""),
|
|
"risk_score": token_data.get("risk_score", 0),
|
|
}
|
|
except Exception as e:
|
|
logger.warning(f"GoPlus check failed for {address}: {e}")
|
|
return {}
|
|
|
|
|
|
# ══════════════════════════════════════════════════════════════════════
|
|
# KNOWN SCAM DATABASE - Manual curated patterns (expanded)
|
|
# ══════════════════════════════════════════════════════════════════════
|
|
|
|
KNOWN_SCAMS_EXPANDED = [
|
|
# From crypto_embeddings.py KNOWN_SCAM_PATTERNS
|
|
{
|
|
"name": "Honeypot - Sell Disabled",
|
|
"description": "Token where only the creator can sell. maxSellAmount=0, tradingEnabled=false, blacklist of non-owner addresses.",
|
|
"indicators": [
|
|
"maxSellAmount=0",
|
|
"tradingEnabled=false",
|
|
"onlyOwner transfer",
|
|
"blacklist all",
|
|
],
|
|
"severity": "critical",
|
|
"source": "rmi-curated",
|
|
"code_snippets": ["maxSellAmount = 0", "tradingEnabled = false"],
|
|
},
|
|
{
|
|
"name": "Unlimited Mint / Rug Pull",
|
|
"description": "Owner can mint unlimited tokens, diluting holders to zero before draining liquidity.",
|
|
"indicators": ["unrestricted mint", "onlyOwner mint", "no supply cap"],
|
|
"severity": "critical",
|
|
"source": "rmi-curated",
|
|
"code_snippets": ["function mint(address to, uint256 amount) external onlyOwner"],
|
|
},
|
|
{
|
|
"name": "Hidden Fee Manipulation",
|
|
"description": "Owner sets fees to 99% post-launch, making sells impossible or stealing all transfers.",
|
|
"indicators": ["setTax(99)", "dynamic fee > 50%", "no max fee cap"],
|
|
"severity": "high",
|
|
"source": "rmi-curated",
|
|
"code_snippets": ["function setFee(uint256 _fee) external onlyOwner", "taxFee = 99"],
|
|
},
|
|
{
|
|
"name": "Liquidity Drain Backdoor",
|
|
"description": "Hidden function allowing owner to remove all liquidity via manualSwap or rescueToken.",
|
|
"indicators": ["manualSwap", "removeLiquidity all", "drain", "rescueToken onlyOwner"],
|
|
"severity": "critical",
|
|
"source": "rmi-curated",
|
|
"code_snippets": [
|
|
"function manualSwap() external onlyOwner",
|
|
"uniswapV2Router.removeLiquidity",
|
|
],
|
|
},
|
|
{
|
|
"name": "Proxy Upgrade Trap",
|
|
"description": "Upgradeable proxy where admin can swap implementation to malicious contract post-launch.",
|
|
"indicators": ["upgradeTo", "upgradeToAndCall", "UUPS", "changeAdmin"],
|
|
"severity": "high",
|
|
"source": "rmi-curated",
|
|
"code_snippets": ["function upgradeTo(address newImplementation) external onlyOwner"],
|
|
},
|
|
{
|
|
"name": "Wallet Drainer / Approval Scam",
|
|
"description": "Contract tricks users into unlimited approval, then drains via transferFrom.",
|
|
"indicators": ["unlimited approval", "transferFrom", "approveAndCall", "permit exploit"],
|
|
"severity": "critical",
|
|
"source": "rmi-curated",
|
|
"code_snippets": ["IERC20(token).transferFrom(victim, attacker, balance)"],
|
|
},
|
|
# Additional patterns from real-world incidents
|
|
{
|
|
"name": "Flash Loan Attack",
|
|
"description": "Exploiter uses flash loan to manipulate price oracle or governance, then extracts value.",
|
|
"indicators": ["flashLoan", "getFlashLoan", "onFlashLoan", "price oracle manipulation"],
|
|
"severity": "critical",
|
|
"source": "rmi-curated",
|
|
"code_snippets": ["function flashLoan(", "function onFlashLoan("],
|
|
},
|
|
{
|
|
"name": "Reentrancy Attack",
|
|
"description": "Contract calls external contract before updating state, allowing recursive withdrawal.",
|
|
"indicators": [
|
|
"reentrancy",
|
|
"call before state update",
|
|
"no reentrancy guard",
|
|
"checks-effects-interactions violated",
|
|
],
|
|
"severity": "critical",
|
|
"source": "rmi-curated",
|
|
"code_snippets": ["msg.sender.call{value:", "nonReentrant", "ReentrancyGuard"],
|
|
},
|
|
{
|
|
"name": "Unchecked Return Value",
|
|
"description": "Contract ignores return value of external call, assuming success. Classic King of the Ether pattern.",
|
|
"indicators": ["unchecked send", "call without require", ".send( without check"],
|
|
"severity": "high",
|
|
"source": "rmi-curated",
|
|
"code_snippets": ["address(msg.sender).send(", "address(this).balance"],
|
|
},
|
|
{
|
|
"name": "Access Control Missing",
|
|
"description": "Critical functions lack access control (no onlyOwner/modifier), allowing anyone to drain/modify.",
|
|
"indicators": [
|
|
"no onlyOwner",
|
|
"public init function",
|
|
"unprotected withdraw",
|
|
"no modifier",
|
|
],
|
|
"severity": "critical",
|
|
"source": "rmi-curated",
|
|
"code_snippets": ["function withdraw() public {", "function init("],
|
|
},
|
|
{
|
|
"name": "MEV Sandwich Vulnerability",
|
|
"description": "High-slippage token that sandwich bots exploit repeatedly. Creator often runs the bots.",
|
|
"indicators": [
|
|
"slippage > 10%",
|
|
"sandwich pattern",
|
|
"same wallet MEV + deployer",
|
|
"high tax token",
|
|
],
|
|
"severity": "medium",
|
|
"source": "rmi-curated",
|
|
"code_snippets": [],
|
|
},
|
|
{
|
|
"name": "Fake Token - Name Squatting",
|
|
"description": "Token with identical name/symbol to legitimate project. Trick users into buying the wrong token.",
|
|
"indicators": [
|
|
"identical name to top-100",
|
|
"different contract from official",
|
|
"recently created",
|
|
"no liquidity",
|
|
],
|
|
"severity": "medium",
|
|
"source": "rmi-curated",
|
|
"code_snippets": [],
|
|
},
|
|
{
|
|
"name": "Anti-Bot Launch Scam",
|
|
"description": "Claims 'anti-bot' but actually blocks ALL buys except creator addresses. Classic honeypot variant.",
|
|
"indicators": ["antiBot", "isBot", "botDetection", "onlyAllowed", "_isExcluded"],
|
|
"severity": "high",
|
|
"source": "rmi-curated",
|
|
"code_snippets": ["modifier antiBot(", "require(!isBot(msg.sender)"],
|
|
},
|
|
{
|
|
"name": "Cloned Contract with Modified Tax",
|
|
"description": "Copy of popular token contract with tax parameters changed to steal from unaware investors.",
|
|
"indicators": [
|
|
"identical bytecode except tax",
|
|
"same name as verified",
|
|
"different deployer",
|
|
"tax > 20%",
|
|
],
|
|
"severity": "high",
|
|
"source": "rmi-curated",
|
|
"code_snippets": [],
|
|
},
|
|
{
|
|
"name": "Fake Liquidity Lock",
|
|
"description": "Claims liquidity locked but LP tokens held by creator wallet with no time-lock or fake lock contract.",
|
|
"indicators": [
|
|
"fake lock",
|
|
"LP in EOA",
|
|
"no unicrypt/teamfinance",
|
|
"liquidity lock claim unverified",
|
|
],
|
|
"severity": "high",
|
|
"source": "rmi-curated",
|
|
"code_snippets": [],
|
|
},
|
|
]
|
|
|
|
|
|
# ══════════════════════════════════════════════════════════════════════
|
|
# INGESTION ORCHESTRATOR
|
|
# ══════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
class ScamIngestionPipeline:
|
|
"""
|
|
Automated pipeline for building the scam knowledge base.
|
|
|
|
Usage:
|
|
pipeline = ScamIngestionPipeline()
|
|
stats = await pipeline.run_full_ingestion()
|
|
"""
|
|
|
|
def __init__(self):
|
|
self._last_run: datetime | None = None
|
|
self._total_ingested = 0
|
|
# Primary: RSS feeds (always work)
|
|
self.connectors = [
|
|
("curated", self._ingest_curated),
|
|
("rss_feeds", self._ingest_all_rss),
|
|
]
|
|
|
|
async def run_full_ingestion(self, force: bool = False) -> dict[str, Any]:
|
|
"""Run all ingestion sources and return stats."""
|
|
if not force and self._last_run and (datetime.now(UTC) - self._last_run).seconds < 300:
|
|
return {
|
|
"status": "skipped",
|
|
"reason": "rate limited",
|
|
"last_run": self._last_run.isoformat(),
|
|
}
|
|
|
|
from app.crypto_embeddings import get_embedder
|
|
|
|
embedder = await get_embedder()
|
|
|
|
stats = {"sources": {}, "total": 0, "errors": 0}
|
|
|
|
for source_name, ingest_fn in self.connectors:
|
|
try:
|
|
docs = await ingest_fn(embedder)
|
|
if docs:
|
|
# Ingest via Redis + FAISS (pgvector removed)
|
|
from app.rag_service import ingest_document
|
|
|
|
count = 0
|
|
for doc in docs:
|
|
try:
|
|
meta = doc.get("metadata", {})
|
|
meta["source"] = doc.get("source", source_name)
|
|
await ingest_document(
|
|
collection=doc.get("collection", "known_scams"),
|
|
content=doc.get("content", ""),
|
|
metadata=meta,
|
|
)
|
|
count += 1
|
|
except Exception:
|
|
pass
|
|
stats["sources"][source_name] = count
|
|
stats["total"] += count
|
|
logger.info(f"Ingested {count} docs from {source_name}")
|
|
except Exception as e:
|
|
logger.error(f"Ingestion failed for {source_name}: {e}")
|
|
stats["errors"] += 1
|
|
stats["sources"][source_name] = 0
|
|
|
|
self._last_run = datetime.now(UTC)
|
|
self._total_ingested += stats["total"]
|
|
|
|
# FAISS indexes are built from Redis automatically
|
|
# when first search hits each collection - no manual build needed.
|
|
|
|
return {
|
|
"status": "completed",
|
|
"total_ingested": stats["total"],
|
|
"cumulative_total": self._total_ingested,
|
|
"sources": stats["sources"],
|
|
"errors": stats["errors"],
|
|
"timestamp": self._last_run.isoformat(),
|
|
}
|
|
|
|
async def _ingest_all_rss(self, embedder) -> list[dict]:
|
|
"""Ingest from all security RSS feeds in parallel."""
|
|
all_docs = []
|
|
tasks = [RSSFeedConnector.fetch_feed(url, name, limit=15) for url, name in SECURITY_RSS_FEEDS]
|
|
results = await asyncio.gather(*tasks, return_exceptions=True)
|
|
|
|
for i, articles in enumerate(results):
|
|
if isinstance(articles, Exception):
|
|
logger.warning(f"RSS feed {SECURITY_RSS_FEEDS[i][1]} error: {articles}")
|
|
continue
|
|
if not isinstance(articles, list):
|
|
continue
|
|
|
|
source_name = SECURITY_RSS_FEEDS[i][1]
|
|
for article in articles:
|
|
try:
|
|
content = f"SECURITY INTEL [{source_name}]: {article['name']}. {article.get('description', '')}"
|
|
result = await embedder.embed_scam_pattern(
|
|
pattern_name=article["name"],
|
|
description=content,
|
|
severity="medium",
|
|
)
|
|
doc_id = hashlib.sha256(f"rss:{source_name}:{article['name']}".encode()).hexdigest()[:16]
|
|
all_docs.append(
|
|
{
|
|
"id": doc_id,
|
|
"collection": "market_intel",
|
|
"embedding": result.vector,
|
|
"content": content,
|
|
"metadata": article,
|
|
"source": f"rss-{source_name.lower()}",
|
|
"severity": "medium",
|
|
}
|
|
)
|
|
except Exception as e:
|
|
logger.warning(f"Failed to embed RSS article {article.get('name')}: {e}")
|
|
|
|
logger.info(f"RSS ingestion: {len(all_docs)} docs from {len(SECURITY_RSS_FEEDS)} feeds")
|
|
return all_docs
|
|
|
|
async def _ingest_curated(self, embedder) -> list[dict]:
|
|
"""Ingest manually curated scam patterns (always available)."""
|
|
docs = []
|
|
for pattern in KNOWN_SCAMS_EXPANDED:
|
|
try:
|
|
result = await embedder.embed_scam_pattern(
|
|
pattern_name=pattern["name"],
|
|
description=pattern["description"],
|
|
code_snippets=pattern.get("code_snippets", []),
|
|
indicators=pattern.get("indicators", []),
|
|
severity=pattern["severity"],
|
|
)
|
|
doc_id = hashlib.sha256(f"curated:{pattern['name']}".encode()).hexdigest()[:16]
|
|
docs.append(
|
|
{
|
|
"id": doc_id,
|
|
"collection": "known_scams",
|
|
"embedding": result.vector,
|
|
"content": f"{pattern['name']}: {pattern['description']}",
|
|
"metadata": pattern,
|
|
"source": pattern["source"],
|
|
"severity": pattern["severity"],
|
|
}
|
|
)
|
|
except Exception as e:
|
|
logger.warning(f"Failed to embed curated pattern {pattern['name']}: {e}")
|
|
return docs
|
|
|
|
|
|
# ══════════════════════════════════════════════════════════════════════
|
|
# SINGLETON
|
|
# ══════════════════════════════════════════════════════════════════════
|
|
|
|
_pipeline: ScamIngestionPipeline | None = None
|
|
|
|
|
|
async def get_ingestion_pipeline() -> ScamIngestionPipeline:
|
|
global _pipeline
|
|
if _pipeline is None:
|
|
_pipeline = ScamIngestionPipeline()
|
|
return _pipeline
|