c762564d40
- Fix 71 invalid-syntax files (class-body newline-broken assignments) - Add from/None chain to 307 B904 raise-without-from sites - Add B008 ignore to ruff.toml (already in pyproject.toml) - Noqa F401 on __init__.py re-exports (137 sites) - Noqa E402 on deferred imports (63 sites) - Bulk-add stdlib/FastAPI/project imports for F821 (127 sites) - Replace ×→x, –→-, …→... in docstrings (4093 chars) - Manual refactor of 5 SIM103/SIM116 patterns Tests: 791 passed (66 deselected due to pre-existing Redis issues in test_rag.py) Co-authored-by: opencode <opencode@rugmunch.io>
746 lines
28 KiB
Python
746 lines
28 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Multi-Chain Scam Address Ingestion
|
|
===================================
|
|
Aggregate blockchain scam addresses from multiple sources into RAG known_scams.
|
|
|
|
Sources (address-level, chain-specific):
|
|
1. Etherscan phish-hack.csv - 110 phishing/hack addresses (ETH/BSC/Polygon/etc)
|
|
2. Etherscan combined_labels.csv - 3,472 scam-labeled addresses across 7 chains
|
|
3. Etherscan combined_labels.csv - 46K+ labeled wallets → wallet_profiles collection
|
|
4. Solana token-registry JSON - 55 explicitly flagged scam tokens (Solana)
|
|
5. Rekt hack database (JSON) - DeFi exploit addresses (ETH/BSC/etc)
|
|
6. MetaMask phishing domains - 198K phishing domains (domain-level, batched)
|
|
|
|
Strategy:
|
|
- ADDRESS-LEVEL data goes 1:1 into known_scams (high value for our scanners)
|
|
- DOMAIN-LEVEL data gets batched 50/domain per doc (supplementary, lower value)
|
|
- Wallet profile data goes to wallet_profiles collection (non-scam labels)
|
|
|
|
This replaces the old single-source scripts with a unified approach.
|
|
"""
|
|
|
|
import asyncio
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
|
|
import httpx
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s")
|
|
logger = logging.getLogger("ingest_multi_chain_scams")
|
|
|
|
DATA_DIR = os.path.join(os.path.dirname(__file__), "..", "data")
|
|
RAG_API = "http://localhost:8000/api/v1/rag/ingest"
|
|
|
|
SCAM_LABEL_KEYWORDS = [
|
|
"phish",
|
|
"hack",
|
|
"exploit",
|
|
"scam",
|
|
"drain",
|
|
"attack",
|
|
"heist",
|
|
"malicious",
|
|
"fraud",
|
|
"rug",
|
|
"compromis",
|
|
"whale-alert",
|
|
]
|
|
|
|
|
|
def make_doc_id(collection: str, key: str) -> str:
|
|
return hashlib.sha256(f"{collection}:{key.lower()}".encode()).hexdigest()[:16]
|
|
|
|
|
|
async def ingest_doc(collection: str, content: str, metadata: dict) -> bool:
|
|
try:
|
|
async with httpx.AsyncClient(timeout=30) as client:
|
|
resp = await client.post(
|
|
RAG_API,
|
|
json={
|
|
"collection": collection,
|
|
"content": content,
|
|
"metadata": metadata,
|
|
},
|
|
)
|
|
return resp.status_code == 200
|
|
except Exception as e:
|
|
logger.debug(f"Ingest error: {e}")
|
|
return False
|
|
|
|
|
|
# ─── Source 1: Etherscan Phish/Hack CSV ─────────────────────────────
|
|
|
|
|
|
async def ingest_etherscan_phish_hack() -> dict[str, int]:
|
|
csv_path = os.path.join(DATA_DIR, "etherscan_phish_hack.csv")
|
|
if not os.path.exists(csv_path):
|
|
logger.warning(f"Missing: {csv_path}")
|
|
return {"ingested": 0, "errors": 0}
|
|
|
|
ingested = 0
|
|
errors = 0
|
|
with open(csv_path, newline="", encoding="utf-8") as f:
|
|
rows = list(csv.DictReader(f))
|
|
|
|
logger.info(f"Etherscan phish/hack: {len(rows)} addresses")
|
|
for i, row in enumerate(rows):
|
|
address = row["address"].strip()
|
|
if not address:
|
|
continue
|
|
name_tag = row.get("name_tag", "").strip()
|
|
label_type = row.get("label_type", "phish-hack").strip()
|
|
chain = row.get("chain", "Ethereum").strip()
|
|
balance = row.get("balance", "").strip()
|
|
txn_count = row.get("txn_count", "").strip()
|
|
|
|
doc_id = make_doc_id("known_scams", f"eth_phish:{address}:{chain}")
|
|
content = (
|
|
f"Etherscan {label_type} label: {address} on {chain}. "
|
|
f"Tag: {name_tag}. Balance: {balance}. Txns: {txn_count}."
|
|
)
|
|
metadata = {
|
|
"address": address.lower(),
|
|
"label_type": label_type,
|
|
"name_tag": name_tag,
|
|
"chain": chain.lower(),
|
|
"balance": balance,
|
|
"txn_count": txn_count,
|
|
"source": "etherscan_phish_hack",
|
|
"severity": "critical" if "phish" in label_type.lower() else "high",
|
|
"doc_id": doc_id,
|
|
}
|
|
|
|
if await ingest_doc("known_scams", content, metadata):
|
|
ingested += 1
|
|
else:
|
|
errors += 1
|
|
|
|
if (i + 1) % 20 == 0:
|
|
logger.info(f" Phish/hack: {i + 1}/{len(rows)} | ingested: {ingested}")
|
|
|
|
return {"ingested": ingested, "errors": errors}
|
|
|
|
|
|
# ─── Source 2: Etherscan Combined Labels CSV ─────────────────────────
|
|
|
|
|
|
async def ingest_etherscan_combined() -> dict[str, int]:
|
|
csv_path = os.path.join(DATA_DIR, "etherscan_combined_labels.csv")
|
|
if not os.path.exists(csv_path):
|
|
logger.warning(f"Missing: {csv_path}")
|
|
return {"known_scams": 0, "wallet_profiles": 0, "errors": 0}
|
|
|
|
stats = {"known_scams": 0, "wallet_profiles": 0, "errors": 0}
|
|
with open(csv_path, newline="", encoding="utf-8") as f:
|
|
rows = list(csv.DictReader(f))
|
|
|
|
scam_rows = [r for r in rows if any(kw in r.get("label", "").lower() for kw in SCAM_LABEL_KEYWORDS)]
|
|
wallet_rows = [r for r in rows if not any(kw in r.get("label", "").lower() for kw in SCAM_LABEL_KEYWORDS)]
|
|
|
|
logger.info(
|
|
f"Etherscan combined: {len(rows)} total → {len(scam_rows)} scam → known_scams, {len(wallet_rows)} → wallet_profiles"
|
|
)
|
|
|
|
for i, row in enumerate(rows):
|
|
address = row["address"].strip()
|
|
name = row.get("name", "").strip()
|
|
label = row.get("label", "").strip()
|
|
chain = row.get("chain", "Ethereum").strip()
|
|
if not address or not label:
|
|
continue
|
|
|
|
is_scam = any(kw in label.lower() for kw in SCAM_LABEL_KEYWORDS)
|
|
|
|
if is_scam:
|
|
collection = "known_scams"
|
|
doc_id = make_doc_id("known_scams", f"eth_combined:{address}:{chain}")
|
|
content = (
|
|
f"Etherscan scam label: {address} ({name}) on {chain}. "
|
|
f"Category: {label}. Flagged as scam/phishing/exploit address."
|
|
)
|
|
metadata = {
|
|
"address": address.lower(),
|
|
"label": label,
|
|
"name": name,
|
|
"chain": chain.lower(),
|
|
"source": "etherscan_combined",
|
|
"severity": "high",
|
|
"label_type": "scam",
|
|
"doc_id": doc_id,
|
|
}
|
|
else:
|
|
collection = "wallet_profiles"
|
|
doc_id = make_doc_id("wallet_profiles", f"eth_combined:{address}:{chain}")
|
|
content = f"Etherscan labeled address: {address} ({name}) on {chain}. Category: {label}."
|
|
metadata = {
|
|
"address": address.lower(),
|
|
"label": label,
|
|
"name": name,
|
|
"chain": chain.lower(),
|
|
"source": "etherscan_combined",
|
|
"label_type": "profile",
|
|
"doc_id": doc_id,
|
|
}
|
|
|
|
if await ingest_doc(collection, content, metadata):
|
|
stats[collection] += 1
|
|
else:
|
|
stats["errors"] += 1
|
|
|
|
if (i + 1) % 1000 == 0:
|
|
logger.info(
|
|
f" Combined: {i + 1}/{len(rows)} | scams: {stats['known_scams']}, profiles: {stats['wallet_profiles']}, errors: {stats['errors']}"
|
|
)
|
|
if (i + 1) % 500 == 0:
|
|
await asyncio.sleep(0)
|
|
|
|
return stats
|
|
|
|
|
|
# ─── Source 3: Rekt Hack Database ────────────────────────────────────
|
|
|
|
|
|
async def ingest_rekt_hacks() -> dict[str, int]:
|
|
json_path = os.path.join(DATA_DIR, "rekt_hacks.json")
|
|
if not os.path.exists(json_path):
|
|
# Try raw
|
|
json_path = os.path.join(DATA_DIR, "rekt_hacks_raw.json")
|
|
if not os.path.exists(json_path):
|
|
logger.warning("Missing rekt_hacks JSON")
|
|
return {"ingested": 0, "errors": 0}
|
|
|
|
with open(json_path) as f:
|
|
hacks = json.load(f)
|
|
|
|
if isinstance(hacks, dict):
|
|
hacks = hacks.get("hacks", hacks.get("data", []))
|
|
if not isinstance(hacks, list):
|
|
logger.warning(f"Unexpected rekt format: {type(hacks)}")
|
|
return {"ingested": 0, "errors": 0}
|
|
|
|
logger.info(f"Rekt hacks: {len(hacks)} entries")
|
|
ingested = 0
|
|
errors = 0
|
|
|
|
for i, hack in enumerate(hacks):
|
|
if not isinstance(hack, dict):
|
|
continue
|
|
name = hack.get("project", hack.get("name", ""))
|
|
chain = hack.get("chain", "ethereum")
|
|
attacker = hack.get("attacker_address", hack.get("attacker", ""))
|
|
contract = hack.get("contract_address", hack.get("contract", ""))
|
|
amount = hack.get("amount", hack.get("funds_lost", ""))
|
|
attack_type = hack.get("attack_type", hack.get("type", ""))
|
|
description = hack.get("description", "")
|
|
date = hack.get("date", "")
|
|
|
|
# Ingest attacker address
|
|
if attacker and len(str(attacker)) > 10:
|
|
doc_id = make_doc_id("known_scams", f"rekt_attacker:{attacker}")
|
|
content = (
|
|
f"DeFi hack attacker address: {attacker} on {chain}. "
|
|
f"Project: {name}. Loss: ${amount}. Type: {attack_type}. "
|
|
f"Date: {date}. {description[:200]}"
|
|
)
|
|
metadata = {
|
|
"address": str(attacker).lower(),
|
|
"chain": chain.lower(),
|
|
"source": "rekt_hacks",
|
|
"name": name,
|
|
"attack_type": attack_type,
|
|
"amount": str(amount),
|
|
"severity": "critical",
|
|
"doc_id": doc_id,
|
|
}
|
|
if await ingest_doc("known_scams", content, metadata):
|
|
ingested += 1
|
|
else:
|
|
errors += 1
|
|
|
|
# Ingest exploited contract
|
|
if contract and len(str(contract)) > 10 and str(contract) != str(attacker):
|
|
doc_id = make_doc_id("known_scams", f"rekt_contract:{contract}")
|
|
content = (
|
|
f"DeFi hack exploited contract: {contract} on {chain}. "
|
|
f"Project: {name}. Loss: ${amount}. Type: {attack_type}. "
|
|
f"Date: {date}. {description[:200]}"
|
|
)
|
|
metadata = {
|
|
"address": str(contract).lower(),
|
|
"chain": chain.lower(),
|
|
"source": "rekt_hacks",
|
|
"name": name,
|
|
"attack_type": attack_type,
|
|
"amount": str(amount),
|
|
"severity": "high",
|
|
"doc_id": doc_id,
|
|
}
|
|
if await ingest_doc("known_scams", content, metadata):
|
|
ingested += 1
|
|
else:
|
|
errors += 1
|
|
|
|
if (i + 1) % 50 == 0:
|
|
logger.info(f" Rekt: {i + 1}/{len(hacks)} | ingested: {ingested}")
|
|
|
|
return {"ingested": ingested, "errors": errors}
|
|
|
|
|
|
# ─── Source 4: Solana Token Registry (flagged scams) ──────────────────
|
|
|
|
|
|
async def ingest_solana_flagged_tokens() -> dict[str, int]:
|
|
"""Download Solana token list and ingest explicitly flagged scam tokens."""
|
|
token_list_url = "https://raw.githubusercontent.com/solana-labs/token-list/main/src/tokens/solana.tokenlist.json"
|
|
|
|
try:
|
|
async with httpx.AsyncClient(timeout=60) as client:
|
|
resp = await client.get(token_list_url)
|
|
if resp.status_code != 200:
|
|
logger.error(f"Solana token list fetch failed: {resp.status_code}")
|
|
return {"ingested": 0, "errors": 0}
|
|
data = resp.json()
|
|
except Exception as e:
|
|
logger.error(f"Solana token list error: {e}")
|
|
return {"ingested": 0, "errors": 0}
|
|
|
|
tokens = data.get("tokens", [])
|
|
# Filter for scam/spam/suspicious flags in name, symbol, or tags
|
|
scam_keywords = ["scam", "spam", "phishing", "fake", "fraud", "rug", "exploit"]
|
|
flagged = []
|
|
for t in tokens:
|
|
name_lower = (t.get("name", "") + " " + t.get("symbol", "")).lower()
|
|
tags_lower = [str(tag).lower() for tag in t.get("tags", [])]
|
|
if any(kw in name_lower for kw in scam_keywords) or any(kw in " ".join(tags_lower) for kw in scam_keywords):
|
|
flagged.append(t)
|
|
|
|
logger.info(f"Solana token list: {len(tokens)} total, {len(flagged)} flagged as scam/spam")
|
|
ingested = 0
|
|
errors = 0
|
|
|
|
for t in flagged:
|
|
address = t.get("address", "")
|
|
name = t.get("name", "")
|
|
symbol = t.get("symbol", "")
|
|
if not address or len(address) < 20:
|
|
continue
|
|
|
|
doc_id = make_doc_id("known_scams", f"sol_flagged:{address}")
|
|
content = (
|
|
f"Solana flagged scam token: {name} ({symbol}). "
|
|
f"Mint address: {address}. Flagged in Solana token registry as scam/spam/phishing."
|
|
)
|
|
metadata = {
|
|
"address": address,
|
|
"chain": "solana",
|
|
"source": "solana_token_registry",
|
|
"name": name,
|
|
"symbol": symbol,
|
|
"severity": "high",
|
|
"category": "scam_token",
|
|
"doc_id": doc_id,
|
|
}
|
|
if await ingest_doc("known_scams", content, metadata):
|
|
ingested += 1
|
|
else:
|
|
errors += 1
|
|
|
|
return {"ingested": ingested, "errors": errors}
|
|
|
|
|
|
# ─── Source 5: MetaMask Phishing Domains (batched, supplementary) ────
|
|
|
|
|
|
async def ingest_metamask_domains() -> dict[str, int]:
|
|
"""Ingest MetaMask phishing domain blacklist in batches of 50."""
|
|
config_url = "https://raw.githubusercontent.com/MetaMask/eth-phishing-detect/master/src/config.json"
|
|
|
|
try:
|
|
async with httpx.AsyncClient(timeout=60) as client:
|
|
resp = await client.get(config_url)
|
|
if resp.status_code != 200:
|
|
logger.error(f"MetaMask list fetch failed: {resp.status_code}")
|
|
return {"ingested": 0, "errors": 0}
|
|
data = resp.json()
|
|
except Exception as e:
|
|
logger.error(f"MetaMask list error: {e}")
|
|
return {"ingested": 0, "errors": 0}
|
|
|
|
blacklist = data.get("blacklist", [])
|
|
logger.info(f"MetaMask blacklist: {len(blacklist)} domains (batching 50/doc)")
|
|
|
|
batch_size = 50
|
|
total_batches = (len(blacklist) + batch_size - 1) // batch_size
|
|
ingested = 0
|
|
errors = 0
|
|
|
|
for batch_idx in range(total_batches):
|
|
batch = blacklist[batch_idx * batch_size : (batch_idx + 1) * batch_size]
|
|
domain_list = "\n".join(batch)
|
|
doc_id = make_doc_id("known_scams", f"mm_batch:{batch_idx}")
|
|
content = (
|
|
f"MetaMask Confirmed Phishing Domains (Batch {batch_idx + 1}/{total_batches}):\n"
|
|
f"{domain_list}\n"
|
|
f"Source: MetaMask eth-phishing-detect. Confirmed phishing/scam websites targeting crypto users."
|
|
)
|
|
metadata = {
|
|
"source": "metamask_phishing_detect",
|
|
"batch_index": batch_idx,
|
|
"total_batches": total_batches,
|
|
"domain_count": len(batch),
|
|
"category": "phishing_domain",
|
|
"severity": "medium",
|
|
"doc_id": doc_id,
|
|
}
|
|
|
|
if await ingest_doc("known_scams", content, metadata):
|
|
ingested += 1
|
|
else:
|
|
errors += 1
|
|
|
|
if (batch_idx + 1) % 100 == 0:
|
|
logger.info(f" MetaMask: {batch_idx + 1}/{total_batches} batches | ingested: {ingested}")
|
|
await asyncio.sleep(0.05)
|
|
|
|
return {"ingested": ingested, "errors": errors}
|
|
|
|
|
|
# ─── Source 6: ScamSniffer GitHub Repo (phishing domains + addresses) ────
|
|
|
|
|
|
async def ingest_scamsniffer_github() -> dict[str, int]:
|
|
"""Ingest ScamSniffer GitHub scam-database repo.
|
|
|
|
Contains blacklists of phishing domains and malicious wallet addresses.
|
|
7-day delay vs real-time but excellent for historical pattern analysis.
|
|
"""
|
|
base_url = "https://raw.githubusercontent.com/scamsniffer/scam-database/main"
|
|
files_to_fetch = [
|
|
"blacklist/domains.json",
|
|
"blacklist/addresses.json",
|
|
"blacklist/urls.json",
|
|
]
|
|
|
|
ingested = 0
|
|
errors = 0
|
|
total_domains = 0
|
|
total_addresses = 0
|
|
|
|
for file_path in files_to_fetch:
|
|
url = f"{base_url}/{file_path}"
|
|
try:
|
|
async with httpx.AsyncClient(timeout=60) as client:
|
|
resp = await client.get(url)
|
|
if resp.status_code != 200:
|
|
logger.warning(f"ScamSniffer {file_path}: HTTP {resp.status_code}")
|
|
continue
|
|
data = resp.json()
|
|
except Exception as e:
|
|
logger.warning(f"ScamSniffer {file_path} error: {e}")
|
|
continue
|
|
|
|
if not isinstance(data, list):
|
|
logger.warning(f"ScamSniffer {file_path}: unexpected format {type(data)}")
|
|
continue
|
|
|
|
item_type = file_path.split("/")[-1].replace(".json", "")
|
|
logger.info(f"ScamSniffer {item_type}: {len(data)} entries")
|
|
|
|
# Batch ingest
|
|
batch_size = 50
|
|
total_batches = (len(data) + batch_size - 1) // batch_size
|
|
|
|
for batch_idx in range(total_batches):
|
|
batch = data[batch_idx * batch_size : (batch_idx + 1) * batch_size]
|
|
|
|
if item_type == "domains":
|
|
content = (
|
|
f"ScamSniffer Confirmed Phishing Domains (Batch {batch_idx + 1}/{total_batches}):\n"
|
|
f"{chr(10).join(batch)}\n"
|
|
f"Source: ScamSniffer scam-database GitHub. 7-day delayed but verified."
|
|
)
|
|
metadata = {
|
|
"source": "scamsniffer_github",
|
|
"item_type": "domain",
|
|
"batch_index": batch_idx,
|
|
"total_batches": total_batches,
|
|
"item_count": len(batch),
|
|
"category": "phishing_domain",
|
|
"severity": "high",
|
|
}
|
|
total_domains += len(batch)
|
|
elif item_type == "addresses":
|
|
# Addresses may be dicts or strings
|
|
addr_list = []
|
|
for item in batch:
|
|
if isinstance(item, dict):
|
|
addr = item.get("address", item.get("addr", ""))
|
|
chain = item.get("chain", "unknown")
|
|
label = item.get("label", "")
|
|
if addr:
|
|
addr_list.append(f"{addr} ({chain}) {label}")
|
|
elif isinstance(item, str):
|
|
addr_list.append(item)
|
|
|
|
content = (
|
|
f"ScamSniffer Malicious Addresses (Batch {batch_idx + 1}/{total_batches}):\n"
|
|
f"{chr(10).join(addr_list)}\n"
|
|
f"Source: ScamSniffer scam-database GitHub. Wallet drainer and phishing addresses."
|
|
)
|
|
metadata = {
|
|
"source": "scamsniffer_github",
|
|
"item_type": "address",
|
|
"batch_index": batch_idx,
|
|
"total_batches": total_batches,
|
|
"item_count": len(batch),
|
|
"category": "malicious_address",
|
|
"severity": "critical",
|
|
}
|
|
total_addresses += len(batch)
|
|
else: # urls
|
|
content = (
|
|
f"ScamSniffer Malicious URLs (Batch {batch_idx + 1}/{total_batches}):\n"
|
|
f"{chr(10).join(batch)}\n"
|
|
f"Source: ScamSniffer scam-database GitHub."
|
|
)
|
|
metadata = {
|
|
"source": "scamsniffer_github",
|
|
"item_type": "url",
|
|
"batch_index": batch_idx,
|
|
"total_batches": total_batches,
|
|
"item_count": len(batch),
|
|
"category": "malicious_url",
|
|
"severity": "high",
|
|
}
|
|
|
|
if await ingest_doc("known_scams", content, metadata):
|
|
ingested += 1
|
|
else:
|
|
errors += 1
|
|
|
|
if (batch_idx + 1) % 50 == 0:
|
|
logger.info(
|
|
f" ScamSniffer {item_type}: {batch_idx + 1}/{total_batches} batches | ingested: {ingested}"
|
|
)
|
|
await asyncio.sleep(0.05)
|
|
|
|
logger.info(f"ScamSniffer total: {total_domains} domains, {total_addresses} addresses")
|
|
return {
|
|
"ingested": ingested,
|
|
"errors": errors,
|
|
"domains": total_domains,
|
|
"addresses": total_addresses,
|
|
}
|
|
|
|
|
|
# ─── Source 7: PhishDestroy Destroylist (140K+ threats) ────
|
|
|
|
|
|
async def ingest_phishdestroy() -> dict[str, int]:
|
|
"""Ingest PhishDestroy destroylist - 140K curated crypto/web3 scam domains.
|
|
|
|
MIT licensed. Available as domains.txt, domains.csv, active_domains.json.
|
|
We fetch the plain text list and batch it.
|
|
"""
|
|
urls = {
|
|
"domains_txt": "https://raw.githubusercontent.com/PhishDestroy/destroylist/main/domains.txt",
|
|
"active_json": "https://raw.githubusercontent.com/PhishDestroy/destroylist/main/active_domains.json",
|
|
}
|
|
|
|
ingested = 0
|
|
errors = 0
|
|
total_domains = 0
|
|
|
|
# Try active_domains.json first (structured)
|
|
try:
|
|
async with httpx.AsyncClient(timeout=60) as client:
|
|
resp = await client.get(urls["active_json"])
|
|
if resp.status_code == 200:
|
|
data = resp.json()
|
|
if isinstance(data, list):
|
|
domains = data
|
|
elif isinstance(data, dict):
|
|
domains = list(data.keys())
|
|
else:
|
|
domains = []
|
|
|
|
logger.info(f"PhishDestroy active_domains.json: {len(domains)} domains")
|
|
|
|
batch_size = 50
|
|
total_batches = (len(domains) + batch_size - 1) // batch_size
|
|
|
|
for batch_idx in range(total_batches):
|
|
batch = domains[batch_idx * batch_size : (batch_idx + 1) * batch_size]
|
|
domain_list = "\n".join(batch)
|
|
doc_id = make_doc_id("known_scams", f"pd_active:{batch_idx}")
|
|
content = (
|
|
f"PhishDestroy Active Malicious Domains (Batch {batch_idx + 1}/{total_batches}):\n"
|
|
f"{domain_list}\n"
|
|
f"Source: PhishDestroy destroylist (MIT License). DNS-validated active crypto/web3 scam domains."
|
|
)
|
|
metadata = {
|
|
"source": "phishdestroy_destroylist",
|
|
"batch_index": batch_idx,
|
|
"total_batches": total_batches,
|
|
"domain_count": len(batch),
|
|
"category": "phishing_domain",
|
|
"severity": "high",
|
|
"license": "MIT",
|
|
"doc_id": doc_id,
|
|
}
|
|
|
|
if await ingest_doc("known_scams", content, metadata):
|
|
ingested += 1
|
|
total_domains += len(batch)
|
|
else:
|
|
errors += 1
|
|
|
|
if (batch_idx + 1) % 100 == 0:
|
|
logger.info(f" PhishDestroy: {batch_idx + 1}/{total_batches} | ingested: {ingested}")
|
|
await asyncio.sleep(0.05)
|
|
except Exception as e:
|
|
logger.warning(f"PhishDestroy active_domains.json failed: {e}")
|
|
|
|
# Fallback to domains.txt
|
|
if total_domains == 0:
|
|
try:
|
|
async with httpx.AsyncClient(timeout=60) as client:
|
|
resp = await client.get(urls["domains_txt"])
|
|
if resp.status_code == 200:
|
|
domains = [d.strip() for d in resp.text.splitlines() if d.strip()]
|
|
logger.info(f"PhishDestroy domains.txt: {len(domains)} domains")
|
|
|
|
batch_size = 50
|
|
total_batches = (len(domains) + batch_size - 1) // batch_size
|
|
|
|
for batch_idx in range(total_batches):
|
|
batch = domains[batch_idx * batch_size : (batch_idx + 1) * batch_size]
|
|
domain_list = "\n".join(batch)
|
|
doc_id = make_doc_id("known_scams", f"pd_txt:{batch_idx}")
|
|
content = (
|
|
f"PhishDestroy Malicious Domains (Batch {batch_idx + 1}/{total_batches}):\n"
|
|
f"{domain_list}\n"
|
|
f"Source: PhishDestroy destroylist (MIT License)."
|
|
)
|
|
metadata = {
|
|
"source": "phishdestroy_destroylist",
|
|
"batch_index": batch_idx,
|
|
"total_batches": total_batches,
|
|
"domain_count": len(batch),
|
|
"category": "phishing_domain",
|
|
"severity": "high",
|
|
"license": "MIT",
|
|
"doc_id": doc_id,
|
|
}
|
|
|
|
if await ingest_doc("known_scams", content, metadata):
|
|
ingested += 1
|
|
total_domains += len(batch)
|
|
else:
|
|
errors += 1
|
|
|
|
if (batch_idx + 1) % 100 == 0:
|
|
logger.info(f" PhishDestroy: {batch_idx + 1}/{total_batches} | ingested: {ingested}")
|
|
await asyncio.sleep(0.05)
|
|
except Exception as e:
|
|
logger.warning(f"PhishDestroy domains.txt failed: {e}")
|
|
|
|
logger.info(f"PhishDestroy total: {total_domains} domains ingested")
|
|
return {"ingested": ingested, "errors": errors, "domains": total_domains}
|
|
|
|
|
|
# ─── Main ─────────────────────────────────────────────────────────────
|
|
|
|
|
|
async def main():
|
|
total_stats = {
|
|
"known_scams_added": 0,
|
|
"wallet_profiles_added": 0,
|
|
"errors": 0,
|
|
"sources": {},
|
|
}
|
|
|
|
# Source 1: Etherscan phish/hack (110 addresses, HIGH VALUE)
|
|
logger.info("=" * 60)
|
|
logger.info("SOURCE 1: Etherscan Phish/Hack Addresses")
|
|
logger.info("=" * 60)
|
|
r = await ingest_etherscan_phish_hack()
|
|
total_stats["known_scams_added"] += r["ingested"]
|
|
total_stats["errors"] += r["errors"]
|
|
total_stats["sources"]["etherscan_phish_hack"] = r
|
|
logger.info(f" Result: {r}")
|
|
|
|
# Source 2: Etherscan combined labels (3,472 scam + 46K profiles)
|
|
logger.info("=" * 60)
|
|
logger.info("SOURCE 2: Etherscan Combined Labels")
|
|
logger.info("=" * 60)
|
|
r = await ingest_etherscan_combined()
|
|
total_stats["known_scams_added"] += r["known_scams"]
|
|
total_stats["wallet_profiles_added"] += r["wallet_profiles"]
|
|
total_stats["errors"] += r["errors"]
|
|
total_stats["sources"]["etherscan_combined"] = r
|
|
logger.info(f" Result: {r}")
|
|
|
|
# Source 3: Rekt hack database (attacker + exploited contract addresses)
|
|
logger.info("=" * 60)
|
|
logger.info("SOURCE 3: Rekt DeFi Hack Database")
|
|
logger.info("=" * 60)
|
|
r = await ingest_rekt_hacks()
|
|
total_stats["known_scams_added"] += r["ingested"]
|
|
total_stats["errors"] += r["errors"]
|
|
total_stats["sources"]["rekt_hacks"] = r
|
|
logger.info(f" Result: {r}")
|
|
|
|
# Source 4: Solana flagged scam tokens
|
|
logger.info("=" * 60)
|
|
logger.info("SOURCE 4: Solana Token Registry Flagged Scams")
|
|
logger.info("=" * 60)
|
|
r = await ingest_solana_flagged_tokens()
|
|
total_stats["known_scams_added"] += r["ingested"]
|
|
total_stats["errors"] += r["errors"]
|
|
total_stats["sources"]["solana_flagged"] = r
|
|
logger.info(f" Result: {r}")
|
|
|
|
# Source 5: MetaMask phishing domains (supplementary, low priority)
|
|
logger.info("=" * 60)
|
|
logger.info("SOURCE 5: MetaMask Phishing Domain Blacklist")
|
|
logger.info("=" * 60)
|
|
r = await ingest_metamask_domains()
|
|
total_stats["known_scams_added"] += r["ingested"]
|
|
total_stats["errors"] += r["errors"]
|
|
total_stats["sources"]["metamask_domains"] = r
|
|
logger.info(f" Result: {r}")
|
|
|
|
# Source 6: ScamSniffer GitHub repo (phishing domains + addresses)
|
|
logger.info("=" * 60)
|
|
logger.info("SOURCE 6: ScamSniffer GitHub Scam Database")
|
|
logger.info("=" * 60)
|
|
r = await ingest_scamsniffer_github()
|
|
total_stats["known_scams_added"] += r["ingested"]
|
|
total_stats["errors"] += r["errors"]
|
|
total_stats["sources"]["scamsniffer_github"] = r
|
|
logger.info(f" Result: {r}")
|
|
|
|
# Source 7: PhishDestroy destroylist (140K+ curated threats)
|
|
logger.info("=" * 60)
|
|
logger.info("SOURCE 7: PhishDestroy Destroylist")
|
|
logger.info("=" * 60)
|
|
r = await ingest_phishdestroy()
|
|
total_stats["known_scams_added"] += r["ingested"]
|
|
total_stats["errors"] += r["errors"]
|
|
total_stats["sources"]["phishdestroy"] = r
|
|
logger.info(f" Result: {r}")
|
|
|
|
logger.info("=" * 60)
|
|
logger.info(f"FINAL SUMMARY: {total_stats}")
|
|
logger.info("=" * 60)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|