""" RMI MEGA SCRAPER v3 - Substack, Mirror.xyz, Medium, Blog Scrapers Grabs EVERYTHING crypto. Biggest free news DB on the internet. """ import hashlib import json import logging import time from xml.etree import ElementTree as ET import httpx logger = logging.getLogger("rmi.scraper") # ═══════════════════════════════════════════════════════ # SUBSTACK - 50+ top crypto newsletters (free RSS) # ═══════════════════════════════════════════════════════ SUBSTACK_FEEDS = [ ("substack-bankless", "https://substack.com/@bankless/feed"), ("substack-milkroad", "https://substack.com/@milkroad/feed"), ("substack-messaricrypto", "https://substack.com/@messari/feed"), ("substack-defiweekly", "https://substack.com/@defiweekly/feed"), ("substack-thedefiedge", "https://substack.com/@thedefiedge/feed"), ("substack-cryptopragmatist", "https://substack.com/@cryptopragmatist/feed"), ("substack-blockworks", "https://substack.com/@blockworks/feed"), ("substack-coindesk", "https://substack.com/@coindesk/feed"), ("substack-reflexivity", "https://substack.com/@reflexivityresearch/feed"), ("substack-delphi", "https://substack.com/@delphidigital/feed"), ("substack-0xresearch", "https://substack.com/@0xresearch/feed"), ("substack-decentralised", "https://substack.com/@decentralisedco/feed"), ("substack-cryptoventure", "https://substack.com/@cryptoventure/feed"), ("substack-onchaintimes", "https://substack.com/@onchaintimes/feed"), ("substack-web3isgoinggreat", "https://substack.com/@web3isgreat/feed"), ("substack-dirtroads", "https://substack.com/@dirtroads/feed"), ("substack-frontiertech", "https://substack.com/@frontiertech/feed"), ("substack-chainalysis", "https://substack.com/@chainalysis/feed"), ("substack-elliptic", "https://substack.com/@elliptic/feed"), ("substack-trmlabs", "https://substack.com/@trmlabs/feed"), ("substack-thetie", "https://substack.com/@thetie/feed"), ("substack-glassnode", "https://substack.com/@glassnode/feed"), ("substack-nansen", "https://substack.com/@nansen/feed"), ("substack-arkham", "https://substack.com/@arkhamintel/feed"), ("substack-paradigm", "https://substack.com/@paradigm/feed"), ("substack-a16zcrypto", "https://substack.com/@a16zcrypto/feed"), ("substack-dragonfly", "https://substack.com/@dragonfly/feed"), ("substack-pantera", "https://substack.com/@panteracapital/feed"), ("substack-multicoin", "https://substack.com/@multicoincap/feed"), ("substack-variant", "https://substack.com/@variantfund/feed"), ("substack-electriccapital", "https://substack.com/@electriccapital/feed"), ("substack-coinfund", "https://substack.com/@coinfund/feed"), ("substack-framework", "https://substack.com/@frameworkventures/feed"), ("substack-1confirmation", "https://substack.com/@1confirmation/feed"), ("substack-placeholder", "https://substack.com/@placeholder/feed"), ("substack-union-square", "https://substack.com/@usv/feed"), ] # ═══════════════════════════════════════════════════════ # MIRROR.XYZ - Decentralized crypto publishing # ═══════════════════════════════════════════════════════ MIRROR_FEEDS = [ ("mirror-weekly", "https://mirror.xyz/0x/feed"), # Individual writers (their RSS) ("mirror-zoranjoc", "https://zoranjoc.mirror.xyz/feed"), ("mirror-liam", "https://liam.mirror.xyz/feed"), ("mirror-dcbuilder", "https://dcbuilder.mirror.xyz/feed"), ("mirror-pacman", "https://pacman.mirror.xyz/feed"), ("mirror-nick", "https://nick.mirror.xyz/feed"), ] # ═══════════════════════════════════════════════════════ # MEDIUM - Crypto publications # ═══════════════════════════════════════════════════════ MEDIUM_FEEDS = [ ("medium-coinfund", "https://blog.coinfund.io/feed"), ("medium-a16z", "https://a16zcrypto.com/feed/"), ("medium-paradigm", "https://medium.com/feed/paradigm"), ("medium-dragonfly", "https://medium.com/feed/dragonfly-research"), ("medium-multicoin", "https://multicoin.capital/feed/"), ("medium-electric", "https://medium.com/feed/electric-capital"), ("medium-coindesk", "https://www.coindesk.com/arc/outboundfeeds/rss/"), ] # ═══════════════════════════════════════════════════════ # CRYPTO RESEARCH / TECH BLOGS # ═══════════════════════════════════════════════════════ RESEARCH_FEEDS = [ ("arxiv-crypto", "https://rss.arxiv.org/rss/cs.CR"), ("github-trending-crypto", "https://github.com/trending?l=solidity&since=daily"), ("cryptopanic-news", "https://cryptopanic.com/news/rss/"), ("coingecko-research", "https://www.coingecko.com/en/research/rss"), ("thelafeed", "https://www.thela.news/feed"), ("cryptonewsbtc", "https://www.newsbtc.com/feed/"), ("cryptodailyuk", "https://cryptodaily.co.uk/feed/"), ("cryptoglobe", "https://www.cryptoglobe.com/feed/"), ("coinculture", "https://coinculture.com/au/feed/"), ("cryptopress", "https://cryptopress.com/feed/"), ("cryptopurview", "https://cryptopurview.com/feed/"), ("coinchapter", "https://coinchapter.com/feed/"), ("cryptowisser", "https://www.cryptowisser.com/feed/"), ("blockster", "https://blockster.com/feed/"), ("crypto-news-feed", "https://www.crypto-news-feed.com/feed/"), ("coininsider", "https://www.coininsider.com/feed/"), ("dailycoin", "https://dailycoin.com/feed/"), ("thecryptoupdates", "https://thecryptoupdates.com/feed/"), ("cryptoflies", "https://cryptoflies.com/feed/"), ("coincentral", "https://coincentral.com/feed/"), ] def fetch_generic_feeds(feed_list, prefix="", label="generic"): """Fetch any list of RSS feeds. Returns articles.""" results = [] for source, url in feed_list: try: resp = httpx.get(url, timeout=15, headers={"User-Agent": "RMI/4.0 MegaScraper"}) if resp.status_code != 200: continue root = ET.fromstring(resp.content) items = root.findall(".//item") or root.findall(".//{http://www.w3.org/2005/Atom}entry") for item in items[:20]: title = (item.findtext("title", "") or "").strip() content = ( item.findtext("description", "") or item.findtext("content", "") or item.findtext("{http://www.w3.org/2005/Atom}summary", "") or "" ).strip() if title and len(title) > 5: doc_id = ( f"{prefix}{source}:" + hashlib.sha256((source + title).encode()).hexdigest()[:16] ) results.append( { "id": doc_id, "title": title, "content": content[:3000], "url": url, "source": source, "sentiment": 0.0, "tickers": [], "published": "", "ingested_at": time.time(), } ) logger.info(f" {prefix}{source}: {len(results)} articles") except Exception as e: logger.debug(f" {prefix}{source}: {str(e)[:60]}") return results def store_all(): """Fetch all sources and store in Redis + Postgres.""" all_articles = [] all_articles.extend(fetch_generic_feeds(SUBSTACK_FEEDS, "[Substack] ", "substack")) all_articles.extend(fetch_generic_feeds(MIRROR_FEEDS, "[Mirror] ", "mirror")) all_articles.extend(fetch_generic_feeds(MEDIUM_FEEDS, "[Medium] ", "medium")) all_articles.extend(fetch_generic_feeds(RESEARCH_FEEDS, "[Research] ", "research")) try: from dotenv import load_dotenv load_dotenv("/app/.env", override=True) import os import psycopg2 import redis r = redis.Redis( host="rmi-redis", port=6379, password=os.getenv("REDIS_PASSWORD"), decode_responses=True ) # Dedup and store count = 0 for a in all_articles: if not r.exists(f"rmi:news:article:{a['id']}"): r.zadd("rmi:news:substack:index", {a["id"]: a["ingested_at"]}) r.set(f"rmi:news:article:{a['id']}", json.dumps(a)) count += 1 # Update stats existing = json.loads(r.get("rmi:news:stats") or "{}") existing["substack_articles"] = count existing["total_sources"] = ( existing.get("total_sources", 0) + len(SUBSTACK_FEEDS) + len(MIRROR_FEEDS) + len(MEDIUM_FEEDS) + len(RESEARCH_FEEDS) ) r.set("rmi:news:stats", json.dumps(existing)) # Also Postgres try: pg = psycopg2.connect( host="rmi-postgres", port=5432, user="rmi", password=os.getenv("POSTGRES_PASSWORD"), dbname="rmi", ) cur = pg.cursor() cur.execute( "CREATE TABLE IF NOT EXISTS crypto_news (id TEXT PRIMARY KEY, title TEXT, content TEXT, source TEXT, sentiment REAL, ingested_at DOUBLE PRECISION, category TEXT DEFAULT 'research')" ) for a in all_articles: cur.execute( "INSERT INTO crypto_news (id, title, content, source, sentiment, ingested_at, category) VALUES (%s,%s,%s,%s,%s,%s,'research') ON CONFLICT (id) DO NOTHING", (a["id"], a["title"], a["content"], a["source"], 0.0, a["ingested_at"]), ) pg.commit() cur.close() pg.close() except Exception as e: logger.error(f"Postgres: {e}") return { "substack": len(SUBSTACK_FEEDS), "mirror": len(MIRROR_FEEDS), "medium": len(MEDIUM_FEEDS), "research": len(RESEARCH_FEEDS), "new_articles": count, } except Exception as e: logger.error(f"Store failed: {e}") return {"error": str(e), "articles_collected": len(all_articles)} if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format="%(asctime)s [mega] %(message)s") result = store_all() logger.info(json.dumps(result, indent=2))