233 lines
11 KiB
Python
233 lines
11 KiB
Python
"""
|
|
RMI MEGA SCRAPER v3 - Substack, Mirror.xyz, Medium, Blog Scrapers
|
|
Grabs EVERYTHING crypto. Biggest free news DB on the internet.
|
|
"""
|
|
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import time
|
|
from xml.etree import ElementTree as ET
|
|
|
|
import httpx
|
|
|
|
logger = logging.getLogger("rmi.scraper")
|
|
|
|
# ═══════════════════════════════════════════════════════
|
|
# SUBSTACK - 50+ top crypto newsletters (free RSS)
|
|
# ═══════════════════════════════════════════════════════
|
|
SUBSTACK_FEEDS = [
|
|
("substack-bankless", "https://substack.com/@bankless/feed"),
|
|
("substack-milkroad", "https://substack.com/@milkroad/feed"),
|
|
("substack-messaricrypto", "https://substack.com/@messari/feed"),
|
|
("substack-defiweekly", "https://substack.com/@defiweekly/feed"),
|
|
("substack-thedefiedge", "https://substack.com/@thedefiedge/feed"),
|
|
("substack-cryptopragmatist", "https://substack.com/@cryptopragmatist/feed"),
|
|
("substack-blockworks", "https://substack.com/@blockworks/feed"),
|
|
("substack-coindesk", "https://substack.com/@coindesk/feed"),
|
|
("substack-reflexivity", "https://substack.com/@reflexivityresearch/feed"),
|
|
("substack-delphi", "https://substack.com/@delphidigital/feed"),
|
|
("substack-0xresearch", "https://substack.com/@0xresearch/feed"),
|
|
("substack-decentralised", "https://substack.com/@decentralisedco/feed"),
|
|
("substack-cryptoventure", "https://substack.com/@cryptoventure/feed"),
|
|
("substack-onchaintimes", "https://substack.com/@onchaintimes/feed"),
|
|
("substack-web3isgoinggreat", "https://substack.com/@web3isgreat/feed"),
|
|
("substack-dirtroads", "https://substack.com/@dirtroads/feed"),
|
|
("substack-frontiertech", "https://substack.com/@frontiertech/feed"),
|
|
("substack-chainalysis", "https://substack.com/@chainalysis/feed"),
|
|
("substack-elliptic", "https://substack.com/@elliptic/feed"),
|
|
("substack-trmlabs", "https://substack.com/@trmlabs/feed"),
|
|
("substack-thetie", "https://substack.com/@thetie/feed"),
|
|
("substack-glassnode", "https://substack.com/@glassnode/feed"),
|
|
("substack-nansen", "https://substack.com/@nansen/feed"),
|
|
("substack-arkham", "https://substack.com/@arkhamintel/feed"),
|
|
("substack-paradigm", "https://substack.com/@paradigm/feed"),
|
|
("substack-a16zcrypto", "https://substack.com/@a16zcrypto/feed"),
|
|
("substack-dragonfly", "https://substack.com/@dragonfly/feed"),
|
|
("substack-pantera", "https://substack.com/@panteracapital/feed"),
|
|
("substack-multicoin", "https://substack.com/@multicoincap/feed"),
|
|
("substack-variant", "https://substack.com/@variantfund/feed"),
|
|
("substack-electriccapital", "https://substack.com/@electriccapital/feed"),
|
|
("substack-coinfund", "https://substack.com/@coinfund/feed"),
|
|
("substack-framework", "https://substack.com/@frameworkventures/feed"),
|
|
("substack-1confirmation", "https://substack.com/@1confirmation/feed"),
|
|
("substack-placeholder", "https://substack.com/@placeholder/feed"),
|
|
("substack-union-square", "https://substack.com/@usv/feed"),
|
|
]
|
|
|
|
# ═══════════════════════════════════════════════════════
|
|
# MIRROR.XYZ - Decentralized crypto publishing
|
|
# ═══════════════════════════════════════════════════════
|
|
MIRROR_FEEDS = [
|
|
("mirror-weekly", "https://mirror.xyz/0x/feed"),
|
|
# Individual writers (their RSS)
|
|
("mirror-zoranjoc", "https://zoranjoc.mirror.xyz/feed"),
|
|
("mirror-liam", "https://liam.mirror.xyz/feed"),
|
|
("mirror-dcbuilder", "https://dcbuilder.mirror.xyz/feed"),
|
|
("mirror-pacman", "https://pacman.mirror.xyz/feed"),
|
|
("mirror-nick", "https://nick.mirror.xyz/feed"),
|
|
]
|
|
|
|
# ═══════════════════════════════════════════════════════
|
|
# MEDIUM - Crypto publications
|
|
# ═══════════════════════════════════════════════════════
|
|
MEDIUM_FEEDS = [
|
|
("medium-coinfund", "https://blog.coinfund.io/feed"),
|
|
("medium-a16z", "https://a16zcrypto.com/feed/"),
|
|
("medium-paradigm", "https://medium.com/feed/paradigm"),
|
|
("medium-dragonfly", "https://medium.com/feed/dragonfly-research"),
|
|
("medium-multicoin", "https://multicoin.capital/feed/"),
|
|
("medium-electric", "https://medium.com/feed/electric-capital"),
|
|
("medium-coindesk", "https://www.coindesk.com/arc/outboundfeeds/rss/"),
|
|
]
|
|
|
|
# ═══════════════════════════════════════════════════════
|
|
# CRYPTO RESEARCH / TECH BLOGS
|
|
# ═══════════════════════════════════════════════════════
|
|
RESEARCH_FEEDS = [
|
|
("arxiv-crypto", "https://rss.arxiv.org/rss/cs.CR"),
|
|
("github-trending-crypto", "https://github.com/trending?l=solidity&since=daily"),
|
|
("cryptopanic-news", "https://cryptopanic.com/news/rss/"),
|
|
("coingecko-research", "https://www.coingecko.com/en/research/rss"),
|
|
("thelafeed", "https://www.thela.news/feed"),
|
|
("cryptonewsbtc", "https://www.newsbtc.com/feed/"),
|
|
("cryptodailyuk", "https://cryptodaily.co.uk/feed/"),
|
|
("cryptoglobe", "https://www.cryptoglobe.com/feed/"),
|
|
("coinculture", "https://coinculture.com/au/feed/"),
|
|
("cryptopress", "https://cryptopress.com/feed/"),
|
|
("cryptopurview", "https://cryptopurview.com/feed/"),
|
|
("coinchapter", "https://coinchapter.com/feed/"),
|
|
("cryptowisser", "https://www.cryptowisser.com/feed/"),
|
|
("blockster", "https://blockster.com/feed/"),
|
|
("crypto-news-feed", "https://www.crypto-news-feed.com/feed/"),
|
|
("coininsider", "https://www.coininsider.com/feed/"),
|
|
("dailycoin", "https://dailycoin.com/feed/"),
|
|
("thecryptoupdates", "https://thecryptoupdates.com/feed/"),
|
|
("cryptoflies", "https://cryptoflies.com/feed/"),
|
|
("coincentral", "https://coincentral.com/feed/"),
|
|
]
|
|
|
|
|
|
def fetch_generic_feeds(feed_list, prefix="", label="generic"):
|
|
"""Fetch any list of RSS feeds. Returns articles."""
|
|
results = []
|
|
for source, url in feed_list:
|
|
try:
|
|
resp = httpx.get(url, timeout=15, headers={"User-Agent": "RMI/4.0 MegaScraper"})
|
|
if resp.status_code != 200:
|
|
continue
|
|
root = ET.fromstring(resp.content)
|
|
items = root.findall(".//item") or root.findall(".//{http://www.w3.org/2005/Atom}entry")
|
|
for item in items[:20]:
|
|
title = (item.findtext("title", "") or "").strip()
|
|
content = (
|
|
item.findtext("description", "")
|
|
or item.findtext("content", "")
|
|
or item.findtext("{http://www.w3.org/2005/Atom}summary", "")
|
|
or ""
|
|
).strip()
|
|
if title and len(title) > 5:
|
|
doc_id = (
|
|
f"{prefix}{source}:"
|
|
+ hashlib.sha256((source + title).encode()).hexdigest()[:16]
|
|
)
|
|
results.append(
|
|
{
|
|
"id": doc_id,
|
|
"title": title,
|
|
"content": content[:3000],
|
|
"url": url,
|
|
"source": source,
|
|
"sentiment": 0.0,
|
|
"tickers": [],
|
|
"published": "",
|
|
"ingested_at": time.time(),
|
|
}
|
|
)
|
|
logger.info(f" {prefix}{source}: {len(results)} articles")
|
|
except Exception as e:
|
|
logger.debug(f" {prefix}{source}: {str(e)[:60]}")
|
|
return results
|
|
|
|
|
|
def store_all():
|
|
"""Fetch all sources and store in Redis + Postgres."""
|
|
all_articles = []
|
|
all_articles.extend(fetch_generic_feeds(SUBSTACK_FEEDS, "[Substack] ", "substack"))
|
|
all_articles.extend(fetch_generic_feeds(MIRROR_FEEDS, "[Mirror] ", "mirror"))
|
|
all_articles.extend(fetch_generic_feeds(MEDIUM_FEEDS, "[Medium] ", "medium"))
|
|
all_articles.extend(fetch_generic_feeds(RESEARCH_FEEDS, "[Research] ", "research"))
|
|
|
|
try:
|
|
from dotenv import load_dotenv
|
|
|
|
load_dotenv("/app/.env", override=True)
|
|
import os
|
|
|
|
import psycopg2
|
|
import redis
|
|
|
|
r = redis.Redis(
|
|
host="rmi-redis", port=6379, password=os.getenv("REDIS_PASSWORD"), decode_responses=True
|
|
)
|
|
|
|
# Dedup and store
|
|
count = 0
|
|
for a in all_articles:
|
|
if not r.exists(f"rmi:news:article:{a['id']}"):
|
|
r.zadd("rmi:news:substack:index", {a["id"]: a["ingested_at"]})
|
|
r.set(f"rmi:news:article:{a['id']}", json.dumps(a))
|
|
count += 1
|
|
|
|
# Update stats
|
|
existing = json.loads(r.get("rmi:news:stats") or "{}")
|
|
existing["substack_articles"] = count
|
|
existing["total_sources"] = (
|
|
existing.get("total_sources", 0)
|
|
+ len(SUBSTACK_FEEDS)
|
|
+ len(MIRROR_FEEDS)
|
|
+ len(MEDIUM_FEEDS)
|
|
+ len(RESEARCH_FEEDS)
|
|
)
|
|
r.set("rmi:news:stats", json.dumps(existing))
|
|
|
|
# Also Postgres
|
|
try:
|
|
pg = psycopg2.connect(
|
|
host="rmi-postgres",
|
|
port=5432,
|
|
user="rmi",
|
|
password=os.getenv("POSTGRES_PASSWORD"),
|
|
dbname="rmi",
|
|
)
|
|
cur = pg.cursor()
|
|
cur.execute(
|
|
"CREATE TABLE IF NOT EXISTS crypto_news (id TEXT PRIMARY KEY, title TEXT, content TEXT, source TEXT, sentiment REAL, ingested_at DOUBLE PRECISION, category TEXT DEFAULT 'research')"
|
|
)
|
|
for a in all_articles:
|
|
cur.execute(
|
|
"INSERT INTO crypto_news (id, title, content, source, sentiment, ingested_at, category) VALUES (%s,%s,%s,%s,%s,%s,'research') ON CONFLICT (id) DO NOTHING",
|
|
(a["id"], a["title"], a["content"], a["source"], 0.0, a["ingested_at"]),
|
|
)
|
|
pg.commit()
|
|
cur.close()
|
|
pg.close()
|
|
except Exception as e:
|
|
logger.error(f"Postgres: {e}")
|
|
|
|
return {
|
|
"substack": len(SUBSTACK_FEEDS),
|
|
"mirror": len(MIRROR_FEEDS),
|
|
"medium": len(MEDIUM_FEEDS),
|
|
"research": len(RESEARCH_FEEDS),
|
|
"new_articles": count,
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"Store failed: {e}")
|
|
return {"error": str(e), "articles_collected": len(all_articles)}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s [mega] %(message)s")
|
|
result = store_all()
|
|
logger.info(json.dumps(result, indent=2))
|