RugMunchMedia/rmi-backend
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
rmi-backend/app/databus/mega_scraper.py
opencode c762564d40 style(rmi-backend): complete lint cleanup — 1175→0 ruff errors 
- Fix 71 invalid-syntax files (class-body newline-broken assignments)
- Add from/None chain to 307 B904 raise-without-from sites
- Add B008 ignore to ruff.toml (already in pyproject.toml)
- Noqa F401 on __init__.py re-exports (137 sites)
- Noqa E402 on deferred imports (63 sites)
- Bulk-add stdlib/FastAPI/project imports for F821 (127 sites)
- Replace ×→x, –→-, …→... in docstrings (4093 chars)
- Manual refactor of 5 SIM103/SIM116 patterns

Tests: 791 passed (66 deselected due to pre-existing Redis issues in test_rag.py)
Co-authored-by: opencode <opencode@rugmunch.io>
2026-07-06 15:43:20 +02:00

233 lines
11 KiB
Python
Raw Permalink Blame History

"""
RMI MEGA SCRAPER v3 - Substack, Mirror.xyz, Medium, Blog Scrapers
Grabs EVERYTHING crypto. Biggest free news DB on the internet.
"""
import hashlib
import json
import logging
import time
from xml.etree import ElementTree as ET
import httpx
logger = logging.getLogger("rmi.scraper")
# ═══════════════════════════════════════════════════════
# SUBSTACK - 50+ top crypto newsletters (free RSS)
# ═══════════════════════════════════════════════════════
SUBSTACK_FEEDS = [
("substack-bankless", "https://substack.com/@bankless/feed"),
("substack-milkroad", "https://substack.com/@milkroad/feed"),
("substack-messaricrypto", "https://substack.com/@messari/feed"),
("substack-defiweekly", "https://substack.com/@defiweekly/feed"),
("substack-thedefiedge", "https://substack.com/@thedefiedge/feed"),
("substack-cryptopragmatist", "https://substack.com/@cryptopragmatist/feed"),
("substack-blockworks", "https://substack.com/@blockworks/feed"),
("substack-coindesk", "https://substack.com/@coindesk/feed"),
("substack-reflexivity", "https://substack.com/@reflexivityresearch/feed"),
("substack-delphi", "https://substack.com/@delphidigital/feed"),
("substack-0xresearch", "https://substack.com/@0xresearch/feed"),
("substack-decentralised", "https://substack.com/@decentralisedco/feed"),
("substack-cryptoventure", "https://substack.com/@cryptoventure/feed"),
("substack-onchaintimes", "https://substack.com/@onchaintimes/feed"),
("substack-web3isgoinggreat", "https://substack.com/@web3isgreat/feed"),
("substack-dirtroads", "https://substack.com/@dirtroads/feed"),
("substack-frontiertech", "https://substack.com/@frontiertech/feed"),
("substack-chainalysis", "https://substack.com/@chainalysis/feed"),
("substack-elliptic", "https://substack.com/@elliptic/feed"),
("substack-trmlabs", "https://substack.com/@trmlabs/feed"),
("substack-thetie", "https://substack.com/@thetie/feed"),
("substack-glassnode", "https://substack.com/@glassnode/feed"),
("substack-nansen", "https://substack.com/@nansen/feed"),
("substack-arkham", "https://substack.com/@arkhamintel/feed"),
("substack-paradigm", "https://substack.com/@paradigm/feed"),
("substack-a16zcrypto", "https://substack.com/@a16zcrypto/feed"),
("substack-dragonfly", "https://substack.com/@dragonfly/feed"),
("substack-pantera", "https://substack.com/@panteracapital/feed"),
("substack-multicoin", "https://substack.com/@multicoincap/feed"),
("substack-variant", "https://substack.com/@variantfund/feed"),
("substack-electriccapital", "https://substack.com/@electriccapital/feed"),
("substack-coinfund", "https://substack.com/@coinfund/feed"),
("substack-framework", "https://substack.com/@frameworkventures/feed"),
("substack-1confirmation", "https://substack.com/@1confirmation/feed"),
("substack-placeholder", "https://substack.com/@placeholder/feed"),
("substack-union-square", "https://substack.com/@usv/feed"),
]
# ═══════════════════════════════════════════════════════
# MIRROR.XYZ - Decentralized crypto publishing
# ═══════════════════════════════════════════════════════
MIRROR_FEEDS = [
("mirror-weekly", "https://mirror.xyz/0x/feed"),
# Individual writers (their RSS)
("mirror-zoranjoc", "https://zoranjoc.mirror.xyz/feed"),
("mirror-liam", "https://liam.mirror.xyz/feed"),
("mirror-dcbuilder", "https://dcbuilder.mirror.xyz/feed"),
("mirror-pacman", "https://pacman.mirror.xyz/feed"),
("mirror-nick", "https://nick.mirror.xyz/feed"),
]
# ═══════════════════════════════════════════════════════
# MEDIUM - Crypto publications
# ═══════════════════════════════════════════════════════
MEDIUM_FEEDS = [
("medium-coinfund", "https://blog.coinfund.io/feed"),
("medium-a16z", "https://a16zcrypto.com/feed/"),
("medium-paradigm", "https://medium.com/feed/paradigm"),
("medium-dragonfly", "https://medium.com/feed/dragonfly-research"),
("medium-multicoin", "https://multicoin.capital/feed/"),
("medium-electric", "https://medium.com/feed/electric-capital"),
("medium-coindesk", "https://www.coindesk.com/arc/outboundfeeds/rss/"),
]
# ═══════════════════════════════════════════════════════
# CRYPTO RESEARCH / TECH BLOGS
# ═══════════════════════════════════════════════════════
RESEARCH_FEEDS = [
("arxiv-crypto", "https://rss.arxiv.org/rss/cs.CR"),
("github-trending-crypto", "https://github.com/trending?l=solidity&since=daily"),
("cryptopanic-news", "https://cryptopanic.com/news/rss/"),
("coingecko-research", "https://www.coingecko.com/en/research/rss"),
("thelafeed", "https://www.thela.news/feed"),
("cryptonewsbtc", "https://www.newsbtc.com/feed/"),
("cryptodailyuk", "https://cryptodaily.co.uk/feed/"),
("cryptoglobe", "https://www.cryptoglobe.com/feed/"),
("coinculture", "https://coinculture.com/au/feed/"),
("cryptopress", "https://cryptopress.com/feed/"),
("cryptopurview", "https://cryptopurview.com/feed/"),
("coinchapter", "https://coinchapter.com/feed/"),
("cryptowisser", "https://www.cryptowisser.com/feed/"),
("blockster", "https://blockster.com/feed/"),
("crypto-news-feed", "https://www.crypto-news-feed.com/feed/"),
("coininsider", "https://www.coininsider.com/feed/"),
("dailycoin", "https://dailycoin.com/feed/"),
("thecryptoupdates", "https://thecryptoupdates.com/feed/"),
("cryptoflies", "https://cryptoflies.com/feed/"),
("coincentral", "https://coincentral.com/feed/"),
]
def fetch_generic_feeds(feed_list, prefix="", label="generic"):
"""Fetch any list of RSS feeds. Returns articles."""
results = []
for source, url in feed_list:
try:
resp = httpx.get(url, timeout=15, headers={"User-Agent": "RMI/4.0 MegaScraper"})
if resp.status_code != 200:
continue
root = ET.fromstring(resp.content)
items = root.findall(".//item") or root.findall(".//{http://www.w3.org/2005/Atom}entry")
for item in items[:20]:
title = (item.findtext("title", "") or "").strip()
content = (
item.findtext("description", "")
or item.findtext("content", "")
or item.findtext("{http://www.w3.org/2005/Atom}summary", "")
or ""
).strip()
if title and len(title) > 5:
doc_id = (
f"{prefix}{source}:"
+ hashlib.sha256((source + title).encode()).hexdigest()[:16]
)
results.append(
{
"id": doc_id,
"title": title,
"content": content[:3000],
"url": url,
"source": source,
"sentiment": 0.0,
"tickers": [],
"published": "",
"ingested_at": time.time(),
}
)
logger.info(f" {prefix}{source}: {len(results)} articles")
except Exception as e:
logger.debug(f" {prefix}{source}: {str(e)[:60]}")
return results
def store_all():
"""Fetch all sources and store in Redis + Postgres."""
all_articles = []
all_articles.extend(fetch_generic_feeds(SUBSTACK_FEEDS, "[Substack] ", "substack"))
all_articles.extend(fetch_generic_feeds(MIRROR_FEEDS, "[Mirror] ", "mirror"))
all_articles.extend(fetch_generic_feeds(MEDIUM_FEEDS, "[Medium] ", "medium"))
all_articles.extend(fetch_generic_feeds(RESEARCH_FEEDS, "[Research] ", "research"))
try:
from dotenv import load_dotenv
load_dotenv("/app/.env", override=True)
import os
import psycopg2
import redis
r = redis.Redis(
host="rmi-redis", port=6379, password=os.getenv("REDIS_PASSWORD"), decode_responses=True
)
# Dedup and store
count = 0
for a in all_articles:
if not r.exists(f"rmi:news:article:{a['id']}"):
r.zadd("rmi:news:substack:index", {a["id"]: a["ingested_at"]})
r.set(f"rmi:news:article:{a['id']}", json.dumps(a))
count += 1
# Update stats
existing = json.loads(r.get("rmi:news:stats") or "{}")
existing["substack_articles"] = count
existing["total_sources"] = (
existing.get("total_sources", 0)
+ len(SUBSTACK_FEEDS)
+ len(MIRROR_FEEDS)
+ len(MEDIUM_FEEDS)
+ len(RESEARCH_FEEDS)
)
r.set("rmi:news:stats", json.dumps(existing))
# Also Postgres
try:
pg = psycopg2.connect(
host="rmi-postgres",
port=5432,
user="rmi",
password=os.getenv("POSTGRES_PASSWORD"),
dbname="rmi",
)
cur = pg.cursor()
cur.execute(
"CREATE TABLE IF NOT EXISTS crypto_news (id TEXT PRIMARY KEY, title TEXT, content TEXT, source TEXT, sentiment REAL, ingested_at DOUBLE PRECISION, category TEXT DEFAULT 'research')"
)
for a in all_articles:
cur.execute(
"INSERT INTO crypto_news (id, title, content, source, sentiment, ingested_at, category) VALUES (%s,%s,%s,%s,%s,%s,'research') ON CONFLICT (id) DO NOTHING",
(a["id"], a["title"], a["content"], a["source"], 0.0, a["ingested_at"]),
)
pg.commit()
cur.close()
pg.close()
except Exception as e:
logger.error(f"Postgres: {e}")
return {
"substack": len(SUBSTACK_FEEDS),
"mirror": len(MIRROR_FEEDS),
"medium": len(MEDIUM_FEEDS),
"research": len(RESEARCH_FEEDS),
"new_articles": count,
}
except Exception as e:
logger.error(f"Store failed: {e}")
return {"error": str(e), "articles_collected": len(all_articles)}
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO, format="%(asctime)s [mega] %(message)s")
result = store_all()
logger.info(json.dumps(result, indent=2))
Reference in a new issue View git blame Copy permalink