109 lines
4 KiB
Python
109 lines
4 KiB
Python
#!/usr/bin/env python3
|
|
"""Convert and ingest additional arxiv papers into RAG."""
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
import pymupdf4llm
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
|
PAPERS_DIR = os.path.dirname(os.path.abspath(__file__)).rstrip("/scripts") + "/data/papers"
|
|
|
|
with open(os.path.join(PAPERS_DIR, "papers_metadata.json")) as f:
|
|
existing = json.load(f)
|
|
|
|
new_papers = {
|
|
"2109.05940": {
|
|
"title": "Smart Contract Vulnerability Detection: Graph Feature + Expert Pattern Fusion",
|
|
"authors": [],
|
|
"year": 2021,
|
|
"arxiv_id": "2109.05940",
|
|
"attack_types": ["vulnerability", "static-analysis"],
|
|
"collection": "forensic_reports",
|
|
"description": "Improving smart contract vulnerability detection using graph features and expert patterns.",
|
|
},
|
|
"2006.06116": {
|
|
"title": "Temporal-Amount Snapshot MultiGraph for Ethereum Transaction Tracking",
|
|
"authors": [],
|
|
"year": 2020,
|
|
"arxiv_id": "2006.06116",
|
|
"attack_types": ["transaction-graph", "tracking"],
|
|
"collection": "forensic_reports",
|
|
"description": "Multi-graph snapshot approach for tracking Ethereum transactions.",
|
|
},
|
|
"1905.00346": {
|
|
"title": "Reentrancy Vulnerability Identification in Ethereum Smart Contracts",
|
|
"authors": [],
|
|
"year": 2019,
|
|
"arxiv_id": "1905.00346",
|
|
"attack_types": ["reentrancy", "vulnerability"],
|
|
"collection": "forensic_reports",
|
|
"description": "Detection of reentrancy vulnerabilities in Ethereum smart contracts.",
|
|
},
|
|
"2207.01247": {
|
|
"title": "How to Design Tokenomics: Framework from 20+ Projects",
|
|
"authors": [],
|
|
"year": 2022,
|
|
"arxiv_id": "2207.01247",
|
|
"attack_types": ["tokenomics", "framework"],
|
|
"collection": "market_intel",
|
|
"description": "Tokenomics design framework from analyzing 20+ blockchain projects.",
|
|
},
|
|
"2202.05146": {
|
|
"title": "Empirical Evidence from Governance Token Distributions",
|
|
"authors": [],
|
|
"year": 2022,
|
|
"arxiv_id": "2202.05146",
|
|
"attack_types": ["governance", "token-distribution"],
|
|
"collection": "market_intel",
|
|
"description": "Empirical analysis of governance token distributions and DAO implications.",
|
|
},
|
|
"2004.02968": {
|
|
"title": "Ponzi Scheme Detection in Ethereum Transaction Network",
|
|
"authors": [],
|
|
"year": 2020,
|
|
"arxiv_id": "2004.02968",
|
|
"attack_types": ["ponzi", "detection"],
|
|
"collection": "forensic_reports",
|
|
"description": "Graph-based Ponzi scheme detection in Ethereum transactions.",
|
|
},
|
|
"2104.02368": {
|
|
"title": "EtherClue: Digital Investigation of Attacks on Ethereum Smart Contracts",
|
|
"authors": [],
|
|
"year": 2021,
|
|
"arxiv_id": "2104.02368",
|
|
"attack_types": ["forensics", "attack-detection"],
|
|
"collection": "forensic_reports",
|
|
"description": "EtherClue digital forensic investigation of smart contract attacks.",
|
|
},
|
|
}
|
|
|
|
converted = 0
|
|
for arxiv_id, info in new_papers.items():
|
|
pdf_path = os.path.join(PAPERS_DIR, f"{arxiv_id}.pdf")
|
|
txt_path = os.path.join(PAPERS_DIR, f"{arxiv_id}.txt")
|
|
if not os.path.exists(pdf_path):
|
|
print(f"SKIP {arxiv_id}: PDF not found")
|
|
continue
|
|
if arxiv_id in existing:
|
|
print(f"SKIP {arxiv_id}: already in metadata")
|
|
continue
|
|
sz = os.path.getsize(pdf_path)
|
|
if sz < 5000:
|
|
print(f"INVALID {arxiv_id}: PDF too small ({sz} bytes)")
|
|
continue
|
|
try:
|
|
text = pymupdf4llm.to_markdown(pdf_path)
|
|
with open(txt_path, "w") as f:
|
|
f.write(text)
|
|
words = len(text.split())
|
|
print(f"OK {arxiv_id}: {words} words")
|
|
existing[arxiv_id] = info
|
|
converted += 1
|
|
except Exception as e:
|
|
print(f"ERROR {arxiv_id}: {e}")
|
|
|
|
with open(os.path.join(PAPERS_DIR, "papers_metadata.json"), "w") as f:
|
|
json.dump(existing, f, indent=2)
|
|
print(f"Converted: {converted}, Total papers: {len(existing)}")
|