merge: chore/cleanup-remove-bloat-and-secrets into main
This commit is contained in:
commit
bde2f3a97d
1173 changed files with 437609 additions and 0 deletions
256
worker.py
Normal file
256
worker.py
Normal file
|
|
@ -0,0 +1,256 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
RMI Worker - Background job processor for token scanning and queue processing
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
handlers=[logging.StreamHandler(sys.stdout)],
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
REDIS_HOST = os.getenv("REDIS_HOST", "dragonfly")
|
||||
REDIS_PORT = int(os.getenv("REDIS_PORT", "6379"))
|
||||
REDIS_PASSWORD = os.getenv("REDIS_PASSWORD", "")
|
||||
REDIS_DB = int(os.getenv("REDIS_DB", "0"))
|
||||
|
||||
SCAN_QUEUE = "token_scan_queue"
|
||||
NEWS_QUEUE = "news_queue"
|
||||
ALERT_QUEUE = "alert_queue"
|
||||
|
||||
|
||||
async def connect_redis():
|
||||
"""Connect to Redis/Dragonfly with retry logic"""
|
||||
import redis.asyncio as redis
|
||||
|
||||
retries = 0
|
||||
max_retries = 30
|
||||
|
||||
while retries < max_retries:
|
||||
try:
|
||||
r = redis.Redis(
|
||||
host=REDIS_HOST,
|
||||
port=REDIS_PORT,
|
||||
password=REDIS_PASSWORD,
|
||||
db=REDIS_DB,
|
||||
decode_responses=True,
|
||||
)
|
||||
await r.ping()
|
||||
logger.info(f"Connected to Redis at {REDIS_HOST}:{REDIS_PORT}")
|
||||
return r
|
||||
except Exception as e:
|
||||
retries += 1
|
||||
logger.warning(f"Redis connection attempt {retries}/{max_retries} failed: {e}")
|
||||
if retries >= max_retries:
|
||||
logger.error("Max Redis retries reached, exiting")
|
||||
sys.exit(1)
|
||||
await asyncio.sleep(2)
|
||||
|
||||
|
||||
async def process_scan_job(r, job_data):
|
||||
"""Process a token scan job from the queue"""
|
||||
try:
|
||||
import json
|
||||
|
||||
import httpx
|
||||
|
||||
token_address = job_data.get("token_address")
|
||||
chain = job_data.get("chain", "solana")
|
||||
|
||||
logger.info(f"Scanning token: {token_address} on {chain}")
|
||||
|
||||
# Fetch token data from DexScreener
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
try:
|
||||
response = await client.get(f"https://api.dexscreener.com/latest/dex/tokens/{token_address}")
|
||||
if response.status_code == 200:
|
||||
token_data = response.json()
|
||||
logger.info(f"Token data fetched: {token_data.get('baseToken', {}).get('name', 'Unknown')}")
|
||||
else:
|
||||
logger.warning(f"DexScreener returned {response.status_code}")
|
||||
token_data = {}
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch token data: {e}")
|
||||
token_data = {}
|
||||
|
||||
# Store result in Redis cache
|
||||
scan_result = {
|
||||
"token_address": token_address,
|
||||
"chain": chain,
|
||||
"scanned_at": datetime.now(UTC).isoformat(),
|
||||
"status": "completed",
|
||||
"data": token_data,
|
||||
"risk_score": 0, # TODO: Implement actual risk scoring
|
||||
"flags": [], # TODO: Implement flag detection
|
||||
}
|
||||
|
||||
await r.set(f"scan_result:{token_address}", json.dumps(scan_result), ex=3600) # Cache for 1 hour
|
||||
await r.lpush("scan_results", json.dumps(scan_result)) # Add to results list
|
||||
await r.ltrim("scan_results", 0, 99) # Keep only last 100 results
|
||||
|
||||
logger.info(f"Scan completed for {token_address}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Scan job failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def process_news_job(r, job_data):
|
||||
"""Process a news aggregation job"""
|
||||
try:
|
||||
logger.info(f"Processing news job: {job_data}")
|
||||
# TODO: Implement news aggregation
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"News job failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def process_alert_job(r, job_data):
|
||||
"""Process an alert generation job"""
|
||||
try:
|
||||
logger.info(f"Processing alert job: {job_data}")
|
||||
# TODO: Implement alert generation
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Alert job failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def worker_loop(r):
|
||||
"""Main worker loop - process jobs from queues"""
|
||||
logger.info("Starting worker loop...")
|
||||
|
||||
while True:
|
||||
try:
|
||||
# Try to get a job from scan queue first
|
||||
result = await r.brpop(SCAN_QUEUE, timeout=1)
|
||||
if result:
|
||||
_, job_json = result
|
||||
import json
|
||||
|
||||
job_data = json.loads(job_json)
|
||||
await process_scan_job(r, job_data)
|
||||
continue
|
||||
|
||||
# Check news queue
|
||||
result = await r.brpop(NEWS_QUEUE, timeout=1)
|
||||
if result:
|
||||
_, job_json = result
|
||||
import json
|
||||
|
||||
job_data = json.loads(job_json)
|
||||
await process_news_job(r, job_data)
|
||||
continue
|
||||
|
||||
# Check alert queue
|
||||
result = await r.brpop(ALERT_QUEUE, timeout=1)
|
||||
if result:
|
||||
_, job_json = result
|
||||
import json
|
||||
|
||||
job_data = json.loads(job_json)
|
||||
await process_alert_job(r, job_data)
|
||||
continue
|
||||
|
||||
# No jobs, small sleep before next iteration
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
logger.info("Worker loop cancelled")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Worker loop error: {e}")
|
||||
await asyncio.sleep(5)
|
||||
|
||||
|
||||
async def periodic_token_scan(r):
|
||||
"""Periodically scan new tokens from DEXs using token_discovery engine."""
|
||||
logger.info("Starting periodic token scanner...")
|
||||
|
||||
try:
|
||||
from app.token_discovery import MONITORED_CHAINS, discover_new_tokens
|
||||
|
||||
DISCOVERY_AVAILABLE = True
|
||||
except ImportError:
|
||||
logger.warning("token_discovery module not available, using stub mode")
|
||||
DISCOVERY_AVAILABLE = False
|
||||
|
||||
while True:
|
||||
try:
|
||||
if DISCOVERY_AVAILABLE:
|
||||
# Discover new tokens across all monitored chains
|
||||
discovered = await discover_new_tokens(
|
||||
r,
|
||||
chains=list(MONITORED_CHAINS.keys()),
|
||||
scan_security=True,
|
||||
)
|
||||
|
||||
total = sum(len(tokens) for tokens in discovered.values())
|
||||
if total > 0:
|
||||
# Queue new tokens for deep scanning
|
||||
for chain, tokens in discovered.items():
|
||||
for token in tokens:
|
||||
await r.lpush(
|
||||
SCAN_QUEUE,
|
||||
json.dumps(
|
||||
{
|
||||
"token_address": token["address"],
|
||||
"chain": chain,
|
||||
"symbol": token.get("symbol", ""),
|
||||
"name": token.get("name", ""),
|
||||
"liquidity_usd": token.get("liquidity_usd", 0),
|
||||
"risk_score": token.get("risk_score", 0),
|
||||
"discovered_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
logger.info(f"🔍 Queued {total} new tokens for deep scanning")
|
||||
else:
|
||||
logger.debug("No new tokens discovered this cycle")
|
||||
else:
|
||||
logger.warning("Periodic scan tick - token discovery not available")
|
||||
|
||||
# Run every 5 minutes
|
||||
await asyncio.sleep(300)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
logger.info("Periodic scanner cancelled")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Periodic scanner error: {e}")
|
||||
await asyncio.sleep(60)
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main entry point"""
|
||||
logger.info("RMI Worker starting...")
|
||||
logger.info(f"Config: REDIS_HOST={REDIS_HOST}, REDIS_PORT={REDIS_PORT}")
|
||||
|
||||
r = await connect_redis()
|
||||
|
||||
# Run worker loop and periodic scanner concurrently
|
||||
await asyncio.gather(worker_loop(r), periodic_token_scan(r), return_exceptions=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
asyncio.run(main())
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Worker stopped by user")
|
||||
except Exception as e:
|
||||
logger.error(f"Worker crashed: {e}")
|
||||
sys.exit(1)
|
||||
Loading…
Add table
Add a link
Reference in a new issue