feat(rag): extract RAG to dedicated rmi-rag package (#1)

39 modules (14,026 LOC) moved from rmi-backend into rmi-rag/src/rmi_rag/.
17 files in rmi-backend updated to import from rmi_rag.* instead of
local app.rag* / app.*_rag modules.

New package: rmi-rag (git.rugmunch.io/RugMunchMedia/rmi-rag)
Contains all RAG code: embeddings, chunking, hybrid search,
cross-encoder, Qdrant store, incremental indexing, parent retrieval,
query rewriting, BM25, streaming, multimodal, agentic, observability.

rmi-backend added rmi-rag as git dependency in requirements.txt.

app/api/v1/rag/search.py

@@ -10,6 +10,8 @@ from typing import Annotated, Any
 
 from fastapi import APIRouter, Depends, HTTPException
 from pydantic import BaseModel, Field
+from rmi_rag.engine import bulk_ingest as engine_bulk_ingest, get_stats as engine_get_stats
+from rmi_rag.incremental_indexer import DeltaTracker
 
 from app.rag import (
     FeedbackRecord,
@@ -19,8 +21,6 @@ from app.rag import (
     SearchRequest,
     SearchResponse,
 )
-from app.rag.engine import bulk_ingest as engine_bulk_ingest, get_stats as engine_get_stats
-from app.rag.incremental_indexer import DeltaTracker
 
 router = APIRouter(prefix="/api/v1/rag/v2", tags=["rag"])
 

app/bundle_cluster_rag.py

@@ -489,7 +489,7 @@ async def find_similar_clusters(
     Find clusters similar to a target cluster using behavioral vector similarity.
     "This cluster looks like the Wintermute cluster from March"
     """
-    from app.rag.qdrant_store import get_qdrant_store
+    from rmi_rag.qdrant_store import get_qdrant_store
 
     # Embed the target cluster
     target_vec = embed_cluster_profile(target_cluster)
@@ -513,7 +513,7 @@ async def find_similar_bundles(
     limit: int = 10,
 ) -> list[dict[str, Any]]:
     """Find bundles similar to a target bundle."""
-    from app.rag.qdrant_store import get_qdrant_store
+    from rmi_rag.qdrant_store import get_qdrant_store
 
     target_vec = embed_bundle_profile(target_bundle)
     store = await get_qdrant_store()
@@ -540,8 +540,9 @@ async def search_clusters_by_description(
     "Show me all wash trading clusters from Solana in the last month"
     → embeds the query, searches against cluster behavioral vectors
     """
+    from rmi_rag.qdrant_store import get_qdrant_store
+
     from app.domains.intelligence.crypto_embeddings import get_embedder
-    from app.rag.qdrant_store import get_qdrant_store
 
     embedder = await get_embedder()
 
@@ -583,7 +584,7 @@ async def index_bundle_detection(bundle: dict[str, Any]) -> str:
     After bundle detection runs, index the result in RAG.
     Store bundle behavioral vector + metadata for future similarity search.
     """
-    from app.rag.qdrant_store import get_qdrant_store
+    from rmi_rag.qdrant_store import get_qdrant_store
 
     vec = embed_bundle_profile(bundle)
     token = bundle.get("token_address", "unknown")
@@ -623,7 +624,7 @@ async def index_cluster_detection(cluster: dict[str, Any]) -> dict[str, Any]:
     """
     After cluster detection runs, index + auto-label + store.
     """
-    from app.rag.qdrant_store import get_qdrant_store
+    from rmi_rag.qdrant_store import get_qdrant_store
 
     vec = embed_cluster_profile(cluster)
 
@@ -677,8 +678,9 @@ All labels: {json.dumps(labels["all_labels"])}"""
 
 async def backfill_label_templates():
     """Index all cluster label templates into pgvector for auto-labeling."""
+    from rmi_rag.qdrant_store import get_qdrant_store
+
     from app.domains.intelligence.crypto_embeddings import get_embedder
-    from app.rag.qdrant_store import get_qdrant_store
 
     embedder = await get_embedder()
     store = await get_qdrant_store()

app/catalog/service.py

@@ -28,6 +28,10 @@ import time
 from typing import Any
 
 import httpx
+from rmi_rag.engine import (
+    ingest_document as rag_ingest_document,
+    three_pillar_search as rag_three_pillar_search,
+)
 
 from app.catalog.models import (
     COLLECTIONS,
@@ -36,10 +40,6 @@ from app.catalog.models import (
     Wallet,
     utcnow,
 )
-from app.rag.engine import (
-    ingest_document as rag_ingest_document,
-    three_pillar_search as rag_three_pillar_search,
-)
 
 log = logging.getLogger(__name__)
 

app/contextual_chunking.py

@@ -120,7 +120,7 @@ def chunk_document(
     # ── Solidity AST-aware chunking ──
     if _is_solidity(text):
         try:
-            from app.solidity_chunker import chunk_solidity_ast
+            from rmi_rag.solidity_chunker import chunk_solidity_ast
 
             ast_chunks = chunk_solidity_ast(text)
             if ast_chunks:

app/domains/databus/rag_ingestion.py

@@ -173,7 +173,7 @@ async def _embed_batch(docs: list[dict], collection: str) -> int:
 
             # Store parent document for context expansion
             try:
-                from app.rag.parent_retriever import get_parent_retriever
+                from rmi_rag.parent_retriever import get_parent_retriever
 
                 pr = await get_parent_retriever()
                 await pr.store_parent(
@@ -185,7 +185,7 @@ async def _embed_batch(docs: list[dict], collection: str) -> int:
                 logger.debug("parent_store_failed doc_id=%s err=%s", doc_id, e)
 
             try:
-                from app.rag.incremental_indexer import DeltaTracker
+                from rmi_rag.incremental_indexer import DeltaTracker
 
                 DeltaTracker.track_add(doc_id, collection)
             except Exception as e:

app/domains/intelligence/rag_service.py

@@ -211,7 +211,7 @@ async def ingest_document(
     # ── Update secondary indexes ──
     # BM25 index: invalidate cache so it gets rebuilt
     try:
-        from app.splade_bm25 import _bm25_built_at, _bm25_index
+        from rmi_rag.splade_bm25 import _bm25_built_at, _bm25_index
 
         if _bm25_index is not None:
             # Invalidate cache so next search rebuilds
@@ -255,7 +255,7 @@ async def ingest_document(
 
     # Store parent document for context expansion
     try:
-        from app.rag.parent_retriever import get_parent_retriever
+        from rmi_rag.parent_retriever import get_parent_retriever
 
         pr = await get_parent_retriever()
         await pr.store_parent(doc_id=doc_id, content=content, metadata=metadata)
@@ -350,7 +350,7 @@ async def search_similar(
             logger.info(f"Semantic cache hit for query: {query[:60]}")
             # Trace observability
             try:
-                from app.rag_observability import trace_retrieval
+                from rmi_rag.observability import trace_retrieval
 
                 trace_retrieval((time.time() - t0) * 1000, collection, len(cached), cache_hit=True)
             except Exception:
@@ -410,7 +410,7 @@ async def search_similar(
 
     # Trace observability
     try:
-        from app.rag_observability import trace_retrieval
+        from rmi_rag.observability import trace_retrieval
 
         trace_retrieval((time.time() - t0) * 1000, collection, len(results), cache_hit=cache_hit)
     except Exception:
@@ -592,7 +592,7 @@ async def three_pillar_search(
     # ── Pillar 2: Sparse text search (BM25 + SPLADE) ──
     pillar2_results: list[dict[str, Any]] = []
     try:
-        from app.splade_bm25 import splade_search
+        from rmi_rag.splade_bm25 import splade_search
 
         pillar2_results = await splade_search(query, collections, limit=limit * 2)
         for r in pillar2_results:
@@ -748,7 +748,7 @@ async def three_pillar_search(
 
         # Stage 1: Ollama bge-m3 cosine approximation
         try:
-            from app.rag.cross_encoder import get_reranker as _get_ollama_reranker
+            from rmi_rag.cross_encoder import get_reranker as _get_ollama_reranker
 
             ollama_reranker = await _get_ollama_reranker()
             fused = await ollama_reranker.rerank(query, fused, top_k=limit * 3)
@@ -787,7 +787,7 @@ async def three_pillar_search(
     parent_child_applied = False
     if use_parent_child and fused:
         try:
-            from app.contextual_chunking import parent_child_chunk  # noqa: F401
+            from rmi_rag.contextual_chunking import parent_child_chunk  # noqa: F401
 
             expanded_results = []
             for r in fused[: limit * 2]:
@@ -826,7 +826,7 @@ async def three_pillar_search(
 
     # ── Parent document context expansion (ParentRetriever) ──
     try:
-        from app.rag.parent_retriever import get_parent_retriever
+        from rmi_rag.parent_retriever import get_parent_retriever
 
         pr = await get_parent_retriever()
         fused = await pr.expand_context(fused)

app/multimodal_rag.py

@@ -222,7 +222,7 @@ async def describe_image(
     task: "describe" | "extract_addresses" | "analyze_chart" | "detect_scam"
     """
     try:
-        from app.rag_agentic import AI_BASE, LLM_API_KEY
+        from rmi_rag.agentic import AI_BASE, LLM_API_KEY
 
         if not LLM_API_KEY:
             logger.warning("No LLM key available for image description")

app/rag/__init__.py

@@ -16,7 +16,7 @@ from app.core.health import DomainHealth
 async def _health_check() -> DomainHealth:
     """RAG health: v3 engine present + Redis reachable."""
     try:
-        from app.rag.engine import get_stats
+        from rmi_rag.engine import get_stats
         stats = get_stats()
         return DomainHealth(
             name="rag",
@@ -35,7 +35,7 @@ health_mod.register_health_check("rag", _health_check)
 
 
 # Public API
-from app.rag.models import (  # noqa: E402
+from rmi_rag.models import (  # noqa: E402
     COLLECTIONS,
     EmbeddingProvider,
     FeedbackRecord,
@@ -45,7 +45,7 @@ from app.rag.models import (  # noqa: E402
     SearchRequest,
     SearchResponse,
 )
-from app.rag.service import RAGService, init_rag  # noqa: E402
+from rmi_rag.service import RAGService, init_rag  # noqa: E402
 
 __all__ = [
     "COLLECTIONS",

app/rag/ann_index.py

@@ -19,8 +19,7 @@ from dataclasses import dataclass
 from typing import Any
 
 import numpy as np
-
-from app.rag.embeddings import EMBEDDING_DIM
+from rmi_rag.embeddings import EMBEDDING_DIM
 
 log = logging.getLogger(__name__)
 

app/rag/cross_encoder.py

@@ -174,7 +174,7 @@ class OllamaReranker:
 
         # ── observability ──
         try:
-            from app.rag_observability import trace_reranker
+            from rmi_rag.observability import trace_reranker
 
             trace_reranker(
                 duration_ms=duration_ms,

app/rag/engine.py

@@ -21,9 +21,9 @@ import logging
 import time
 from typing import Any
 
-from app.rag.ann_index import Hit, get_index
-from app.rag.chunking import chunk_text, content_hash, is_duplicate, mark_ingested
-from app.rag.embeddings import current_backend, get_embedding
+from rmi_rag.ann_index import Hit, get_index
+from rmi_rag.chunking import chunk_text, content_hash, is_duplicate, mark_ingested
+from rmi_rag.embeddings import current_backend, get_embedding
 
 log = logging.getLogger(__name__)
 
@@ -272,7 +272,7 @@ def get_collection_stats(collection: str) -> dict:
 
 def get_stats(collections: list[str] | None = None) -> dict:
     """Get stats for all (or specified) collections + the active embedder backend."""
-    from app.rag.models import COLLECTIONS as DEFAULT_COLLECTIONS
+    from rmi_rag.models import COLLECTIONS as DEFAULT_COLLECTIONS
 
     cols = collections or DEFAULT_COLLECTIONS
     return {

app/rag/service.py

@@ -9,13 +9,12 @@ from __future__ import annotations
 import time
 from typing import Any
 
-from app.core.logging import get_logger
-from app.rag.engine import (
+from rmi_rag.engine import (
     get_stats as engine_get_stats,
     ingest_document as engine_ingest_document,
     three_pillar_search as engine_three_pillar_search,
 )
-from app.rag.models import (
+from rmi_rag.models import (
     FeedbackRecord,
     IngestRequest,
     IngestResult,
@@ -24,6 +23,8 @@ from app.rag.models import (
     SearchResponse,
 )
 
+from app.core.logging import get_logger
+
 log = get_logger(__name__)
 
 
@@ -77,13 +78,13 @@ class RAGService:
             )
             if r.get("status") == "ok":
                 try:
-                    from app.rag.incremental_indexer import DeltaTracker
+                    from rmi_rag.incremental_indexer import DeltaTracker
 
                     DeltaTracker.track_add(doc_id, req.collection)
                 except Exception as e:
                     log.debug("delta_track_failed", error=str(e))
                 try:
-                    from app.rag.parent_retriever import get_parent_retriever
+                    from rmi_rag.parent_retriever import get_parent_retriever
 
                     pr = await get_parent_retriever()
                     await pr.store_parent(

app/rag_endpoints.py

@@ -26,7 +26,7 @@ async def rag_chunk_solidity(request: Request, data: dict):
     source = data.get("source", "")
     if not source:
         return {"error": "source is required"}
-    from app.solidity_chunker import chunk_solidity_ast
+    from rmi_rag.solidity_chunker import chunk_solidity_ast
 
     chunks = chunk_solidity_ast(source)
     return {"chunks": chunks, "count": len(chunks)}
@@ -61,7 +61,7 @@ async def rag_cross_chain(request: Request, data: dict):
 
 @router.post("/rag/image-analysis")
 async def rag_image_analysis(request: Request, data: dict):
-    from app.multimodal_rag import describe_image
+    from rmi_rag.multimodal import describe_image
 
     return {
         "task": data.get("task", "describe"),
@@ -75,7 +75,7 @@ async def rag_logo_check(request: Request, data: dict):
     if not url:
         return {"error": "image_url is required"}
     r = await httpx.AsyncClient(timeout=15).get(url)
-    from app.multimodal_rag import compute_image_hash, get_logo_db
+    from rmi_rag.multimodal import compute_image_hash, get_logo_db
 
     h = compute_image_hash(r.content)
     return {**get_logo_db().check_theft(h), "computed_hash": h}
@@ -93,21 +93,21 @@ async def rag_investigate_endpoint(request: Request, data: dict):
 
 @router.get("/rag/graph-communities")
 async def rag_graph_communities(request: Request):
-    from app.graph_rag import graph_rag_search
+    from rmi_rag.graph_rag import graph_rag_search
 
     return await graph_rag_search(query="")
 
 
 @router.post("/rag/watch")
 async def rag_create_watch(request: Request, data: dict):
-    from app.streaming_rag import create_watch
+    from rmi_rag.streaming import create_watch
 
     return await create_watch(token_address=data.get("token_address", ""), chain=data.get("chain", "ethereum"))
 
 
 @router.get("/rag/watches")
 async def rag_list_watches(request: Request):
-    from app.streaming_rag import list_active_watches
+    from rmi_rag.streaming import list_active_watches
 
     return await list_active_watches()
 

app/rag_historical.py

@@ -342,7 +342,7 @@ async def ingest_historical_source(source_id: str) -> dict:
         return {"source": source_id, "docs_scraped": 0, "status": "empty"}
 
     # Chunk and dedup
-    from app.rag_chunking import chunk_batch
+    from rmi_rag.app_chunking import chunk_batch
 
     chunks = chunk_batch(docs, source["content_type"])
 

app/rag_metrics_api.py

@@ -4,8 +4,7 @@ GET /api/v1/rag/metrics - embedding/retrieval latency, cache hit rate, errors.
 """
 
 from fastapi import APIRouter, Request
-
-from app.rag_observability import get_metrics
+from rmi_rag.observability import get_metrics
 
 router = APIRouter(prefix="/api/v1/rag", tags=["rag-metrics"])
 

app/rate_limiter.py

@@ -458,7 +458,7 @@ class EmbeddingDispatcher:
         task: 'scam_detection', 'user_search', 'news_ingestion', 'bulk_ingestion', etc.
         Routes to appropriate provider tier - saves premium credits for critical tasks.
         """
-        from app.embed_tiers import get_allowed_providers, get_min_dims, get_tier
+        from rmi_rag.embed_tiers import get_allowed_providers, get_min_dims, get_tier
 
         get_tier(task)
         allowed = get_allowed_providers(task)

app/services/supabase_rag.py

@@ -49,7 +49,7 @@ async def search_similar(
         similarity_threshold: Minimum cosine similarity (0-1)
     """
     # Pad/truncate to match the embedding model dimension (bge-m3)
-    from app.rag.embeddings import _resize
+    from rmi_rag.embeddings import _resize
 
     table_dim = 1024
     padded_embedding = _resize(query_embedding, table_dim)

pyproject.toml

@@ -22,7 +22,7 @@ dependencies = [
     "langfuse>=3.0.0",
     "PyJWT>=2.10.0",
     "python-multipart>=0.0.17",
-    "duckdb>=0.10.0",
+    "duckdb>=0.10.0", "rmi-rag",
 ]
 
 [project.optional-dependencies]

requirements.txt

@@ -175,3 +175,4 @@ x25519==0.0.2
 yarl==1.24.2
 aiofiles>=24.1.0
 pyotp>=2.9.0
+rmi-rag @ git+ssh://git@git.rugmunch.io:2222/RugMunchMedia/rmi-rag.git