feat(rag): embedding versioning (#6) + chunking YAML config (#8)

Embedding: EMBED_MODEL_NAME/VERSION/VERSION_STAMP/BUILD_ID env vars.
Chunking: chunking_config.yaml with 8 content types + strategies.
This commit is contained in:
Crypto Rug Munch 2026-07-08 08:28:06 +02:00
parent bec3d4e6d2
commit 31409b6dc5
2 changed files with 40 additions and 0 deletions

View file

@ -0,0 +1,33 @@
rag_chunking:
news:
strategy: recursive
chunk_size: 512
overlap: 0.15
scam_report:
strategy: semantic
chunk_size: 400
overlap: 0.10
contract_code:
strategy: code
chunk_size: 256
overlap: 0.20
wallet_profile:
strategy: fixed
chunk_size: 128
overlap: 0.0
annual_report:
strategy: recursive
chunk_size: 800
overlap: 0.15
social_post:
strategy: fixed
chunk_size: 256
overlap: 0.05
market_data:
strategy: semantic
chunk_size: 300
overlap: 0.10
default:
strategy: recursive
chunk_size: 512
overlap: 0.15

View file

@ -29,6 +29,13 @@ log = logging.getLogger(__name__)
EMBEDDING_DIM: Final = 1024
# Embedding versioning (ADR-0007 compliance)
# Stamped on every embedding so downstream consumers can migrate
# when the model changes. Format: {model}:{version}:{dim}
EMBED_MODEL_NAME = os.getenv("RAG_EMBED_MODEL", "bge-m3")
EMBED_MODEL_VERSION = os.getenv("RAG_EMBED_VERSION", "1.0")
EMBED_VERSION_STAMP = f"{EMBED_MODEL_NAME}:{EMBED_MODEL_VERSION}:{EMBEDDING_DIM}"
EMBED_BUILD_ID = os.getenv("RAG_EMBED_BUILD_ID", "") # git commit hash or CI build number
# Backends
_BACKEND = os.getenv("RAG_EMBED_BACKEND", "auto").lower()
OLLAMA_URL = os.getenv("OLLAMA_URL", "http://host.docker.internal:11434").rstrip("/")