merge: chore/cleanup-remove-bloat-and-secrets into main

app/routers/ai_stream.py

@@ -0,0 +1,96 @@
+"""SSE Response Streaming — real-time token streaming for AI endpoints."""
+
+import asyncio
+import json
+
+from fastapi import APIRouter
+from fastapi.responses import StreamingResponse
+from pydantic import BaseModel
+
+from app.core.model_router import TaskType, route_task
+
+router = APIRouter(prefix="/api/v1/ai", tags=["ai-streaming"])
+
+
+class StreamRequest(BaseModel):
+    prompt: str
+    task: str = "fast"  # fast|cheap|complex|bulk|code
+    prefer: str = "fast"  # fast|cheap
+
+
+async def _stream_ollama(prompt: str, model: str):
+    """Stream from Ollama."""
+    import httpx
+
+    async with httpx.AsyncClient(timeout=120) as c, c.stream(
+        "POST", "http://localhost:11434/api/generate", json={"model": model, "prompt": prompt}
+    ) as r:
+        async for line in r.aiter_lines():
+            if line:
+                try:
+                    chunk = json.loads(line)
+                    if chunk.get("done"):
+                        yield f"data: {json.dumps({'done': True, 'model': model})}\n\n"
+                        break
+                    yield f"data: {json.dumps({'token': chunk.get('response', '')})}\n\n"
+                except json.JSONDecodeError:
+                    continue
+
+
+@router.post("/stream")
+async def stream_ai(req: StreamRequest):
+    """Stream AI response in real-time. Tokens appear as generated."""
+    decision = route_task(TaskType(req.task), req.prefer)
+
+    if decision.provider == "ollama":
+        return StreamingResponse(
+            _stream_ollama(req.prompt, decision.model),
+            media_type="text/event-stream",
+            headers={
+                "X-Model": decision.model,
+                "X-Provider": decision.provider,
+                "X-Latency-Ms": str(decision.estimated_latency_ms),
+            },
+        )
+
+    # For non-Ollama providers, fall back to blocking + stream result
+    async def _blocking_stream():
+        from app.core.model_router import smart_route
+
+        result = await smart_route(req.prompt, req.task, req.prefer)
+        if result and "response" in result:
+            words = result["response"].split()
+            for word in words:
+                yield f"data: {json.dumps({'token': word + ' '})}\n\n"
+                await asyncio.sleep(0.05)
+        yield f"data: {json.dumps({'done': True, 'model': decision.model})}\n\n"
+
+    return StreamingResponse(
+        _blocking_stream(),
+        media_type="text/event-stream",
+        headers={"X-Model": decision.model, "X-Provider": decision.provider},
+    )
+
+
+@router.post("/route")
+async def ai_route(req: StreamRequest):
+    """Non-streaming: auto-route to best model, return complete response."""
+    from app.core.model_router import smart_route
+
+    result = await smart_route(req.prompt, req.task, req.prefer)
+    return result if result else {"error": "All providers failed"}
+
+
+@router.get("/providers")
+async def list_providers():
+    """List all available AI providers with capabilities."""
+    from app.core.model_router import ROUTING_TABLE
+
+    return {
+        "providers": {
+            task.value: [
+                {"model": m[0], "provider": m[1], "latency_ms": m[2] * 1000, "cost_per_1M_input": m[3]} for m in models
+            ]
+            for task, models in ROUTING_TABLE.items()
+        }
+    }