From 9bf04d921d81b6bdb13528137ae9226bfad8d9d2 Mon Sep 17 00:00:00 2001 From: gramps Date: Thu, 9 Jul 2026 09:59:04 -0700 Subject: [PATCH] fix: route FIM passthrough to llama-server native /completion endpoint (supports suffix param) --- routers/completions.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/routers/completions.py b/routers/completions.py index 41c700a..d167375 100644 --- a/routers/completions.py +++ b/routers/completions.py @@ -255,11 +255,20 @@ async def _fim_passthrough(body: dict) -> JSONResponse: async with httpx.AsyncClient() as client: try: resp = await client.post( - f"{LLAMA_SERVER_BASE}/v1/completions", + f"{LLAMA_SERVER_BASE}/completion", json=body, timeout=httpx.Timeout(30.0, connect=5.0), ) - return JSONResponse(content=resp.json(), status_code=resp.status_code) + data = resp.json() + wrapped = { + "choices": [{ + "text": data.get("content", ""), + "index": 0, + "finish_reason": data.get("stop", False), + }], + "usage": {}, + } + return JSONResponse(content=wrapped, status_code=resp.status_code) except httpx.ConnectError: raise HTTPException(status_code=503, detail="Cannot connect to inference server") except Exception as e: