gramps/cAIc
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: route FIM passthrough to llama-server native /completion endpoint (supports suffix param)

Browse Source
This commit is contained in:
gramps
2026-07-09 09:59:04 -07:00
parent fbacb1861d
commit 9bf04d921d
1 changed files with 11 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
routers/completions.py
+11 -2
View File
@@ -255,11 +255,20 @@ async def _fim_passthrough(body: dict) -> JSONResponse:
async with httpx.AsyncClient() as client: async with httpx.AsyncClient() as client:
try: try:
resp = await client.post( resp = await client.post(
f"{LLAMA_SERVER_BASE}/v1/completions", f"{LLAMA_SERVER_BASE}/completion",
json=body, json=body,
timeout=httpx.Timeout(30.0, connect=5.0), timeout=httpx.Timeout(30.0, connect=5.0),
) )
return JSONResponse(content=resp.json(), status_code=resp.status_code) data = resp.json()
wrapped = {
"choices": [{
"text": data.get("content", ""),
"index": 0,
"finish_reason": data.get("stop", False),
}],
"usage": {},
}
return JSONResponse(content=wrapped, status_code=resp.status_code)
except httpx.ConnectError: except httpx.ConnectError:
raise HTTPException(status_code=503, detail="Cannot connect to inference server") raise HTTPException(status_code=503, detail="Cannot connect to inference server")
except Exception as e: except Exception as e: