v1.10.0 -> v1.11.0: terminal command RAG hook (Roadmap I)

- POST /api/ingest with Bearer token auth, chunk_text, embed, Qdrant upsert - docs/jc-ingest.sh — shell script for PROMPT_COMMAND hook on jarvis - /api/ingest exempted from session middleware (self-authenticating) - 5 tests: missing/wrong key, empty/missing content, success path
2026-07-03 12:35:21 -07:00
parent 04fbe90f08
commit 1ac21ad13f
5 changed files with 203 additions and 3 deletions
@@ -34,6 +34,7 @@ import routers.chat as chat
 import routers.search_route as search_route
 import routers.completions as completions
 import routers.upload as upload
+import routers.ingest as ingest

 # --- Logging ---
 log = logging.getLogger("jarvischat")
@@ -102,7 +103,7 @@ async def session_auth_middleware(request: Request, call_next):

    unauth_paths = {
        "/api/auth/login", "/api/auth/logout", "/api/auth/session",
-        "/api/auth/heartbeat", "/api/auth/guest",
+        "/api/auth/heartbeat", "/api/auth/guest", "/api/ingest",
    }

    if path.startswith("/api/"):
@@ -141,7 +142,7 @@ async def index(request: Request):
 for router_module in [
    auth_router, conversations.router, memories.router, models.router,
    presets.router, profile.router, settings.router, skills.router,
-    chat.router, search_route.router, completions.router, upload.router,
+    chat.router, search_route.router, completions.router, upload.router, ingest.router,
 ]:
    app.include_router(router_module)

@@ -9,7 +9,7 @@ import logging

 log = logging.getLogger("jarvischat")

-VERSION = "v1.10.0"
+VERSION = "v1.11.0"
 OLLAMA_BASE = os.environ.get("OLLAMA_BASE", "http://localhost:11434")
 LLAMA_SERVER_BASE = os.environ.get("LLAMA_SERVER_BASE", "http://192.168.50.108:8081")
 SEARXNG_BASE = "http://localhost:8888"
@@ -0,0 +1,31 @@
+#!/bin/bash
+# jc-ingest.sh — pipe terminal commands into jarvisChat RAG
+# Deploy to /home/gramps/bin/jc-ingest.sh on jarvis (192.168.50.210)
+#
+# Usage:
+#   1. chmod +x /home/gramps/bin/jc-ingest.sh
+#   2. Add to ~/.bashrc:
+#        export JARVISCHAT_COMPLETIONS_API_KEY="$(cat /opt/jarvischat/.completions_key)"
+#        export PROMPT_COMMAND="jc_capture"
+#        source /home/gramps/bin/jc-ingest.sh
+#
+#   The PROMPT_COMMAND hook runs jc_capture() after each command.
+#   Only commands matching the filter pattern are ingested.
+#
+# Filter: currently captures git, pip, systemctl, sudo, vi/vim, curl,
+# wget, apt, python, pytest commands. Edit the grep pattern to adjust.
+
+JC_URL="http://192.168.50.210:8080/api/ingest"
+JC_TOKEN="${JARVISCHAT_COMPLETIONS_API_KEY}"
+
+jc_capture() {
+    local cmd
+    cmd=$(history 1 | sed 's/^[ ]*[0-9]*[ ]*//')
+    if echo "$cmd" | grep -qE '^(git|pip|systemctl|sudo|vi|vim|curl|wget|apt|python|pytest)'; then
+        curl -s -X POST "$JC_URL" \
+            -H "Authorization: Bearer $JC_TOKEN" \
+            -H "Content-Type: application/json" \
+            -d "{\"content\": $(echo "$cmd" | python3 -c 'import json,sys; print(json.dumps(sys.stdin.read().strip()))'), \"source\": \"terminal\"}" \
+            > /dev/null 2>&1 &
+    fi
+}
@@ -0,0 +1,64 @@
+"""JarvisChat routers - /api/ingest terminal command RAG hook."""
+import logging
+from datetime import datetime, timezone
+
+import httpx
+from fastapi import APIRouter, HTTPException, Request
+from fastapi.responses import JSONResponse
+
+from config import COMPLETIONS_API_KEY
+from rag import chunk_text, QDRANT_URL, EMBED_URL, EMBED_MODEL, RAG_COLLECTION
+
+log = logging.getLogger("jarvischat")
+router = APIRouter()
+
+
+def _check_api_key(request: Request):
+    auth = request.headers.get("Authorization", "")
+    if not auth.startswith("Bearer "):
+        raise HTTPException(status_code=401, detail="Missing Bearer token")
+    token = auth[7:].strip()
+    if token != COMPLETIONS_API_KEY:
+        raise HTTPException(status_code=401, detail="Invalid API key")
+
+
+@router.post("/api/ingest")
+async def ingest_content(request: Request):
+    _check_api_key(request)
+    body = await request.json()
+    content = (body.get("content") or "").strip()
+    if not content:
+        raise HTTPException(status_code=422, detail="content is required")
+    source = str(body.get("source", "external")).strip() or "external"
+    metadata = body.get("metadata") or {}
+
+    chunks = chunk_text(content)
+    if not chunks:
+        raise HTTPException(status_code=422, detail="content produced no chunks")
+
+    ingested = 0
+    async with httpx.AsyncClient() as client:
+        for i, chunk in enumerate(chunks):
+            embed_resp = await client.post(
+                f"{EMBED_URL}/api/embeddings",
+                json={"model": EMBED_MODEL, "prompt": chunk},
+                timeout=30.0,
+            )
+            if embed_resp.status_code != 200:
+                log.warning(f"Ingest embedding failed for chunk {i}: {embed_resp.status_code}")
+                continue
+            vector = embed_resp.json()["embedding"]
+            point_id = f"ingest-{source}-{datetime.now(timezone.utc).timestamp()}-{i}"
+            payload = {"text": chunk, "source": source, "ingest_date": datetime.now(timezone.utc).isoformat(), "type": "ingest"}
+            payload.update(metadata)
+            upsert_resp = await client.put(
+                f"{QDRANT_URL}/collections/{RAG_COLLECTION}/points?wait=true",
+                json={"points": [{"id": point_id, "vector": vector, "payload": payload}]},
+                timeout=30.0,
+            )
+            if upsert_resp.status_code in (200, 201):
+                ingested += 1
+            else:
+                log.warning(f"Ingest Qdrant upsert failed for chunk {i}: {upsert_resp.status_code}")
+
+    return {"chunks_ingested": ingested, "source": source, "message": f"Ingested {ingested} chunks from {source}"}
@@ -0,0 +1,104 @@
+import json
+import os
+from pathlib import Path
+
+import httpx
+from fastapi.testclient import TestClient
+
+import app
+import db
+import routers.ingest as ingest_route
+from security import SESSIONS, PIN_ATTEMPTS, RATE_EVENTS
+
+
+def make_client(tmp_path: Path) -> TestClient:
+    os.environ["JARVISCHAT_ADMIN_PIN"] = "1234"
+    db.DB_PATH = tmp_path / "jarvischat-ingest.db"
+    SESSIONS.clear()
+    PIN_ATTEMPTS.clear()
+    RATE_EVENTS.clear()
+    db.init_db()
+    return TestClient(app.app, raise_server_exceptions=False)
+
+
+TEST_API_KEY = "test-sk-jarvischat-ingest"
+
+
+def _auth_headers() -> dict:
+    return {"Authorization": f"Bearer {TEST_API_KEY}", "Content-Type": "application/json", "Origin": "http://testserver"}
+
+
+def test_ingest_missing_api_key(tmp_path: Path):
+    with make_client(tmp_path) as client:
+        resp = client.post("/api/ingest", json={"content": "test"}, headers={"Origin": "http://testserver"})
+        assert resp.status_code == 401
+
+
+def test_ingest_wrong_api_key(tmp_path: Path):
+    with make_client(tmp_path) as client:
+        resp = client.post("/api/ingest", json={"content": "test"},
+                           headers={"Authorization": "Bearer wrong", "Origin": "http://testserver"})
+        assert resp.status_code == 401
+
+
+def test_ingest_empty_content(tmp_path: Path):
+    monkeypatch = __import__('pytest').MonkeyPatch()
+    monkeypatch.setattr(ingest_route, "COMPLETIONS_API_KEY", TEST_API_KEY)
+    with make_client(tmp_path) as client:
+        resp = client.post("/api/ingest", json={"content": ""}, headers=_auth_headers())
+        assert resp.status_code == 422
+    monkeypatch.undo()
+
+
+def test_ingest_missing_content(tmp_path: Path):
+    monkeypatch = __import__('pytest').MonkeyPatch()
+    monkeypatch.setattr(ingest_route, "COMPLETIONS_API_KEY", TEST_API_KEY)
+    with make_client(tmp_path) as client:
+        resp = client.post("/api/ingest", json={}, headers=_auth_headers())
+        assert resp.status_code == 422
+    monkeypatch.undo()
+
+
+def test_ingest_success(tmp_path: Path, monkeypatch):
+    monkeypatch.setattr(ingest_route, "COMPLETIONS_API_KEY", TEST_API_KEY)
+
+    embed_count = 0
+
+    class FakeAsyncClient:
+        class FakeResponse:
+            def __init__(self, status, json_data=None):
+                self.status_code = status
+                self._json = json_data or {}
+
+            def json(self):
+                return self._json
+
+        def __init__(self, *a, **kw):
+            pass
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *a):
+            pass
+
+        async def post(self, url, **kw):
+            nonlocal embed_count
+            if "/api/embeddings" in url:
+                embed_count += 1
+                return self.FakeResponse(200, {"embedding": [0.1] * 768})
+            return self.FakeResponse(200)
+
+        async def put(self, url, **kw):
+            return self.FakeResponse(200)
+
+    monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **kw: FakeAsyncClient())
+
+    with make_client(tmp_path) as client:
+        resp = client.post("/api/ingest", json={"content": "test " * 1000, "source": "terminal"}, headers=_auth_headers())
+        assert resp.status_code == 200
+        data = resp.json()
+        assert data["source"] == "terminal"
+        assert data["chunks_ingested"] > 0
+        assert embed_count == data["chunks_ingested"]
+        assert "message" in data