diff --git a/app.py b/app.py index b378fd2..b3cb059 100644 --- a/app.py +++ b/app.py @@ -34,6 +34,7 @@ import routers.chat as chat import routers.search_route as search_route import routers.completions as completions import routers.upload as upload +import routers.ingest as ingest # --- Logging --- log = logging.getLogger("jarvischat") @@ -102,7 +103,7 @@ async def session_auth_middleware(request: Request, call_next): unauth_paths = { "/api/auth/login", "/api/auth/logout", "/api/auth/session", - "/api/auth/heartbeat", "/api/auth/guest", + "/api/auth/heartbeat", "/api/auth/guest", "/api/ingest", } if path.startswith("/api/"): @@ -141,7 +142,7 @@ async def index(request: Request): for router_module in [ auth_router, conversations.router, memories.router, models.router, presets.router, profile.router, settings.router, skills.router, - chat.router, search_route.router, completions.router, upload.router, + chat.router, search_route.router, completions.router, upload.router, ingest.router, ]: app.include_router(router_module) diff --git a/config.py b/config.py index f9a4bec..75c6a25 100644 --- a/config.py +++ b/config.py @@ -9,7 +9,7 @@ import logging log = logging.getLogger("jarvischat") -VERSION = "v1.10.0" +VERSION = "v1.11.0" OLLAMA_BASE = os.environ.get("OLLAMA_BASE", "http://localhost:11434") LLAMA_SERVER_BASE = os.environ.get("LLAMA_SERVER_BASE", "http://192.168.50.108:8081") SEARXNG_BASE = "http://localhost:8888" diff --git a/docs/jc-ingest.sh b/docs/jc-ingest.sh new file mode 100644 index 0000000..6460508 --- /dev/null +++ b/docs/jc-ingest.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# jc-ingest.sh — pipe terminal commands into jarvisChat RAG +# Deploy to /home/gramps/bin/jc-ingest.sh on jarvis (192.168.50.210) +# +# Usage: +# 1. chmod +x /home/gramps/bin/jc-ingest.sh +# 2. Add to ~/.bashrc: +# export JARVISCHAT_COMPLETIONS_API_KEY="$(cat /opt/jarvischat/.completions_key)" +# export PROMPT_COMMAND="jc_capture" +# source /home/gramps/bin/jc-ingest.sh +# +# The PROMPT_COMMAND hook runs jc_capture() after each command. +# Only commands matching the filter pattern are ingested. +# +# Filter: currently captures git, pip, systemctl, sudo, vi/vim, curl, +# wget, apt, python, pytest commands. Edit the grep pattern to adjust. + +JC_URL="http://192.168.50.210:8080/api/ingest" +JC_TOKEN="${JARVISCHAT_COMPLETIONS_API_KEY}" + +jc_capture() { + local cmd + cmd=$(history 1 | sed 's/^[ ]*[0-9]*[ ]*//') + if echo "$cmd" | grep -qE '^(git|pip|systemctl|sudo|vi|vim|curl|wget|apt|python|pytest)'; then + curl -s -X POST "$JC_URL" \ + -H "Authorization: Bearer $JC_TOKEN" \ + -H "Content-Type: application/json" \ + -d "{\"content\": $(echo "$cmd" | python3 -c 'import json,sys; print(json.dumps(sys.stdin.read().strip()))'), \"source\": \"terminal\"}" \ + > /dev/null 2>&1 & + fi +} diff --git a/routers/ingest.py b/routers/ingest.py new file mode 100644 index 0000000..82aa5d5 --- /dev/null +++ b/routers/ingest.py @@ -0,0 +1,64 @@ +"""JarvisChat routers - /api/ingest terminal command RAG hook.""" +import logging +from datetime import datetime, timezone + +import httpx +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import JSONResponse + +from config import COMPLETIONS_API_KEY +from rag import chunk_text, QDRANT_URL, EMBED_URL, EMBED_MODEL, RAG_COLLECTION + +log = logging.getLogger("jarvischat") +router = APIRouter() + + +def _check_api_key(request: Request): + auth = request.headers.get("Authorization", "") + if not auth.startswith("Bearer "): + raise HTTPException(status_code=401, detail="Missing Bearer token") + token = auth[7:].strip() + if token != COMPLETIONS_API_KEY: + raise HTTPException(status_code=401, detail="Invalid API key") + + +@router.post("/api/ingest") +async def ingest_content(request: Request): + _check_api_key(request) + body = await request.json() + content = (body.get("content") or "").strip() + if not content: + raise HTTPException(status_code=422, detail="content is required") + source = str(body.get("source", "external")).strip() or "external" + metadata = body.get("metadata") or {} + + chunks = chunk_text(content) + if not chunks: + raise HTTPException(status_code=422, detail="content produced no chunks") + + ingested = 0 + async with httpx.AsyncClient() as client: + for i, chunk in enumerate(chunks): + embed_resp = await client.post( + f"{EMBED_URL}/api/embeddings", + json={"model": EMBED_MODEL, "prompt": chunk}, + timeout=30.0, + ) + if embed_resp.status_code != 200: + log.warning(f"Ingest embedding failed for chunk {i}: {embed_resp.status_code}") + continue + vector = embed_resp.json()["embedding"] + point_id = f"ingest-{source}-{datetime.now(timezone.utc).timestamp()}-{i}" + payload = {"text": chunk, "source": source, "ingest_date": datetime.now(timezone.utc).isoformat(), "type": "ingest"} + payload.update(metadata) + upsert_resp = await client.put( + f"{QDRANT_URL}/collections/{RAG_COLLECTION}/points?wait=true", + json={"points": [{"id": point_id, "vector": vector, "payload": payload}]}, + timeout=30.0, + ) + if upsert_resp.status_code in (200, 201): + ingested += 1 + else: + log.warning(f"Ingest Qdrant upsert failed for chunk {i}: {upsert_resp.status_code}") + + return {"chunks_ingested": ingested, "source": source, "message": f"Ingested {ingested} chunks from {source}"} diff --git a/tests/test_ingest.py b/tests/test_ingest.py new file mode 100644 index 0000000..dcaa22b --- /dev/null +++ b/tests/test_ingest.py @@ -0,0 +1,104 @@ +import json +import os +from pathlib import Path + +import httpx +from fastapi.testclient import TestClient + +import app +import db +import routers.ingest as ingest_route +from security import SESSIONS, PIN_ATTEMPTS, RATE_EVENTS + + +def make_client(tmp_path: Path) -> TestClient: + os.environ["JARVISCHAT_ADMIN_PIN"] = "1234" + db.DB_PATH = tmp_path / "jarvischat-ingest.db" + SESSIONS.clear() + PIN_ATTEMPTS.clear() + RATE_EVENTS.clear() + db.init_db() + return TestClient(app.app, raise_server_exceptions=False) + + +TEST_API_KEY = "test-sk-jarvischat-ingest" + + +def _auth_headers() -> dict: + return {"Authorization": f"Bearer {TEST_API_KEY}", "Content-Type": "application/json", "Origin": "http://testserver"} + + +def test_ingest_missing_api_key(tmp_path: Path): + with make_client(tmp_path) as client: + resp = client.post("/api/ingest", json={"content": "test"}, headers={"Origin": "http://testserver"}) + assert resp.status_code == 401 + + +def test_ingest_wrong_api_key(tmp_path: Path): + with make_client(tmp_path) as client: + resp = client.post("/api/ingest", json={"content": "test"}, + headers={"Authorization": "Bearer wrong", "Origin": "http://testserver"}) + assert resp.status_code == 401 + + +def test_ingest_empty_content(tmp_path: Path): + monkeypatch = __import__('pytest').MonkeyPatch() + monkeypatch.setattr(ingest_route, "COMPLETIONS_API_KEY", TEST_API_KEY) + with make_client(tmp_path) as client: + resp = client.post("/api/ingest", json={"content": ""}, headers=_auth_headers()) + assert resp.status_code == 422 + monkeypatch.undo() + + +def test_ingest_missing_content(tmp_path: Path): + monkeypatch = __import__('pytest').MonkeyPatch() + monkeypatch.setattr(ingest_route, "COMPLETIONS_API_KEY", TEST_API_KEY) + with make_client(tmp_path) as client: + resp = client.post("/api/ingest", json={}, headers=_auth_headers()) + assert resp.status_code == 422 + monkeypatch.undo() + + +def test_ingest_success(tmp_path: Path, monkeypatch): + monkeypatch.setattr(ingest_route, "COMPLETIONS_API_KEY", TEST_API_KEY) + + embed_count = 0 + + class FakeAsyncClient: + class FakeResponse: + def __init__(self, status, json_data=None): + self.status_code = status + self._json = json_data or {} + + def json(self): + return self._json + + def __init__(self, *a, **kw): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + pass + + async def post(self, url, **kw): + nonlocal embed_count + if "/api/embeddings" in url: + embed_count += 1 + return self.FakeResponse(200, {"embedding": [0.1] * 768}) + return self.FakeResponse(200) + + async def put(self, url, **kw): + return self.FakeResponse(200) + + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **kw: FakeAsyncClient()) + + with make_client(tmp_path) as client: + resp = client.post("/api/ingest", json={"content": "test " * 1000, "source": "terminal"}, headers=_auth_headers()) + assert resp.status_code == 200 + data = resp.json() + assert data["source"] == "terminal" + assert data["chunks_ingested"] > 0 + assert embed_count == data["chunks_ingested"] + assert "message" in data