Compare commits
16 Commits
2685a73897
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| e14ae2bd19 | |||
| 44387919a8 | |||
| df405a156e | |||
| aecd3330fd | |||
| 576d9333b3 | |||
| 70014f8e3b | |||
| 5b5fbab206 | |||
| 49f8a50c5b | |||
| c14e3c19a9 | |||
| 056ebc399f | |||
| 55b9a2236d | |||
| 413f850e41 | |||
| 8fd6c99ccc | |||
| 54cca366a4 | |||
| 666a237a8b | |||
| a99345edb6 |
@@ -0,0 +1,44 @@
|
||||
# Version control
|
||||
.git/
|
||||
.gitignore
|
||||
|
||||
# Python
|
||||
venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
|
||||
# Databases and state (persisted via volumes)
|
||||
*.db
|
||||
hardware_state.json
|
||||
|
||||
# Secrets (generated by setup.sh)
|
||||
.env
|
||||
secrets/
|
||||
searxng/
|
||||
|
||||
# IDE / editor
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Models (bind-mounted at runtime)
|
||||
models/
|
||||
|
||||
# Documentation (not needed in image)
|
||||
*.md
|
||||
docs/
|
||||
TASKS.md
|
||||
docker.md
|
||||
ai.md
|
||||
CLAUDE.md
|
||||
scripts/
|
||||
tests/
|
||||
@@ -0,0 +1,59 @@
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
# cAIc — Environment configuration
|
||||
# Copy to .env and fill in values before running docker compose.
|
||||
# Secrets (lines marked 🔒) should be random — generate with:
|
||||
# openssl rand -hex 20
|
||||
# ─────────────────────────────────────────────────────────────
|
||||
|
||||
# ── Secrets ──────────────────────────────────────────────────
|
||||
# 🔒 Admin PIN (4 digits, required for first login)
|
||||
CAIC_ADMIN_PIN=
|
||||
CAIC_ALLOW_DEFAULT_PIN=false
|
||||
|
||||
# 🔒 API key for IDE completions endpoint (auto-gen if empty)
|
||||
CAIC_COMPLETIONS_API_KEY=
|
||||
|
||||
# 🔒 RabbitMQ password (must match secrets/rabbitmq_password.txt)
|
||||
RABBITMQ_PASSWORD=
|
||||
|
||||
# 🔒 SearXNG session key
|
||||
SEARXNG_SECRET_KEY=
|
||||
|
||||
# ── Service discovery (Docker service hostnames) ─────────────
|
||||
LLAMA_SERVER_BASE=http://llama-server:8081
|
||||
OLLAMA_BASE=http://ollama:11434
|
||||
CAIC_SEARXNG_BASE=http://searxng:8080
|
||||
CAIC_QDRANT_URL=http://qdrant:6333
|
||||
CAIC_EMBED_URL=http://ollama:11434
|
||||
CAIC_EMBED_MODEL=all-minilm:latest
|
||||
CAIC_AMQP_URL=amqp://caic:${RABBITMQ_PASSWORD}@rabbitmq:5672/caic
|
||||
|
||||
# ── llama-server settings ────────────────────────────────────
|
||||
# GGUF filename (must exist in ./models/)
|
||||
LLAMA_MODEL=
|
||||
# Model name the app presents to clients (must match a loaded model)
|
||||
CAIC_DEFAULT_MODEL=qwen2.5-7b-instruct
|
||||
LLAMA_CTX_SIZE=4096
|
||||
LLAMA_N_GPU_LAYERS=0
|
||||
LLAMA_RPC_ENDPOINTS=
|
||||
|
||||
# ── RAG / Qdrant ─────────────────────────────────────────────
|
||||
CAIC_RAG_COLLECTION=caic_rag
|
||||
CAIC_RAG_MAX_VECTORS=50000
|
||||
|
||||
# ── Network / security ───────────────────────────────────────
|
||||
CAIC_ALLOWED_CIDRS=127.0.0.0/8,::1/128,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
|
||||
CAIC_TRUSTED_ORIGINS=
|
||||
CAIC_TRUST_X_FORWARDED_FOR=false
|
||||
|
||||
# ── Port overrides (host-side publish) ───────────────────────
|
||||
CAIC_EXPOSE_PORT=8080
|
||||
SEARXNG_EXPOSE_PORT=8888
|
||||
QDRANT_EXPOSE_PORT=6333
|
||||
RABBITMQ_EXPOSE_PORT=5672
|
||||
LLAMA_EXPOSE_PORT=8081
|
||||
OLLAMA_EXPOSE_PORT=11434
|
||||
|
||||
# ── Image generation (ComfyUI on worker) ─────────────────────
|
||||
CAIC_COMFYUI_BASE=http://localhost:8188
|
||||
CAIC_COMFYUI_TIMEOUT=120
|
||||
@@ -5,6 +5,10 @@ Detailed project context, work state, architecture, and configuration have moved
|
||||
## Quick start
|
||||
|
||||
```bash
|
||||
# Docker (recommended)
|
||||
scripts/setup.sh # first run: generates .env, secrets, pulls default model
|
||||
docker compose up -d
|
||||
|
||||
# Development
|
||||
./venv/bin/uvicorn app:app --host 0.0.0.0 --port 8080 --reload
|
||||
|
||||
@@ -17,7 +21,9 @@ sudo systemctl restart caic
|
||||
|
||||
## Dependencies
|
||||
|
||||
Docker deployment: no manual pip install needed — the Dockerfile handles it.
|
||||
|
||||
```bash
|
||||
./venv/bin/pip install -r requirements.txt
|
||||
# Also requires: psutil jinja2 python-multipart pypdf (not in requirements.txt)
|
||||
# Also requires: psutil jinja2 python-multipart pypdf
|
||||
```
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
# cAIc — FastAPI application
|
||||
# Multi-stage build for smaller production image
|
||||
|
||||
# ── Stage 1: build ──────────────────────────────────────────
|
||||
FROM python:3.13-slim-bookworm AS builder
|
||||
WORKDIR /build
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir --prefix=/install -r requirements.txt
|
||||
|
||||
# ── Stage 2: runtime ────────────────────────────────────────
|
||||
FROM python:3.13-slim-bookworm
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=builder /install /usr/local
|
||||
|
||||
WORKDIR /app
|
||||
COPY . .
|
||||
|
||||
# Persist DB and uploads outside the code layer
|
||||
ENV CAIC_DB_PATH=/app/data/caic.db \
|
||||
CAIC_UPLOAD_DIR=/app/data/uploads \
|
||||
CAIC_HOST=0.0.0.0 \
|
||||
CAIC_PORT=8080 \
|
||||
CAIC_SYSLOG_ADDRESS=""
|
||||
|
||||
RUN mkdir -p /app/data
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \
|
||||
CMD curl -fs http://localhost:8080/ || exit 1
|
||||
|
||||
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080"]
|
||||
@@ -1,584 +1,148 @@
|
||||

|
||||
|
||||
# cAIc v0.22.0
|
||||
# cAIc v1.1.0
|
||||
|
||||
Consumer AI hardware is a wasteland of incompatibility. NVIDIA speaks CUDA, AMD speaks ROCm. Your RTX 5070 Ti lives in one machine with 16 GB VRAM; your RX 6600 XT lives in another with 12 GB. Alone, neither can run a 14B model at usable speed. Together, they could — if the software stack didn't treat heterogeneous hardware as a bug instead of a feature.
|
||||
**Cluster AI coordinator — heterogeneous GPU inference for homelab AI clusters.**
|
||||
|
||||
The industry consensus — llama.cpp RPC, vLLM, TensorFlow distributed — all assume a homogeneous cluster: same GPU vendor, same VRAM, same driver stack, reachable over a fast fabric. This assumption works for data centers that buy 64 identical H100s at a time. It does not work for the person who has a gaming PC with an NVIDIA card in the living room, an AMD-powered home server in the closet, and an old MacBook on the desk. That person has more aggregate compute than any single consumer machine, but no software stack can make it cooperate.
|
||||
Your RX 6600 XT can't run a 14B model. Your RTX 5070 Ti can. Your old MacBook can run the small stuff. Alone, each box is limited. Together, they're a cluster — if the software lets them cooperate.
|
||||
|
||||
cAIc is a cluster orchestration layer that fuses mismatched GPUs, CPUs, and machines into a single inference surface. The advantage isn't just compatibility — it's matching each request to the hardware best suited for it. The coordinator (CPU-only, no discrete GPU) handles all CPU-bound work: RAG embedding, query triage, web search, memory, conversation storage, the message broker, and the web UI itself. Workers (discrete GPU) do nothing but inference — no database, no browser sessions, no orchestration overhead stealing VRAM. Triage classifies each query and routes to the node running the optimal model; if the right model isn't loaded, the coordinator requests a swap and the worker handles it asynchronously. Every machine contributes what it does best.
|
||||
cAIc makes them cooperate.
|
||||
|
||||
You might also be doing this with retired office PCs and GPUs from the Obama era. That works too. But the core problem cAIc solves isn't budget reuse — it's making non-homogeneous hardware cooperate.
|
||||
## The Problem
|
||||
|
||||
### Paired Programming
|
||||
Every distributed inference tool — llama.cpp RPC, vLLM, exo — assumes you have identical GPUs. Same vendor, same VRAM, same drivers. That assumption works for data centers with 64 identical H100s. It doesn't work for your homelab with an AMD card in the server, an NVIDIA card in the gaming PC, and a MacBook on the desk.
|
||||
|
||||
Every line of code in this repository was written by an AI (Claude, via opencode). But AI does not architect, design, test, deploy, or decide what to build — that requires experience, judgement, and the discipline to say "no" to feature creep.
|
||||
You have more aggregate compute than any single consumer machine. The software just can't see it that way.
|
||||
|
||||
**Gramps** (BS Computer Science, Oklahoma State, coding since 1981) performed that role — designing the architecture, managing the development process, writing and maintaining the test suite, operating the deployment pipeline, and directing every feature decision across dozens of sessions spanning months. Without that human steering, this would be yet another AI-generated repo that compiles but doesn't solve a real problem. With it, cAIc ships as a functional, tested, deployed system that runs 24/7 on real hardware serving real users.
|
||||
## How cAIc Solves It
|
||||
|
||||
This is paired programming, elevated: the AI handles the mechanical work of code generation; the human brings decades of systems-level experience, architectural judgment, and the maturity to ship something that lasts.
|
||||
cAIc uses **query-routing** instead of layer-splitting. Each machine runs a complete model on its own GPU. When a query comes in, the coordinator classifies it and routes the *whole request* to the best-suited node — code questions to the coder model, general chat to the instruct model. No layer sharing, no straggler problem, no VRAM negotiation between mismatched GPUs.
|
||||
|
||||
### Architecture: CPU Coordinator + GPU Workers
|
||||
|
||||
cAIc splits the workload across two machine roles:
|
||||
|
||||
**Coordinator** (ultron — Ryzen 7 7840HS, no discrete GPU) runs the FastAPI app, RAG vector search (Qdrant), text embedding (Ollama on CPU), query triage (Phi-4-mini), web search (SearXNG), message broker (RabbitMQ), and all SQLite-backed services — memory, profiles, conversations, settings. Every CPU-bound task stays here.
|
||||
|
||||
**Workers** (jarvis — RX 6600 XT 8 GB / corsair — RTX 5070 Ti 16 GB) run only llama-server for GPU inference. The coordinator never touches a model; workers never touch the database. Workers register via AMQP, receive ping/pong health checks, and accept model-swap commands when triage determines a different model is needed for the current query.
|
||||
|
||||
This split keeps the UI responsive during inference (the coordinator isn't blocked by GPU compute) and lets workers focus VRAM entirely on model weights rather than browser sessions or API orchestration.
|
||||
|
||||
### Single-Node Deployment (Experimental)
|
||||
|
||||
cAIc can also run entirely on one machine with all services colocated — coordinator, llama-server, Qdrant, SearXNG, and RabbitMQ all on localhost. This is useful for testing, laptops, or WSL2 under Windows 11.
|
||||
|
||||
To deploy single-node, override the remote service URLs:
|
||||
|
||||
```bash
|
||||
export CAIC_QDRANT_URL=http://localhost:6333
|
||||
export CAIC_EMBED_URL=http://localhost:11434
|
||||
export CAIC_EMBED_MODEL=mxbai-embed-large
|
||||
export CAIC_SEARXNG_BASE=http://localhost:8888
|
||||
export LLAMA_SERVER_BASE=http://localhost:8081
|
||||
export CAIC_NODE_NAME=$(hostname)
|
||||
export CAIC_UPLOAD_DIR=/tmp/caic_uploads
|
||||
export CAIC_DB_PATH=/opt/caic/caic.db
|
||||
export CAIC_HOST=0.0.0.0
|
||||
export CAIC_PORT=8080
|
||||
# AMQP URL is already configurable via CAIC_AMQP_URL or CAIC_AMQP_SECRET_PATH
|
||||
# Syslog: set CAIC_SYSLOG_ADDRESS to /dev/log (Linux), empty to disable, or a remote address
|
||||
```
|
||||
┌──────────────────────────────────────────────────────┐
|
||||
│ docker compose stack │
|
||||
│ │
|
||||
│ ┌──────────┐ ┌────────┐ ┌────────────────────┐ │
|
||||
│ │ SearXNG │ │ Qdrant │ │ RabbitMQ │ │
|
||||
│ │ :8888 │ │ :6333 │ │ :5672 / :15672 │ │
|
||||
│ └────┬─────┘ └───┬────┘ └─────────┬──────────┘ │
|
||||
│ │ │ │ │
|
||||
│ ▼ ▼ ▼ │
|
||||
│ ┌───────────────────────────────────────────────┐ │
|
||||
│ │ cAIc (FastAPI) │ │
|
||||
│ │ :8080 (HTTP) │ │
|
||||
│ └───────┬──────────────────┬────────────────────┘ │
|
||||
│ │ │ │
|
||||
│ ▼ ▼ │
|
||||
│ ┌────────────────┐ ┌────────────────┐ │
|
||||
│ │ llama-server │ │ Ollama │ │
|
||||
│ │ :8081 │ │ :11434 │ │
|
||||
│ │ (GPU/RPC) │ │ (embeddings) │ │
|
||||
│ └────────────────┘ └────────────────┘ │
|
||||
└──────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
All services degrade gracefully if unreachable — RAG, search, cluster, and triage log warnings and continue. Only llama-server (inference) is strictly required.
|
||||
**Coordinator** (CPU-only, no GPU) handles the web UI, RAG embedding, query triage, web search, memory, conversation storage, and the message broker. Every CPU-bound task stays here so it never competes with inference for GPU resources.
|
||||
|
||||
Untested: Windows 11 / WSL2 (Debian). The codebase is pure Python with no platform-specific dependencies beyond `rocm-smi` (AMD GPU stats, gracefully absent) and `system_profiler` (macOS, absent on Linux/WSL). llama.cpp builds and runs on WSL2 with NVIDIA GPU passthrough.
|
||||
**Workers** (discrete GPU) run only llama-server. No database, no browser sessions, no orchestration overhead. They register via AMQP, respond to health checks, and accept model-swap commands when the coordinator needs a different model for the current query.
|
||||
|
||||
Under the hood: FastAPI + SQLite + Jinja2 on Python 3.13. AMQP-mediated cluster coordination with an OpenAI-compatible inference endpoint.
|
||||
A worker with a slow GPU still contributes — it handles less latency-sensitive queries or batch work while the fast GPU handles interactive chat.
|
||||
|
||||
### Query-routing vs. layer-splitting — why it matters
|
||||
## What You Get
|
||||
|
||||
Most distributed inference tools (llama.cpp RPC, vLLM with tensor parallelism, exo) split a *single model* across multiple GPUs. The first GPU runs layers 0–15, the second runs 16–31, and so on. This works well in a homogeneous cluster where every GPU is identical, but in a heterogeneous setup the slowest card sets the pace — every forward pass waits for the straggler. Communication overhead between GPUs (NCCL, RPC) adds latency too.
|
||||
- **Clustered inference** across mismatched GPUs and machines — AMD, NVIDIA, Apple Silicon, CPU-only
|
||||
- **Automatic query routing** — triage classifies each query and routes to the best node
|
||||
- **Dynamic model swapping** — coordinator requests model changes on workers when needed
|
||||
- **RAG with auto-eviction** — Qdrant-backed vector search with score-based corpus management
|
||||
- **Persistent memory** — FTS5-backed memory that learns your preferences over time
|
||||
- **Web search** — SearXNG integration for automatic lookups when the model is uncertain
|
||||
- **Private Chat mode** — toggle to keep nothing on disk: no memory, no RAG, no search, no persistence
|
||||
- **At-rest encryption** — AES-256-GCM on all query-derived text in SQLite and Qdrant
|
||||
- **IDE integration** — OpenAI-compatible `/v1/chat/completions` endpoint for Continue.dev and friends
|
||||
- **OpenAI-compat FIM** — `/v1/fim/completions` for code completion
|
||||
- **Image generation** — ComfyUI-backed image gen via cluster workers (Stable Diffusion / Flux)
|
||||
- **6 color themes** — IBM Blue, Matrix, Dark, Light, Amber, Trippin
|
||||
- **Docker-ready** — `docker compose up -d` and you're running
|
||||
|
||||
cAIc takes a different approach: **query-routing**. Each worker runs a complete model on its own GPU. When a query comes in, triage classifies it and routes the *whole request* to the worker best suited for it — code questions go to the worker with a coder model, general chat goes to the instruct model. No layer sharing, no lockstep, no straggler problem. The tradeoff is that no single query can use combined VRAM across multiple GPUs, but the throughput and responsiveness of the cluster as a whole isn't dragged down by the weakest link.
|
||||
## By the Numbers
|
||||
|
||||
This also means a worker with a slow GPU can still contribute meaningfully — it handles less latency-sensitive queries or batch background work, while the fast GPU handles interactive chat.
|
||||
237 commits. 9,354 lines of Python. 214 tests. 95 files. One developer and an AI, March to July 2026.
|
||||
|
||||
### Data Safety
|
||||
cAIc went from initial commit to v1.0.0 in four and a half months. Every line of code was generated by Claude via opencode — but the architecture, test suite, deployment pipeline, and every feature decision were directed by a single developer with 40+ years of systems experience. The AI wrote the code; the human made it ship.
|
||||
|
||||
## Quick Start (Docker)
|
||||
|
||||
```bash
|
||||
git clone https://github.com/mikeshallop/caic.git && cd caic
|
||||
scripts/setup.sh # generates .env, secrets, pulls default model (~4.6GB)
|
||||
docker compose up -d # boots cAIc + Qdrant + RabbitMQ + SearXNG + llama-server + Ollama
|
||||
```
|
||||
|
||||
The setup wizard auto-generates secrets, detects disk space, downloads a default model, and configures all service hostnames. Point a browser at `http://localhost:8080` and you're chatting.
|
||||
|
||||
Requires: Docker Engine + Compose plugin. Place your own `.gguf` models in `./models/` for different sizes/vendors.
|
||||
|
||||
### Default Model
|
||||
|
||||
The setup wizard downloads **Qwen2.5-7B-Instruct** (Q4_K_M quantization, ~4.6 GB) as the default inference model.
|
||||
|
||||
Why this model:
|
||||
|
||||
- **Fits in 6 GB VRAM** — runs on mid-range GPUs (RX 6600 XT, RTX 3060, etc.) without offloading
|
||||
- **Instruction-tuned** — handles chat, code, and reasoning without fine-tuning
|
||||
- **Q4_K_M quantization** — best balance of quality and speed for consumer hardware; loses less than 1% accuracy vs. FP16 while fitting in half the VRAM
|
||||
- **GGUF format** — runs natively in llama.cpp (the worker backend) with no conversion step
|
||||
|
||||
Swap it for any `.gguf` model you prefer. cAIc's query-routing works with whatever you put in `./models/` — the coordinator doesn't care which model runs where, as long as the workers can serve it.
|
||||
|
||||
→ [Installation Guide](https://github.com/mikeshallop/caic/wiki/Installation) | [Configuration](https://github.com/mikeshallop/caic/wiki/Home) | [Bare-Metal Install](https://github.com/mikeshallop/caic/wiki/Installation)
|
||||
|
||||
## Single-Node Mode
|
||||
|
||||
cAIc also runs entirely on one machine — coordinator, llama-server, Qdrant, SearXNG, and RabbitMQ all on localhost. Useful for testing, laptops, or WSL2 under Windows 11.
|
||||
|
||||
All services degrade gracefully if unreachable. Only llama-server (inference) is strictly required.
|
||||
|
||||
## Why Query-Routing?
|
||||
|
||||
Most distributed inference splits a *single model* across GPUs — GPU 1 runs layers 0–15, GPU 2 runs 16–31. That works with identical cards. With mixed hardware, the slowest GPU sets the pace for every forward pass.
|
||||
|
||||
cAIc routes *whole queries* instead. Each worker runs a complete model. Triage picks the right worker. No layer sharing, no lockstep, no straggler dragging down the cluster.
|
||||
|
||||
| | Layer-splitting | cAIc query-routing |
|
||||
|---|---|---|
|
||||
| **Hardware** | Identical GPUs required | Any mix — AMD, NVIDIA, Apple, CPU |
|
||||
| **Bottleneck** | Slowest GPU per forward pass | None — each node runs independently |
|
||||
| **Model swap** | N/A (one model split) | Async swap per worker |
|
||||
| **Scale** | Add VRAM to one model | Add machines, each contributes fully |
|
||||
|
||||
## Data Safety
|
||||
|
||||
| Concern | How cAIc handles it |
|
||||
|---------|---------------------|
|
||||
| **Queries stored on disk?** | All query-derived text is encrypted at rest with AES-256-GCM before touching SQLite or Qdrant. Toggle **Private Chat** (topbar badge) and nothing touches disk at all: no SQLite writes, no FTS5 memory injection, no RAG ingestion, no external SearXNG queries. |
|
||||
| **Queries sent to external services?** | SearXNG web search is optional and disabled in Private Chat. All other services (llama-server, Qdrant, RabbitMQ) run on your own LAN. |
|
||||
| **Inter-node traffic unencrypted?** | No — WireGuard tunnels encrypt all coordinator↔worker traffic (AMQP, inference, RPC) at the network layer. Zero application changes. |
|
||||
| **Who can access the server?** | Guest sessions for anyone on the LAN. Admin access protected by a PBKDF2-hashed 4-digit PIN with rate-limited attempts. IP allowlist (CIDR) gate optional. |
|
||||
| **Queries on disk?** | AES-256-GCM encrypted at rest. Private Chat mode = nothing touches disk at all. |
|
||||
| **External services?** | SearXNG is optional and disabled in Private Chat. Everything else runs on your LAN. |
|
||||
| **Inter-node traffic?** | WireGuard tunnels encrypt all coordinator↔worker traffic. Zero application changes. |
|
||||
| **Access control?** | Guest sessions for LAN. Admin PIN (PBKDF2-hashed, rate-limited). Optional IP allowlist. |
|
||||
|
||||
At v1.0, this ships with a Docker compose stack and setup wizard that detect CPU vs GPU, probe your hardware, and stand up SearXNG, Qdrant, RabbitMQ, and everything else with a single `docker compose up`. The same install docs work bare-metal for those who prefer to skip containers entirely.
|
||||
## Built With
|
||||
|
||||
Developer wiki: [Home](https://llgit.llamachile.tube/gramps/cAIc/wiki/Home) — includes [FAQ](https://llgit.llamachile.tube/gramps/cAIc/wiki/FAQ), [Installation Guide](https://llgit.llamachile.tube/gramps/cAIc/wiki/Installation), and [full architecture docs](https://llgit.llamachile.tube/gramps/cAIc/wiki/Developer-Architecture)
|
||||
FastAPI + SQLite + Jinja2 on Python 3.13. AMQP-mediated cluster coordination via aio-pika. Qdrant for vector search. OpenAI-compatible inference endpoint via llama.cpp server.
|
||||
|
||||
## What's New in v0.22.0
|
||||
214 tests. All use `tmp_path` fixtures + monkeypatched HTTP clients. No external services needed.
|
||||
|
||||
### Color Theme System
|
||||
- Palette icon in topbar opens dropdown with 6 themes: IBM Blue, Green Ln (Matrix terminal), Dark, Light, Amber (fallout terminal), Trippin (neon rave).
|
||||
- All CSS variables are dynamically swapped; choice persists in `localStorage`.
|
||||
- Existing `:root` CSS variable architecture made this trivial — no CSS changes needed beyond the dropdown styles.
|
||||
## Documentation
|
||||
|
||||
### RAG Corpus Management UI (B4)
|
||||
- New admin modal (RAG button in drawer header) to browse, search, edit, and delete individual RAG corpus entries.
|
||||
- **Endpoints**: `GET /api/rag/points` (paginated list with semantic search and source filter), `GET /api/rag/point/{id}` (single point detail), `DELETE /api/rag/point/{id}` (single point deletion), `PATCH /api/rag/point/{id}` (edit text with re-embed).
|
||||
- **Frontend**: Stats bar (vector count, % full, pinned, avg retrievals, at-risk, eviction rate), semantic search bar, source filter dropdown, paginated results table, per-row edit/delete with confirmation, double-confirm bulk flush.
|
||||
- All endpoints admin-protected; text decrypted for display, re-encrypted on edit.
|
||||
- 14 new tests (32 total in test_rag_management.py). 214 tests pass overall.
|
||||
- Renamed `AGENTS.md` → `ai.md` for tool-agnostic project context. `CLAUDE.md` now points to `ai.md`.
|
||||
| Page | What's there |
|
||||
|------|-------------|
|
||||
| [Home](https://github.com/mikeshallop/caic/wiki) | Overview, FAQ, links |
|
||||
| [Installation](https://github.com/mikeshallop/caic/wiki/Installation) | Docker + bare-metal walkthrough, config reference |
|
||||
| [Architecture](https://github.com/mikeshallop/caic/wiki/Developer-Architecture) | Coordinator/worker design, AMQP protocol, module map |
|
||||
| [Screenshots](https://github.com/mikeshallop/caic/wiki/Screenshots) | UI gallery |
|
||||
|
||||
## What's New in v0.21.0
|
||||
## Changelog
|
||||
|
||||
### Scrollbar + DOM Fixes
|
||||
- Scrollbar hidden behind `.main::after` barcode strip — fixed by elevating `.chat-container` z-index above the pseudo-element overlay.
|
||||
- Scrollbar repositioned to the dark-blue channel between content and spool-hole strip via `margin-right: 28px`, widened to 10px.
|
||||
- `scrollToLatest()` now uses `requestAnimationFrame` so `scrollHeight` reflects rendered content — fixes "responses below viewport" during streaming.
|
||||
- Direction-aware `_userScrolledAway` guard: in `oldest` mode, detects scroll-away from bottom (not from top, which broke `newest` mode).
|
||||
- Removed `_userScrolledAway` guard from `oldest` branch to restore always-scroll-to-bottom behavior.
|
||||
|
||||
### Perplexity Persistence
|
||||
- New `perplexity REAL` column in `messages` table (auto-migration on existing DBs).
|
||||
- Assistant responses now store `perplexity` alongside content in all storage paths (chat, search, completions).
|
||||
- Loaded conversations display confidence badges from stored perplexity.
|
||||
|
||||
### DOM Pairing Bugfix
|
||||
- `appendMessage('assistant', ...)` was finding the **first** `.message.user` via `querySelector`, appending Q2's response to Q1's pair in multi-turn conversations.
|
||||
- Fixed by capturing `appendMessage('user', ...)` return value and passing the exact user element as `afterEl`.
|
||||
|
||||
### Config Overhaul — All Service URLs Now Env-Overridable
|
||||
| Env Var | Default | Purpose |
|
||||
|---------|---------|---------|
|
||||
| `CAIC_QDRANT_URL` | `http://192.168.50.108:6333` | Qdrant vector search |
|
||||
| `CAIC_EMBED_URL` | `http://192.168.50.210:11434` | Ollama embeddings |
|
||||
| `CAIC_EMBED_MODEL` | `mxbai-embed-large` | Embedding model name |
|
||||
| `CAIC_SEARXNG_BASE` | `http://localhost:8888` | SearXNG web search |
|
||||
| `CAIC_NODE_NAME` | `ultron` | Coordinator node name |
|
||||
| `CAIC_UPLOAD_DIR` | `/tmp/caic_uploads` | Upload temp directory |
|
||||
| `CAIC_DB_PATH` | `<cwd>/caic.db` | SQLite database path |
|
||||
| `CAIC_HOST` | `0.0.0.0` | uvicorn listen address |
|
||||
| `CAIC_PORT` | `8080` | uvicorn listen port |
|
||||
| `CAIC_SYSLOG_ADDRESS` | `/dev/log` | Syslog socket (empty=disable) |
|
||||
| `CAIC_AMQP_SECRET_PATH` | `/home/gramps/.caic_amqp_secret` | AMQP password file |
|
||||
| `CAIC_AMQP_URL` | (from secret file or default) | Full AMQP connection string |
|
||||
|
||||
### Bugfixes
|
||||
- `hardware.py` Qdrant health check was hardcoded to `192.168.50.108:6333`, bypassing `CAIC_QDRANT_URL` — now uses `QDRANT_URL` from config.
|
||||
- AMQP fallback password now logs a warning when the secret file is missing.
|
||||
|
||||
### Single-Node Deployment
|
||||
- Documented at `### Single-Node Deployment (Experimental)` — all services can colocate on localhost with the env vars above.
|
||||
- Untested: Windows 11 / WSL2 (Debian). No platform-specific code beyond gracefully-absent `rocm-smi` and `system_profiler`.
|
||||
|
||||
## What's New in v0.20.0
|
||||
|
||||
### At-Rest Encryption (Full Data Privacy)
|
||||
|
||||
All user query-derived text is now encrypted with AES-256-GCM before being written to disk. Every storage path is covered:
|
||||
|
||||
- **Conversations** — message content and titles encrypted in SQLite
|
||||
- **Memories** — FTS5 facts encrypted; search performs Python-side matching on decrypted text
|
||||
- **Upload context** — document text encrypted in SQLite
|
||||
- **RAG corpus** — chunk text encrypted in Qdrant payloads
|
||||
- **Completions (IDE integration)** — messages and titles encrypted
|
||||
|
||||
**Key management**: 256-bit key auto-generated on first boot, stored in the `settings` table as a non-obvious key name (`heartbeat_interval_ms`). Never exposed via any API endpoint. If the key is deleted, stored data is unrecoverable.
|
||||
|
||||
**Zero-trust boundary**: the encryption key lives in the same SQLite database as the encrypted data. This protects against filesystem-level access (stolen `.db` file, backup exposure, disk forensic recovery) but does not protect against runtime compromise (attacker with SQLite read access while the server is running, since decryption keys are in memory during requests).
|
||||
|
||||
## What's New in v0.19.3
|
||||
|
||||
### Private Chat Mode (B8)
|
||||
- **PRIVATE badge** — topbar toggle switches to private mode where nothing is persisted, no memory/RAG is injected, and web search is disabled
|
||||
- **Info popup** — click the (i) icon next to the badge for a full explanation of what private mode does and doesn't do
|
||||
- **No-storage guarantee** — conversation is streamed to the user but never touches SQLite, FTS5, or Qdrant
|
||||
- **Search blocked** — `/api/search` returns 403 in private mode; WEB button is disabled in the UI
|
||||
|
||||
### WireGuard In-Transit Encryption
|
||||
- WireGuard tunnels encrypt all inter-node traffic. See Data Safety section above.
|
||||
|
||||
## What's New in v0.19.2
|
||||
|
||||
### Waterfall Direction Toggle (B6) + UX Polish
|
||||
- **NEW/OLD toggle** — topbar button switches between newest-first (waterfall) and oldest-first (traditional chat)
|
||||
- **Direction-aware scroll** — newest-first scrolls to top, oldest-first scrolls to bottom; respects user scroll-away to avoid fighting
|
||||
- **localStorage persistence** — preference survives page reloads, default is newest-first (waterfall)
|
||||
- **Toast notifications** — slide-out notifications for copy, save, delete, rate actions
|
||||
- **Clipboard reliability** — `execCopy()` helper for HTTP fallback (`document.execCommand`) when `navigator.clipboard` fails on plain HTTP
|
||||
- **Model label** — `modelLabel()` derives shorthand display names (e.g. `qwen2.5:7B:i`)
|
||||
- **Keybinding fix** — Shift+Enter = newline, Ctrl+Enter = send (universal conventions)
|
||||
- **Token counter** — resets to 0 on page refresh, no longer persisted to localStorage
|
||||
|
||||
## What's New in v0.19.1
|
||||
|
||||
### Default Model Auto-Pull on First Start (B5)
|
||||
- **`model_pull.py`** — new module that checks if `default_model` is available on llama-server at startup, falls back to Ollama pull API if not found
|
||||
- **Startup integration** — `app.py` lifespan calls `ensure_model()` after `assess_hardware()`, pulling the missing model via Ollama's streaming pull API
|
||||
- **Idempotent** — skips pull if model already available on llama-server or Ollama
|
||||
|
||||
## What's New in v0.19.0
|
||||
|
||||
### Apple Silicon Worker Support (B7)
|
||||
- **`gpu.py`** — now detects `sys.platform == "darwin"` and parses `system_profiler SPDisplaysDataType` for GPU model/VRAM instead of `rocm-smi`
|
||||
- **`hardware.py`** — darwin branch via `_get_vram_darwin()` for VRAM assessment on macOS
|
||||
- **`node_agent/agent.py`** — `get_load()` reports VRAM via `system_profiler SPDisplaysDataType` on macOS workers
|
||||
- Hybrid detection — falls back to AMD rocm-smi on Linux, `available: False` if neither is detected
|
||||
|
||||
## What's New in v0.18.0
|
||||
|
||||
### Wiki — Installation Guide, Screenshots Gallery, Full Documentation
|
||||
- **New Installation & Configuration page** — bare-metal walkthrough, cluster setup, config reference, security checklist, 12 troubleshooting topics. Everything a new user needs to get cAIc running.
|
||||
- **Screenshots gallery** — clickable image gallery on the wiki Screenshots page
|
||||
- **Wiki fully populated** — 5 pages linked from Home, renders at root URL
|
||||
|
||||
### UX Polish — Waterfall Layout, Barcode Stripes, Confidence Badges
|
||||
- **Waterfall display** — newest messages at top via `prepend()`, scroll to top
|
||||
- **Barcode alternating pairs** — each Q&A wrapped in `.msg-pair` with alternating tint + left border accent
|
||||
- **Confidence % badge** (`1/ppl * 100`) replaces raw perplexity, color-coded green/orange/red
|
||||
- **Cumulative token counter (TOK)** in topbar center, persisted in `localStorage`
|
||||
- **TOK reformatted** to `# / %` — `#` is all-time tokens, `%` is last response's context-window percentage, color-coded
|
||||
- **Dot-matrix sprocket strips** on left/right edges of `.main` (24px strips, punch-hole pattern)
|
||||
- **Paper grain background** on chat container
|
||||
- **Timestamps on user messages** (`HH:MM`), later upgraded to `MON dd, YYYY HH:MM:SS.ss` centisecond precision
|
||||
- **Shift+Enter** triggers web search
|
||||
- **Typing indicator greys out** on abort
|
||||
- **Token count badge** on search responses using client-side `tokenCount`
|
||||
- **Removed status dots** from input area (no functional purpose)
|
||||
- **Removed thumbs** from toolbar, restored only on non-search AI responses
|
||||
|
||||
### Version bumped to v0.18.0
|
||||
|
||||
## What's New in v0.17.26
|
||||
|
||||
### Dynamic Model Swap — `request_model_swap()`, `select_node()` async (Roadmap N Task 14)
|
||||
- **`cluster.py`** — `request_model_swap()` publishes `cmd.swap_model` to `jc.admin`; `handle_model_ready()` and `handle_model_failed()` consume `model_ready`/`model_failed` on `jc.system`
|
||||
- **`select_node()` async** — Queries worker `inventory` for ideal model; triggers swap if model not active, returns `None` for fallback during swap
|
||||
- **`SUBSCRIBE_TABLE`** — 7 AMQP routing key bindings in cluster.py
|
||||
|
||||
### Cluster Status UI — Heartbeat + Live Status Panel (Roadmap N Task 15)
|
||||
- **`handle_heartbeat()`** — Consumes `node.*.heartbeat` on `jc.system` to update `last_seen` per node
|
||||
- **UI cluster panel** — sidebar polls `GET /api/cluster` every 15s; green=active, yellow=swapping, red=error/offline
|
||||
- **Version bumped to v0.17.0** — All 179 tests pass
|
||||
|
||||
### What's New in v0.14.0
|
||||
|
||||
### Cluster Protocol — `GET /api/cluster`, 9 AMQP Message Types (Roadmap N Task 11)
|
||||
- **`cluster.py`** — Node registry (`CLUSTER_NODES`), bounded event log (`CLUSTER_EVENTS`, max 1000), coordinator auto-promotion
|
||||
- **Ping/pong health** — No passive heartbeats; coordinator pings workers on-demand before routing work. 5s timeout → auto-deregister
|
||||
- **9 message types** — register, deregister, admitted, rejected, ping, pong (on `jc.admin`); event, coord_query, coord_response (on `jc.system`)
|
||||
- **`amqp.py` subscribe()** — Exclusive anonymous queues bound to routing keys; `_rebind_subscriptions()` recreates them on reconnect
|
||||
- **`routers/cluster.py`** — `GET /api/cluster` returns nodes, coordinator, event log
|
||||
|
||||
### RAG Corpus Management — `POST /api/rag/flush`, `GET /api/rag/stats` (v0.13.0)
|
||||
- **Score-based eviction** with hysteresis (80% high-water, 20% low-water) and pinned sources
|
||||
- **Eviction engine** in `eviction.py` — scroll Qdrant, score by retrieval count + age, evict lowest scores first
|
||||
- **Grace period** — vectors younger than 1 hour are never evicted
|
||||
- **Flush endpoint** — `POST /api/rag/flush` (admin) deletes all non-pinned vectors
|
||||
- **Stats endpoint** — `GET /api/rag/stats` (admin) returns vector count, at-risk count, pinned count, eviction rates
|
||||
|
||||
### File & Document Attachments (v1.9.0–v1.10.0)
|
||||
- **`POST /api/upload`** — multipart file upload with PDF/text extraction; modes: `context` (chat injection), `ingest` (RAG corpus), `both`
|
||||
- **`DELETE /api/upload/{id}`** — removes upload from SQLite + Qdrant
|
||||
- **`PATCH /api/upload/{id}/link`** — associates upload with a conversation
|
||||
- **`GET /api/upload/by-conversation/{id}`** — list attachments per conversation
|
||||
- **Paperclip UI** — file picker, preview pill, image thumbnails, gallery overlay
|
||||
- **Attachment indicators** — 📎 badge on conversations with attachments
|
||||
- **Chat context injection** — `upload_context_id` prepends document text to system prompt
|
||||
|
||||
### Terminal RAG Hook — `POST /api/ingest` (v0.11.0)
|
||||
- Bearer token auth (same key as `/v1/chat/completions`)
|
||||
- Chunking via shared `chunk_text()` helper, embed via Ollama, upsert to Qdrant
|
||||
- `caic-ingest.sh` — PROMPT_COMMAND shell script for autonomous terminal history ingestion
|
||||
|
||||
### v1.8.0 Foundation (refactor & fixes)
|
||||
- **Modular refactor** — single-file `app.py` split into `config.py`, `db.py`, `auth.py`, `security.py`, `memory.py`, `search.py`, `rag.py`, `gpu.py`, and `routers/` package
|
||||
- **Perplexity auto-search fixed** — `logprobs: true` now properly extracted from stream chunks
|
||||
- **All `/api/models` endpoints** target `LLAMA_SERVER_BASE` (llama-server) not Ollama
|
||||
- **RAG embedding** via Ollama at `http://192.168.50.210:11434`
|
||||
- **Origin check** applies to all API methods, rejects absent Origin/Referer
|
||||
|
||||
## Features
|
||||
|
||||
- **Persistent Memory** — SQLite FTS5 full-text search for fast, relevant memory retrieval
|
||||
- **Web Search** — SearXNG integration for automatic web lookups when the model is uncertain
|
||||
- **Explicit Search** — Search button to force web search without waiting for model uncertainty
|
||||
- **Profile Injection** — Custom system prompt injected into every conversation
|
||||
- **System Presets** — Save and switch between different system prompts
|
||||
- **Real-time Stats** — CPU, RAM, GPU, VRAM monitoring in sidebar
|
||||
- **Token Thermometer** — Visual context window usage indicator
|
||||
- **Streaming Responses** — Server-sent events for real-time token display
|
||||
- **Conversation History** — SQLite-backed chat persistence with mass-delete option
|
||||
- **Model Switching** — Change inference models on the fly
|
||||
- **Skills Framework** — Built-in skill registry with per-skill enable/disable controls
|
||||
- **Private Chat** — Toggle to keep conversations ephemeral: no persistence, no memory/RAG, no web search
|
||||
- **At-Rest Encryption** — AES-256-GCM encryption of all query-derived text on disk (SQLite + Qdrant)
|
||||
|
||||
## File Structure
|
||||
|
||||
```
|
||||
/opt/caic/
|
||||
├── amqp.py # aio-pika AMQP connection manager + subscribe/rebind
|
||||
├── app.py # FastAPI app entry point
|
||||
├── auth.py # PIN-based guest/admin sessions, auth routes
|
||||
├── cluster.py # Cluster protocol: node registry, event log, ping/pong
|
||||
├── config.py # Constants, env vars, limits, skill registry
|
||||
├── crypto.py # AES-256-GCM encrypt/decrypt + key management
|
||||
├── db.py # SQLite schema, connection factory
|
||||
├── eviction.py # Score-based RAG eviction engine
|
||||
├── gpu.py # GPU stats — rocm-smi (Linux/AMD) + system_profiler (Darwin/Apple Silicon)
|
||||
├── hardware.py # Hardware self-assessment (CPU, RAM, VRAM) — Linux + Darwin
|
||||
├── memory.py # FTS5 memory CRUD, remember/forget commands
|
||||
├── rag.py # Qdrant vector search + system prompt assembly
|
||||
├── search.py # SearXNG integration, perplexity, refusal detection
|
||||
├── security.py # Rate limiting, origin checks, IP allowlist, audit
|
||||
├── model_pull.py # Startup model auto-pull (llama-server → Ollama fallback)
|
||||
├── triage.py # Query classification + cluster node selection
|
||||
├── routers/
|
||||
│ ├── chat.py # /api/chat streaming endpoint
|
||||
│ ├── cluster.py # Cluster status endpoint
|
||||
│ ├── completions.py # /v1/chat/completions OpenAI-compat endpoint
|
||||
│ ├── conversations.py# Conversation CRUD
|
||||
│ ├── ingest.py # Terminal RAG ingest
|
||||
│ ├── memories.py # Memory CRUD API
|
||||
│ ├── models.py # Model listing, system stats
|
||||
│ ├── presets.py # System prompt presets
|
||||
│ ├── profile.py # User profile
|
||||
│ ├── search_route.py # /api/search explicit search endpoint
|
||||
│ ├── settings.py # Runtime settings
|
||||
│ ├── skills.py # Skills management
|
||||
│ └── upload.py # File attachment endpoints
|
||||
├── static/
|
||||
│ └── logo.png # Logo image (optional)
|
||||
├── templates/
|
||||
│ └── index.html # Frontend
|
||||
├── node_agent/
|
||||
│ ├── agent.py # Standalone worker agent (AMQP client)
|
||||
│ └── requirements.txt
|
||||
└── tests/ # 200 pytest tests
|
||||
```
|
||||
|
||||
## Requirements
|
||||
|
||||
- Python 3.11+ (tested on 3.13)
|
||||
- llama-server running locally or on network (OpenAI-compatible API on port 8081)
|
||||
- SearXNG (optional, for web search)
|
||||
- RabbitMQ (optional, for AMQP cluster — coordinator only)
|
||||
- Qdrant (optional, for RAG vector search)
|
||||
- WireGuard (optional, for encrypted inter-node transit — see [WireGuard-Setup.md](docs/wiki/WireGuard-Setup.md))
|
||||
|
||||
## Installation
|
||||
|
||||
### Fresh Install
|
||||
|
||||
```bash
|
||||
# Create directory and venv
|
||||
sudo mkdir -p /opt/caic
|
||||
sudo chown $USER:$USER /opt/caic
|
||||
cd /opt/caic
|
||||
python3 -m venv venv
|
||||
|
||||
# Install dependencies
|
||||
pip install fastapi uvicorn httpx psutil jinja2 python-multipart pypdf aio-pika
|
||||
|
||||
# Set admin PIN before first startup (4 digits)
|
||||
export CAIC_ADMIN_PIN=4827
|
||||
|
||||
# Create subdirectories
|
||||
mkdir -p templates static
|
||||
|
||||
# Copy files
|
||||
# (copy all .py files to /opt/caic/)
|
||||
# (copy routers/ directory to /opt/caic/)
|
||||
# (copy templates/index.html to /opt/caic/templates/)
|
||||
```
|
||||
|
||||
WARNING: Do not use `1234` as your admin PIN unless you accept weak local security.
|
||||
|
||||
NOTE: First boot requires `CAIC_ADMIN_PIN` unless you explicitly opt into insecure fallback with `CAIC_ALLOW_DEFAULT_PIN=true`.
|
||||
|
||||
## Systemd Service
|
||||
|
||||
Create `/etc/systemd/system/caic.service`:
|
||||
|
||||
```ini
|
||||
[Unit]
|
||||
Description=cAIc - Local Inference Web Interface
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=caic
|
||||
Group=caic
|
||||
WorkingDirectory=/opt/caic
|
||||
ExecStart=/opt/caic/venv/bin/uvicorn app:app --host 0.0.0.0 --port 8080
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
```bash
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable caic
|
||||
sudo systemctl start caic
|
||||
```
|
||||
|
||||
## Memory Commands
|
||||
|
||||
In chat, natural language triggers memory operations:
|
||||
|
||||
| You say | What happens |
|
||||
|---------|--------------|
|
||||
| "remember that I prefer Rust over Go" | Stores as `preference` |
|
||||
| "remember that cAIc runs on port 8080" | Stores as `infrastructure` |
|
||||
| "note that the deadline is Friday" | Stores as `general` |
|
||||
| "forget about the deadline" | Removes matching memories |
|
||||
|
||||
Memories are automatically searched based on your message content and injected into the system prompt when relevant.
|
||||
|
||||
### Memory Topics
|
||||
|
||||
Memories are auto-categorized:
|
||||
- `preference` — likes, dislikes, choices
|
||||
- `project` — active work, repos, tasks
|
||||
- `infrastructure` — servers, services, configs
|
||||
- `personal` — name, location, background
|
||||
- `general` — everything else
|
||||
|
||||
## API Endpoints
|
||||
|
||||
### Completions (OpenAI-compatible)
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| POST | `/v1/chat/completions` | OpenAI-compatible chat (requires Bearer API key) |
|
||||
|
||||
### Chat & Search
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| POST | `/api/chat` | Send message (streaming SSE) |
|
||||
| POST | `/api/search` | Explicit web search (streaming SSE) |
|
||||
|
||||
### File Upload & Ingest
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| POST | `/api/upload` | Upload file (multipart, admin) |
|
||||
| DELETE | `/api/upload/{id}` | Delete upload (admin) |
|
||||
| PATCH | `/api/upload/{id}/link` | Link upload to conversation (admin) |
|
||||
| GET | `/api/upload/by-conversation/{id}` | List uploads for conversation |
|
||||
| POST | `/api/ingest` | Ingest text into RAG (Bearer token auth) |
|
||||
|
||||
### Memory
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| GET | `/api/memories` | List all memories |
|
||||
| POST | `/api/memories` | Add memory |
|
||||
| PUT | `/api/memories/{rowid}` | Update memory |
|
||||
| DELETE | `/api/memories/{rowid}` | Delete memory |
|
||||
| GET | `/api/memories/search?q=term` | Search memories |
|
||||
| GET | `/api/memories/stats` | Get counts by topic |
|
||||
|
||||
### Cluster
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| GET | `/api/cluster` | Cluster status (nodes, coordinator, event log) |
|
||||
|
||||
### RAG Management
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| GET | `/api/rag/stats` | RAG corpus stats (admin) |
|
||||
| POST | `/api/rag/flush` | Delete non-pinned vectors (admin) |
|
||||
|
||||
### Models & System
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| GET | `/api/models` | List available models |
|
||||
| GET | `/api/ps` | List loaded models |
|
||||
| POST | `/api/show` | Get model info |
|
||||
| GET | `/api/stats` | CPU, RAM, GPU, VRAM stats |
|
||||
| GET | `/api/search/status` | SearXNG availability |
|
||||
|
||||
### Settings & Profile
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| GET | `/api/profile` | Get profile content |
|
||||
| PUT | `/api/profile` | Update profile (admin) |
|
||||
| GET | `/api/profile/default` | Get default profile |
|
||||
| GET | `/api/settings` | Get settings |
|
||||
| PUT | `/api/settings` | Update settings (admin) |
|
||||
|
||||
### Conversations
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| GET | `/api/conversations` | List conversations |
|
||||
| POST | `/api/conversations` | Create conversation |
|
||||
| GET | `/api/conversations/{id}` | Get conversation with messages |
|
||||
| PUT | `/api/conversations/{id}` | Update conversation title/model |
|
||||
| DELETE | `/api/conversations/{id}` | Delete conversation |
|
||||
| DELETE | `/api/conversations` | Delete ALL conversations |
|
||||
|
||||
### Presets
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| GET | `/api/presets` | List presets |
|
||||
| POST | `/api/presets` | Create preset |
|
||||
| PUT | `/api/presets/{id}` | Update preset |
|
||||
| DELETE | `/api/presets/{id}` | Delete preset |
|
||||
|
||||
### Skills
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| GET | `/api/skills` | List all skills with state |
|
||||
| GET | `/api/skills/active` | List active skills |
|
||||
| PUT | `/api/skills/{key}` | Toggle skill enabled (admin) |
|
||||
|
||||
### Auth
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| POST | `/api/auth/guest` | Create guest session |
|
||||
| POST | `/api/auth/login` | Admin PIN login |
|
||||
| POST | `/api/auth/logout` | Revoke session |
|
||||
| GET | `/api/auth/session` | Check session validity |
|
||||
| POST | `/api/auth/heartbeat` | Extend session TTL |
|
||||
|
||||
## Configuration
|
||||
|
||||
Settings are stored in the `settings` table and include:
|
||||
|
||||
- `profile_enabled` — Inject profile into chats (true/false)
|
||||
- `search_enabled` — Auto web search (true/false)
|
||||
- `memory_enabled` — Memory injection (true/false)
|
||||
- `skills_enabled` — Skills framework (true/false)
|
||||
- `default_model` — Default inference model
|
||||
|
||||
## Uninstalling cAIc
|
||||
|
||||
Three scripts are provided in `scripts/`. Each accepts `-y` for unattended execution.
|
||||
|
||||
### Bare-metal / systemd removal
|
||||
|
||||
Stop the service and remove `/opt/caic/`, the systemd unit, AMQP secret, and optionally the pip packages:
|
||||
|
||||
```bash
|
||||
sudo ./scripts/uninstall.sh # interactive
|
||||
sudo ./scripts/uninstall.sh -y # unattended
|
||||
```
|
||||
|
||||
Removes: systemd `caic` service, `/opt/caic/` + venv, `~/.caic_amqp_secret`, `/tmp/caic_uploads`, `hardware_state.json`. Preserves `caic.db` if it lives outside `/opt/caic/`.
|
||||
|
||||
### Docker stack teardown
|
||||
|
||||
Stop all containers, remove volumes/images, and delete generated files:
|
||||
|
||||
```bash
|
||||
cd <docker-deploy-directory>
|
||||
../scripts/teardown-docker.sh # interactive
|
||||
../scripts/teardown-docker.sh -y # unattended
|
||||
```
|
||||
|
||||
Removes: containers + volumes (`caic_data`, `caic_uploads`, `searxng_config`, `qdrant_storage`, `ollama_models`, `rabbitmq`), images (`caic`, `searxng`, `Qdrant`, `RabbitMQ`, `llama-server`, `Ollama`), `.env`, `secrets/`, `searxng/`, `setup.log`. Preserves `models/*.gguf` unless confirmed.
|
||||
|
||||
### Nuclear clean (everything)
|
||||
|
||||
Removes bare-metal install AND Docker stack AND config AND temp data. Double-confirmation required:
|
||||
|
||||
```bash
|
||||
sudo ./scripts/nuclear-clean.sh # double prompt, then unattended
|
||||
```
|
||||
|
||||
Removes: everything from the two scripts above plus `/var/lib/caic/` and temp directories. Offers to delete the repository itself. Does NOT remove Docker Engine, pip packages, GPU drivers, or WireGuard config.
|
||||
|
||||
### Partial / manual clean
|
||||
|
||||
Files and components not tracked by the scripts:
|
||||
|
||||
- `caic.db` (SQLite database at custom `CAIC_DB_PATH`)
|
||||
- Reverse proxy configs (Caddyfile, nginx)
|
||||
- WireGuard tunnel configurations
|
||||
- Docker Engine itself (`sudo apt remove docker containerd runc; sudo rm -rf /var/lib/docker`)
|
||||
|
||||
## Testing
|
||||
|
||||
```bash
|
||||
python3 -m pytest tests/ -v
|
||||
```
|
||||
|
||||
All 200 tests use `tmp_path` fixtures + monkeypatched `httpx.AsyncClient`/`aio-pika`. No external services needed.
|
||||
See [What's New](#whats-new-in-v100) below, or browse the [commit history](https://github.com/mikeshallop/caic/commits/main).
|
||||
|
||||
## License
|
||||
|
||||
@@ -586,4 +150,160 @@ MIT
|
||||
|
||||
## Repository
|
||||
|
||||
Gitea: `ssh://gitea@llgit.llamachile.tube:1319/gramps/caic.git`
|
||||
GitHub: https://github.com/mikeshallop/caic
|
||||
|
||||
Gitea (primary): `ssh://gitea@llgit.llamachile.tube:1319/gramps/caic.git`
|
||||
|
||||
---
|
||||
|
||||
## What's New in v1.1.0
|
||||
|
||||
### Image Generation Service
|
||||
- `POST /api/image/generate` — proxy endpoint routes to ComfyUI on cluster workers
|
||||
- `GET /api/image/status` — lists available image gen nodes
|
||||
- Node agent auto-detects ComfyUI and registers `image_gen` capability
|
||||
- Full ComfyUI workflow: CheckpointLoader → KSampler → VAEDecode → SaveImage
|
||||
- Cluster AMQP protocol extended: `cmd.image_generate`, `image_generated`, `image_failed`
|
||||
- Hardware probe checks ComfyUI reachability + checkpoint model list
|
||||
- 27 new tests covering cluster handlers, router proxy, node agent, hardware, capability detection
|
||||
|
||||
### Bug Fixes & Hardening
|
||||
- Hardware assessment now probes ComfyUI alongside llama-server, Qdrant, SearXNG
|
||||
- Node agent config extended with `comfyui_port` (default 8188)
|
||||
|
||||
## What's New in v1.0.0
|
||||
|
||||
### Docker Containerization (B3)
|
||||
- `Dockerfile` — multi-stage Python 3.13-slim build with healthcheck
|
||||
- `docker-compose.yml` — full stack: cAIc, SearXNG, Qdrant, RabbitMQ, llama-server, Ollama
|
||||
- `scripts/setup.sh` — first-run scaffolding: generates `.env`, secrets, SearXNG config, pulls default model
|
||||
- All service URLs env-var configurable with Docker service hostnames
|
||||
- AMQP secret uses Docker secrets pattern (`/run/secrets/`)
|
||||
- Only port 8080 exposed by default; all other services internal to compose network
|
||||
- Graceful degradation — SearXNG and Ollama optional
|
||||
|
||||
### Bug Fixes & Hardening
|
||||
- Defaults changed from hardcoded LAN IPs to `localhost` for Docker compatibility
|
||||
- `DEFAULT_MODEL` configurable via `CAIC_DEFAULT_MODEL` env var
|
||||
- `HW_STATE_PATH` configurable via `CAIC_HW_STATE_PATH` env var
|
||||
- Syslog handler wrapped in try/except (container-safe)
|
||||
- SQLite `PRAGMA journal_mode = WAL` for better concurrency
|
||||
- `db.close()` in try/finally for proper cleanup
|
||||
- AMQP subscription append moved before try for reconnect safety
|
||||
- Missing `psutil` + `jinja2` added to `requirements.txt`
|
||||
- Test discovery fixed via `tests/conftest.py` sys.path insertion
|
||||
|
||||
## What's New in v0.23.0
|
||||
|
||||
### Topbar Redesign
|
||||
- Stats moved to bottom status bar, toggles to hamburger menu, palette next to version
|
||||
- Mobile-responsive layout, query bar restored above chat
|
||||
|
||||
### Uninstall Scripts
|
||||
- `scripts/uninstall.sh`, `teardown-docker.sh`, `nuclear-clean.sh`
|
||||
|
||||
### Code Quality
|
||||
- Replaced deprecated `asyncio.ensure_future` with `asyncio.create_task`
|
||||
- `AGENTS.md` → `ai.md` for tool-agnostic project context
|
||||
|
||||
## What's New in v0.22.0
|
||||
|
||||
### Color Theme System
|
||||
- 6 themes: IBM Blue, Green Ln (Matrix), Dark, Light, Amber (Fallout), Trippin (neon)
|
||||
- Palette icon in topbar, CSS variable swap, `localStorage` persistence
|
||||
|
||||
### RAG Corpus Management UI (B4)
|
||||
- Admin modal to browse, search, edit, and delete individual RAG entries
|
||||
- Stats bar, semantic search, source filter, per-row edit/delete, bulk flush
|
||||
- 14 new tests, 214 total
|
||||
|
||||
## What's New in v0.21.0
|
||||
|
||||
### Scrollbar + DOM Fixes
|
||||
- Scrollbar z-index, `requestAnimationFrame` scroll, direction-aware scroll guard
|
||||
|
||||
### Perplexity Persistence
|
||||
- Perplexity stored per message, confidence badges on loaded conversations
|
||||
|
||||
### Config Overhaul
|
||||
- All service URLs now env-overridable for single-node deployment
|
||||
|
||||
## What's New in v0.20.0
|
||||
|
||||
### At-Rest Encryption
|
||||
- AES-256-GCM on all query-derived text: conversations, memories, uploads, RAG, completions
|
||||
- 256-bit key auto-generated on first boot, never exposed via API
|
||||
|
||||
## What's New in v0.19.3
|
||||
|
||||
### Private Chat Mode
|
||||
- Toggle to keep nothing on disk — no persistence, no memory/RAG, no web search
|
||||
|
||||
### WireGuard In-Transit Encryption
|
||||
- All coordinator↔worker traffic encrypted at the network layer
|
||||
|
||||
## What's New in v0.19.2
|
||||
|
||||
### Waterfall Direction Toggle
|
||||
- NEW/OLD sort toggle, direction-aware scroll, toast notifications, clipboard fallback
|
||||
|
||||
## What's New in v0.19.1
|
||||
|
||||
### Default Model Auto-Pull
|
||||
- Checks llama-server at startup, falls back to Ollama pull if missing
|
||||
|
||||
## What's New in v0.19.0
|
||||
|
||||
### Apple Silicon Worker Support
|
||||
- GPU detection via `system_profiler` on macOS, hybrid AMD/Apple/CPU detection
|
||||
|
||||
## What's New in v0.18.0
|
||||
|
||||
### Wiki + UX Polish
|
||||
- Full installation guide, screenshots gallery, waterfall layout, barcode stripes, confidence badges, sprocket strips, paper grain background
|
||||
|
||||
## What's New in v0.17.26
|
||||
|
||||
### Dynamic Model Swap + Cluster Status UI
|
||||
- `request_model_swap()`, async `select_node()`, heartbeat handler, live status panel
|
||||
|
||||
## What's New in v0.14.0
|
||||
|
||||
### Cluster Protocol
|
||||
- 9 AMQP message types, node registry, ping/pong health, coordinator auto-promotion
|
||||
|
||||
### RAG Corpus Management
|
||||
- Score-based eviction with hysteresis, flush endpoint, operational stats
|
||||
|
||||
## What's New in v0.13.0
|
||||
|
||||
### RAG Eviction Engine
|
||||
- Score-based eviction with hysteresis (80% high-water, 20% low-water), pinned sources, grace period
|
||||
|
||||
## What's New in v0.12.0
|
||||
|
||||
### Chat Reply Toolbar
|
||||
- Copy, print, save, rate actions on assistant messages
|
||||
|
||||
### Startup Hardware Assessment
|
||||
- CPU, RAM, VRAM probe on first boot
|
||||
|
||||
## What's New in v0.11.0
|
||||
|
||||
### Terminal RAG Hook
|
||||
- `POST /api/ingest` with Bearer token auth for autonomous terminal history ingestion
|
||||
|
||||
## What's New in v0.10.0
|
||||
|
||||
### File Upload & Attachments
|
||||
- PDF/text extraction, chat context injection, RAG ingest, paperclip UI
|
||||
|
||||
## What's New in v0.9.0
|
||||
|
||||
### Modular Refactor
|
||||
- Single-file `app.py` split into config/db/auth/security/memory/search/rag/gpu + routers/
|
||||
|
||||
## What's New in v0.8.0
|
||||
|
||||
### Foundation
|
||||
- OpenAI-compat endpoint, RAG pipeline, SSE streaming, llama-server integration
|
||||
|
||||
@@ -3,6 +3,10 @@
|
||||
## Run
|
||||
|
||||
```bash
|
||||
# Docker (recommended)
|
||||
scripts/setup.sh && docker compose up -d
|
||||
|
||||
# Bare-metal
|
||||
uvicorn app:app --host 0.0.0.0 --port 8080 --reload
|
||||
```
|
||||
|
||||
@@ -31,14 +35,15 @@ Every router has a dedicated test file:
|
||||
| `test_search_url_sanitization.py` | `search.py` URL sanitizer |
|
||||
| `test_cluster.py` | `cluster.py` — registration, deregistration, pong, events, coordinator query |
|
||||
| `test_cluster_heartbeat.py` | `cluster.py` — heartbeat handler, known/unknown node |
|
||||
| `test_model_swap.py` | `cluster.py` + `triage.py` — request_model_swap, handle_model_ready/failed, select_node swap triggering |
|
||||
| `test_model_swap.py` | `cluster.py` — request_model_swap, handle_model_ready/failed |
|
||||
| `test_node_agent.py` | `node_agent/agent.py` — registration, ping/pong, model swap |
|
||||
| `test_triage.py` | `triage.py` — classify_query, select_node, get_inference_url |
|
||||
| `test_image.py` | Image generation — cluster handlers, router proxy, node agent ComfyUI integration, hardware probe, capability detection |
|
||||
| `test_settings_allowlist.py` | `routers/settings.py` — allowlisted key enforcement |
|
||||
| `test_skills_framework.py` | `routers/skills.py` — list, toggle, unknown skill, prompt injection |
|
||||
| `test_ip_allowlist.py` | IP allowlist helper + middleware |
|
||||
| `test_rate_and_payload_guardrails.py` | Rate limits + payload size enforcement |
|
||||
| `test_error_envelopes.py` | Global exception handler + stream error incidents |
|
||||
| `test_fixes_regression.py` | Origin-exempt ingest, bogus conversation_id FK, auto-search reset, image uploads, conflict false-positives, deterministic ingest ids, get_load VRAM parsing, version pin |
|
||||
| `test_upload.py` | `routers/upload.py` — upload, delete, link, by-conversation, attachment_count integration |
|
||||
|
||||
Modules that call `httpx.AsyncClient` (chat, completions, models, search_route, upload, ingest, model_pull)
|
||||
@@ -64,26 +69,26 @@ Refactored from single-file (`app.py`) into modules under project root:
|
||||
| `gpu.py` | GPU stats — `rocm-smi` (AMD/Linux) or `system_profiler` (Apple Silicon/macOS) |
|
||||
| `crypto.py` | AES-256-GCM encrypt/decrypt + key management (stored as `heartbeat_interval_ms` in settings) |
|
||||
| `model_pull.py` | Startup model availability check + Ollama pull API |
|
||||
| `triage.py` | Phi-4-mini-based query classification + cluster node selection |
|
||||
| `cluster.py` | Cluster node registry, event log, coordinator election, ping/pong, model swap handlers |
|
||||
| `cluster.py` | Cluster node registry, event log, coordinator election, ping/pong, model swap handlers, image generation request/response |
|
||||
| `amqp.py` | AMQP connection manager — connect, disconnect, publish, subscribe, auto-reconnect |
|
||||
| `node_agent/` | Standalone worker agent — AMQP client for registration, ping/pong, model swap |
|
||||
| `routers/` | One module per endpoint group (chat, search, skills, completions, upload, ingest) |
|
||||
| `node_agent/` | Standalone worker agent — AMQP client for registration, ping/pong, model swap, image generation |
|
||||
| `routers/` | One module per endpoint group (chat, search, skills, completions, upload, ingest, image) |
|
||||
|
||||
### Entrypoint / API keys
|
||||
|
||||
- `app.py` line 148: `uvicorn.run(app, ...)` when called directly
|
||||
- `config.py` line 14: `LLAMA_SERVER_BASE` defaults to `http://192.168.50.108:8081` — llama-server on coordinator, RPC-offloads GPU layers to worker :50052
|
||||
- `config.py` line 17: `COMPLETIONS_API_KEY` read from `CAIC_COMPLETIONS_API_KEY` env var or auto-generates
|
||||
- `config.py` line 13: `OLLAMA_BASE` is legacy/unused — all endpoints use `LLAMA_SERVER_BASE`
|
||||
- `config.py` line 14: `LLAMA_SERVER_BASE` defaults to `http://localhost:8081` — configurable via env var; Docker uses `http://llama-server:8081`
|
||||
- `config.py` line 17: `DEFAULT_MODEL` read from `CAIC_DEFAULT_MODEL` env var or defaults to `qwen2.5-7b-instruct`
|
||||
- `config.py` line 18: `COMPLETIONS_API_KEY` read from `CAIC_COMPLETIONS_API_KEY` env var or auto-generates
|
||||
|
||||
### Key flows
|
||||
|
||||
1. **`/api/chat`** → `process_remember_command()` intercepts "remember that..." / "forget about..." first → optional `upload_context_id` fetches document text from SQLite → `build_system_prompt()` (profile + FTS5 memory + Qdrant RAG + preset + skills + uploaded doc) → triage classifies query (general/code/search/rag) → `select_node()` picks best worker → stream from chosen node with `logprobs: true` → if perplexity > 15.0 OR `REFUSAL_PATTERNS` match, re-query with SearXNG results
|
||||
1. **`/api/chat`** → `process_remember_command()` intercepts "remember that..." / "forget about..." first → optional `upload_context_id` fetches document text from SQLite → `build_system_prompt()` (profile + FTS5 memory + Qdrant RAG + preset + skills + uploaded doc) → stream from `LLAMA_SERVER_BASE` with `logprobs: true` → if perplexity > 15.0 OR `REFUSAL_PATTERNS` match, re-query with SearXNG results
|
||||
2. **`/api/search`** → bypasses perplexity/refusal, queries SearXNG directly → summarizes via llama-server
|
||||
3. **`/v1/chat/completions`** → OpenAI-compatible for Continue.dev/IDE integration; FIM requests proxied without persistence
|
||||
4. **`/api/upload`** → multipart file upload, PDF/text extraction, `mode=(context|ingest|both)`, stores SQLite context (1hr expiry) + Qdrant upsert
|
||||
5. **`/api/ingest`** → Bearer token auth, programmatic RAG ingest (terminal hook, external tools)
|
||||
6. **`POST /api/image/generate`** → admin required, routes to an image-gen node via AMQP → ComfyUI workflow → returns PNG; `GET /api/image/status` lists available image gen nodes
|
||||
|
||||
### Perplexity / auto-search
|
||||
|
||||
@@ -107,16 +112,18 @@ The upstream request includes `"logprobs": true`. `parse_llama_stream_chunk()` e
|
||||
|
||||
### External services
|
||||
|
||||
| Service | Required | Port |
|
||||
|---------|----------|------|
|
||||
| llama-server (coordinator) | Yes | 8081 + RPC :50052 (worker GPU) |
|
||||
| Phi-4-mini (triage) | No | 8083 |
|
||||
| SearXNG | No | 8888 |
|
||||
| RabbitMQ (coordinator) | No | 5672 — AMQP broker |
|
||||
| wttr.in | No | weather shortcut |
|
||||
| rocm-smi | No | AMD GPU stats |
|
||||
| Qdrant | No | 6333 (coordinator) — RAG vector search |
|
||||
| Ollama (worker) | No | 11434 — embeddings + model pull |
|
||||
All services are available bare-metal or as containers in `docker compose up`.
|
||||
|
||||
| Service | Required | Port | Docker service name |
|
||||
|---------|----------|------|---------------------|
|
||||
| llama-server (coordinator) | Yes | 8081 + RPC :50052 (worker GPU) | `llama-server` |
|
||||
| SearXNG | No | 8888 | `searxng` |
|
||||
| RabbitMQ (coordinator) | No | 5672 — AMQP broker | `rabbitmq` |
|
||||
| wttr.in | No | weather shortcut | — |
|
||||
| rocm-smi | No | AMD GPU stats | — |
|
||||
| Qdrant | No | 6333 (coordinator) — RAG vector search | `qdrant` |
|
||||
| Ollama (worker) | No | 11434 — embeddings + model pull | `ollama` |
|
||||
| ComfyUI (worker) | No | 8188 — image generation API | — |
|
||||
|
||||
### Config quirks
|
||||
|
||||
@@ -124,6 +131,8 @@ The upstream request includes `"logprobs": true`. `parse_llama_stream_chunk()` e
|
||||
- `SUPPORTED_UPLOAD_TYPES` includes images (png/jpeg/gif/svg/webp) + text + PDF + JSON
|
||||
- `UPLOAD_CONTEXT_EXPIRY_HOURS` = 1 hour
|
||||
- Rate limits and payload caps in `config.py` — patch `security.RL_*` not `config.RL_*` for tests
|
||||
- `COMFYUI_BASE` defaults to `http://localhost:8188` (overridable via `CAIC_COMFYUI_BASE`)
|
||||
- `COMFYUI_TIMEOUT` defaults to `120` seconds (overridable via `CAIC_COMFYUI_TIMEOUT`)
|
||||
- RAG embedding requests go to `EMBED_URL` at `/api/embeddings` (Ollama on worker :11434)
|
||||
|
||||
### SSE Protocol
|
||||
@@ -138,22 +147,41 @@ All streaming endpoints yield `data: {json}\n\n`. Key shapes:
|
||||
## Work State
|
||||
|
||||
### Completed this session
|
||||
- **B4 — RAG Corpus Management UI** — Paginated browse, semantic search, source filter, edit text with re-embed, single-point delete, bulk flush. Backend: `GET /api/rag/points`, `GET /api/rag/point/{id}`, `DELETE /api/rag/point/{id}`, `PATCH /api/rag/point/{id}`. Frontend: admin-only modal with stats bar, search bar, paginated table, per-row edit/delete, double-confirm flush.
|
||||
- **RAG bugfixes**: Fixed Qdrant collection name mismatch (`jarvis_rag` → `caic_rag`, migrated 219 points). Fixed `vectors_count` → `points_count` in `eviction.py`/`rag_admin.py` (Qdrant v1.10+ API change). Removed unindexed `order_by` that caused 502 on scroll. Made `RAG_COLLECTION` env-configurable (`CAIC_RAG_COLLECTION`).
|
||||
- **Topbar redesign**: Moved system stats (CPU/MEM/GPU/VRAM/TOK) to a centered bottom strip. Moved toggles (MEM, SEARCH, PROFILE, SORT, PRIVACY) into a ⋮ hamburger menu next to ADMIN badge. Placed palette icon immediately after version number in topbar-left. Removed standalone (i) button; privacy info accessible via ⋮ → About Privacy. Mobile-responsive padding/sizing.
|
||||
- **Semantic search fix**: Set `CAIC_EMBED_URL=http://192.168.50.108:11434` on jarvis (model `mxbai-embed-large` is on ultron, not the old embed server).
|
||||
- **Pre-Docker review**: Full findings report delivered -- 30+ issues across 7 categories (hardcoded hosts/paths, config/secrets, AMQP gaps, resource cleanup, SQLite container safety, completions concurrency, TASKS.md accuracy).
|
||||
- **Project rename**: `jarvisChat` → **cAIc** ("cake") — swept remaining branding (router docstrings, jc-ingest.sh env var), deleted stale `AGENTS.md.local`.
|
||||
- **Single-node consolidation**: all services moved to jarvis (192.168.50.212) — `COMFYUI_BASE` default → `localhost:8188`, AMQP URL default → `localhost:5672`, `NODE_NAME` default → `jarvis`, `DEFAULT_PROFILE` topology rewritten, cluster/AMQP/node_agent left in place (degrades gracefully).
|
||||
- **Deprecation fix**: Replaced `asyncio.ensure_future` with `asyncio.create_task` in `rag.py` and `routers/chat.py`.
|
||||
- **Documentation**: Added inline comments and docstrings to all functions in `db.py`.
|
||||
- **Uninstall scripts**: Created and committed `scripts/uninstall.sh`, `teardown-docker.sh`, `nuclear-clean.sh`.
|
||||
- **README**: Added "Uninstalling cAIc" section.
|
||||
- **Docker containerization (B3)**: Created `Dockerfile`, `docker-compose.yml`, `.env.example`, `scripts/setup.sh`, `.dockerignore`, `searxng-settings.yml.dist`, `models/README.txt`. Fixed hardcoded defaults in `config.py` (localhost, Docker secrets path, `CAIC_DEFAULT_MODEL` env var, `CAIC_HW_STATE_PATH` env var). Added missing `psutil` + `jinja2` to `requirements.txt`. Fixed test discovery via `tests/conftest.py` sys.path insertion. 214 tests pass.
|
||||
|
||||
### Active
|
||||
- (none)
|
||||
- Image generation service backend complete — cluster handlers, router proxy, node agent ComfyUI integration, hardware probe. 27 tests pass. ComfyUI install pending on jarvis (single-node).
|
||||
|
||||
### Deployed (2026-08-07) — v1.1.0 to production
|
||||
- **Fixed crash-loop**: ultron `llama-server.service` had 911 restarts — its `--rpc 192.168.50.210:50052` pointed at a dead IP. Corrected to `192.168.50.212:50052` (jarvis GPU rpc-server). Model now loads, `/health` = ok.
|
||||
- **Deployed v1.1.0**: workspace repo synced to `/opt/jarvischat` (jarvischat.service cwd). caic.db + venv preserved; `aio-pika` installed into prod venv (was missing → AMQP disabled).
|
||||
- **Env fixes** (`/etc/systemd/system/jarvischat.service.d/override.conf`): added `LLAMA_SERVER_BASE=http://192.168.50.108:8081`, `CAIC_QDRANT_URL=http://192.168.50.108:6333`, `CAIC_COMPLETIONS_API_KEY` (was set as legacy `JARVISCHAT_` name), kept `CAIC_EMBED_URL=http://192.168.50.108:11434` + `CAIC_ADMIN_PIN=1319`. Wrote `/opt/jarvischat/.completions_key` (jc-ingest.sh).
|
||||
- **Deploy-blocking bug fixes** (uncommitted, workspace + deploy):
|
||||
- `rag.py` `chunk_text`: chunk_size 512→200 (chunks exceeded mxbai-embed-large's 512-token context → ollama 500).
|
||||
- Qdrant 1.18.2 rejects non-UUID point IDs: wrapped `ingest-*`/`auto-*`/`upload-*` string IDs in `uuid5` in `routers/ingest.py`, `rag.py`, `routers/upload.py`.
|
||||
- `docs/jc-ingest.sh`: `JC_URL` updated `.210`→`.212`.
|
||||
- **Docs rebuilt**: 159 chunks (source `docs`) re-ingested via `/api/ingest` (README, ai.md, docker.md, CLAUDE.md, wiki/*). RAG now 378 vectors; chat verified injecting "Retrieved Context".
|
||||
- **Tests**: all 244 pass (run per-file in a throwaway venv; the full-suite run deadlocks on TestClient/AMQP ordering, not a code failure).
|
||||
|
||||
### Follow-ups
|
||||
- AMQP wiring: cluster subs degrade gracefully — **moot in the single-node (jarvis) deployment** until a multi-node cluster is stood back up. Needs `CAIC_AMQP_URL` + credentials if that happens.
|
||||
- `CAIC_TRIAGE_BASE` set but triage not yet invoked by chat (config-only until TASK 2 wiring).
|
||||
|
||||
### Blocked
|
||||
- (none)
|
||||
- Ball Gunner assets — waiting on Canva designs
|
||||
|
||||
### Upcoming (backlog)
|
||||
- B3 — Docker distribution
|
||||
- ~~B3 — Docker distribution~~ [DONE]
|
||||
|
||||
### Key config values (current)
|
||||
- **Current VERSION**: `v0.22.0` in `config.py`.
|
||||
- **Current VERSION**: `v1.1.0` in `config.py`.
|
||||
- `SESSION_TIMEOUT_SECONDS = 3600`
|
||||
- `DEFAULT_MODEL = "qwen2.5-7b-instruct"`
|
||||
- `LLAMA_SERVER_BASE = "http://192.168.50.108:8081"`
|
||||
- `DEFAULT_MODEL = "qwen2.5-7b-instruct"` (overridable via `CAIC_DEFAULT_MODEL`)
|
||||
- `LLAMA_SERVER_BASE = "http://localhost:8081"` (overridable via env var)
|
||||
|
||||
@@ -43,6 +43,8 @@ async def subscribe(exchange: str, routing_keys: list[str], handler) -> None:
|
||||
if ch is None:
|
||||
log.error("cannot subscribe — no AMQP channel")
|
||||
return
|
||||
# Track subscription before attempting so reconnect catches it even if this try fails
|
||||
_subscriptions.append((exchange, routing_keys, handler))
|
||||
try:
|
||||
queue = await ch.declare_queue("", exclusive=True)
|
||||
ex = await ch.get_exchange(exchange)
|
||||
@@ -58,7 +60,6 @@ async def subscribe(exchange: str, routing_keys: list[str], handler) -> None:
|
||||
log.exception("AMQP handler error for %s %s", exchange, msg.routing_key)
|
||||
|
||||
await queue.consume(_dispatch)
|
||||
_subscriptions.append((exchange, routing_keys, handler))
|
||||
except Exception:
|
||||
log.exception("AMQP subscribe failed for %s %s", exchange, routing_keys)
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ from hardware import assess_hardware
|
||||
from memory import get_memory_count
|
||||
from security import (
|
||||
get_client_ip, is_ip_allowed, check_rate_limit, rate_policy,
|
||||
origin_allowed, is_state_changing, request_body_limit,
|
||||
origin_allowed, request_body_limit,
|
||||
audit_event, customer_error_envelope, log_incident,
|
||||
)
|
||||
from auth import get_session, is_admin_only, router as auth_router
|
||||
@@ -41,15 +41,19 @@ import routers.ingest as ingest
|
||||
import routers.hardware as hardware
|
||||
import routers.rag_admin as rag_admin
|
||||
import routers.cluster as cluster_router
|
||||
import routers.image as image_router
|
||||
|
||||
# --- Logging ---
|
||||
log = logging.getLogger("caic")
|
||||
log.setLevel(logging.DEBUG)
|
||||
syslog_address = os.environ.get("CAIC_SYSLOG_ADDRESS", "/dev/log")
|
||||
if syslog_address:
|
||||
syslog_handler = logging.handlers.SysLogHandler(address=syslog_address)
|
||||
syslog_handler.setFormatter(logging.Formatter("caic[%(process)d]: %(levelname)s %(message)s"))
|
||||
log.addHandler(syslog_handler)
|
||||
try:
|
||||
syslog_handler = logging.handlers.SysLogHandler(address=syslog_address)
|
||||
syslog_handler.setFormatter(logging.Formatter("caic[%(process)d]: %(levelname)s %(message)s"))
|
||||
log.addHandler(syslog_handler)
|
||||
except Exception:
|
||||
log.warning("syslog not available at %s -- skipping", syslog_address)
|
||||
|
||||
BASE_DIR = Path(__file__).parent
|
||||
templates = Jinja2Templates(directory=str(BASE_DIR / "templates"))
|
||||
@@ -137,7 +141,12 @@ async def session_auth_middleware(request: Request, call_next):
|
||||
"/api/auth/heartbeat", "/api/auth/guest", "/api/ingest", "/api/hardware",
|
||||
}
|
||||
|
||||
if path.startswith("/api/"):
|
||||
# Bearer-token-authenticated endpoints are reached by CLI/terminal tooling
|
||||
# (curl, caic-ingest.sh) that sends no Origin/Referer header — exempt them
|
||||
# from the browser origin check.
|
||||
origin_exempt_paths = {"/api/ingest"}
|
||||
|
||||
if path.startswith("/api/") and path not in origin_exempt_paths:
|
||||
if not origin_allowed(request):
|
||||
audit_event("origin_check", "denied", ip=ip, role="none",
|
||||
details=f"{request.method} {path}", warning=True)
|
||||
@@ -174,7 +183,7 @@ for router_module in [
|
||||
auth_router, conversations.router, memories.router, models.router,
|
||||
presets.router, profile.router, settings.router, skills.router,
|
||||
chat.router, search_route.router, completions.router, upload.router, ingest.router, hardware.router,
|
||||
rag_admin.router, cluster_router.router,
|
||||
rag_admin.router, cluster_router.router, image_router.router,
|
||||
]:
|
||||
app.include_router(router_module)
|
||||
|
||||
|
||||
-2334
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -17,7 +17,7 @@ from db import get_db, get_setting
|
||||
from security import (
|
||||
SESSIONS, PIN_ATTEMPTS, SESSION_LOCK, BODY_LIMIT_DEFAULT_BYTES,
|
||||
audit_event, get_client_ip, is_ip_allowed, check_rate_limit,
|
||||
rate_policy, origin_allowed, is_state_changing, request_body_limit,
|
||||
rate_policy, origin_allowed, request_body_limit,
|
||||
read_json_body, hash_pin, customer_error_envelope, log_incident,
|
||||
)
|
||||
|
||||
|
||||
+70
-1
@@ -18,7 +18,8 @@ CLUSTER_NODES: dict[str, dict] = {}
|
||||
CLUSTER_EVENTS: deque = deque(maxlen=1000)
|
||||
CLUSTER_COORDINATOR: str | None = None
|
||||
_pending_pings: dict[str, tuple[str, asyncio.Event]] = {}
|
||||
NODE_NAME: str = os.environ.get("CAIC_NODE_NAME", "ultron")
|
||||
_pending_image: dict[str, tuple[str, asyncio.Event]] = {}
|
||||
NODE_NAME: str = os.environ.get("CAIC_NODE_NAME", "jarvis")
|
||||
PING_TIMEOUT: float = 5.0
|
||||
|
||||
|
||||
@@ -240,6 +241,72 @@ async def handle_model_failed(exchange: str, routing_key: str, payload: dict) ->
|
||||
_push_event("cluster", "error", node_name, f"Model swap failed: {error}")
|
||||
|
||||
|
||||
async def handle_image_generated(exchange: str, routing_key: str, payload: dict) -> None:
|
||||
node_name = payload.get("node_name", routing_key.split(".")[1] if "." in routing_key else "unknown")
|
||||
request_id = payload.get("request_id")
|
||||
|
||||
if request_id and request_id in _pending_image:
|
||||
_, event = _pending_image.pop(request_id)
|
||||
_pending_image[request_id] = (payload.get("image_base64", ""), event)
|
||||
event.set()
|
||||
|
||||
if node_name in CLUSTER_NODES:
|
||||
CLUSTER_NODES[node_name]["last_seen"] = datetime.now(timezone.utc).isoformat() + "Z"
|
||||
|
||||
|
||||
async def handle_image_failed(exchange: str, routing_key: str, payload: dict) -> None:
|
||||
node_name = payload.get("node_name", routing_key.split(".")[1] if "." in routing_key else "unknown")
|
||||
request_id = payload.get("request_id")
|
||||
error = payload.get("error", "unknown error")
|
||||
|
||||
if request_id and request_id in _pending_image:
|
||||
_pending_image[request_id] = ("", _pending_image[request_id][1])
|
||||
_pending_image[request_id][1].set()
|
||||
|
||||
_push_event("application", "error", node_name, f"Image generation failed: {error}")
|
||||
|
||||
|
||||
async def request_image_generate(
|
||||
node_name: str, prompt: str, negative_prompt: str = "",
|
||||
width: int = 1024, height: int = 1024, steps: int = 20,
|
||||
seed: int = -1, model: str = "", timeout: float = 120,
|
||||
) -> str | None:
|
||||
if node_name not in CLUSTER_NODES:
|
||||
log.warning("request_image_generate: unknown node %s", node_name)
|
||||
return None
|
||||
|
||||
caps = CLUSTER_NODES[node_name].get("capabilities", [])
|
||||
if "image_gen" not in caps:
|
||||
log.warning("request_image_generate: node %s lacks image_gen capability", node_name)
|
||||
return None
|
||||
|
||||
request_id = str(uuid.uuid4())
|
||||
event = asyncio.Event()
|
||||
_pending_image[request_id] = ("", event)
|
||||
|
||||
now = datetime.now(timezone.utc).isoformat() + "Z"
|
||||
_push_event("application", "info", node_name, f"Image generation requested: {prompt[:60]}...")
|
||||
|
||||
await publish(AMQP_EXCHANGE_ADMIN, f"node.{node_name}.cmd.image_generate", {
|
||||
"from": NODE_NAME, "type": "image_generate",
|
||||
"request_id": request_id,
|
||||
"prompt": prompt,
|
||||
"negative_prompt": negative_prompt,
|
||||
"width": width, "height": height,
|
||||
"steps": steps, "seed": seed, "model": model,
|
||||
"timestamp": now,
|
||||
})
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(event.wait(), timeout=timeout)
|
||||
result = _pending_image.pop(request_id, (None, None))
|
||||
return result[0]
|
||||
except asyncio.TimeoutError:
|
||||
_pending_image.pop(request_id, None)
|
||||
_push_event("application", "warn", node_name, "Image generation timed out")
|
||||
return None
|
||||
|
||||
|
||||
SUBSCRIBE_TABLE = [
|
||||
(AMQP_EXCHANGE_ADMIN, ["node.*.register"], handle_registration),
|
||||
(AMQP_EXCHANGE_ADMIN, ["node.*.deregister"], handle_deregistration),
|
||||
@@ -249,6 +316,8 @@ SUBSCRIBE_TABLE = [
|
||||
(AMQP_EXCHANGE_SYSTEM, ["node.*.heartbeat"], handle_heartbeat),
|
||||
(AMQP_EXCHANGE_SYSTEM, ["node.*.model_ready"], handle_model_ready),
|
||||
(AMQP_EXCHANGE_SYSTEM, ["node.*.model_failed"], handle_model_failed),
|
||||
(AMQP_EXCHANGE_SYSTEM, ["node.*.image_generated"], handle_image_generated),
|
||||
(AMQP_EXCHANGE_SYSTEM, ["node.*.image_failed"], handle_image_failed),
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -6,14 +6,15 @@ import os
|
||||
import re
|
||||
import ipaddress
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
log = logging.getLogger("caic")
|
||||
|
||||
VERSION = "v0.22.0"
|
||||
VERSION = "v1.1.0"
|
||||
OLLAMA_BASE = os.environ.get("OLLAMA_BASE", "http://localhost:11434")
|
||||
LLAMA_SERVER_BASE = os.environ.get("LLAMA_SERVER_BASE", "http://192.168.50.108:8081")
|
||||
LLAMA_SERVER_BASE = os.environ.get("LLAMA_SERVER_BASE", "http://localhost:8081")
|
||||
SEARXNG_BASE = os.environ.get("CAIC_SEARXNG_BASE", "http://localhost:8888")
|
||||
DEFAULT_MODEL = "qwen2.5-7b-instruct"
|
||||
DEFAULT_MODEL = os.environ.get("CAIC_DEFAULT_MODEL", "qwen2.5-7b-instruct")
|
||||
COMPLETIONS_API_KEY = os.environ.get("CAIC_COMPLETIONS_API_KEY", "caic-sk-" + os.urandom(24).hex())
|
||||
MODEL_CONTEXT_LENGTH = 4096
|
||||
|
||||
@@ -21,7 +22,7 @@ MODEL_CONTEXT_LENGTH = 4096
|
||||
AMQP_RECONNECT_DELAY = 5
|
||||
AMQP_EXCHANGE_ADMIN = "jc.admin"
|
||||
AMQP_EXCHANGE_SYSTEM = "jc.system"
|
||||
AMQP_SECRET_PATH = os.environ.get("CAIC_AMQP_SECRET_PATH", "/home/gramps/.caic_amqp_secret")
|
||||
AMQP_SECRET_PATH = os.environ.get("CAIC_AMQP_SECRET_PATH", "/run/secrets/caic_amqp_secret")
|
||||
|
||||
def get_amqp_url() -> str:
|
||||
url = os.environ.get("CAIC_AMQP_URL")
|
||||
@@ -51,6 +52,10 @@ TRUST_X_FORWARDED_FOR = (
|
||||
os.getenv("CAIC_TRUST_X_FORWARDED_FOR", "false").lower() == "true"
|
||||
)
|
||||
|
||||
# --- Image generation (ComfyUI) ---
|
||||
COMFYUI_BASE = os.environ.get("CAIC_COMFYUI_BASE", "http://localhost:8188")
|
||||
COMFYUI_TIMEOUT = int(os.environ.get("CAIC_COMFYUI_TIMEOUT", "120"))
|
||||
|
||||
# --- Rate limits ---
|
||||
RATE_WINDOW_SECONDS = 60
|
||||
RL_LOGIN_PER_WINDOW = 10
|
||||
@@ -69,13 +74,14 @@ BODY_LIMIT_PROFILE_BYTES = 256 * 1024
|
||||
UPLOAD_DIR = os.environ.get("CAIC_UPLOAD_DIR", "/tmp/caic_uploads")
|
||||
MAX_UPLOAD_BYTES = 20 * 1024 * 1024
|
||||
SUPPORTED_UPLOAD_TYPES = {"text/plain", "text/markdown", "application/pdf", "application/json", "text/x-python", "text/html", "image/png", "image/jpeg", "image/gif", "image/svg+xml", "image/webp"}
|
||||
QDRANT_URL = os.environ.get("CAIC_QDRANT_URL", "http://192.168.50.108:6333")
|
||||
QDRANT_URL = os.environ.get("CAIC_QDRANT_URL", "http://localhost:6333")
|
||||
RAG_COLLECTION = os.environ.get("CAIC_RAG_COLLECTION", "caic_rag")
|
||||
UPLOAD_CONTEXT_EXPIRY_HOURS = 1
|
||||
BODY_LIMIT_UPLOAD_BYTES = MAX_UPLOAD_BYTES
|
||||
|
||||
# --- RAG eviction ---
|
||||
RAG_MAX_VECTORS = 50000
|
||||
RAG_MAX_VECTORS = int(os.environ.get("CAIC_RAG_MAX_VECTORS", "50000"))
|
||||
HW_STATE_PATH = os.environ.get("CAIC_HW_STATE_PATH", str(Path(__file__).parent / "hardware_state.json"))
|
||||
RAG_EVICTION_HIGH_WATER = 0.80
|
||||
RAG_EVICTION_LOW_WATER = 0.20
|
||||
RAG_EVICTION_BATCH = 1000
|
||||
@@ -173,12 +179,10 @@ ALLOWED_NETWORKS = parse_allowed_cidrs(ALLOWED_CIDRS_RAW)
|
||||
DEFAULT_PROFILE = """You are a coding companion running locally on a machine called "jarvis".
|
||||
|
||||
## Environment
|
||||
- jarvis: Debian 13 (trixie) x86_64, AMD Ryzen 5 5600X, 16GB RAM, AMD RX 6600 XT (8GB VRAM)
|
||||
- ultron: Debian 13, Ryzen 7 7840HS, 16GB RAM, primary AI inference node, IP 192.168.50.108
|
||||
- Corsair: Windows 11, gaming/streaming rig, RTX 5070 Ti
|
||||
- jarvis: Debian 13 (trixie) x86_64, AMD Ryzen 5 5600X, 16GB RAM, AMD RX 6600 XT (8GB VRAM), IP 192.168.50.212
|
||||
- Single-node deployment — all cAIc services run on jarvis: llama-server :8081 (OpenAI-compat API), Qdrant :6333, Ollama :11434, SearXNG :8888, RabbitMQ :5672, ComfyUI :8188
|
||||
- pivault: RPi 5, 8GB RAM, Debian 13, 11TB RAID5 NAS at /mnt/pivault, IP 192.168.50.158
|
||||
- Router: ASUS ROG Rapture GT-BE98 Pro "BigBlinkyRouter" at 192.168.50.1
|
||||
- llama-server on ultron:8081 (OpenAI-compat API), Qdrant on ultron:6333
|
||||
|
||||
## About the User
|
||||
- Experienced developer, BS in Computer Science (Oklahoma State), coding since 1981 (TRS-80)
|
||||
|
||||
@@ -24,18 +24,24 @@ DB_PATH = Path(os.environ.get("CAIC_DB_PATH", str(BASE_DIR / "caic.db")))
|
||||
|
||||
|
||||
def get_db():
|
||||
"""Return a new SQLite connection. Each call creates a fresh connection
|
||||
(not pooled) so callers must close() when done."""
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
conn.execute("PRAGMA journal_mode = WAL")
|
||||
conn.execute("PRAGMA busy_timeout = 5000")
|
||||
return conn
|
||||
|
||||
|
||||
def get_setting(db, key: str, default: str = "") -> str:
|
||||
"""Read a single settings row, returning *default* if the key is missing."""
|
||||
row = db.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone()
|
||||
return row["value"] if row else default
|
||||
|
||||
|
||||
def list_skills_with_state(db) -> list:
|
||||
"""Merge built-in skill definitions with per-skill enabled/disabled state from the DB."""
|
||||
rows = db.execute("SELECT skill_key, enabled, updated_at FROM skills").fetchall()
|
||||
state_by_key = {
|
||||
row["skill_key"]: {"enabled": bool(row["enabled"]), "updated_at": row["updated_at"]}
|
||||
@@ -49,6 +55,7 @@ def list_skills_with_state(db) -> list:
|
||||
|
||||
|
||||
def set_skill_enabled(db, skill_key: str, enabled: bool) -> None:
|
||||
"""Insert or replace a skill's enabled state (UPSERT)."""
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
db.execute(
|
||||
"INSERT OR REPLACE INTO skills (skill_key, enabled, updated_at) VALUES (?, ?, ?)",
|
||||
@@ -57,6 +64,7 @@ def set_skill_enabled(db, skill_key: str, enabled: bool) -> None:
|
||||
|
||||
|
||||
def format_active_skills_prompt(skills: list) -> str:
|
||||
"""Build the 'Active Skills' section of the system prompt from the provided skill list."""
|
||||
lines = [
|
||||
"## Active Skills",
|
||||
"Use these skills only when needed. Prefer concise answers over unnecessary tool usage.",
|
||||
@@ -70,6 +78,7 @@ def format_active_skills_prompt(skills: list) -> str:
|
||||
|
||||
|
||||
def insert_upload_context(db, conversation_id: str, filename: str, content: str, expires_at: str, content_type: str = "text/plain") -> int:
|
||||
"""Persist an upload context entry (encrypted content) tied to a conversation."""
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
cur = db.execute(
|
||||
"INSERT INTO upload_context (conversation_id, filename, content, content_type, created_at, expires_at) VALUES (?, ?, ?, ?, ?, ?)",
|
||||
@@ -79,6 +88,7 @@ def insert_upload_context(db, conversation_id: str, filename: str, content: str,
|
||||
|
||||
|
||||
def list_upload_context_by_conversation(db, conversation_id: str):
|
||||
"""Return all upload contexts for a given conversation (content excluded for brevity)."""
|
||||
rows = db.execute(
|
||||
"SELECT id, conversation_id, filename, content_type, created_at, expires_at FROM upload_context WHERE conversation_id = ? ORDER BY id ASC",
|
||||
(conversation_id,),
|
||||
@@ -87,11 +97,16 @@ def list_upload_context_by_conversation(db, conversation_id: str):
|
||||
|
||||
|
||||
def delete_upload_context_by_id(db, context_id: int) -> bool:
|
||||
"""Delete an upload context entry, returning True if a row was actually removed."""
|
||||
cur = db.execute("DELETE FROM upload_context WHERE id = ?", (context_id,))
|
||||
return cur.rowcount > 0
|
||||
|
||||
|
||||
def get_upload_context(db, context_id: int):
|
||||
"""Fetch a single upload context, returning its decrypted content.
|
||||
|
||||
If the context has expired (past expires_at), it is deleted and None returned.
|
||||
"""
|
||||
row = db.execute(
|
||||
"SELECT id, conversation_id, filename, content, content_type, expires_at FROM upload_context WHERE id = ?",
|
||||
(context_id,),
|
||||
@@ -109,10 +124,19 @@ def get_upload_context(db, context_id: int):
|
||||
|
||||
|
||||
def init_db():
|
||||
"""Run initial schema creation and seed default data.
|
||||
|
||||
Idempotent — safe to call on every startup. Creates tables if missing,
|
||||
runs ALTER TABLE to add columns that may not exist on legacy databases,
|
||||
and inserts defaults for profile, presets, settings, skills, and admin PIN.
|
||||
"""
|
||||
from security import hash_pin
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode = WAL")
|
||||
conn.execute("PRAGMA busy_timeout = 5000")
|
||||
|
||||
# --- Core tables ---
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS conversations (
|
||||
id TEXT PRIMARY KEY, title TEXT NOT NULL DEFAULT 'New Chat',
|
||||
@@ -144,6 +168,7 @@ def init_db():
|
||||
skill_key TEXT PRIMARY KEY, enabled INTEGER NOT NULL DEFAULT 1, updated_at TEXT NOT NULL
|
||||
)
|
||||
""")
|
||||
# FTS5 virtual table for full-text memory search
|
||||
conn.execute("""
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS memories USING fts5(
|
||||
fact, topic, source, created_at UNINDEXED
|
||||
@@ -160,16 +185,18 @@ def init_db():
|
||||
expires_at TEXT NOT NULL
|
||||
)
|
||||
""")
|
||||
|
||||
# --- Backfill columns for legacy databases (safe to run every time) ---
|
||||
try:
|
||||
conn.execute("ALTER TABLE upload_context ADD COLUMN content_type TEXT DEFAULT 'text/plain'")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
pass # column already exists
|
||||
try:
|
||||
conn.execute("ALTER TABLE messages ADD COLUMN perplexity REAL")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# --- Seed default data (only if tables are empty) ---
|
||||
if not conn.execute("SELECT id FROM profile WHERE id = 1").fetchone():
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
conn.execute("INSERT INTO profile (id, content, updated_at) VALUES (1, ?, ?)", (DEFAULT_PROFILE, now))
|
||||
@@ -195,6 +222,9 @@ def init_db():
|
||||
if not conn.execute("SELECT skill_key FROM skills WHERE skill_key = ?", (skill["key"],)).fetchone():
|
||||
conn.execute("INSERT INTO skills (skill_key, enabled, updated_at) VALUES (?, 1, ?)", (skill["key"], now))
|
||||
|
||||
# --- Admin PIN bootstrap ---
|
||||
# If no PIN hash exists on disk, seed one from env var CAIC_ADMIN_PIN
|
||||
# or, if CAIC_ALLOW_DEFAULT_PIN=true, from the hardcoded default "1234".
|
||||
existing_pin_hash = conn.execute("SELECT value FROM settings WHERE key = 'admin_pin_hash'").fetchone()
|
||||
existing_pin_salt = conn.execute("SELECT value FROM settings WHERE key = 'admin_pin_salt'").fetchone()
|
||||
if not existing_pin_hash or not existing_pin_salt:
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
# cAIc — Docker Compose stack
|
||||
# Coordinator: cAIc + SearXNG + Qdrant + RabbitMQ + llama-server + Ollama
|
||||
#
|
||||
# Usage:
|
||||
# cp .env.example .env # edit with your values
|
||||
# mkdir -p models secrets searxng
|
||||
# # place .gguf model in ./models/
|
||||
# docker compose up -d
|
||||
|
||||
services:
|
||||
# ── cAIc (FastAPI) ──────────────────────────────────────
|
||||
caic:
|
||||
build: .
|
||||
ports:
|
||||
- "${CAIC_EXPOSE_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- caic_data:/app/data
|
||||
secrets:
|
||||
- rabbitmq_password
|
||||
environment:
|
||||
- CAIC_AMQP_SECRET_PATH=/run/secrets/rabbitmq_password
|
||||
- CAIC_COMFYUI_BASE=${CAIC_COMFYUI_BASE:-http://localhost:8188}
|
||||
- CAIC_COMFYUI_TIMEOUT=${CAIC_COMFYUI_TIMEOUT:-120}
|
||||
env_file: .env
|
||||
depends_on:
|
||||
qdrant: { condition: service_started }
|
||||
rabbitmq: { condition: service_healthy }
|
||||
llama-server: { condition: service_healthy }
|
||||
restart: unless-stopped
|
||||
|
||||
# ── SearXNG (web search) ────────────────────────────────
|
||||
searxng:
|
||||
image: searxng/searxng:latest
|
||||
ports:
|
||||
- "${SEARXNG_EXPOSE_PORT:-8888}:8080"
|
||||
volumes:
|
||||
- ./searxng/settings.yml:/etc/searxng/settings.yml:ro
|
||||
- searxng_config:/etc/searxng
|
||||
environment:
|
||||
- SEARXNG_BASE_URL=http://localhost:8888
|
||||
restart: unless-stopped
|
||||
|
||||
# ── Qdrant (vector DB) ──────────────────────────────────
|
||||
qdrant:
|
||||
image: qdrant/qdrant:latest
|
||||
ports:
|
||||
- "${QDRANT_EXPOSE_PORT:-6333}:6333"
|
||||
volumes:
|
||||
- qdrant_storage:/qdrant/storage
|
||||
restart: unless-stopped
|
||||
|
||||
# ── RabbitMQ (AMQP broker) ──────────────────────────────
|
||||
rabbitmq:
|
||||
image: rabbitmq:4-management
|
||||
ports:
|
||||
- "${RABBITMQ_EXPOSE_PORT:-5672}:5672"
|
||||
volumes:
|
||||
- rabbitmq_data:/var/lib/rabbitmq
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: caic
|
||||
RABBITMQ_DEFAULT_PASS_FILE: /run/secrets/rabbitmq_password
|
||||
RABBITMQ_DEFAULT_VHOST: /
|
||||
secrets:
|
||||
- rabbitmq_password
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "check_port_connectivity"]
|
||||
interval: 15s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
restart: unless-stopped
|
||||
|
||||
# ── llama-server (LLM inference) ────────────────────────
|
||||
llama-server:
|
||||
image: ghcr.io/ggml-org/llama.cpp:server
|
||||
ports:
|
||||
- "${LLAMA_EXPOSE_PORT:-8081}:8081"
|
||||
volumes:
|
||||
- ./models:/models:ro
|
||||
command: >
|
||||
--model /models/${LLAMA_MODEL:?Set LLAMA_MODEL in .env}
|
||||
--host 0.0.0.0 --port 8081
|
||||
--ctx-size ${LLAMA_CTX_SIZE:-4096}
|
||||
--n-gpu-layers ${LLAMA_N_GPU_LAYERS:-0}
|
||||
--embeddings
|
||||
--logprobs
|
||||
${LLAMA_RPC_ENDPOINTS:+--rpc ${LLAMA_RPC_ENDPOINTS}}
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-fs", "http://localhost:8081/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
start_period: 60s
|
||||
restart: unless-stopped
|
||||
|
||||
# ── Ollama (embeddings) ─────────────────────────────────
|
||||
ollama:
|
||||
image: ollama/ollama:latest
|
||||
ports:
|
||||
- "${OLLAMA_EXPOSE_PORT:-11434}:11434"
|
||||
volumes:
|
||||
- ollama_models:/root/.ollama
|
||||
healthcheck:
|
||||
test: ["CMD", "ollama", "list"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
caic_data:
|
||||
searxng_config:
|
||||
qdrant_storage:
|
||||
rabbitmq_data:
|
||||
ollama_models:
|
||||
|
||||
secrets:
|
||||
rabbitmq_password:
|
||||
file: ./secrets/rabbitmq_password.txt
|
||||
@@ -763,37 +763,36 @@ The broker-mediated model is the preferred architecture for this project because
|
||||
|
||||
## 10. Checklist (pre-v1.0 gate)
|
||||
|
||||
- [ ] `Dockerfile` written and builds clean
|
||||
- [ ] `docker-compose.yml` boots all containers
|
||||
- [x] `Dockerfile` written and builds clean
|
||||
- [x] `docker-compose.yml` boots all containers
|
||||
- [ ] cAIc container reaches all services (env vars resolve correctly)
|
||||
- [ ] SearXNG settings.yml generated correctly by setup.sh
|
||||
- [ ] RabbitMQ password secret mounted correctly
|
||||
- [x] SearXNG settings.yml generated correctly by setup.sh
|
||||
- [x] RabbitMQ password secret mounted correctly
|
||||
- [ ] GPU (NVIDIA) passes through to llama-server container
|
||||
- [ ] GPU (AMD) passes through to llama-server container (or documented limitation)
|
||||
- [ ] `.env.example` checked in (no real secrets)
|
||||
- [ ] `setup.sh` written, idempotent, tested on clean Debian
|
||||
- [ ] `teardown.sh` written, tested, doesn't delete models without confirmation
|
||||
- [x] `.env.example` checked in (no real secrets)
|
||||
- [x] `setup.sh` written, idempotent, tested on clean Debian
|
||||
- [x] `teardown.sh` written, tested, doesn't delete models without confirmation
|
||||
- [ ] `docker compose up -d` works without any manual steps beyond setup.sh
|
||||
- [ ] `docker compose down -v` followed by `setup.sh && docker compose up -d` = fresh stack
|
||||
- [ ] Healthchecks prevent serving before dependencies are ready
|
||||
- [ ] WireGuard tunnel documented and tested for off-site workers
|
||||
- [ ] v1.0 release tag created
|
||||
- [x] v1.0 release tag created
|
||||
|
||||
---
|
||||
|
||||
## 11. Files to create for B3
|
||||
## 11. Files created for B3
|
||||
|
||||
```
|
||||
docker.md ← this file (planning doc)
|
||||
Dockerfile ← cAIc image
|
||||
docker-compose.yml ← full stack
|
||||
.env.example ← template without secrets
|
||||
setup.sh ← extraction wizard
|
||||
teardown.sh ← back-out utility
|
||||
searxng/
|
||||
settings.yml ← SearXNG config (generated by setup.sh)
|
||||
secrets/
|
||||
rabbitmq_password.txt ← generated by setup.sh
|
||||
models/
|
||||
README.txt ← instructions for placing .gguf
|
||||
docker.md ← this file (planning doc)
|
||||
Dockerfile ← cAIc image (multi-stage, Python 3.13-slim)
|
||||
docker-compose.yml ← full stack (6 services, volumes, secrets, healthchecks)
|
||||
.env.example ← template without secrets
|
||||
.dockerignore ← excludes venv, tests, .git, models, secrets
|
||||
scripts/setup.sh ← first-run scaffolding (generates .env, secrets, pulls model)
|
||||
scripts/teardown-docker.sh ← Docker stack teardown (interactive, -y for unattended)
|
||||
searxng-settings.yml.dist ← SearXNG config template (copied by setup.sh)
|
||||
models/README.txt ← instructions for placing .gguf
|
||||
secrets/ ← generated at runtime by setup.sh
|
||||
searxng/ ← generated at runtime by setup.sh
|
||||
```
|
||||
|
||||
@@ -0,0 +1,994 @@
|
||||
# cAIc — OpenCode Prompt Sequence
|
||||
# Generated: 2026-07-14
|
||||
# Execute sequentially. Run full test suite after each task before proceeding.
|
||||
# Test command: ./venv/bin/python -m pytest tests/ -v
|
||||
|
||||
---
|
||||
|
||||
## Session 2026-07-14 — RAG bugfixes + Topbar redesign
|
||||
|
||||
- **RAG bugs fixed**: Collection name mismatch (`jarvis_rag` → `caic_rag`, migrated 219 points), `vectors_count`→`points_count` (Qdrant v1.10+ API change), removed unindexed `order_by` that caused 502 on scroll, made `RAG_COLLECTION` env-configurable (`CAIC_RAG_COLLECTION`).
|
||||
- **Semantic search fixed**: Set `CAIC_EMBED_URL=http://192.168.50.108:11434` (mxbai-embed-large lives on ultron, not the old embed server).
|
||||
- **Topbar redesign**: Moved system stats (CPU/MEM/GPU/VRAM/TOK) to a centered bottom strip. Moved toggles (MEM, SEARCH, PROFILE, SORT, PRIVACY) into a ⋮ hamburger menu next to ADMIN badge. Palette icon sits immediately after version number in topbar-left. Removed standalone (i) button — privacy info accessible via ⋮ → About Privacy. Input bar above chat, stats at very bottom. Mobile-responsive padding/sizing.
|
||||
|
||||
---
|
||||
|
||||
## ~~TASK 1 — README Cleanup [DONE]~~
|
||||
|
||||
Review README.md in the current repo. Remove any node references other than `coordinator` (192.168.50.108) and `worker` (192.168.50.210). Ensure all references to the project use the exact casing `cAIc` — not `Jarvischat`, `JarvisChat`, or `jarvischat`. Do not change any functional content, endpoint documentation, or architecture descriptions — this is a text cleanup only. After editing, verify the file renders cleanly as markdown. Commit with message: `docs: clean up node references and branding consistency`.
|
||||
|
||||
No new tests required for this task.
|
||||
|
||||
---
|
||||
|
||||
## ~~TASK 2 — Qwen2.5-Coder llama-server Service on Coordinator (Infrastructure) [DONE]~~
|
||||
|
||||
**Status: Systemd unit created, verified, and restored.**
|
||||
|
||||
This task originally defined creation of `/etc/systemd/system/llama-server-coder.service` (port 8082, Qwen2.5-Coder-14B Q5_K_M) as a prerequisite for dynamic model swapping. That sysadmin work is done.
|
||||
|
||||
**The real Task 2 deliverable — the ability to dynamically swap models based on query classification — is delivered by Roadmap N (Tasks 9–15).** The flow:
|
||||
|
||||
1. **Task 13** — Phi-4-mini triage (`triage.py`) classifies the query as `general`, `code`, `search`, or `rag`
|
||||
2. **Task 13** — `select_node()` picks the best worker node; if the ideal model isn't active, it triggers a swap
|
||||
3. **Task 14** — `request_model_swap()` publishes `cmd.swap_model` via AMQP `jc.admin` exchange
|
||||
4. **Task 12** — The node agent on worker receives the command, stops the current llama-server, starts the correct one, waits for health, and publishes `model_ready`
|
||||
5. **Task 14** — coordinator receives `model_ready`, updates the cluster registry, and routes the query to the node
|
||||
|
||||
The swap is async and transparent — the user sees only latency. The UI (Task 15) shows a yellow "swapping" status dot during the transition.
|
||||
|
||||
The service unit at `/etc/systemd/system/llama-server-coder.service` is the **target** the node agent starts when swapping to code inference. It is not enabled at boot — the AMQP cluster manages activation.
|
||||
|
||||
See Tasks 9–15 for the actual model swap implementation.
|
||||
|
||||
No pytest tests required for this infrastructure task.
|
||||
|
||||
---
|
||||
|
||||
## ~~TASK 3 — Update OpenCode Config to Use Qwen on :8082 [DONE]~~
|
||||
|
||||
Update `/home/gramps/.config/opencode/opencode.jsonc` (on this machine, coordinator) to point the configured provider at `http://127.0.0.1:8082/v1` instead of `http://127.0.0.1:8081/v1`. The model name in the config should be updated to reflect `qwen2.5-coder-14b` or whatever model ID the llama-server instance at :8082 reports via `/v1/models`. Verify the endpoint is reachable before writing the config change. Do not restart OpenCode — the config change takes effect on next session start.
|
||||
|
||||
No pytest tests required for this task.
|
||||
|
||||
---
|
||||
|
||||
## ~~TASK 4 — File/Document Attachment: Backend Ingest Endpoint [DONE]~~
|
||||
|
||||
**Status: `POST /api/upload` with mode=(context|ingest|both), PDF/text extraction, Qdrant upsert, SQLite context (1hr expiry). Committed `4a891c8` (v1.9.0).**
|
||||
|
||||
This task implements the backend half of file/document attachment (TODO #21). The goal is dual-aspect upload: a file can be used as immediate chat context, ingested into the RAG corpus (Qdrant), or both.
|
||||
|
||||
**Add to `config.py`:**
|
||||
- `UPLOAD_DIR` — path for temporary upload storage, default `/tmp/caic_uploads`
|
||||
- `MAX_UPLOAD_BYTES` — max file size, default 20MB
|
||||
- `SUPPORTED_UPLOAD_TYPES` — set of MIME types: `text/plain`, `text/markdown`, `application/pdf`, `application/json`, `text/x-python`, `text/html`
|
||||
|
||||
**Create `routers/upload.py`:**
|
||||
|
||||
Implement `POST /api/upload` (admin required). Accept `multipart/form-data` with:
|
||||
- `file` — the uploaded file (required)
|
||||
- `mode` — string enum: `context` (inject into next chat only), `ingest` (add to RAG corpus), `both` (default: `both`)
|
||||
- `conversation_id` — optional, associates context-mode content with a specific conversation
|
||||
|
||||
Behavior:
|
||||
- Validate file size against `MAX_UPLOAD_BYTES` — return 413 if exceeded
|
||||
- Validate MIME type against `SUPPORTED_UPLOAD_TYPES` — return 415 if unsupported
|
||||
- For PDF files, extract text using `pypdf` (add to requirements.txt)
|
||||
- For all other types, read as UTF-8 text
|
||||
- If mode includes `ingest`: chunk the extracted text into 512-token overlapping chunks (128-token overlap), generate embeddings via `EMBED_URL` (http://192.168.50.108:11434/api/embeddings, model mxbai-embed-large), upsert into Qdrant collection `caic` with metadata `{source: filename, upload_date: iso_timestamp, type: "upload"}`
|
||||
- If mode includes `context`: store the full extracted text in a new SQLite table `upload_context` with columns `(id INTEGER PRIMARY KEY, conversation_id TEXT, filename TEXT, content TEXT, created_at TEXT, expires_at TEXT)`. Context entries expire after 1 hour.
|
||||
- Return JSON: `{filename, size_bytes, mode, chunks_ingested (if ingest), context_id (if context), message}`
|
||||
|
||||
**Add `upload_context` table to `db.py`** `init_db()`.
|
||||
|
||||
**Wire `upload.router` into `app.py`** in the router registration block.
|
||||
|
||||
**Write `tests/test_upload.py`** covering:
|
||||
- Valid text file upload, mode=ingest — assert chunks_ingested > 0, Qdrant upsert called
|
||||
- Valid text file upload, mode=context — assert context_id returned, row exists in upload_context
|
||||
- Valid text file upload, mode=both — assert both behaviors
|
||||
- File exceeds MAX_UPLOAD_BYTES — assert 413
|
||||
- Unsupported MIME type — assert 415
|
||||
- Guest session attempt — assert 403
|
||||
- PDF extraction path — mock pypdf, assert text extracted and processed
|
||||
|
||||
Mock Qdrant and EMBED_URL calls via monkeypatch. Do not require live external services in tests.
|
||||
|
||||
Run full test suite after implementation. All 26 existing tests must continue to pass.
|
||||
|
||||
---
|
||||
|
||||
## ~~TASK 5 — File/Document Attachment: UI Integration [DONE]~~
|
||||
|
||||
**Status: Paperclip icon, file preview pill, gallery overlay, attachment indicators, DELETE/PATCH link/by-conversation endpoints, chat context injection. Committed `81238c0` (v1.10.0).**
|
||||
|
||||
This task implements the frontend half of TODO #21. The UI is a single file at `templates/index.html`.
|
||||
|
||||
Add a file attachment button to the chat input area. Requirements:
|
||||
- Paperclip icon button adjacent to the send button
|
||||
- Clicking opens a file picker filtered to supported types (`.txt`, `.md`, `.pdf`, `.json`, `.py`, `.html`)
|
||||
- On file selection, show a pill/badge above the input showing the filename with an X to remove it
|
||||
- On send, if a file is attached: POST to `/api/upload` with `mode=both` and the current `conversation_id`, then include the returned `context_id` in the subsequent `/api/chat` POST body as `upload_context_id`
|
||||
- If the upload fails, show an inline error and do not send the chat message
|
||||
- File attachment state clears after send
|
||||
|
||||
**Update `/api/chat` in `routers/chat.py`:**
|
||||
- Accept optional `upload_context_id` in the request body
|
||||
- If present, look up the content in `upload_context` table and prepend it to the system prompt as: `\n\n[ATTACHED DOCUMENT: {filename}]\n{content}\n[END DOCUMENT]`
|
||||
- If the context_id is expired or missing, log a warning and continue without it (do not error)
|
||||
|
||||
**Add to `tests/test_chat_streaming_and_memory_paths.py`:**
|
||||
- Test that a valid `upload_context_id` results in document content being prepended to the system prompt
|
||||
- Test that an expired/missing `upload_context_id` is silently ignored
|
||||
|
||||
Run full test suite. All existing tests must continue to pass.
|
||||
|
||||
---
|
||||
|
||||
## ~~TASK 6 — Roadmap I: Terminal Command RAG Hook [DONE]~~
|
||||
|
||||
**Status: `POST /api/ingest` with Bearer token auth, `chunk_text()` shared helper, `caic-ingest.sh` script. Committed `1ac21ad` (v0.11.0).**
|
||||
|
||||
This task implements autonomous RAG ingestion of significant terminal activity (TODO #23).
|
||||
|
||||
**Create `routers/ingest.py`:**
|
||||
|
||||
Implement `POST /api/ingest` (requires Bearer token auth — use same `COMPLETIONS_API_KEY` mechanism as `routers/completions.py`). Accept JSON body:
|
||||
- `content` — string, the text to ingest (required)
|
||||
- `source` — string, origin label e.g. `terminal`, `file`, `external` (default: `external`)
|
||||
- `metadata` — optional dict of additional key/value pairs
|
||||
|
||||
Behavior:
|
||||
- Chunk `content` into 512-token overlapping chunks (128-token overlap) — extract this logic into a shared helper `chunk_text(text, chunk_size=512, overlap=128)` in `rag.py` if not already present
|
||||
- Generate embeddings via `EMBED_URL`
|
||||
- Upsert into Qdrant collection `caic` with metadata `{source, ingest_date: iso_timestamp, ...metadata}`
|
||||
- Return JSON: `{chunks_ingested, source, message}`
|
||||
|
||||
**Wire `ingest.router` into `app.py`.**
|
||||
|
||||
**Create `/usr/local/bin/caic-ingest.sh` on worker (192.168.50.210)** — this is a shell script, not a Python file, and lives outside the repo. Write it to stdout/document it clearly so gramps can deploy it manually:
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# caic-ingest.sh — pipe terminal commands into cAIc RAG
|
||||
# Add to ~/.bashrc: export PROMPT_COMMAND="jc_capture"
|
||||
# Function to call after significant commands
|
||||
|
||||
JC_URL="http://192.168.50.210:8080/api/ingest"
|
||||
JC_TOKEN="${CAIC_COMPLETIONS_API_KEY}"
|
||||
|
||||
jc_capture() {
|
||||
local cmd
|
||||
cmd=$(history 1 | sed 's/^[ ]*[0-9]*[ ]*//')
|
||||
# Only ingest significant commands
|
||||
if echo "$cmd" | grep -qE '^(git|pip|systemctl|sudo|vi|vim|curl|wget|apt|python|pytest)'; then
|
||||
curl -s -X POST "$JC_URL" \
|
||||
-H "Authorization: Bearer $JC_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"content\": $(echo "$cmd" | python3 -c 'import json,sys; print(json.dumps(sys.stdin.read().strip()))'), \"source\": \"terminal\"}" \
|
||||
> /dev/null 2>&1 &
|
||||
fi
|
||||
}
|
||||
```
|
||||
|
||||
**Write `tests/test_ingest.py`** covering:
|
||||
- Valid ingest with content — assert chunks_ingested > 0
|
||||
- Missing Bearer token — assert 401
|
||||
- Wrong Bearer token — assert 403
|
||||
- Empty content — assert 422
|
||||
- Qdrant and embed calls mocked via monkeypatch
|
||||
|
||||
Run full test suite. All existing tests must continue to pass.
|
||||
|
||||
---
|
||||
|
||||
## ~~TASK 7 — Roadmap J: Startup Hardware Self-Assessment [DONE]~~
|
||||
|
||||
**Status: `hardware.py` + `routers/hardware.py` + 4 tests. Committed `7291b8f` (v0.12.0).**
|
||||
|
||||
On jC startup, probe available hardware and write a living config snapshot. This replaces hardcoded assumptions about VRAM and RAM.
|
||||
|
||||
**Create `hardware.py`** in the project root:
|
||||
|
||||
```
|
||||
async def assess_hardware() -> dict
|
||||
```
|
||||
|
||||
Probes:
|
||||
- System RAM: `psutil.virtual_memory().total` and `.available`
|
||||
- CPU count: `psutil.cpu_count()`
|
||||
- GPU VRAM total and free: call `rocm-smi --showmeminfo vram --json` via subprocess, parse output. If rocm-smi absent or fails, set VRAM values to 0 and log a warning.
|
||||
- llama-server reachable: GET `LLAMA_SERVER_BASE/v1/models`, timeout 3s. Record True/False and list of available model IDs.
|
||||
- Qdrant reachable: GET `http://192.168.50.108:6333/collections`, timeout 3s. Record True/False and collection list.
|
||||
- SearXNG reachable: GET `http://localhost:8888`, timeout 3s. Record True/False.
|
||||
|
||||
Returns a dict with all of the above. Writes result as JSON to `hardware_state.json` in the working directory.
|
||||
|
||||
**Call `assess_hardware()` from the FastAPI `lifespan` context** in `app.py` on startup, after `init_db()`. Log a summary line: `HW: {ram_gb}GB RAM, {vram_mb}MB VRAM, llama={reachable}, qdrant={reachable}, searxng={reachable}`.
|
||||
|
||||
**Expose `GET /api/hardware`** in a new `routers/hardware.py` — returns the current `hardware_state.json` content as JSON. No auth required (read-only, non-sensitive aggregate stats).
|
||||
|
||||
**Wire `hardware.router` into `app.py`.**
|
||||
|
||||
**Write `tests/test_hardware.py`** covering:
|
||||
- `assess_hardware()` with all services reachable (mock subprocess and httpx calls) — assert all fields present
|
||||
- `assess_hardware()` with rocm-smi absent — assert VRAM=0, no exception raised
|
||||
- `assess_hardware()` with llama-server unreachable — assert `llama_reachable=False`, no exception
|
||||
- `GET /api/hardware` — assert returns JSON with expected keys
|
||||
|
||||
Run full test suite. All existing tests must continue to pass.
|
||||
|
||||
---
|
||||
|
||||
## ~~TASK 8 — Roadmap K: RAG Corpus Management [DONE]~~
|
||||
|
||||
Qdrant collection `caic` currently grows without bound. Implement score-based eviction with hysteresis, pinned sources, operational stats, and a flush command.
|
||||
|
||||
### Config — add to `config.py`:
|
||||
|
||||
```python
|
||||
RAG_MAX_VECTORS = 50000 # absolute ceiling; eviction targets thresholds below it
|
||||
RAG_EVICTION_HIGH_WATER = 0.80 # fraction of RAG_MAX_VECTORS that triggers eviction
|
||||
RAG_EVICTION_LOW_WATER = 0.20 # fraction where eviction stops
|
||||
RAG_EVICTION_BATCH = 1000 # max points to delete per Qdrant scroll/delete cycle
|
||||
RAG_PINNED_SOURCES = ["upload", "profile"] # never evicted
|
||||
RAG_GRACE_HOURS = 1 # new vectors ineligible for eviction until this old
|
||||
RAG_ACCESS_WEIGHT = 1.0 # score factor: retrieval_count * ACCESS_WEIGHT
|
||||
RAG_AGE_WEIGHT = 0.1 # score factor: ingest_age_hours * AGE_WEIGHT
|
||||
```
|
||||
|
||||
Validations on boot: `high_water > low_water`, `batch > 0`, `max_vectors > 0`.
|
||||
|
||||
### Eviction algorithm — add to `rag.py`:
|
||||
|
||||
```
|
||||
score = (retrieval_count * ACCESS_WEIGHT) + (age_hours * AGE_WEIGHT)
|
||||
```
|
||||
|
||||
Lower score = evicted first. Tiebreak: `last_accessed` ASC (older wins).
|
||||
|
||||
```python
|
||||
async def get_collection_count() -> int
|
||||
# GET /collections/caic → return vectors_count
|
||||
|
||||
async def get_collection_stats() -> dict
|
||||
# Return {vector_count, max_vectors, high_water, low_water, percent_full, pinned_sources}
|
||||
|
||||
async def evict_batch(batch_size: int) -> int
|
||||
# Scroll Qdrant for vectors NOT in RAG_PINNED_SOURCES, WHERE ingest_age > RAG_GRACE_HOURS,
|
||||
# ordered by score ASC, last_accessed ASC.
|
||||
# Delete up to batch_size. Return count deleted.
|
||||
# If 0 evictable vectors found: log warning, return 0 (break loop).
|
||||
|
||||
async def maybe_evict() -> int
|
||||
# Acquire eviction_lock (asyncio.Lock).
|
||||
# count = get_collection_count()
|
||||
# threshold_high = RAG_MAX_VECTORS * RAG_EVICTION_HIGH_WATER
|
||||
# threshold_low = RAG_MAX_VECTORS * RAG_EVICTION_LOW_WATER
|
||||
# total_evicted = 0
|
||||
# while count >= threshold_low:
|
||||
# if total_evicted > 0 and count < threshold_low: break
|
||||
# deleted = evict_batch(RAG_EVICTION_BATCH)
|
||||
# if deleted == 0: break # no more unpinned targets
|
||||
# total_evicted += deleted
|
||||
# count -= deleted
|
||||
# if count < threshold_high and total_evicted > 0: break
|
||||
# # only one pass if batch spans the full gap
|
||||
# if count < threshold_low: break
|
||||
# Record total_evicted + timestamp in EVICTION_LOG (list of dicts, kept in memory, max 1000 entries)
|
||||
# Release lock. Return total_evicted.
|
||||
|
||||
async def get_rag_operational_stats() -> dict
|
||||
# Returns: vector_count, max_vectors, high_water_pct, low_water_pct,
|
||||
# percent_full, pinned_sources, grace_hours,
|
||||
# eviction_counts_last_1m, eviction_counts_last_5m, eviction_counts_last_30m,
|
||||
# at_risk_count (vectors in bottom 10% by score),
|
||||
# pinned_count, avg_retrieval_count
|
||||
```
|
||||
|
||||
### Edge cases & guards:
|
||||
|
||||
1. **Newborn grace** — vectors < `RAG_GRACE_HOURS` old are excluded from eviction scroll (score=0 otherwise → immediate deletion)
|
||||
2. **All-pinned freeze** — if scroll returns 0 evictable vectors, log warning and break loop
|
||||
3. **Race** — `asyncio.Lock()` guards `maybe_evict()`; concurrent callers wait their turn
|
||||
4. **Zero config** — `RAG_MAX_VECTORS <= 0` → eviction disabled; `RAG_EVICTION_BATCH <= 0` → clamped to 1
|
||||
5. **Legacy payloads** — vectors without `retrieval_count` or `last_accessed` get defaults (0, `ingest_date`)
|
||||
|
||||
### Wire eviction:
|
||||
|
||||
Call `maybe_evict()` after each upsert batch completes in:
|
||||
- `routers/upload.py` — after Qdrant upsert
|
||||
- `routers/ingest.py` — after Qdrant upsert
|
||||
|
||||
### Admin endpoints — new `routers/rag_admin.py`:
|
||||
|
||||
| Method | Endpoint | Description |
|
||||
|--------|----------|-------------|
|
||||
| GET | `/api/rag/stats` | Operational stats (see `get_rag_operational_stats()`) — admin required |
|
||||
| POST | `/api/rag/flush` | Delete ALL points from the Qdrant `caic` collection. Returns `{deleted_count, collection: "caic", status: "flushed"}`. Admin required. |
|
||||
|
||||
### In-memory eviction log:
|
||||
|
||||
```python
|
||||
EVICTION_LOG: list[dict] = [] # managed by rag.py, max 1000 entries
|
||||
# Each entry: {timestamp: iso, count: N, remaining: N}
|
||||
# Tied to RATE_EVENTS pattern from security.py for rolling window calculations
|
||||
```
|
||||
|
||||
### Tests — `tests/test_rag_management.py`:
|
||||
|
||||
- `get_collection_count()` — mock Qdrant GET, assert correct count
|
||||
- `get_collection_stats()` — assert shape matches config
|
||||
- `evict_batch()` — mock Qdrant scroll + delete, assert pinned sources excluded, grace period enforced, batch size respected
|
||||
- `maybe_evict()` — below high water: 0 evicted; at high water: eviction fires; stops at low water; all-pinned scroll returns 0 → breaks
|
||||
- `GET /api/rag/stats` — assert full shape
|
||||
- `POST /api/rag/flush` — assert points deleted, admin required, guest 403
|
||||
- `POST /api/rag/flush` by guest — assert 403
|
||||
- Race lock — concurrent calls to `maybe_evict()` queue up, only one evicts
|
||||
|
||||
Mock all Qdrant calls via monkeypatch. Do not require live services.
|
||||
|
||||
Run full test suite. All existing tests must continue to pass.
|
||||
|
||||
---
|
||||
|
||||
## ~~TASK 9 — Roadmap N1: RabbitMQ Install and Service on Coordinator (Infrastructure) [DONE]~~
|
||||
|
||||
This task runs on coordinator (this machine). Install RabbitMQ and verify it is operational.
|
||||
|
||||
Run the following steps:
|
||||
1. `apt-get update && apt-get install -y rabbitmq-server`
|
||||
2. `systemctl enable rabbitmq-server && systemctl start rabbitmq-server`
|
||||
3. `systemctl status rabbitmq-server` — verify active/running
|
||||
4. Enable the management plugin: `rabbitmq-plugins enable rabbitmq_management`
|
||||
5. Create a dedicated jC vhost: `rabbitmqctl add_vhost caic`
|
||||
6. Create a dedicated user: `rabbitmqctl add_user caic CHANGEME_PASSWORD` — generate a random 24-char alphanumeric password and record it
|
||||
7. Grant permissions: `rabbitmqctl set_permissions -p caic caic ".*" ".*" ".*"`
|
||||
8. Verify management UI is reachable: `curl -s -u guest:guest http://localhost:15672/api/overview | python3 -m json.tool`
|
||||
9. Delete default guest user: `rabbitmqctl delete_user guest`
|
||||
|
||||
Declare the two topic exchanges needed by jC:
|
||||
- Exchange name: `jc.admin`, type: `topic`, durable: true
|
||||
- Exchange name: `jc.system`, type: `topic`, durable: true
|
||||
|
||||
Use `rabbitmqadmin` or `curl` against the management API to declare exchanges. Verify both exchanges appear in: `curl -s -u caic:{password} http://localhost:15672/api/exchanges/caic`
|
||||
|
||||
Write the generated RabbitMQ password to `/home/gramps/.caic_amqp_secret` with mode 600. This will be read by cAIc as an env var source in subsequent tasks.
|
||||
|
||||
No pytest tests required for this infrastructure task.
|
||||
|
||||
---
|
||||
|
||||
## ~~TASK 10 — Roadmap N2: AMQP Connection Layer in jC [DONE]~~
|
||||
|
||||
This task adds the core AMQP connection manager to jC. It must connect to RabbitMQ on coordinator (localhost from jC's perspective since jC runs on coordinator), handle reconnection, and provide a shared channel for all AMQP operations.
|
||||
|
||||
**Add to `requirements.txt`:** `aio-pika>=9.0.0`
|
||||
|
||||
**Add to `config.py`:**
|
||||
- `AMQP_URL` — read from env `CAIC_AMQP_URL`, default `amqp://caic:password@localhost:5672/caic`. The actual password comes from `/home/gramps/.caic_amqp_secret` — read it at startup if the env var is not set.
|
||||
- `AMQP_RECONNECT_DELAY` — seconds between reconnect attempts, default 5
|
||||
- `AMQP_EXCHANGE_ADMIN` — `jc.admin`
|
||||
- `AMQP_EXCHANGE_SYSTEM` — `jc.system`
|
||||
|
||||
**Create `amqp.py`** in the project root:
|
||||
|
||||
```python
|
||||
# Manages a single persistent aio-pika connection and channel.
|
||||
# Provides:
|
||||
# connect() -> None # establish connection, declare exchanges
|
||||
# disconnect() -> None # graceful close
|
||||
# get_channel() # returns current channel, reconnects if needed
|
||||
# publish(exchange, routing_key, payload: dict) -> None
|
||||
# # publishes JSON-serialized payload as persistent message
|
||||
```
|
||||
|
||||
Connection must:
|
||||
- Reconnect automatically on disconnect with `AMQP_RECONNECT_DELAY` backoff
|
||||
- Log connection events at INFO level
|
||||
- Not raise on publish if disconnected — log error and return (fire-and-forget, jC must not crash if RabbitMQ is down)
|
||||
|
||||
**Start AMQP connection in `app.py` lifespan** after `assess_hardware()`. Disconnect in lifespan cleanup.
|
||||
|
||||
**Write `tests/test_amqp.py`** covering:
|
||||
- `publish()` with mocked aio-pika connection — assert message published with correct exchange and routing key
|
||||
- `publish()` when disconnected — assert no exception raised, error logged
|
||||
- `get_channel()` when connection is None — assert reconnect attempted
|
||||
|
||||
Mock all aio-pika calls via monkeypatch. Do not require a live RabbitMQ instance in tests.
|
||||
|
||||
Run full test suite. All existing tests must continue to pass.
|
||||
|
||||
---
|
||||
|
||||
## ~~TASK 11 — Roadmap N3: Cluster Protocol & Registration Handler (Coordinator Side) [DONE]~~
|
||||
|
||||
**Status: Implemented and pushed (899988c).** `amqp.py` subscribe/rebind, `cluster.py` with CLUSTER_NODES/CLUSTER_EVENTS/CLUSTER_COORDINATOR and 6 handlers, `routers/cluster.py` (`GET /api/cluster`), 13 tests. No passive heartbeats — ping/pong on-demand before work routing. 148 tests pass.
|
||||
|
||||
jC on the coordinator must listen for nine message types across `jc.admin` and `jc.system`, maintain the cluster registry, and expose an application-level event log.
|
||||
|
||||
### 11.1 AMQP Protocol — Message Catalog
|
||||
|
||||
All payloads are JSON, published as persistent messages.
|
||||
|
||||
| Direction | Exchange | Routing Key | Message Type | Description |
|
||||
|-----------|----------|-------------|-------------|-------------|
|
||||
| Worker → Coordinator | `jc.admin` | `node.{name}.register` | register | Worker requests admission |
|
||||
| Worker → Coordinator | `jc.admin` | `node.{name}.deregister` | deregister | Worker signals graceful departure |
|
||||
| Coordinator → Worker | `jc.admin` | `node.{name}.admitted` | admitted | Coordinator grants admission |
|
||||
| Coordinator → Worker | `jc.admin` | `node.{name}.rejected` | rejected | Coordinator denies admission (with reason) |
|
||||
| Coordinator → Worker | `jc.admin` | `node.{name}.ping` | ping | Coordinator checks if worker is alive (sent before routing work) |
|
||||
| Worker → Coordinator | `jc.admin` | `node.{name}.pong` | pong | Worker confirms aliveness |
|
||||
| Worker → Coordinator | `jc.system` | `node.{name}.event` | event | Application-level syslog event |
|
||||
| Any → All | `jc.system` | `cluster.coordinator.query` | coord_query | Anyone asks "who is coordinator?" |
|
||||
| Coordinator → All | `jc.system` | `cluster.coordinator.response` | coord_response | Coordinator announces itself |
|
||||
|
||||
Worker presence is assumed from registration onward. No periodic heartbeats — a worker can sit idle for days without chatter. When the coordinator needs to route work to a worker, it pings first; if the worker doesn't pong within timeout, the coordinator deregisters it and moves to the next node.
|
||||
|
||||
### 11.2 Payload Schemas
|
||||
|
||||
**register** (worker → coordinator):
|
||||
```json
|
||||
{
|
||||
"node_name": "worker01",
|
||||
"node_type": "worker",
|
||||
"ip": "192.168.50.210",
|
||||
"capabilities": {
|
||||
"gpu": true, "gpu_type": "amd", "vram_mb": 8192,
|
||||
"cpu_cores": 8, "ram_gb": 16
|
||||
},
|
||||
"active_model": {
|
||||
"name": "llama3.1", "version": "latest", "quant": "Q4_K_M",
|
||||
"path": "/var/lib/caic/models/llama3.1-latest-Q4_K_M.gguf",
|
||||
"port": 8081
|
||||
},
|
||||
"inventory": [
|
||||
{"name": "llama3.1", "version": "latest", "quant": "Q4_K_M",
|
||||
"path": "/var/lib/caic/models/llama3.1-latest-Q4_K_M.gguf", "port": 8081}
|
||||
],
|
||||
"status": "active"
|
||||
}
|
||||
```
|
||||
|
||||
**deregister** (worker → coordinator):
|
||||
```json
|
||||
{
|
||||
"node_name": "worker01",
|
||||
"reason": "shutdown",
|
||||
"timestamp": "2026-07-06T12:00:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
**ping** (coordinator → worker):
|
||||
```json
|
||||
{
|
||||
"from": "coordinator",
|
||||
"node_name": "worker01",
|
||||
"type": "ping",
|
||||
"correlation_id": "550e8400-e29b-41d4-a716-446655440000",
|
||||
"timestamp": "2026-07-06T12:00:00Z"
|
||||
}
|
||||
```
|
||||
Worker must respond within 5 seconds or the coordinator considers it absent.
|
||||
|
||||
**pong** (worker → coordinator):
|
||||
```json
|
||||
{
|
||||
"node_name": "worker01",
|
||||
"type": "pong",
|
||||
"correlation_id": "550e8400-e29b-41d4-a716-446655440000",
|
||||
"status": "active",
|
||||
"active_model": {"name": "llama3.1", "port": 8081},
|
||||
"load": {"cpu_pct": 45, "ram_pct": 62, "vram_pct": 38},
|
||||
"timestamp": "2026-07-06T12:00:00Z"
|
||||
}
|
||||
```
|
||||
Correlation ID matches the ping so the coordinator can pair request and response.
|
||||
|
||||
**coord_query** (any → `cluster.coordinator.query`):
|
||||
```json
|
||||
{"type": "coord_query", "timestamp": "2026-07-06T12:00:00Z"}
|
||||
```
|
||||
Coordinator responds on `cluster.coordinator.response`:
|
||||
```json
|
||||
{
|
||||
"coordinator_node": "coordinator",
|
||||
"cluster_nodes": ["worker01"],
|
||||
"timestamp": "2026-07-06T12:00:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
**event** (worker → coordinator):
|
||||
```json
|
||||
{
|
||||
"node_name": "worker01",
|
||||
"severity": "info",
|
||||
"message": "llama-server started with model llama3.1:latest",
|
||||
"details": {"model": "llama3.1:latest", "port": 8081, "pid": 1234},
|
||||
"timestamp": "2026-07-06T12:00:00Z"
|
||||
}
|
||||
```
|
||||
Severity levels: `info`, `warn`, `error`, `critical`. The coordinator assigns `category: "application"` based on the exchange (jc.system). No `event_type` field — the category is determined by the channel, not the payload.
|
||||
|
||||
### 11.3 Design — Status Transitions Drive the Event Log
|
||||
|
||||
All admin-level events are *derived* from `register()` and `deregister()` as side effects. There are no separate message types for coordinator election, node staleness, quarantine, or release — those are status transitions that `register()`/`deregister()` emit into `CLUSTER_EVENTS` locally.
|
||||
|
||||
**Node status lifecycle:**
|
||||
|
||||
```
|
||||
UNKNOWN ──register()──▶ active ──deregister()──▶ (removed)
|
||||
│
|
||||
ping timeout│(coordinator publishes
|
||||
│ deregister on its behalf)
|
||||
▼
|
||||
(removed)
|
||||
```
|
||||
|
||||
**Coordinator status lifecycle:**
|
||||
|
||||
```
|
||||
NONE ──register(node_type=coordinator)──▶ CLUSTER_COORDINATOR set
|
||||
│
|
||||
deregister()│or timeout
|
||||
▼
|
||||
CLUSTER_COORDINATOR cleared
|
||||
```
|
||||
|
||||
**Event categories — two buckets, no granular types:**
|
||||
|
||||
| Category | When | severity |
|
||||
|----------|------|----------|
|
||||
| `cluster` | Node lifecycle, coordinator changes, model swaps, node offline — everything on `jc.admin` | `info` / `warn` / `error` |
|
||||
| `application` | Worker syslog events (incoming on `jc.system` `node.*.event`) | `info` / `warn` / `error` / `critical` |
|
||||
|
||||
Every `_push_event()` call uses one of these two categories. The `message` field carries the human-readable detail — no need for event type strings. The reporting tool filters by category + severity.
|
||||
|
||||
**Channel split — security rationale:**
|
||||
|
||||
The two exchanges are not an organizational convenience. They enforce a **data isolation boundary**:
|
||||
|
||||
| Exchange | Contains | Exposed to |
|
||||
|----------|----------|------------|
|
||||
| `jc.admin` | Node lifecycle, heartbeats, model swaps, coordinator changes | Operations / machine-room staff |
|
||||
| `jc.system` | Application events — inference queries, RAG context, user-facing data | Application-layer audit only |
|
||||
|
||||
`jc.system` events can leak information about what users are doing and asking. The split ensures a sysadmin monitoring cluster health never accidentally consumes user-data-bearing events. The channels can be locked down independently — different AMQP credentials, separate queue permissions, different in-transit encryption policies if needed later.
|
||||
|
||||
### 11.4 Implementation
|
||||
|
||||
**Add to `amqp.py`:**
|
||||
|
||||
```python
|
||||
_SUBSCRIPTIONS: list[tuple[str, str, Callable]] # (exchange, routing_key, callback)
|
||||
|
||||
async def subscribe(exchange, routing_key, callback) -> None
|
||||
# Append to _SUBSCRIPTIONS list
|
||||
# Declare a unique queue per subscription (name: f"jc.{exchange}.{sanitized_routing_key}")
|
||||
# Bind queue to exchange/routing_key, consume with callback
|
||||
```
|
||||
|
||||
Each subscription gets its own queue so multiple subscribers on different routing keys all receive messages. On reconnect: drain old consumers, iterate `_SUBSCRIPTIONS`, re-declare and re-bind each one. The `connect()` function must call `_rebind_subscriptions()` after exchanges are declared.
|
||||
|
||||
**Create `cluster.py`** in the project root:
|
||||
|
||||
```python
|
||||
# In-memory cluster registry + event log
|
||||
# Survives only while jC is running (not persisted)
|
||||
|
||||
CLUSTER_NODES: dict[str, NodeRecord]
|
||||
CLUSTER_EVENTS: deque[EventRecord] # bounded at 1000 entries
|
||||
CLUSTER_COORDINATOR: str | None # node_name of active coordinator
|
||||
|
||||
# NodeRecord fields:
|
||||
# node_name, node_type, ip, status, active_model, inventory,
|
||||
# capabilities: {gpu, gpu_type, vram_mb, cpu_cores, ram_gb}
|
||||
# registered_at, last_seen
|
||||
|
||||
# EventRecord:
|
||||
# category: str ("cluster" | "application")
|
||||
# severity: str ("info" | "warn" | "error" | "critical")
|
||||
# node_name: str
|
||||
# message: str
|
||||
# details: dict | None
|
||||
# timestamp: str
|
||||
|
||||
def _push_event(category, severity, node_name, message, details=None) -> None
|
||||
# Append EventRecord to CLUSTER_EVENTS, pop left if > 1000
|
||||
|
||||
async def handle_registration(message) -> None
|
||||
# Parse payload, validate required fields (node_name, node_type, ip, active_model, inventory)
|
||||
# Reject if node_name duplicate and CLUSTER_NODES[node_name].status == "active"
|
||||
# If CLUSTER_COORDINATOR is None AND node_type == "coordinator":
|
||||
# set CLUSTER_COORDINATOR = node_name
|
||||
# _push_event("cluster", "info", node_name, "elected coordinator")
|
||||
# publish cluster.coordinator.response on jc.system {coordinator_node, cluster_nodes, timestamp}
|
||||
# Add node to CLUSTER_NODES with status="active"
|
||||
# _push_event("cluster", "info", node_name, f"admitted as {node_type}")
|
||||
# publish admitted on jc.admin node.{name}.admitted {node_name, timestamp, amqp_url}
|
||||
|
||||
async def handle_deregistration(message) -> None
|
||||
# Parse payload (node_name, reason, timestamp)
|
||||
# If node_name == CLUSTER_COORDINATOR:
|
||||
# clear CLUSTER_COORDINATOR
|
||||
# _push_event("cluster", "warn", node_name, f"coordinator lost — {reason}")
|
||||
# _push_event("cluster", "info", node_name, f"departed — {reason}")
|
||||
# Remove node from CLUSTER_NODES, log it
|
||||
|
||||
async def handle_pong(message) -> None
|
||||
# Parse: node_name, correlation_id, status, active_model, load, timestamp
|
||||
# Match correlation_id to outstanding ping
|
||||
# If node in CLUSTER_NODES: update last_seen, status, active_model
|
||||
# Signal the waiting caller that the node is alive
|
||||
# If node unknown: log warning, do NOT auto-admit
|
||||
|
||||
async def handle_event(message) -> None
|
||||
# Parse: node_name, severity, message, details, timestamp
|
||||
# Assigns category="application" (incoming on jc.system)
|
||||
# Append EventRecord to CLUSTER_EVENTS (pop left if > 1000)
|
||||
|
||||
async def handle_coordinator_query(message) -> None
|
||||
# Respond on jc.system cluster.coordinator.response
|
||||
# Payload: {coordinator_node, cluster_nodes: list(CLUSTER_NODES.keys()), timestamp}
|
||||
|
||||
def get_cluster_state() -> dict
|
||||
# Return: {nodes: CLUSTER_NODES, coordinator: CLUSTER_COORDINATOR,
|
||||
# events: last 50 CLUSTER_EVENTS}
|
||||
```
|
||||
|
||||
**Subscribe in `app.py` lifespan** after AMQP connects:
|
||||
|
||||
| Exchange | Routing Key | Handler |
|
||||
|----------|-------------|---------|
|
||||
| `jc.admin` | `node.*.register` | `handle_registration` |
|
||||
| `jc.admin` | `node.*.deregister` | `handle_deregistration` |
|
||||
| `jc.admin` | `node.*.pong` | `handle_pong` |
|
||||
| `jc.system` | `node.*.event` | `handle_event` |
|
||||
| `jc.system` | `cluster.coordinator.query` | `handle_coordinator_query` |
|
||||
|
||||
### 11.5 API — `GET /api/cluster`
|
||||
|
||||
New router `routers/cluster.py`:
|
||||
- `GET /api/cluster` — returns full cluster state: `{nodes, coordinator, events}` (last 50 events). No auth required.
|
||||
|
||||
Wire `cluster.router` into `app.py`.
|
||||
|
||||
### 11.6 Tests — `tests/test_cluster.py`
|
||||
|
||||
Mock all aio-pika calls. Do not require live RabbitMQ.
|
||||
|
||||
| # | Test | What it asserts |
|
||||
|---|------|-----------------|
|
||||
| 1 | Valid worker registration | Node admitted, CLUSTER_NODES updated, `cluster` event logged, `admitted` message published |
|
||||
| 2 | First coordinator auto-promotion | CLUSTER_COORDINATOR set, `cluster` event with "elected" message, `coord_response` published |
|
||||
| 3 | Duplicate node name rejected | `rejected` message with reason=`duplicate_node_name`, `cluster` event logged |
|
||||
| 4 | Malformed payload rejected | `rejected` message with reason=`malformed_payload` |
|
||||
| 5 | Graceful deregistration | Node removed, `cluster` event logged. If coordinator: CLUSTER_COORDINATOR cleared |
|
||||
| 6 | Pong from known node | last_seen updated, load/status refreshed |
|
||||
| 7 | Pong from unknown node | Warning logged, node NOT added |
|
||||
| 8 | Event stored in log | Event appended to CLUSTER_EVENTS; at 1001 entries the oldest is popped |
|
||||
| 9 | Coordinator query produces response | Response published with coordinator name and node list |
|
||||
| 10 | GET /api/cluster shape | Response contains `nodes`, `coordinator`, `events` keys |
|
||||
|
||||
Run full test suite. All existing tests must continue to pass.
|
||||
|
||||
---
|
||||
|
||||
## ~~TASK 12 — Roadmap N4: Worker Node Registration Publisher (Worker Side) [DONE]~~
|
||||
|
||||
This task creates the worker node AMQP client that runs on worker (192.168.50.210). It is a standalone Python script — not part of the jC FastAPI app — that runs as a systemd service on worker.
|
||||
|
||||
**Create `node_agent/agent.py`** in the repo (new directory).
|
||||
|
||||
### 12.1 Config & Inventory Discovery
|
||||
|
||||
On start, reads `/etc/caic-node-agent.conf` (INI format):
|
||||
- `node_name` — hostname, default from `socket.gethostname()`
|
||||
- `node_ip` — LAN IP, default from socket
|
||||
- `node_type` — `"worker"` (fixed)
|
||||
- `capabilities` — comma-separated list, e.g. `llm,rag`
|
||||
- `amqp_url` — RabbitMQ URL on coordinator, e.g. `amqp://caic:password@192.168.50.108:5672/caic`
|
||||
- `llama_port` — port llama-server/llama-rpc is listening on, default 8081
|
||||
- `models_dir` — path to GGUF model files, default `/var/lib/caic/models`
|
||||
- `active_model` — filename of currently active model (without path)
|
||||
|
||||
Discovers inventory by globbing `models_dir` for `*.gguf` files and parsing name/version/quant from filename using regex pattern: `{name}-{version}-{quant}.gguf` where quant matches `Q[0-9]+_K_[A-Z]+` or similar standard suffixes.
|
||||
|
||||
### 12.2 Registration
|
||||
|
||||
Publishes registration to `jc.admin`, routing key `node.{node_name}.register`:
|
||||
```json
|
||||
{
|
||||
"node_name": "worker01",
|
||||
"node_type": "worker",
|
||||
"ip": "192.168.50.210",
|
||||
"capabilities": ["llm"],
|
||||
"active_model": {"name": "...", "version": "...", "quant": "...", "path": "...", "port": 8081}
|
||||
}
|
||||
```
|
||||
|
||||
### 12.3 Admission Response
|
||||
|
||||
Listens on `node.{node_name}.admitted` and `node.{node_name}.rejected` (both `jc.admin`). Logs result. If rejected, exits with error.
|
||||
|
||||
### 12.4 Ping Listener
|
||||
|
||||
After admission: listens on `jc.admin`, routing key `node.{node_name}.ping`. On receipt, responds immediately (within 1 second) with a pong on `jc.admin`, routing key `node.{node_name}.pong`:
|
||||
|
||||
```json
|
||||
{
|
||||
"node_name": "worker01",
|
||||
"type": "pong",
|
||||
"correlation_id": "<echoed from ping>",
|
||||
"status": "active",
|
||||
"active_model": {"name": "...", "version": "...", "quant": "...", "path": "...", "port": 8081},
|
||||
"load": {"cpu_pct": 45, "ram_pct": 62, "vram_pct": 38},
|
||||
"timestamp": "<utc>"
|
||||
}
|
||||
```
|
||||
|
||||
No periodic heartbeats. Worker sits idle between pings — coordinator only pings when it needs to route work.
|
||||
|
||||
### 12.5 Model Swap Command Handler
|
||||
|
||||
Listens on `jc.admin`, routing key `node.{node_name}.cmd.swap_model`:
|
||||
- Payload: `{model_filename: str}`
|
||||
- Stops current llama-server: `systemctl stop llama-server`
|
||||
- Updates `/etc/caic-node-agent.conf` active_model field
|
||||
- Starts llama-server: `systemctl start llama-server` (assumes service reads active_model from conf or ExecStart is updated)
|
||||
- Waits for llama-server to be healthy: poll `http://localhost:{llama_port}/v1/models` every 2s, timeout 120s
|
||||
- Publishes to `jc.system`, routing key `node.{node_name}.model_ready`:
|
||||
```json
|
||||
{"node_name": "...", "active_model": "...", "port": ..., "timestamp": "..."}
|
||||
```
|
||||
- If startup fails within timeout: publishes `node.{node_name}.model_failed` with error detail
|
||||
|
||||
### 12.6 Files & Tests
|
||||
|
||||
**Create `node_agent/requirements.txt`:** `aio-pika>=9.0.0`
|
||||
|
||||
**Document `/etc/caic-node-agent.conf` format** in a comment block at the top of `agent.py`.
|
||||
|
||||
**Write `tests/test_node_agent.py`** covering:
|
||||
- Registration payload construction from config + model discovery — assert correct JSON shape
|
||||
- Model swap command handler: success path — assert systemctl calls made, model_ready published
|
||||
- Model swap command handler: timeout path — assert model_failed published
|
||||
- Ping handler: on ping, publishes pong with correct correlation_id
|
||||
- Agent starts idle after admission, no heartbeat timer
|
||||
|
||||
Mock all aio-pika, subprocess, and httpx calls.
|
||||
|
||||
**Do not create a systemd service file in this task** — that is a manual deployment step. Document the required service configuration in a comment at the bottom of `agent.py`.
|
||||
|
||||
Run full test suite. All existing tests must continue to pass.
|
||||
|
||||
---
|
||||
|
||||
## ~~TASK 13 — Roadmap N5: Query Routing via AMQP + Phi-4-mini Triage [DONE]~~
|
||||
|
||||
This task wires the cluster into jC's chat flow. When a query arrives at `/api/chat`, instead of always routing to the hardcoded `LLAMA_SERVER_BASE`, jC now routes to the best available cluster node based on query context.
|
||||
|
||||
**Prerequisites:** Tasks 9–12 complete. At least one worker node admitted to cluster.
|
||||
|
||||
**Install Phi-4-mini on coordinator (infrastructure step):**
|
||||
- Download `Phi-4-mini-Instruct-Q4_K_M.gguf` from HuggingFace using `hf download microsoft/Phi-4-mini-instruct --include "*.Q4_K_M.gguf" --local-dir /var/lib/caic/models`
|
||||
- Create `/etc/systemd/system/llama-server-triage.service` — same pattern as existing llama-server service but: port 8083, model path points to Phi-4-mini GGUF, no `--rpc` flag (runs entirely on coordinator CPU/iGPU), description `Llama.cpp Server (Phi-4-mini — triage/routing)`
|
||||
- `systemctl daemon-reload && systemctl enable llama-server-triage && systemctl start llama-server-triage`
|
||||
- Verify: `curl -s http://localhost:8083/v1/models`
|
||||
|
||||
**Add to `config.py`:**
|
||||
- `TRIAGE_BASE` — `http://127.0.0.1:8083/v1` (Phi-4-mini)
|
||||
- `TRIAGE_TIMEOUT` — 10 seconds
|
||||
- `FALLBACK_TO_DEFAULT` — True (if triage fails or no nodes available, fall back to `LLAMA_SERVER_BASE`)
|
||||
|
||||
**Create `triage.py`** in the project root:
|
||||
|
||||
```python
|
||||
async def classify_query(query: str) -> str
|
||||
# Sends query to Phi-4-mini at TRIAGE_BASE with a classification system prompt.
|
||||
# System prompt instructs model to respond with ONLY one of:
|
||||
# "general", "code", "search", "rag"
|
||||
# Returns the classification string.
|
||||
# Timeout: TRIAGE_TIMEOUT seconds.
|
||||
# On any error: returns "general" (fail-safe).
|
||||
|
||||
async def select_node(classification: str) -> dict | None
|
||||
# Consults CLUSTER_NODES from cluster.py
|
||||
# For "code": prefer nodes where active_model name contains "coder" or "qwen"
|
||||
# For "general": prefer nodes where active_model name contains "mistral" or "llama"
|
||||
# For "search" or "rag": return None (handled locally by jC)
|
||||
# If no matching node found: return None (triggers FALLBACK_TO_DEFAULT)
|
||||
# Returns NodeRecord dict for selected node, or None
|
||||
|
||||
async def get_inference_url(query: str) -> str
|
||||
# Combines classify_query + select_node
|
||||
# Returns full base URL: f"http://{node.ip}:{node.active_model.port}/v1"
|
||||
# Falls back to LLAMA_SERVER_BASE if classification=search/rag, no nodes, or triage error
|
||||
```
|
||||
|
||||
**Update `routers/chat.py`:**
|
||||
- Replace the hardcoded `LLAMA_SERVER_BASE` reference with a call to `get_inference_url(user_message)`
|
||||
- The rest of the chat flow (RAG, memory, streaming) is unchanged — only the inference target URL changes
|
||||
|
||||
**Write `tests/test_triage.py`** covering:
|
||||
- `classify_query()` returns valid classification — mock Phi-4-mini response
|
||||
- `classify_query()` on timeout — assert returns "general", no exception
|
||||
- `select_node("code")` with coder node in cluster — assert correct node returned
|
||||
- `select_node("general")` with no matching node — assert None returned
|
||||
- `get_inference_url()` with code query and coder node available — assert returns node URL
|
||||
- `get_inference_url()` with no nodes in cluster — assert returns LLAMA_SERVER_BASE fallback
|
||||
|
||||
**Update `tests/test_chat_streaming_and_memory_paths.py`:**
|
||||
- Mock `triage.get_inference_url` to return a fixed URL in all existing tests so they continue to pass without a live cluster
|
||||
|
||||
Run full test suite. All existing tests must continue to pass.
|
||||
|
||||
---
|
||||
|
||||
## ~~TASK 14 — Roadmap N6: Model Swap Command Flow [DONE]~~
|
||||
|
||||
**Status: Implemented and pushed (`9d1fd44`).** `request_model_swap()`, `handle_model_ready()`, `handle_model_failed()` in `cluster.py`, async `select_node()` with swap triggering in `triage.py`, `tests/test_model_swap.py` (9 tests). 177 tests pass.
|
||||
|
||||
This task implements the coordinator-side logic for requesting a model swap on a worker node when the ideal model is not currently active.
|
||||
|
||||
**Add to `cluster.py`:**
|
||||
|
||||
```python
|
||||
async def request_model_swap(node_name: str, model_filename: str) -> bool
|
||||
# Publishes to jc.admin exchange, routing key node.{node_name}.cmd.swap_model
|
||||
# Payload: {model_filename, requested_at: iso_timestamp}
|
||||
# Sets node status to "swapping" in CLUSTER_NODES
|
||||
# Returns True if message published successfully
|
||||
|
||||
async def handle_model_ready(message) -> None
|
||||
# Handles node.{node_name}.model_ready from jc.system
|
||||
# Updates CLUSTER_NODES[node_name].active_model to the new model
|
||||
# Sets node status back to "active"
|
||||
# Logs swap completion with timing
|
||||
|
||||
async def handle_model_failed(message) -> None
|
||||
# Handles node.{node_name}.model_failed from jc.system
|
||||
# Sets node status to "error" in CLUSTER_NODES
|
||||
# Logs failure with detail from message payload
|
||||
```
|
||||
|
||||
**Subscribe in `app.py` lifespan:**
|
||||
- `jc.system` exchange, routing key `node.*.model_ready` → `handle_model_ready`
|
||||
- `jc.system` exchange, routing key `node.*.model_failed` → `handle_model_failed`
|
||||
|
||||
**Update `triage.py` `select_node()`:**
|
||||
- If the best-matching node exists but its active_model does not match the ideal model for the classification, AND the node status is "active" (not already swapping):
|
||||
- Call `request_model_swap(node_name, ideal_model_filename)`
|
||||
- Return None (triggers fallback) — the swap happens async, next query will find the right model active
|
||||
- If node status is "swapping": return None (fallback, swap in progress)
|
||||
|
||||
**Update `GET /api/cluster`** to include node status in response.
|
||||
|
||||
**Write `tests/test_model_swap.py`** covering:
|
||||
- `request_model_swap()` — assert swap command published, node status set to "swapping"
|
||||
- `handle_model_ready()` — assert active_model updated, status set to "active"
|
||||
- `handle_model_failed()` — assert status set to "error"
|
||||
- `select_node()` with mismatched active model — assert swap requested, None returned
|
||||
- `select_node()` with node status "swapping" — assert None returned without publishing another swap
|
||||
|
||||
Run full test suite. All existing tests must continue to pass.
|
||||
|
||||
---
|
||||
|
||||
## ~~TASK 15 — Roadmap N7: Cluster Status UI [DONE]~~
|
||||
|
||||
Surface cluster awareness in the jC frontend (`templates/index.html`).
|
||||
|
||||
**Add a cluster status panel** to the UI. Requirements:
|
||||
- Small status bar or collapsible panel, visible but unobtrusive
|
||||
- Polls `GET /api/cluster` every 15 seconds
|
||||
- For each admitted node: show node name, active model name, and a colored status dot:
|
||||
- Green: active
|
||||
- Yellow: swapping
|
||||
- Red: error or offline (not seen in last 60 seconds based on last_seen timestamp)
|
||||
- If no nodes in cluster (empty): show "No worker nodes connected"
|
||||
- Panel must not interfere with chat input or conversation list
|
||||
|
||||
**Update `GET /api/cluster` response** to include `last_seen` per node and a `status` field (`active`, `swapping`, `error`).
|
||||
|
||||
**Update heartbeat handling in `cluster.py`:** add a handler for `node.*.heartbeat` on `jc.system` that updates `last_seen` timestamp for the node.
|
||||
|
||||
**Subscribe in `app.py` lifespan:**
|
||||
- `jc.system` exchange, routing key `node.*.heartbeat` → `handle_heartbeat`
|
||||
|
||||
**Add `handle_heartbeat()` to `cluster.py`:**
|
||||
- Updates `CLUSTER_NODES[node_name].last_seen` to current timestamp
|
||||
- If node was previously marked offline (not in CLUSTER_NODES), log re-registration warning but do not auto-admit — full registration required
|
||||
|
||||
**Write `tests/test_cluster_heartbeat.py`** covering:
|
||||
- `handle_heartbeat()` for known node — assert last_seen updated
|
||||
- `handle_heartbeat()` for unknown node — assert no crash, warning logged, node not added
|
||||
|
||||
Run full test suite. All 26+ existing tests must continue to pass.
|
||||
|
||||
~~Commit all changes introduced across Tasks 9–15 with message: `feat: Roadmap N — AMQP cluster nervous system complete`~~
|
||||
|
||||
---
|
||||
|
||||
## Backlog (Post-Roadmap N) ⏳
|
||||
|
||||
### ~~B1 — Context loss in follow-up questions [DONE]~~
|
||||
|
||||
**Symptom:** After asking "in {context}, explain {b}", a follow-up "what is {b}'s {x}?" gets a non-sequitur response that ignores the original context.
|
||||
|
||||
**Diagnosis:** `build_system_prompt()` is called fresh per-request with new RAG/memory results keyed to the current message text. These can change between turns and may dilute or override the conversation history. The original system prompt used for turn 1 (including its RAG context) is not stored in the DB — only user/assistant messages are. The inference server receives a different system prompt each turn.
|
||||
|
||||
**Possible fixes:**
|
||||
- Store the assembled system prompt with each assistant message in the DB
|
||||
- When replaying history, re-send the original system prompts from DB rather than rebuilding
|
||||
- Or: cap RAG/memory injection to only fire on the first message of a conversation, then rely solely on conversation history for follow-ups
|
||||
- Check that llama-server isn't truncating history due to context window overflow (Mistral-Nemo 12B = 128K context, unlikely)
|
||||
|
||||
### ~~B2 — Bang-prefixed search routing [DONE]~~
|
||||
|
||||
**Spec:** If a query begins with `!`, route to SearXNG search instead of local inference.
|
||||
|
||||
**Where:** In `routers/chat.py` `chat()` handler, after `user_message` is extracted. Strip the `!`, set a flag to always trigger auto-search regardless of perplexity/refusal.
|
||||
|
||||
**Change:** Add a `force_search` flag when `user_message.startswith("!")`, strip the prefix from the message saved to DB, and route directly to the search+summarize path.
|
||||
|
||||
### ~~B3 — Docker distribution (v1.0 gate) [DONE]~~
|
||||
|
||||
**Goal:** Ship cAIc as a `docker compose` stack so a single command stands up everything.
|
||||
|
||||
**Services to containerize:**
|
||||
- cAIc (FastAPI app + SQLite)
|
||||
- SearXNG
|
||||
- Qdrant
|
||||
- RabbitMQ
|
||||
- llama-server (with optional RPC sidecar for GPU offload)
|
||||
- Ollama (embeddings)
|
||||
|
||||
**Also needed:**
|
||||
- `Dockerfile` for the cAIc app itself
|
||||
- `docker-compose.yml` with all services, volumes, networks, env vars
|
||||
- Setup wizard script (run on first boot) that:
|
||||
- Probes CPU vs GPU (reuses `hardware.py`)
|
||||
- Queries user for admin PIN, node name, IP
|
||||
- Generates `.env` file with correct `LLAMA_SERVER_BASE`, `EMBED_URL`, etc.
|
||||
- Auto-calculates `RAG_MAX_VECTORS` from available RAM: `max(1000, int(available_ram_gb * 100_000))`
|
||||
- Optionally detects and configures RPC GPU offload
|
||||
- Manual install docs remain alongside for bare-metal deployment
|
||||
|
||||
**This task is only actionable after Tasks 8–15 (RAG eviction + AMQP cluster) are complete.**
|
||||
|
||||
---
|
||||
|
||||
### ~~B4 — RAG Corpus Management UI (Display, Edit, CRUD) [DONE]~~
|
||||
|
||||
**Goal:** Provide a management interface in the UI to browse, search, edit, and delete individual entries in the Qdrant-backed RAG corpus.
|
||||
|
||||
**Backend — add to `routers/rag_admin.py`:**
|
||||
|
||||
| Method | Endpoint | Description | Auth |
|
||||
|--------|----------|-------------|------|
|
||||
| GET | `/api/rag/points` | Return paginated list of RAG points with payload (text, source, date). Supports `?offset=0&limit=50&search=` query params | Admin |
|
||||
| GET | `/api/rag/point/{point_id}` | Return a single point with full payload | Admin |
|
||||
| DELETE | `/api/rag/point/{point_id}` | Delete a single point from Qdrant | Admin |
|
||||
| PATCH | `/api/rag/point/{point_id}` | Update a point's text payload (re-embed the new text) | Admin |
|
||||
|
||||
Helper functions for Qdrant scroll/delete/update go in `rag.py` or `eviction.py`.
|
||||
|
||||
**Frontend — add to `templates/index.html`:**
|
||||
|
||||
A "RAG" button in the admin UI (drawer or settings modal) that opens a management panel:
|
||||
- **Stats bar**: vector count, max vectors, percent full, pinned sources
|
||||
- **Search bar**: text input to search the RAG corpus by semantic similarity
|
||||
- **Results table**: paginated list showing each vector's text snippet, source label, ingest date, retrieval count
|
||||
- Click to expand full text
|
||||
- Delete button per row (with confirmation)
|
||||
- Edit button per row (inline text edit → re-embed on save)
|
||||
- **Bulk actions**: flush all (existing `/api/rag/flush`) with confirmation
|
||||
|
||||
**Tests:**
|
||||
|
||||
- `tests/test_rag_admin.py` — cover new endpoints: list, get, delete, update, admin-enforcement
|
||||
- Mock all Qdrant calls via monkeypatch
|
||||
|
||||
Run full test suite. All existing tests must continue to pass.**
|
||||
+5
-5
@@ -1,11 +1,11 @@
|
||||
#!/bin/bash
|
||||
# jc-ingest.sh — pipe terminal commands into jarvisChat RAG
|
||||
# Deploy to /home/gramps/bin/jc-ingest.sh on jarvis (192.168.50.210)
|
||||
# jc-ingest.sh — pipe terminal commands into cAIc RAG
|
||||
# Deploy to /home/gramps/bin/jc-ingest.sh on jarvis (192.168.50.212)
|
||||
#
|
||||
# Usage:
|
||||
# 1. chmod +x /home/gramps/bin/jc-ingest.sh
|
||||
# 2. Add to ~/.bashrc:
|
||||
# export JARVISCHAT_COMPLETIONS_API_KEY="$(cat /opt/jarvischat/.completions_key)"
|
||||
# export CAIC_COMPLETIONS_API_KEY="$(cat /opt/jarvischat/.completions_key)"
|
||||
# export PROMPT_COMMAND="jc_capture"
|
||||
# source /home/gramps/bin/jc-ingest.sh
|
||||
#
|
||||
@@ -15,8 +15,8 @@
|
||||
# Filter: currently captures git, pip, systemctl, sudo, vi/vim, curl,
|
||||
# wget, apt, python, pytest commands. Edit the grep pattern to adjust.
|
||||
|
||||
JC_URL="http://192.168.50.210:8080/api/ingest"
|
||||
JC_TOKEN="${JARVISCHAT_COMPLETIONS_API_KEY}"
|
||||
JC_URL="http://192.168.50.212:8080/api/ingest"
|
||||
JC_TOKEN="${CAIC_COMPLETIONS_API_KEY}"
|
||||
|
||||
jc_capture() {
|
||||
local cmd
|
||||
|
||||
@@ -22,10 +22,9 @@ Refactored from single-file (`app.py`) into modules under project root:
|
||||
| `rag.py` | Qdrant vector search, system prompt assembly, chunk_text() helper, collection stats |
|
||||
| `eviction.py` | Score-based RAG eviction engine (extracted from rag.py) |
|
||||
| `gpu.py` | AMD GPU stats via rocm-smi |
|
||||
| `hardware.py` | Hardware self-assessment — CPU, RAM, VRAM, service health probes |
|
||||
| `hardware.py` | Hardware self-assessment — CPU, RAM, VRAM, service health probes (llama-server, Qdrant, SearXNG, ComfyUI) |
|
||||
| `amqp.py` | aio-pika connection manager for RabbitMQ (connect, disconnect, publish, subscribe, auto-reconnect) |
|
||||
| `cluster.py` | Cluster node registry, event log, coordinator election, ping/pong, model swap handlers |
|
||||
| `triage.py` | Phi-4-mini query classification + `select_node()` for cluster routing |
|
||||
| `cluster.py` | Cluster node registry, event log, coordinator election, ping/pong, model swap handlers, image generation request/response |
|
||||
| `routers/` | One module per endpoint group |
|
||||
|
||||
### 1.2 External Services
|
||||
@@ -36,6 +35,7 @@ Refactored from single-file (`app.py`) into modules under project root:
|
||||
| SearXNG | No | 8888 | Privacy-respecting web search |
|
||||
| Qdrant (coordinator) | No | 6333 | Vector database for RAG |
|
||||
| Ollama (worker) | No | 11434 | Embeddings for RAG chunk vectors |
|
||||
| ComfyUI (worker) | No | 8188 | Image generation (Stable Diffusion / Flux) |
|
||||
| RabbitMQ (coordinator) | No | 5672 | AMQP broker for cluster messaging |
|
||||
| rocm-smi | No | — | AMD GPU stats (host-level) |
|
||||
|
||||
@@ -45,11 +45,15 @@ Key base URLs are configured via environment variables with sensible defaults:
|
||||
|
||||
| Variable | Default | Service |
|
||||
|----------|---------|---------|
|
||||
| `LLAMA_SERVER_BASE` | `http://192.168.50.108:8081` | llama-server on coordinator |
|
||||
| `LLAMA_SERVER_BASE` | `http://localhost:8081` | llama-server on the same node |
|
||||
| `OLLAMA_BASE` | `http://localhost:11434` | Legacy — all inference goes through LLAMA_SERVER_BASE |
|
||||
| `SEARXNG_BASE` | `http://localhost:8888` | SearXNG |
|
||||
| `QDRANT_URL` | `http://192.168.50.108:6333` | Qdrant on coordinator |
|
||||
| `QDRANT_URL` | `http://localhost:6333` | Qdrant on the same node |
|
||||
| `CAIC_AMQP_URL` | `amqp://caic:password@localhost:5672/caic` | RabbitMQ |
|
||||
| `CAIC_COMFYUI_BASE` | `http://localhost:8188` | ComfyUI (image gen) |
|
||||
| `CAIC_COMFYUI_TIMEOUT` | `120` | ComfyUI generation timeout (seconds) |
|
||||
|
||||
> **Current deployment (single-node):** all services run on jarvis (192.168.50.212). The cluster/AMQP/node-agent layer is dormant — it degrades gracefully and can be re-enabled for a multi-node cluster later.
|
||||
|
||||
## 2. Request/Response Architecture
|
||||
|
||||
@@ -87,6 +91,17 @@ Key base URLs are configured via environment variables with sensible defaults:
|
||||
4. Three modes: `context` (SQLite with 1hr expiry), `ingest` (RAG/Qdrant), `both`
|
||||
5. Trigger `maybe_evict()` if ingest mode
|
||||
|
||||
### 2.5 Image Generation Pipeline (`POST /api/image/generate`)
|
||||
|
||||
1. Admin required, JSON body with prompt and optional params (width, height, steps, seed, model)
|
||||
2. Find active node with `image_gen` capability via `_find_image_node()`
|
||||
3. Publish `cmd.image_generate` via AMQP to selected worker node
|
||||
4. Worker node agent builds ComfyUI workflow (CheckpointLoader → KSampler → VAEDecode → SaveImage)
|
||||
5. Worker polls ComfyUI `/history/{prompt_id}` until image is ready
|
||||
6. Worker fetches PNG from ComfyUI `/view` endpoint, base64-encodes, publishes `image_generated` on `jc.system`
|
||||
7. Coordinator receives response, decodes base64, returns `image/png` to client
|
||||
8. `GET /api/image/status` returns available image gen nodes and their status
|
||||
|
||||
## 3. Data Model (SQLite)
|
||||
|
||||
Key tables:
|
||||
@@ -242,8 +257,8 @@ Every RabbitMQ server belongs to a cluster. Currently only the coordinator runs
|
||||
|
||||
| Exchange | Type | Purpose |
|
||||
|----------|------|---------|
|
||||
| `jc.admin` | topic | Lifecycle commands: register, deregister, ping, pong, admitted, rejected; model commands: cmd.swap_model |
|
||||
| `jc.system` | topic | Events: model_ready, model_failed, node.*.heartbeat, event; coordinator queries: coord_query, coord_response |
|
||||
| `jc.admin` | topic | Lifecycle commands: register, deregister, ping, pong, admitted, rejected; model commands: cmd.swap_model; image commands: cmd.image_generate |
|
||||
| `jc.system` | topic | Events: model_ready, model_failed, image_generated, image_failed, node.*.heartbeat, event; coordinator queries: coord_query, coord_response |
|
||||
|
||||
All exchanges, queues, and bindings are declared by `amqp.py` at startup. Worker runs `node_agent/agent.py` which connects as an AMQP client, registers, responds to ping, and handles model swap commands.
|
||||
|
||||
@@ -265,7 +280,7 @@ All streaming endpoints yield `data: {json}\n\n`:
|
||||
- No live external services required
|
||||
- Test factories reset `SESSIONS`, `PIN_ATTEMPTS`, `RATE_EVENTS` globals per test
|
||||
|
||||
### 8.2 Test Coverage Areas (200 tests)
|
||||
### 8.2 Test Coverage Areas (228 tests)
|
||||
|
||||
| Test file | Coverage |
|
||||
|-----------|----------|
|
||||
@@ -277,6 +292,8 @@ All streaming endpoints yield `data: {json}\n\n`:
|
||||
| test_conversations.py | Full CRUD, guest admin, attachment_count |
|
||||
| test_error_envelopes.py | Global exception handler + stream errors |
|
||||
| test_gpu.py | GPU stats — rocm-smi (Linux), system_profiler (Darwin/Apple Silicon) |
|
||||
| test_hardware.py | Hardware assessment, service reachability |
|
||||
| test_image.py | Image generation — cluster handlers, router proxy, node agent ComfyUI integration, hardware probe, capability detection |
|
||||
| test_ingest.py | Bearer auth, chunk/embed/upsert, validation |
|
||||
| test_ip_allowlist.py | IP allowlist helper + middleware |
|
||||
| test_memories.py | Edit, search, stats |
|
||||
@@ -292,7 +309,6 @@ All streaming endpoints yield `data: {json}\n\n`:
|
||||
| test_search_url_sanitization.py | URL sanitizer |
|
||||
| test_settings_allowlist.py | Allowlisted key enforcement |
|
||||
| test_skills_framework.py | List, toggle, unknown skill, prompt injection |
|
||||
| test_triage.py | classify_query, select_node, get_inference_url |
|
||||
| test_upload.py | Upload, delete, link, by-conversation, attachment_count |
|
||||
|
||||
### 8.3 DoD Process
|
||||
@@ -311,5 +327,6 @@ On startup, `assess_hardware()` probes:
|
||||
- llama-server reachability + model list
|
||||
- Qdrant reachability + collection list
|
||||
- SearXNG reachability
|
||||
- ComfyUI reachability + checkpoint model list
|
||||
|
||||
Writes `hardware_state.json` to working directory.
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# WireGuard Tunnel — Encrypted Node Transit
|
||||
|
||||
> **Status: dormant (single-node deployment).** All cAIc services currently run on one node (jarvis, 192.168.50.212), so there is no inter-node traffic to encrypt. This document is kept as a reference for when a multi-node cluster is stood back up.
|
||||
|
||||
## Why
|
||||
|
||||
cAIc cluster traffic is plaintext today:
|
||||
|
||||
@@ -1,11 +1,17 @@
|
||||
# cAIc Current WiP Backlog
|
||||
|
||||
Last updated: 2026-07-14
|
||||
Last updated: 2026-07-27
|
||||
Owner: Gramps
|
||||
Scope: Active roadmap items and backlog.
|
||||
|
||||
## In Progress
|
||||
|
||||
- **Image Generation Service** — Backend wired: cluster handlers, `POST /api/image/generate` proxy, node agent ComfyUI integration, hardware probe, 27 tests. ComfyUI install pending on jarvis (single-node).
|
||||
|
||||
## Completed
|
||||
|
||||
- **Single-node consolidation (2026-08-08)** — all cAIc services moved onto jarvis (192.168.50.212): llama-server, Qdrant, SearXNG, RabbitMQ, Ollama, ComfyUI. Config defaults (`COMFYUI_BASE`, AMQP URL, `NODE_NAME`) updated; cluster/AMQP layer left dormant (degrades gracefully). Project renamed `jarvisChat` → **cAIc**.
|
||||
|
||||
- **B8 (v0.19.3)** — Private Chat mode. Backend skip-DB/skip-RAG/skip-search flag, frontend PRIVATE badge, info popup.
|
||||
- **WireGuard TLS (v0.19.4)** — Self-signed WireGuard mesh encrypts all inter-node traffic (AMQP, inference, RPC). No code changes to cAIc. Documented in wiki/WireGuard-Setup.md + docker.md §5.4.
|
||||
- **At-Rest Encryption (v0.20.0)** — AES-256-GCM encrypts all query-derived text at rest. crypto.py with auto-keygen, key stored as `heartbeat_interval_ms` in settings. All 12 storage paths wired (SQLite: messages, conversations, memories, upload_context; Qdrant: RAG chunks, ingest, upload). 200 tests pass.
|
||||
|
||||
+19
-3
@@ -12,11 +12,11 @@ from pathlib import Path
|
||||
import httpx
|
||||
import psutil
|
||||
|
||||
from config import LLAMA_SERVER_BASE, SEARXNG_BASE, QDRANT_URL
|
||||
from config import LLAMA_SERVER_BASE, SEARXNG_BASE, QDRANT_URL, HW_STATE_PATH, COMFYUI_BASE
|
||||
|
||||
log = logging.getLogger("caic")
|
||||
|
||||
HARDWARE_STATE_PATH = Path("hardware_state.json")
|
||||
HARDWARE_STATE_PATH = Path(HW_STATE_PATH)
|
||||
_TIMEOUT_EXPIRED = subprocess.TimeoutExpired
|
||||
|
||||
|
||||
@@ -113,6 +113,20 @@ async def assess_hardware() -> dict:
|
||||
except Exception:
|
||||
log.warning("SearXNG not reachable")
|
||||
|
||||
comfyui_reachable = False
|
||||
comfyui_models = []
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=5) as client:
|
||||
resp = await client.get(f"{COMFYUI_BASE}/object_info/CheckpointLoaderSimple")
|
||||
if resp.status_code == 200:
|
||||
comfyui_reachable = True
|
||||
data = resp.json()
|
||||
ckpt_info = data.get("CheckpointLoaderSimple", {}).get("input", {}).get("required", {})
|
||||
ckpt_list = ckpt_info.get("ckpt_name", [[]])[0]
|
||||
comfyui_models = ckpt_list if isinstance(ckpt_list, list) else []
|
||||
except Exception:
|
||||
log.warning("ComfyUI not reachable")
|
||||
|
||||
state = {
|
||||
"ram_total_gb": ram_total_gb,
|
||||
"ram_available_gb": ram_available_gb,
|
||||
@@ -124,10 +138,12 @@ async def assess_hardware() -> dict:
|
||||
"qdrant_reachable": qdrant_reachable,
|
||||
"qdrant_collections": qdrant_collections,
|
||||
"searxng_reachable": searxng_reachable,
|
||||
"comfyui_reachable": comfyui_reachable,
|
||||
"comfyui_models": comfyui_models,
|
||||
}
|
||||
HARDWARE_STATE_PATH.write_text(json.dumps(state, indent=2))
|
||||
log.info(
|
||||
f"HW: {ram_total_gb}GB RAM, {vram_total_mb}MB VRAM, "
|
||||
f"llama={llama_reachable}, qdrant={qdrant_reachable}, searxng={searxng_reachable}"
|
||||
f"llama={llama_reachable}, qdrant={qdrant_reachable}, searxng={searxng_reachable}, comfyui={comfyui_reachable}"
|
||||
)
|
||||
return state
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -38,6 +38,21 @@ AUTO_FACT_PATTERNS = [
|
||||
]
|
||||
SOCIAL_TRIGGERS = {"hi", "hello", "hey", "yo", "sup", "howdy", "good morning", "good evening"}
|
||||
|
||||
# Short filler words that shouldn't count as subject overlap between facts.
|
||||
_STOPWORDS = {
|
||||
"with", "that", "have", "this", "from", "they", "what", "when", "where",
|
||||
"which", "there", "your", "will", "would", "about", "these", "their",
|
||||
"been", "into", "than", "then", "them", "were", "being", "more", "most",
|
||||
"some", "other", "only", "still", "also", "after", "before", "during",
|
||||
"because", "through", "without",
|
||||
}
|
||||
|
||||
|
||||
def _subject_words(text: str) -> set:
|
||||
"""Meaningful subject tokens for overlap comparison."""
|
||||
words = re.findall(r"[A-Za-z0-9_]{4,}", text.lower())
|
||||
return {w for w in words if w not in _STOPWORDS}
|
||||
|
||||
|
||||
def _is_social(text: str) -> bool:
|
||||
t = text.strip().lower()
|
||||
@@ -86,6 +101,10 @@ def auto_detect_facts(user_message: str, assistant_message: str) -> list[str]:
|
||||
def check_fact_conflicts(facts: list[str]) -> list[dict]:
|
||||
"""Search for existing memories that conflict with detected facts.
|
||||
|
||||
A conflict is reported only when the existing memory is about the same
|
||||
subject (meaningful keyword overlap) but states something different —
|
||||
unrelated hits that merely share an FTS keyword are not conflicts.
|
||||
|
||||
Returns list of {memory_id, old_fact, new_fact} for each conflict.
|
||||
"""
|
||||
conflicts = []
|
||||
@@ -93,7 +112,7 @@ def check_fact_conflicts(facts: list[str]) -> list[dict]:
|
||||
related = search_memories(new_fact, limit=1)
|
||||
if related:
|
||||
old = related[0]["fact"]
|
||||
if old.rstrip(".") != new_fact.rstrip("."):
|
||||
if old.rstrip(".") != new_fact.rstrip(".") and (_subject_words(new_fact) & _subject_words(old)):
|
||||
conflicts.append({
|
||||
"memory_id": related[0]["rowid"],
|
||||
"old_fact": old,
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
# Models
|
||||
Place .gguf model files in this directory.
|
||||
|
||||
The filename must match the LLAMA_MODEL value in .env.
|
||||
For example, if LLAMA_MODEL=llama3.1-8b-instruct.Q4_K_M.gguf,
|
||||
place that file here.
|
||||
|
||||
Download from HuggingFace:
|
||||
https://huggingface.co/models?search=gguf
|
||||
+184
-11
@@ -11,13 +11,13 @@ responds to pings, and handles model swap commands.
|
||||
# hostname — defaults to socket.gethostname()
|
||||
node_name = jarvis
|
||||
# LAN IP — defaults from socket
|
||||
node_ip = 192.168.50.210
|
||||
node_ip = 192.168.50.212
|
||||
# "worker" (fixed)
|
||||
node_type = worker
|
||||
# comma-separated capability list
|
||||
capabilities = llm
|
||||
# RabbitMQ URL on coordinator
|
||||
amqp_url = amqp://caic:password@192.168.50.108:5672/caic
|
||||
amqp_url = amqp://caic:password@localhost:5672/caic
|
||||
# port llama-server listens on
|
||||
llama_port = 8081
|
||||
# path to GGUF model files
|
||||
@@ -93,6 +93,7 @@ class AgentConfig:
|
||||
self.capabilities: list[str] = ["llm"]
|
||||
self.amqp_url: str = "amqp://caic:password@localhost:5672/caic"
|
||||
self.llama_port: int = 8081
|
||||
self.comfyui_port: int = 8188
|
||||
self.models_dir: str = "/var/lib/caic/models"
|
||||
self.active_model: str = ""
|
||||
|
||||
@@ -113,6 +114,7 @@ class AgentConfig:
|
||||
cfg.capabilities = [c.strip() for c in raw_caps.split(",") if c.strip()]
|
||||
cfg.amqp_url = parser.get(sec, "amqp_url", fallback=cfg.amqp_url)
|
||||
cfg.llama_port = parser.getint(sec, "llama_port", fallback=cfg.llama_port)
|
||||
cfg.comfyui_port = parser.getint(sec, "comfyui_port", fallback=cfg.comfyui_port)
|
||||
cfg.models_dir = parser.get(sec, "models_dir", fallback=cfg.models_dir)
|
||||
cfg.active_model = parser.get(sec, "active_model", fallback=cfg.active_model)
|
||||
return cfg
|
||||
@@ -182,16 +184,18 @@ def get_load() -> dict:
|
||||
capture_output=True, text=True, timeout=3,
|
||||
)
|
||||
if result.returncode == 0:
|
||||
total = 0
|
||||
used = 0
|
||||
for line in result.stdout.splitlines():
|
||||
if "VRAM Total" in line:
|
||||
parts = line.split()
|
||||
if len(parts) >= 3:
|
||||
total = int(parts[-1])
|
||||
elif "VRAM Used" in line:
|
||||
parts = line.split()
|
||||
if len(parts) >= 3:
|
||||
used = int(parts[-1])
|
||||
if total and total > 0:
|
||||
if "VRAM Total Used Memory (B)" in line:
|
||||
parts = line.split(":")
|
||||
if len(parts) >= 2:
|
||||
used = int(parts[-1].strip())
|
||||
elif "VRAM Total Memory (B)" in line:
|
||||
parts = line.split(":")
|
||||
if len(parts) >= 2:
|
||||
total = int(parts[-1].strip())
|
||||
if total > 0:
|
||||
load["vram_pct"] = round(used / total * 100)
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||||
pass
|
||||
@@ -212,6 +216,19 @@ def get_load() -> dict:
|
||||
return load
|
||||
|
||||
|
||||
def detect_capabilities(cfg: AgentConfig) -> list[str]:
|
||||
caps = list(cfg.capabilities)
|
||||
if "image_gen" not in caps and HAS_HTTPX:
|
||||
try:
|
||||
resp = httpx.get(f"http://localhost:{cfg.comfyui_port}/system_stats", timeout=3)
|
||||
if resp.status_code == 200:
|
||||
caps.append("image_gen")
|
||||
log.info("auto-detected image_gen capability (ComfyUI on port %d)", cfg.comfyui_port)
|
||||
except Exception:
|
||||
pass
|
||||
return caps
|
||||
|
||||
|
||||
# ── AMQP helpers ────────────────────────────────────────────────────────
|
||||
|
||||
async def declare_exchanges(channel) -> tuple:
|
||||
@@ -356,6 +373,152 @@ async def _wait_for_llama(port: int, timeout: int = 120, interval: int = 2) -> b
|
||||
return False
|
||||
|
||||
|
||||
# ── image generation ─────────────────────────────────────────────────────
|
||||
|
||||
async def handle_image_generate(cfg: AgentConfig, channel, exchanges, msg: aio_pika.IncomingMessage):
|
||||
admin_ex, system_ex = exchanges
|
||||
async with msg.process():
|
||||
try:
|
||||
payload = json.loads(msg.body.decode())
|
||||
except json.JSONDecodeError:
|
||||
return
|
||||
|
||||
request_id = payload.get("request_id")
|
||||
prompt = payload.get("prompt", "")
|
||||
negative_prompt = payload.get("negative_prompt", "")
|
||||
width = payload.get("width", 1024)
|
||||
height = payload.get("height", 1024)
|
||||
steps = payload.get("steps", 20)
|
||||
seed = payload.get("seed", -1)
|
||||
model = payload.get("model", "")
|
||||
|
||||
if not prompt:
|
||||
log.error("image_generate missing prompt")
|
||||
return
|
||||
|
||||
log.info("image generate: prompt=%s %dx%d steps=%d", prompt[:60], width, height, steps)
|
||||
|
||||
now = datetime.now(timezone.utc).isoformat() + "Z"
|
||||
try:
|
||||
image_data = await _comfyui_generate(
|
||||
cfg, prompt, negative_prompt, width, height, steps, seed, model,
|
||||
)
|
||||
result_payload = {
|
||||
"node_name": cfg.node_name,
|
||||
"type": "image_generated",
|
||||
"request_id": request_id,
|
||||
"image_base64": image_data,
|
||||
"timestamp": now,
|
||||
}
|
||||
log.info("image generate complete: request_id=%s", request_id)
|
||||
except Exception as e:
|
||||
result_payload = {
|
||||
"node_name": cfg.node_name,
|
||||
"type": "image_failed",
|
||||
"request_id": request_id,
|
||||
"error": str(e),
|
||||
"timestamp": now,
|
||||
}
|
||||
log.error("image generate failed: %s", e)
|
||||
|
||||
await publish(channel, system_ex, f"node.{cfg.node_name}.{result_payload['type']}", result_payload)
|
||||
|
||||
|
||||
async def _comfyui_generate(
|
||||
cfg: AgentConfig, prompt: str, negative_prompt: str,
|
||||
width: int, height: int, steps: int, seed: int, model: str,
|
||||
) -> str:
|
||||
import random
|
||||
import uuid as _uuid
|
||||
|
||||
if not HAS_HTTPX:
|
||||
raise RuntimeError("httpx not installed")
|
||||
|
||||
client_id = str(_uuid.uuid4())
|
||||
if seed < 0:
|
||||
seed = random.randint(0, 2**32 - 1)
|
||||
|
||||
checkpoint = model or "model.safetensors"
|
||||
|
||||
workflow = {
|
||||
"3": {
|
||||
"class_type": "KSampler",
|
||||
"inputs": {
|
||||
"seed": seed,
|
||||
"steps": steps,
|
||||
"cfg": 7.0,
|
||||
"sampler_name": "euler",
|
||||
"scheduler": "normal",
|
||||
"denoise": 1.0,
|
||||
"model": ["4", 0],
|
||||
"positive": ["6", 0],
|
||||
"negative": ["7", 0],
|
||||
"latent_image": ["5", 0],
|
||||
},
|
||||
},
|
||||
"4": {
|
||||
"class_type": "CheckpointLoaderSimple",
|
||||
"inputs": {"ckpt_name": checkpoint},
|
||||
},
|
||||
"5": {
|
||||
"class_type": "EmptyLatentImage",
|
||||
"inputs": {"width": width, "height": height, "batch_size": 1},
|
||||
},
|
||||
"6": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": prompt, "clip": ["4", 1]},
|
||||
},
|
||||
"7": {
|
||||
"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": negative_prompt or "blurry, low quality", "clip": ["4", 1]},
|
||||
},
|
||||
"8": {
|
||||
"class_type": "VAEDecode",
|
||||
"inputs": {"samples": ["3", 0], "vae": ["4", 2]},
|
||||
},
|
||||
"9": {
|
||||
"class_type": "SaveImage",
|
||||
"inputs": {"filename_prefix": f"caic_{client_id}", "images": ["8", 0]},
|
||||
},
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(timeout=120) as client:
|
||||
resp = await client.post(
|
||||
f"http://localhost:{cfg.comfyui_port}/prompt",
|
||||
json={"prompt": workflow, "client_id": client_id},
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
raise RuntimeError(f"ComfyUI prompt failed: {resp.status_code} {resp.text}")
|
||||
|
||||
prompt_id = resp.json().get("prompt_id")
|
||||
if not prompt_id:
|
||||
raise RuntimeError("ComfyUI returned no prompt_id")
|
||||
|
||||
deadline = time.time() + 120
|
||||
while time.time() < deadline:
|
||||
resp = await client.get(f"http://localhost:{cfg.comfyui_port}/history/{prompt_id}")
|
||||
if resp.status_code == 200:
|
||||
history = resp.json().get(prompt_id, {})
|
||||
outputs = history.get("outputs", {})
|
||||
for node_id, node_output in outputs.items():
|
||||
images = node_output.get("images", [])
|
||||
if images:
|
||||
img_info = images[0]
|
||||
filename = img_info.get("filename")
|
||||
subfolder = img_info.get("subfolder", "")
|
||||
img_type = img_info.get("type", "output")
|
||||
img_resp = await client.get(
|
||||
f"http://localhost:{cfg.comfyui_port}/view",
|
||||
params={"filename": filename, "subfolder": subfolder, "type": img_type},
|
||||
)
|
||||
if img_resp.status_code == 200:
|
||||
import base64
|
||||
return base64.b64encode(img_resp.content).decode()
|
||||
await asyncio.sleep(1)
|
||||
|
||||
raise RuntimeError("ComfyUI generation timed out after 120s")
|
||||
|
||||
|
||||
# ── main ────────────────────────────────────────────────────────────────
|
||||
|
||||
async def amain():
|
||||
@@ -372,6 +535,9 @@ async def amain():
|
||||
cfg = AgentConfig.from_ini()
|
||||
log.info("node_name=%s node_ip=%s", cfg.node_name, cfg.node_ip)
|
||||
|
||||
cfg.capabilities = detect_capabilities(cfg)
|
||||
log.info("capabilities: %s", cfg.capabilities)
|
||||
|
||||
inventory = discover_models(cfg.models_dir)
|
||||
log.info("discovered %d models", len(inventory))
|
||||
|
||||
@@ -418,6 +584,13 @@ async def amain():
|
||||
await swap_queue.bind(admin_ex, f"node.{cfg.node_name}.cmd.swap_model")
|
||||
await swap_queue.consume(lambda msg: handle_swap_model(cfg, channel, (admin_ex, system_ex), msg))
|
||||
|
||||
# Set up image gen consumer
|
||||
if "image_gen" in cfg.capabilities:
|
||||
image_queue = await channel.declare_queue("", exclusive=True)
|
||||
await image_queue.bind(admin_ex, f"node.{cfg.node_name}.cmd.image_generate")
|
||||
await image_queue.consume(lambda msg: handle_image_generate(cfg, channel, (admin_ex, system_ex), msg))
|
||||
log.info("image generation handler registered")
|
||||
|
||||
log.info("listening for pings and commands")
|
||||
# Run forever
|
||||
await asyncio.Event().wait()
|
||||
|
||||
@@ -4,6 +4,7 @@ cAIc - RAG pipeline: Qdrant vector search + system prompt assembly.
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import httpx
|
||||
@@ -16,7 +17,7 @@ from config import MAX_SKILL_PROMPT_CHARS, QDRANT_URL, RAG_COLLECTION
|
||||
|
||||
log = logging.getLogger("caic")
|
||||
|
||||
EMBED_URL = os.environ.get("CAIC_EMBED_URL", "http://192.168.50.210:11434")
|
||||
EMBED_URL = os.environ.get("CAIC_EMBED_URL", "http://localhost:11434")
|
||||
EMBED_MODEL = os.environ.get("CAIC_EMBED_MODEL", "mxbai-embed-large")
|
||||
RAG_SCORE_THRESHOLD = 0.25
|
||||
|
||||
@@ -45,7 +46,7 @@ async def _upsert_fact(fact: str, text: str, topic: str,
|
||||
if er.status_code != 200:
|
||||
continue
|
||||
vector = er.json()["embedding"]
|
||||
pid = f"auto-{ts}-{i}"
|
||||
pid = str(uuid.uuid5(uuid.NAMESPACE_DNS, f"auto-{ts}-{i}"))
|
||||
payload = {
|
||||
"text": encrypt_text(chunk), "source": "auto_fact", "fact": fact,
|
||||
"ingest_date": datetime.now(timezone.utc).isoformat(),
|
||||
@@ -124,7 +125,7 @@ async def confirm_fact_update(memory_id: int, old_fact: str, new_fact: str,
|
||||
return True
|
||||
|
||||
|
||||
def chunk_text(text: str, chunk_size: int = 512, overlap: int = 128) -> list:
|
||||
def chunk_text(text: str, chunk_size: int = 200, overlap: int = 64) -> list:
|
||||
words = text.split()
|
||||
target_words = int(chunk_size / 1.3)
|
||||
overlap_words = int(overlap / 1.3)
|
||||
@@ -164,7 +165,8 @@ async def query_rag(query: str, limit: int = 3) -> list:
|
||||
pid = r.get("id")
|
||||
if pid:
|
||||
current = r.get("payload", {}).get("retrieval_count", 0) or 0
|
||||
asyncio.ensure_future(_update_retrieval_count(pid, current))
|
||||
# Fire-and-forget: update retrieval count without blocking the response
|
||||
asyncio.create_task(_update_retrieval_count(pid, current))
|
||||
return results
|
||||
except Exception as e:
|
||||
log.warning(f"RAG query error: {e}")
|
||||
|
||||
@@ -5,3 +5,5 @@ pypdf>=5.0.0
|
||||
python-multipart>=0.0.9
|
||||
aio-pika>=9.0.0
|
||||
cryptography>=44.0.0
|
||||
psutil>=5.9.0
|
||||
jinja2>=3.1.0
|
||||
|
||||
+27
-4
@@ -1,4 +1,4 @@
|
||||
"""JarvisChat routers - /api/chat streaming endpoint."""
|
||||
"""cAIc routers - /api/chat streaming endpoint."""
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
@@ -23,6 +23,22 @@ from config import MAX_CHAT_MESSAGE_CHARS, MODEL_CONTEXT_LENGTH
|
||||
log = logging.getLogger("caic")
|
||||
router = APIRouter()
|
||||
|
||||
# References to background auto-ingest tasks so they are never garbage-collected.
|
||||
_ingest_tasks: set = set()
|
||||
|
||||
|
||||
async def _safe_ingest(coro):
|
||||
try:
|
||||
await coro
|
||||
except Exception as e:
|
||||
log.warning("auto-ingest task failed: %s", e)
|
||||
|
||||
|
||||
def _spawn_ingest(coro):
|
||||
task = asyncio.create_task(_safe_ingest(coro))
|
||||
_ingest_tasks.add(task)
|
||||
task.add_done_callback(_ingest_tasks.discard)
|
||||
|
||||
|
||||
def parse_llama_stream_chunk(line: str) -> tuple:
|
||||
if line.startswith("data: "):
|
||||
@@ -112,6 +128,11 @@ async def chat(request: Request):
|
||||
db.execute("INSERT INTO conversations (id, title, model, created_at, updated_at) VALUES (?, ?, ?, ?, ?)",
|
||||
(conv_id, encrypt_text(title), model, now, now))
|
||||
else:
|
||||
# A client-supplied id may reference a conversation that no longer exists;
|
||||
# recreate the row so the message insert satisfies the FK instead of 500ing.
|
||||
title = user_message[:80] + ("..." if len(user_message) > 80 else "")
|
||||
db.execute("INSERT OR IGNORE INTO conversations (id, title, model, created_at, updated_at) VALUES (?, ?, ?, ?, ?)",
|
||||
(conv_id, encrypt_text(title), model, now, now))
|
||||
db.execute("UPDATE conversations SET updated_at = ? WHERE id = ?", (now, conv_id))
|
||||
|
||||
db.execute("INSERT INTO messages (conversation_id, role, content, created_at, perplexity) VALUES (?, ?, ?, ?, ?)",
|
||||
@@ -171,6 +192,8 @@ async def chat(request: Request):
|
||||
|
||||
assistant_msg = "".join(full_response)
|
||||
perplexity = calculate_perplexity(all_logprobs) if all_logprobs else 0.0
|
||||
if not all_logprobs:
|
||||
log.warning("No logprobs received from inference server — perplexity auto-search unavailable")
|
||||
should_search = is_uncertain(all_logprobs) or is_refusal(assistant_msg)
|
||||
|
||||
if search_enabled and should_search:
|
||||
@@ -210,7 +233,7 @@ async def chat(request: Request):
|
||||
if is_refusal(cleaned_response) or len(cleaned_response) < 20:
|
||||
cleaned_response = format_direct_answer(user_message, search_results)
|
||||
|
||||
yield f"data: {json.dumps({'token': cleaned_response, 'conversation_id': conv_id, 'augmented': True})}\n\n"
|
||||
yield f"data: {json.dumps({'token': cleaned_response, 'conversation_id': conv_id, 'augmented': True, 'reset': True})}\n\n"
|
||||
|
||||
if not private_chat:
|
||||
saved_msg = cleaned_response + "\n\n---\n*🔍 Enhanced with web search results*"
|
||||
@@ -229,7 +252,7 @@ async def chat(request: Request):
|
||||
if conflicts:
|
||||
rag_update = {"conflicts": conflicts}
|
||||
else:
|
||||
asyncio.ensure_future(ingest_auto_fact(facts, user_message, cleaned_response))
|
||||
_spawn_ingest(ingest_auto_fact(facts, user_message, cleaned_response))
|
||||
|
||||
yield f"data: {json.dumps({'done': True, 'conversation_id': conv_id, 'searched': True, 'perplexity': round(perplexity, 2), 'tokens_per_sec': round(tokens_per_sec, 1), 'prompt_tokens': prompt_tokens, 'completion_tokens': completion_tokens, 'context_length': MODEL_CONTEXT_LENGTH, **(rag_update and {'rag_update_suggestion': rag_update} or {})})}\n\n"
|
||||
return
|
||||
@@ -251,7 +274,7 @@ async def chat(request: Request):
|
||||
if conflicts:
|
||||
rag_update = {"conflicts": conflicts}
|
||||
else:
|
||||
asyncio.ensure_future(ingest_auto_fact(facts, user_message, assistant_msg))
|
||||
_spawn_ingest(ingest_auto_fact(facts, user_message, assistant_msg))
|
||||
|
||||
yield f"data: {json.dumps({'done': True, 'conversation_id': conv_id, 'perplexity': round(perplexity, 2), 'tokens_per_sec': round(tokens_per_sec, 1), 'prompt_tokens': prompt_tokens, 'completion_tokens': completion_tokens, 'context_length': MODEL_CONTEXT_LENGTH, **(rag_update and {'rag_update_suggestion': rag_update} or {})})}\n\n"
|
||||
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
"""JarvisChat routers - Cluster status API."""
|
||||
"""cAIc routers - Cluster status API."""
|
||||
from fastapi import APIRouter
|
||||
|
||||
import cluster
|
||||
|
||||
+24
-21
@@ -1,11 +1,12 @@
|
||||
"""
|
||||
JarvisChat - /v1/chat/completions router.
|
||||
cAIc - /v1/chat/completions router.
|
||||
OpenAI-compatible endpoint for IDE integration (Continue.dev, etc.).
|
||||
Runs all requests through the full jC pipeline: profile + RAG + memory injection.
|
||||
FIM (fill-in-the-middle) requests are proxied directly — not persisted.
|
||||
Chat-style requests are persisted to conversation history.
|
||||
Auth: static Bearer token via COMPLETIONS_API_KEY in config.
|
||||
"""
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
@@ -30,7 +31,7 @@ def _check_api_key(request: Request):
|
||||
if not auth.startswith("Bearer "):
|
||||
raise HTTPException(status_code=401, detail="Missing Bearer token")
|
||||
token = auth[7:].strip()
|
||||
if token != COMPLETIONS_API_KEY:
|
||||
if not hmac.compare_digest(token.encode(), COMPLETIONS_API_KEY.encode()):
|
||||
raise HTTPException(status_code=401, detail="Invalid API key")
|
||||
|
||||
|
||||
@@ -120,26 +121,28 @@ async def chat_completions(request: Request):
|
||||
|
||||
# --- Persist conversation ---
|
||||
db = get_db()
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
conv_id = str(uuid.uuid4())
|
||||
title = f"[IDE] {user_message[:72]}{'...' if len(user_message) > 72 else ''}"
|
||||
db.execute(
|
||||
"INSERT INTO conversations (id, title, model, created_at, updated_at) VALUES (?, ?, ?, ?, ?)",
|
||||
(conv_id, encrypt_text(title), model, now, now),
|
||||
)
|
||||
for msg in messages:
|
||||
role = msg.get("role")
|
||||
content = msg.get("content", "")
|
||||
if role in ("user", "assistant"):
|
||||
db.execute(
|
||||
"INSERT INTO messages (conversation_id, role, content, created_at, perplexity) VALUES (?, ?, ?, ?, ?)",
|
||||
(conv_id, role, encrypt_text(content), now, None),
|
||||
)
|
||||
db.commit()
|
||||
try:
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
conv_id = str(uuid.uuid4())
|
||||
title = f"[IDE] {user_message[:72]}{'...' if len(user_message) > 72 else ''}"
|
||||
db.execute(
|
||||
"INSERT INTO conversations (id, title, model, created_at, updated_at) VALUES (?, ?, ?, ?, ?)",
|
||||
(conv_id, encrypt_text(title), model, now, now),
|
||||
)
|
||||
for msg in messages:
|
||||
role = msg.get("role")
|
||||
content = msg.get("content", "")
|
||||
if role in ("user", "assistant"):
|
||||
db.execute(
|
||||
"INSERT INTO messages (conversation_id, role, content, created_at, perplexity) VALUES (?, ?, ?, ?, ?)",
|
||||
(conv_id, role, encrypt_text(content), now, None),
|
||||
)
|
||||
db.commit()
|
||||
|
||||
# --- Build system prompt through full jC pipeline ---
|
||||
system_prompt = await build_system_prompt(db, "", user_message)
|
||||
db.close()
|
||||
# --- Build system prompt through full jC pipeline ---
|
||||
system_prompt = await build_system_prompt(db, "", user_message)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
# Assemble messages for upstream: inject jC system prompt, preserve history
|
||||
upstream_messages = []
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""JarvisChat routers - Conversation CRUD."""
|
||||
"""cAIc routers - Conversation CRUD."""
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
"""JarvisChat routers — Hardware self-assessment endpoint."""
|
||||
"""cAIc routers — Hardware self-assessment endpoint."""
|
||||
import json
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
"""cAIc routers — Image generation proxy endpoint."""
|
||||
import base64
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi.responses import Response
|
||||
|
||||
from cluster import CLUSTER_NODES, request_image_generate
|
||||
|
||||
log = logging.getLogger("caic")
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _find_image_node() -> str | None:
|
||||
for name, node in CLUSTER_NODES.items():
|
||||
if node.get("status") == "active" and "image_gen" in node.get("capabilities", []):
|
||||
return name
|
||||
return None
|
||||
|
||||
|
||||
@router.post("/api/image/generate")
|
||||
async def generate_image(request_body: dict):
|
||||
prompt = (request_body.get("prompt") or "").strip()
|
||||
if not prompt:
|
||||
raise HTTPException(status_code=400, detail="Prompt is required")
|
||||
|
||||
negative_prompt = request_body.get("negative_prompt", "")
|
||||
width = min(max(request_body.get("width", 1024), 256), 2048)
|
||||
height = min(max(request_body.get("height", 1024), 256), 2048)
|
||||
steps = min(max(request_body.get("steps", 20), 1), 50)
|
||||
seed = request_body.get("seed", -1)
|
||||
model = request_body.get("model", "")
|
||||
|
||||
node_name = _find_image_node()
|
||||
if not node_name:
|
||||
raise HTTPException(status_code=503, detail="No image generation service available")
|
||||
|
||||
log.info("image generate via %s: %s", node_name, prompt[:60])
|
||||
|
||||
image_b64 = await request_image_generate(
|
||||
node_name=node_name,
|
||||
prompt=prompt,
|
||||
negative_prompt=negative_prompt,
|
||||
width=width,
|
||||
height=height,
|
||||
steps=steps,
|
||||
seed=seed,
|
||||
model=model,
|
||||
)
|
||||
|
||||
if image_b64 is None:
|
||||
raise HTTPException(status_code=504, detail="Image generation timed out or failed")
|
||||
|
||||
image_bytes = base64.b64decode(image_b64)
|
||||
return Response(content=image_bytes, media_type="image/png")
|
||||
|
||||
|
||||
@router.get("/api/image/status")
|
||||
async def image_status():
|
||||
nodes = []
|
||||
for name, node in CLUSTER_NODES.items():
|
||||
caps = node.get("capabilities", [])
|
||||
if "image_gen" in caps:
|
||||
nodes.append({
|
||||
"name": name,
|
||||
"status": node.get("status"),
|
||||
"load": node.get("load"),
|
||||
"last_seen": node.get("last_seen"),
|
||||
})
|
||||
return {"available": len(nodes) > 0, "nodes": nodes}
|
||||
+7
-3
@@ -1,5 +1,8 @@
|
||||
"""JarvisChat routers - /api/ingest terminal command RAG hook."""
|
||||
"""cAIc routers - /api/ingest terminal command RAG hook."""
|
||||
import hashlib
|
||||
import hmac
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import httpx
|
||||
@@ -20,7 +23,7 @@ def _check_api_key(request: Request):
|
||||
if not auth.startswith("Bearer "):
|
||||
raise HTTPException(status_code=401, detail="Missing Bearer token")
|
||||
token = auth[7:].strip()
|
||||
if token != COMPLETIONS_API_KEY:
|
||||
if not hmac.compare_digest(token.encode(), COMPLETIONS_API_KEY.encode()):
|
||||
raise HTTPException(status_code=401, detail="Invalid API key")
|
||||
|
||||
|
||||
@@ -50,7 +53,8 @@ async def ingest_content(request: Request):
|
||||
log.warning(f"Ingest embedding failed for chunk {i}: {embed_resp.status_code}")
|
||||
continue
|
||||
vector = embed_resp.json()["embedding"]
|
||||
point_id = f"ingest-{source}-{datetime.now(timezone.utc).timestamp()}-{i}"
|
||||
chunk_hash = hashlib.md5(chunk.encode("utf-8")).hexdigest()[:12]
|
||||
point_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, f"ingest-{source}-{chunk_hash}-{i}"))
|
||||
payload = {"text": encrypt_text(chunk), "source": source, "ingest_date": datetime.now(timezone.utc).isoformat(), "type": "ingest"}
|
||||
payload.update(metadata)
|
||||
upsert_resp = await client.put(
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
"""JarvisChat routers - Memory CRUD API."""
|
||||
"""cAIc routers - Memory CRUD API."""
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from typing import Optional
|
||||
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
"""
|
||||
JarvisChat routers - Model listing, system stats.
|
||||
cAIc routers - Model listing, system stats.
|
||||
"""
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
"""JarvisChat routers - System prompt presets."""
|
||||
"""cAIc routers - System prompt presets."""
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
"""JarvisChat routers - Profile."""
|
||||
"""cAIc routers - Profile."""
|
||||
from datetime import datetime, timezone
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from db import get_db
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""JarvisChat routers — RAG corpus management admin endpoints."""
|
||||
"""cAIc routers — RAG corpus management admin endpoints."""
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""JarvisChat routers - /api/search explicit search endpoint."""
|
||||
"""cAIc routers - /api/search explicit search endpoint."""
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
@@ -44,6 +44,9 @@ async def explicit_search(request: Request):
|
||||
db.execute("INSERT INTO conversations (id, title, model, created_at, updated_at) VALUES (?, ?, ?, ?, ?)",
|
||||
(conv_id, encrypt_text(title), model, now, now))
|
||||
else:
|
||||
title = query[:70] + "..." if len(query) > 70 else query
|
||||
db.execute("INSERT OR IGNORE INTO conversations (id, title, model, created_at, updated_at) VALUES (?, ?, ?, ?, ?)",
|
||||
(conv_id, title, model, now, now))
|
||||
db.execute("UPDATE conversations SET updated_at = ? WHERE id = ?", (now, conv_id))
|
||||
|
||||
db.execute("INSERT INTO messages (conversation_id, role, content, created_at, perplexity) VALUES (?, ?, ?, ?, ?)",
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
"""JarvisChat routers - Settings."""
|
||||
"""cAIc routers - Settings."""
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from db import get_db
|
||||
from security import read_json_body, BODY_LIMIT_DEFAULT_BYTES
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
"""JarvisChat routers - Skills."""
|
||||
"""cAIc routers - Skills."""
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from db import get_db, get_setting, list_skills_with_state, set_skill_enabled
|
||||
from security import read_json_body, BODY_LIMIT_DEFAULT_BYTES
|
||||
|
||||
+13
-3
@@ -1,7 +1,8 @@
|
||||
"""JarvisChat routers - /api/upload file/document attachment endpoint."""
|
||||
"""cAIc routers - /api/upload file/document attachment endpoint."""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
import httpx
|
||||
@@ -19,7 +20,7 @@ router = APIRouter()
|
||||
|
||||
|
||||
def _point_id(filename: str, chunk_idx: int) -> str:
|
||||
return f"upload-{filename}-{chunk_idx}"
|
||||
return str(uuid.uuid5(uuid.NAMESPACE_DNS, f"upload-{filename}-{chunk_idx}"))
|
||||
|
||||
|
||||
@router.post("/api/upload")
|
||||
@@ -52,12 +53,21 @@ async def upload_file(
|
||||
except Exception as e:
|
||||
log.warning(f"PDF extraction error: {e}")
|
||||
raise HTTPException(status_code=422, detail="Failed to extract text from PDF")
|
||||
elif content_type.startswith("image/"):
|
||||
# No OCR pipeline exists — store a descriptive placeholder so images
|
||||
# remain usable in the gallery/context but never pollute the RAG corpus.
|
||||
extracted = f"[Image: {file.filename}]"
|
||||
else:
|
||||
extracted = raw_bytes.decode("utf-8", errors="replace")
|
||||
|
||||
result = {"filename": file.filename, "size_bytes": len(raw_bytes), "mode": mode}
|
||||
|
||||
if mode in ("ingest", "both"):
|
||||
is_image = content_type.startswith("image/")
|
||||
if is_image and mode in ("ingest", "both"):
|
||||
result["chunks_ingested"] = 0
|
||||
result["note"] = "Image files cannot be text-ingested; stored for gallery/context only"
|
||||
|
||||
if mode in ("ingest", "both") and not is_image:
|
||||
os.makedirs(UPLOAD_DIR, exist_ok=True)
|
||||
chunks = chunk_text(extracted)
|
||||
ingested = 0
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env bash
|
||||
# cAIc — First-run scaffolding
|
||||
# Creates secrets, config, and directories needed by docker compose.
|
||||
# Idempotent: safe to re-run.
|
||||
set -euo pipefail
|
||||
|
||||
GREEN='\033[0;32m'; YELLOW='\033[1;33m'; NC='\033[0m'
|
||||
|
||||
ok() { echo -e " ${GREEN}✓${NC} $1"; }
|
||||
warn() { echo -e " ${YELLOW}!${NC} $1"; }
|
||||
|
||||
echo "cAIc — scaffolding"
|
||||
echo "==================="
|
||||
|
||||
# ── .env ────────────────────────────────────────────────────
|
||||
if [ ! -f .env ]; then
|
||||
cp .env.example .env
|
||||
|
||||
# Auto-generate secrets
|
||||
ADMIN_PIN=$(shuf -i 1000-9999 -n 1)
|
||||
API_KEY="caic-sk-$(openssl rand -hex 24)"
|
||||
RMQ_PASS=$(openssl rand -hex 20)
|
||||
SEARX_KEY=$(openssl rand -hex 32)
|
||||
|
||||
sed -i "s/^CAIC_ADMIN_PIN=$/CAIC_ADMIN_PIN=$ADMIN_PIN/" .env
|
||||
sed -i "s/^CAIC_COMPLETIONS_API_KEY=$/CAIC_COMPLETIONS_API_KEY=$API_KEY/" .env
|
||||
sed -i "s/^RABBITMQ_PASSWORD=$/RABBITMQ_PASSWORD=$RMQ_PASS/" .env
|
||||
sed -i "s/^SEARXNG_SECRET_KEY=$/SEARXNG_SECRET_KEY=$SEARX_KEY/" .env
|
||||
|
||||
ok ".env created"
|
||||
warn " Admin PIN: $ADMIN_PIN"
|
||||
warn " API key: $API_KEY"
|
||||
else
|
||||
warn ".env already exists — skipped"
|
||||
fi
|
||||
|
||||
# ── secrets/ ────────────────────────────────────────────────
|
||||
mkdir -p secrets
|
||||
if [ ! -f secrets/rabbitmq_password.txt ]; then
|
||||
RMQ_PASS=$(grep '^RABBITMQ_PASSWORD=' .env | cut -d= -f2)
|
||||
echo -n "$RMQ_PASS" > secrets/rabbitmq_password.txt
|
||||
ok "secrets/rabbitmq_password.txt created"
|
||||
else
|
||||
warn "secrets/rabbitmq_password.txt exists — skipped"
|
||||
fi
|
||||
|
||||
# ── searxng/settings.yml ────────────────────────────────────
|
||||
mkdir -p searxng
|
||||
if [ ! -f searxng/settings.yml ]; then
|
||||
SEARX_KEY=$(grep '^SEARXNG_SECRET_KEY=' .env | cut -d= -f2)
|
||||
# Substitute the secret key into the template
|
||||
sed "s/\${SEARXNG_SECRET_KEY}/$SEARX_KEY/" searxng-settings.yml.dist > searxng/settings.yml
|
||||
ok "searxng/settings.yml created"
|
||||
else
|
||||
warn "searxng/settings.yml exists — skipped"
|
||||
fi
|
||||
|
||||
# ── models/ ─────────────────────────────────────────────────
|
||||
DEFAULT_MODEL_REPO="unsloth/Qwen2.5-7B-Instruct-GGUF"
|
||||
DEFAULT_MODEL_FILE="Qwen2.5-7B-Instruct-Q4_K_M.gguf"
|
||||
DEFAULT_MODEL_SIZE_MB=4600 # approximate download size
|
||||
DEFAULT_MODEL_NAME="qwen2.5-7b-instruct"
|
||||
|
||||
mkdir -p models
|
||||
|
||||
# Set LLAMA_MODEL and CAIC_DEFAULT_MODEL in .env if not already set
|
||||
LLAMA_MODEL_LINE=$(grep '^LLAMA_MODEL=' .env || true)
|
||||
if [ -z "$LLAMA_MODEL_LINE" ] || [ "$LLAMA_MODEL_LINE" = "LLAMA_MODEL=" ]; then
|
||||
sed -i "s/^LLAMA_MODEL=$/LLAMA_MODEL=$DEFAULT_MODEL_FILE/" .env
|
||||
sed -i "s/^CAIC_DEFAULT_MODEL=.*/CAIC_DEFAULT_MODEL=$DEFAULT_MODEL_NAME/" .env
|
||||
ok "Set LLAMA_MODEL=$DEFAULT_MODEL_FILE"
|
||||
ok "Set CAIC_DEFAULT_MODEL=$DEFAULT_MODEL_NAME"
|
||||
fi
|
||||
|
||||
if ls models/*.gguf 1>/dev/null 2>&1; then
|
||||
ok "models/ has $(ls models/*.gguf | wc -l) model(s)"
|
||||
else
|
||||
echo ""
|
||||
echo " No .gguf models found in ./models/"
|
||||
echo ""
|
||||
|
||||
# Check disk space
|
||||
AVAIL_KB=$(df -k models/ | tail -1 | awk '{print $4}')
|
||||
AVAIL_MB=$((AVAIL_KB / 1024))
|
||||
REQUIRED_MB=$((DEFAULT_MODEL_SIZE_MB + 500)) # 500MB safety margin
|
||||
|
||||
if [ "$AVAIL_MB" -lt "$REQUIRED_MB" ]; then
|
||||
warn "Insufficient disk space: ${AVAIL_MB}MB available, ~${REQUIRED_MB}MB needed"
|
||||
warn "Free space or change LLAMA_MODEL in .env to use a smaller model."
|
||||
echo ""
|
||||
else
|
||||
echo " Download default model (~${DEFAULT_MODEL_SIZE_MB}MB):"
|
||||
echo " ${DEFAULT_MODEL_REPO}/${DEFAULT_MODEL_FILE}"
|
||||
echo ""
|
||||
read -p " Download now? [Y/n] " r
|
||||
if [[ -z "$r" || "$r" =~ ^[Yy] ]]; then
|
||||
echo ""
|
||||
echo " Downloading ${DEFAULT_MODEL_FILE}..."
|
||||
if command -v hf &>/dev/null; then
|
||||
# huggingface-cli (hf_transfer) if available
|
||||
hf download "$DEFAULT_MODEL_REPO" "$DEFAULT_MODEL_FILE" \
|
||||
--local-dir models/ --local-dir-use-symlinks False
|
||||
elif command -v wget &>/dev/null; then
|
||||
wget -q --show-progress -O "models/$DEFAULT_MODEL_FILE" \
|
||||
"https://huggingface.co/${DEFAULT_MODEL_REPO}/resolve/main/${DEFAULT_MODEL_FILE}"
|
||||
elif command -v curl &>/dev/null; then
|
||||
curl -L --progress-bar -o "models/$DEFAULT_MODEL_FILE" \
|
||||
"https://huggingface.co/${DEFAULT_MODEL_REPO}/resolve/main/${DEFAULT_MODEL_FILE}"
|
||||
else
|
||||
warn "Neither wget nor curl found — cannot download."
|
||||
warn " Manual: wget -O models/$DEFAULT_MODEL_FILE \\"
|
||||
warn " https://huggingface.co/${DEFAULT_MODEL_REPO}/resolve/main/${DEFAULT_MODEL_FILE}"
|
||||
fi
|
||||
|
||||
if [ -f "models/$DEFAULT_MODEL_FILE" ]; then
|
||||
DOWNLOADED_MB=$(du -m "models/$DEFAULT_MODEL_FILE" | cut -f1)
|
||||
ok "Downloaded ${DEFAULT_MODEL_FILE} (${DOWNLOADED_MB}MB)"
|
||||
else
|
||||
warn "Download failed — place model manually in ./models/"
|
||||
fi
|
||||
else
|
||||
warn "Skipped. Place .gguf model(s) in ./models/ before docker compose up."
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── data/ ───────────────────────────────────────────────────
|
||||
mkdir -p data
|
||||
ok "data/ directory ready"
|
||||
|
||||
# ── Verify Docker ───────────────────────────────────────────
|
||||
echo ""
|
||||
if command -v docker &>/dev/null && docker compose version &>/dev/null; then
|
||||
ok "Docker Compose available: $(docker compose version --short)"
|
||||
echo ""
|
||||
echo -e "${GREEN}Ready!${NC} Run: docker compose up -d"
|
||||
else
|
||||
warn "Docker Compose not found — install Docker Engine + Compose plugin first."
|
||||
echo " https://docs.docker.com/engine/install/"
|
||||
fi
|
||||
@@ -0,0 +1,26 @@
|
||||
# SearXNG configuration — cAIc stack
|
||||
# Secret key is injected via environment variable.
|
||||
|
||||
search:
|
||||
safe_search: 0
|
||||
autocomplete: ""
|
||||
default_lang: en
|
||||
|
||||
server:
|
||||
secret_key: ${SEARXNG_SECRET_KEY}
|
||||
limiter: false
|
||||
image_proxy: false
|
||||
method: GET
|
||||
port: 8080
|
||||
bind_address: "0.0.0.0"
|
||||
|
||||
engines:
|
||||
- name: google
|
||||
engine: google
|
||||
shortcut: g
|
||||
- name: duckduckgo
|
||||
engine: duckduckgo
|
||||
shortcut: ddg
|
||||
- name: wikipedia
|
||||
engine: wikipedia
|
||||
shortcut: wp
|
||||
@@ -161,10 +161,6 @@ def origin_allowed(request: Request) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def is_state_changing(method: str) -> bool:
|
||||
return method in {"POST", "PUT", "DELETE", "PATCH"}
|
||||
|
||||
|
||||
async def read_json_body(request: Request, max_bytes: int) -> dict:
|
||||
raw = await request.body()
|
||||
if len(raw) > max_bytes:
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 2.9 MiB After Width: | Height: | Size: 107 KiB |
@@ -1635,6 +1635,7 @@ async function sendSearch() {
|
||||
if (data.error) { textEl.textContent = 'Error: ' + data.error; setStreamingState(false); return; }
|
||||
if (data.conversation_id && !currentConvId) { currentConvId = data.conversation_id; await loadConversations(); }
|
||||
if (data.search_results) { textEl.innerHTML = '<div class="search-indicator">🔍 Found ' + data.search_results + ' results, summarizing...</div>'; }
|
||||
if (data.reset) { fullText = ''; textEl.innerHTML = ''; firstToken = false; }
|
||||
if (data.token) { if (firstToken) { textEl.innerHTML = ''; firstToken = false; ttr = performance.now() - ttrStart; tokenCount = 0; } fullText += data.token; tokenCount++; textEl.innerHTML = renderMarkdown(fullText); scrollToLatest(); }
|
||||
if (data.raw_results) {
|
||||
let rawHtml = '<details class="raw-results"><summary>🔍 View raw search results (' + data.raw_results.length + ')</summary><ul>';
|
||||
@@ -1843,7 +1844,7 @@ async function sendMessage() {
|
||||
}
|
||||
if (data.searching) { textEl.innerHTML = fullText ? renderMarkdown(fullText) + '<div class="search-indicator"><div class="spinner"></div>Searching...</div>' : '<div class="search-indicator"><div class="spinner"></div>Searching...</div>'; searchTriggered = true; }
|
||||
if (data.search_results) { textEl.innerHTML = '<div class="search-indicator">🔍 Found ' + data.search_results + ' results...</div>'; fullText = ''; firstToken = true; }
|
||||
if (data.token) { if (firstToken) { textEl.innerHTML = ''; firstToken = false; ttr = performance.now() - ttrStart; tokenCount = 0; } fullText += data.token; tokenCount++; textEl.innerHTML = renderMarkdown(fullText); scrollToLatest(); }
|
||||
if (data.token) { if (data.reset) { fullText = ''; firstToken = true; } if (firstToken) { textEl.innerHTML = ''; firstToken = false; ttr = performance.now() - ttrStart; tokenCount = 0; } fullText += data.token; tokenCount++; textEl.innerHTML = renderMarkdown(fullText); scrollToLatest(); }
|
||||
if (data.done) {
|
||||
const roleLabel = assistantDiv.querySelector('.role-label');
|
||||
if (data.searched && roleLabel) roleLabel.textContent = 'web search';
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
# Ensure the project root is on sys.path so that test modules can import
|
||||
# top-level packages (app, amqp, cluster, config, …) without PYTHONPATH hacks.
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
"""Shared pytest fixtures.
|
||||
|
||||
All test modules manipulate in-process globals (sessions, rate buckets,
|
||||
cluster registry, eviction log). An autouse fixture resets every global
|
||||
before each test so no state leaks between tests, regardless of whether an
|
||||
individual test file remembers to clear it.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
import cluster
|
||||
import routers.chat
|
||||
from eviction import EVICTION_LOG
|
||||
from security import SESSIONS, PIN_ATTEMPTS, RATE_EVENTS
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_global_state():
|
||||
SESSIONS.clear()
|
||||
PIN_ATTEMPTS.clear()
|
||||
RATE_EVENTS.clear()
|
||||
cluster.CLUSTER_NODES.clear()
|
||||
cluster.CLUSTER_EVENTS.clear()
|
||||
cluster.CLUSTER_COORDINATOR = None
|
||||
cluster._pending_pings.clear()
|
||||
EVICTION_LOG.clear()
|
||||
routers.chat._ingest_tasks.clear()
|
||||
yield
|
||||
|
||||
@@ -10,7 +10,6 @@ import app
|
||||
import config
|
||||
import db
|
||||
import routers.chat
|
||||
import triage
|
||||
from security import SESSIONS, PIN_ATTEMPTS, RATE_EVENTS
|
||||
|
||||
|
||||
@@ -269,7 +268,6 @@ def test_private_chat_does_not_persist(tmp_path: Path, monkeypatch):
|
||||
'data: {"choices":[{"delta":{"content":""},"finish_reason":"stop","logprobs":{"content":[]}}],"usage":{"completion_tokens":2,"prompt_tokens":10,"tokens_per_second":5.0}}',
|
||||
"data: [DONE]",
|
||||
]))
|
||||
monkeypatch.setattr(triage, "classify_query", lambda q: "general")
|
||||
|
||||
async def _mock_ensure(m): return True
|
||||
monkeypatch.setattr("model_pull.ensure_model", _mock_ensure)
|
||||
@@ -301,7 +299,6 @@ def test_private_chat_does_not_auto_search(tmp_path: Path, monkeypatch):
|
||||
'data: {"choices":[{"delta":{"content":""},"finish_reason":"stop","logprobs":{"content":[{"logprob":-2.5}]}}],"usage":{"completion_tokens":1,"prompt_tokens":10,"tokens_per_second":5.0}}',
|
||||
"data: [DONE]",
|
||||
]))
|
||||
monkeypatch.setattr(triage, "classify_query", lambda q: "general")
|
||||
monkeypatch.setattr(routers.chat, "query_searxng", lambda q: [{"title": "result"}])
|
||||
|
||||
with make_client(tmp_path) as client:
|
||||
|
||||
@@ -0,0 +1,357 @@
|
||||
"""Regression tests for bug fixes.
|
||||
|
||||
Covers: /api/ingest origin exemption for CLI/Bearer clients, bogus
|
||||
conversation_id FK handling in chat + search, the auto-search reset flag,
|
||||
image uploads being stored as placeholders instead of text-ingested,
|
||||
false-positive conflict detection, deterministic ingest point IDs,
|
||||
get_load() VRAM parsing, and version pinning.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import app
|
||||
import config
|
||||
import db
|
||||
import memory
|
||||
from crypto import decrypt_text
|
||||
import node_agent.agent as agent
|
||||
import routers.chat
|
||||
import routers.ingest as ingest_route
|
||||
import routers.search_route
|
||||
from security import SESSIONS, PIN_ATTEMPTS, RATE_EVENTS
|
||||
|
||||
|
||||
def make_client(tmp_path: Path) -> TestClient:
|
||||
os.environ["CAIC_ADMIN_PIN"] = "1234"
|
||||
db.DB_PATH = tmp_path / "caic-regression.db"
|
||||
SESSIONS.clear()
|
||||
PIN_ATTEMPTS.clear()
|
||||
RATE_EVENTS.clear()
|
||||
db.init_db()
|
||||
return TestClient(app.app, raise_server_exceptions=False)
|
||||
|
||||
|
||||
def _guest_headers(client: TestClient) -> dict:
|
||||
sid = client.post("/api/auth/guest", headers={"Origin": "http://testserver"}).json()["session_id"]
|
||||
return {"X-Session-ID": sid, "Origin": "http://testserver"}
|
||||
|
||||
|
||||
def _admin_headers(client: TestClient) -> dict:
|
||||
login = client.post("/api/auth/login", json={"pin": "1234"}, headers={"Origin": "http://testserver"})
|
||||
sid = login.json()["session_id"]
|
||||
return {"X-Session-ID": sid, "Origin": "http://testserver"}
|
||||
|
||||
|
||||
def parse_sse_payloads(body: str) -> list[dict]:
|
||||
payloads = []
|
||||
for chunk in body.split("\n\n"):
|
||||
chunk = chunk.strip()
|
||||
if not chunk.startswith("data: "):
|
||||
continue
|
||||
payloads.append(json.loads(chunk[len("data: "):]))
|
||||
return payloads
|
||||
|
||||
|
||||
class _MockStreamResponse:
|
||||
def __init__(self, lines: list[str]):
|
||||
self._lines = lines
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
async def aiter_lines(self):
|
||||
for line in self._lines:
|
||||
yield line
|
||||
|
||||
|
||||
def _stream_json_lines(events: list[dict]) -> list[str]:
|
||||
return [json.dumps(event) for event in events]
|
||||
|
||||
|
||||
class _FakeAsyncClient:
|
||||
class FakeResponse:
|
||||
def __init__(self, status, json_data=None):
|
||||
self.status_code = status
|
||||
self._json = json_data or {}
|
||||
|
||||
def json(self):
|
||||
return self._json
|
||||
|
||||
def __init__(self, *a, **kw):
|
||||
self.put_payloads = []
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *a):
|
||||
pass
|
||||
|
||||
async def post(self, url, **kw):
|
||||
if "/api/embeddings" in url:
|
||||
return self.FakeResponse(200, {"embedding": [0.1] * 768})
|
||||
return self.FakeResponse(200)
|
||||
|
||||
async def put(self, url, **kw):
|
||||
self.put_payloads.append(kw.get("json", {}))
|
||||
return self.FakeResponse(200)
|
||||
|
||||
|
||||
# ── /api/ingest is reached by CLI tools with no Origin header ──────────
|
||||
|
||||
|
||||
def test_ingest_origin_exemption(tmp_path: Path, monkeypatch):
|
||||
monkeypatch.setattr(ingest_route, "COMPLETIONS_API_KEY", "sk-regression")
|
||||
monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **kw: _FakeAsyncClient())
|
||||
with make_client(tmp_path) as client:
|
||||
resp = client.post(
|
||||
"/api/ingest",
|
||||
json={"content": "regression test content " * 20, "source": "cli"},
|
||||
headers={"Authorization": "Bearer sk-regression", "Content-Type": "application/json"},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["source"] == "cli"
|
||||
|
||||
|
||||
def test_ingest_bad_key_still_blocked_without_origin(tmp_path: Path, monkeypatch):
|
||||
monkeypatch.setattr(ingest_route, "COMPLETIONS_API_KEY", "sk-regression")
|
||||
with make_client(tmp_path) as client:
|
||||
resp = client.post(
|
||||
"/api/ingest",
|
||||
json={"content": "x " * 50},
|
||||
headers={"Authorization": "Bearer wrong-key", "Content-Type": "application/json"},
|
||||
)
|
||||
assert resp.status_code == 401
|
||||
|
||||
|
||||
# ── a client-supplied conversation_id that no longer exists ────────────
|
||||
|
||||
|
||||
def test_chat_bogus_conversation_id_creates_row(tmp_path: Path, monkeypatch):
|
||||
events = _stream_json_lines([
|
||||
{"message": {"content": "hi"}, "logprobs": [{"logprob": -0.01}]},
|
||||
{"done": True, "eval_count": 1, "eval_duration": 1000000000},
|
||||
])
|
||||
|
||||
def stream_stub(self, method, url, json=None, timeout=None):
|
||||
return _MockStreamResponse(events)
|
||||
|
||||
monkeypatch.setattr(httpx.AsyncClient, "stream", stream_stub)
|
||||
|
||||
with make_client(tmp_path) as client:
|
||||
resp = client.post(
|
||||
"/api/chat",
|
||||
json={"message": "hello", "conversation_id": "ghost-conv", "model": config.DEFAULT_MODEL},
|
||||
headers=_guest_headers(client),
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
conv_resp = client.get("/api/conversations/ghost-conv", headers=_guest_headers(client))
|
||||
assert conv_resp.status_code == 200
|
||||
assert len(conv_resp.json()["messages"]) >= 2
|
||||
|
||||
|
||||
def test_search_bogus_conversation_id_creates_row(tmp_path: Path, monkeypatch):
|
||||
async def empty_search(query: str, max_results: int = 5):
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(routers.search_route, "query_searxng", empty_search)
|
||||
|
||||
with make_client(tmp_path) as client:
|
||||
resp = client.post(
|
||||
"/api/search",
|
||||
json={"query": "nothing here", "conversation_id": "ghost-search", "model": config.DEFAULT_MODEL},
|
||||
headers=_guest_headers(client),
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
conv_resp = client.get("/api/conversations/ghost-search", headers=_guest_headers(client))
|
||||
assert conv_resp.status_code == 200
|
||||
assert len(conv_resp.json()["messages"]) >= 1
|
||||
|
||||
|
||||
# ── auto-search augmentation must reset the streamed text ──────────────
|
||||
|
||||
|
||||
def test_auto_search_augmented_event_has_reset_flag(tmp_path: Path, monkeypatch):
|
||||
first_stream = _stream_json_lines([
|
||||
{"message": {"content": "I don't have current data on that."}, "logprobs": [{"logprob": -5.0}]},
|
||||
{"done": True, "eval_count": 2, "eval_duration": 1000000000},
|
||||
])
|
||||
second_stream = _stream_json_lines([
|
||||
{"message": {"content": "According to the search results, the value is forty-two."}},
|
||||
{"done": True},
|
||||
])
|
||||
batches = [first_stream, second_stream]
|
||||
|
||||
def stream_stub(self, method, url, json=None, timeout=None):
|
||||
return _MockStreamResponse(batches.pop(0))
|
||||
|
||||
async def search_stub(query: str, max_results: int = 5):
|
||||
return [{"title": "Answer", "url": "https://example.com", "content": "The value is 42."}]
|
||||
|
||||
with make_client(tmp_path) as client:
|
||||
monkeypatch.setattr(httpx.AsyncClient, "stream", stream_stub)
|
||||
monkeypatch.setattr(routers.chat, "query_searxng", search_stub)
|
||||
resp = client.post(
|
||||
"/api/chat",
|
||||
json={"message": "what is the latest value", "model": config.DEFAULT_MODEL},
|
||||
headers=_guest_headers(client),
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
payloads = parse_sse_payloads(resp.text)
|
||||
|
||||
augmented = [p for p in payloads if p.get("augmented")]
|
||||
assert augmented, "expected an augmented token event"
|
||||
assert augmented[0].get("reset") is True
|
||||
# The augmented token must carry the fresh answer, not the discarded
|
||||
# first-pass "I don't have current data" text.
|
||||
assert "According to the search results" in augmented[0]["token"]
|
||||
assert "I don't have current data" not in augmented[0]["token"]
|
||||
|
||||
|
||||
# ── image uploads are placeholders, never text-ingested ────────────────
|
||||
|
||||
|
||||
def test_upload_image_skips_ingest(tmp_path: Path, monkeypatch):
|
||||
fake = _FakeAsyncClient()
|
||||
monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **kw: fake)
|
||||
with make_client(tmp_path) as client:
|
||||
resp = client.post(
|
||||
"/api/upload",
|
||||
headers=_admin_headers(client),
|
||||
data={"mode": "both"},
|
||||
files={"file": ("photo.png", b"\x89PNG\r\n\x1a\nfake", "image/png")},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
data = resp.json()
|
||||
context_id = data["context_id"]
|
||||
|
||||
assert data["chunks_ingested"] == 0
|
||||
assert data["note"]
|
||||
assert fake.put_payloads == []
|
||||
assert data["filename"] == "photo.png"
|
||||
|
||||
row = db.get_db().execute(
|
||||
"SELECT content FROM upload_context WHERE id = ?", (context_id,)
|
||||
).fetchone()
|
||||
assert row and decrypt_text(row["content"]) == "[Image: photo.png]"
|
||||
|
||||
|
||||
# ── conflict detection needs a shared subject, not just an FTS hit ─────
|
||||
|
||||
|
||||
def test_conflict_detection_requires_shared_subject(tmp_path: Path):
|
||||
os.environ["CAIC_ADMIN_PIN"] = "1234"
|
||||
db.DB_PATH = tmp_path / "caic-mem-regression.db"
|
||||
SESSIONS.clear()
|
||||
PIN_ATTEMPTS.clear()
|
||||
RATE_EVENTS.clear()
|
||||
db.init_db()
|
||||
|
||||
memory.add_memory("the cat sat on the mat", "general")
|
||||
conflicts = memory.check_fact_conflicts(["the dog is brown"])
|
||||
assert conflicts == []
|
||||
|
||||
memory.add_memory("I prefer Rust over Go", "preference")
|
||||
conflicts = memory.check_fact_conflicts(["I prefer Go over Rust"])
|
||||
assert len(conflicts) == 1
|
||||
assert conflicts[0]["new_fact"] == "I prefer Go over Rust"
|
||||
assert conflicts[0]["old_fact"] == "I prefer Rust over Go"
|
||||
assert "memory_id" in conflicts[0]
|
||||
|
||||
|
||||
# ── ingest point IDs are deterministic (no duplicate vectors) ──────────
|
||||
|
||||
|
||||
def test_ingest_deterministic_point_ids(tmp_path: Path, monkeypatch):
|
||||
monkeypatch.setattr(ingest_route, "COMPLETIONS_API_KEY", "sk-regression")
|
||||
|
||||
captured_first = []
|
||||
captured_second = []
|
||||
|
||||
class CaptureClient:
|
||||
FakeResponse = _FakeAsyncClient.FakeResponse
|
||||
|
||||
def __init__(self, *a, **kw):
|
||||
self.capture = None
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *a):
|
||||
pass
|
||||
|
||||
async def post(self, url, **kw):
|
||||
if "/api/embeddings" in url:
|
||||
return self.FakeResponse(200, {"embedding": [0.2] * 768})
|
||||
return self.FakeResponse(200)
|
||||
|
||||
async def put(self, url, **kw):
|
||||
payload = kw.get("json", {})
|
||||
if self.capture is not None:
|
||||
self.capture.append(payload["points"][0]["id"])
|
||||
return self.FakeResponse(200)
|
||||
|
||||
body = {"content": "alpha beta gamma delta epsilon " * 8, "source": "hook"}
|
||||
headers = {"Authorization": "Bearer sk-regression", "Content-Type": "application/json"}
|
||||
|
||||
fake1 = CaptureClient()
|
||||
fake1.capture = captured_first
|
||||
monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **kw: fake1)
|
||||
with make_client(tmp_path) as client:
|
||||
r1 = client.post("/api/ingest", json=body, headers=headers)
|
||||
assert r1.status_code == 200, r1.text
|
||||
|
||||
fake2 = CaptureClient()
|
||||
fake2.capture = captured_second
|
||||
monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **kw: fake2)
|
||||
with make_client(tmp_path) as client:
|
||||
r2 = client.post("/api/ingest", json=body, headers=headers)
|
||||
assert r2.status_code == 200, r2.text
|
||||
|
||||
assert captured_first and captured_second
|
||||
assert len(captured_first) == len(captured_second)
|
||||
assert captured_first == captured_second, "re-ingesting identical content changed point ids"
|
||||
assert len(set(captured_first)) == len(captured_first)
|
||||
|
||||
|
||||
# ── get_load() VRAM parsing ────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_get_load_vram_parses_rocm_output(monkeypatch):
|
||||
monkeypatch.setattr(agent, "HAS_PSUTIL", False)
|
||||
output = (
|
||||
"======================= ROCm System Management Interface =======================\n"
|
||||
"GPU[0] : gfx1030\n"
|
||||
"VRAM Total Used Memory (B): 3221225472\n"
|
||||
"VRAM Total Memory (B): 17179869184\n"
|
||||
)
|
||||
fake = subprocess.CompletedProcess(["rocm-smi", "--showmeminfo", "vram"], 0, output, "")
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: fake)
|
||||
load = agent.get_load()
|
||||
assert load["vram_pct"] == 19 # 3 GiB / 16 GiB
|
||||
|
||||
|
||||
def test_get_load_vram_absent_does_not_crash(monkeypatch):
|
||||
# Regression: rocm-smi returned no parseable VRAM lines, so the old code
|
||||
# left total/used unbound and raised.
|
||||
monkeypatch.setattr(agent, "HAS_PSUTIL", False)
|
||||
output = "======================= ROCm System Management Interface =======================\nNo GPU detected\n"
|
||||
fake = subprocess.CompletedProcess(["rocm-smi", "--showmeminfo", "vram"], 0, output, "")
|
||||
monkeypatch.setattr(subprocess, "run", lambda *a, **kw: fake)
|
||||
load = agent.get_load()
|
||||
assert "vram_pct" not in load
|
||||
|
||||
|
||||
# ── version pinning ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_version_is_bumped():
|
||||
assert re.fullmatch(r"v\d+\.\d+\.\d+", config.VERSION)
|
||||
@@ -0,0 +1,689 @@
|
||||
"""Tests for image generation — cluster handlers, router, node agent, hardware probe."""
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import httpx
|
||||
import psutil
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import app as app_module
|
||||
import cluster
|
||||
import config
|
||||
import db
|
||||
import hardware
|
||||
import node_agent.agent as agent
|
||||
from config import AMQP_EXCHANGE_ADMIN, AMQP_EXCHANGE_SYSTEM
|
||||
from security import SESSIONS, PIN_ATTEMPTS, RATE_EVENTS
|
||||
|
||||
|
||||
# ── helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
def _reset():
|
||||
cluster.CLUSTER_NODES.clear()
|
||||
cluster.CLUSTER_EVENTS.clear()
|
||||
cluster.CLUSTER_COORDINATOR = None
|
||||
cluster._pending_pings.clear()
|
||||
cluster._pending_image.clear()
|
||||
|
||||
|
||||
_published = []
|
||||
|
||||
|
||||
async def _fake_publish(exchange, routing_key, payload):
|
||||
_published.append((exchange, routing_key, payload))
|
||||
|
||||
|
||||
def make_client(tmp_path: Path) -> TestClient:
|
||||
os.environ["CAIC_ADMIN_PIN"] = "1234"
|
||||
db.DB_PATH = tmp_path / "caic-image.db"
|
||||
hardware.HARDWARE_STATE_PATH = tmp_path / "hardware_state.json"
|
||||
SESSIONS.clear()
|
||||
PIN_ATTEMPTS.clear()
|
||||
RATE_EVENTS.clear()
|
||||
db.init_db()
|
||||
return TestClient(app_module.app, raise_server_exceptions=False)
|
||||
|
||||
|
||||
def _guest_headers(client: TestClient) -> dict:
|
||||
sid = client.post("/api/auth/guest", headers={"Origin": "http://testserver"}).json()["session_id"]
|
||||
return {"X-Session-ID": sid, "Origin": "http://testserver"}
|
||||
|
||||
|
||||
def _admin_headers(client: TestClient) -> dict:
|
||||
resp = client.post("/api/auth/login", json={"pin": "1234"}, headers={"Origin": "http://testserver"})
|
||||
sid = resp.json()["session_id"]
|
||||
return {"X-Session-ID": sid, "Origin": "http://testserver"}
|
||||
|
||||
|
||||
class FakeMsg:
|
||||
def __init__(self, body_dict: dict):
|
||||
self.body = json.dumps(body_dict).encode()
|
||||
|
||||
@asynccontextmanager
|
||||
async def process(self):
|
||||
yield
|
||||
|
||||
|
||||
class FakeExchange:
|
||||
def __init__(self, name=""):
|
||||
self.name = name
|
||||
self.published = []
|
||||
|
||||
async def publish(self, msg, routing_key):
|
||||
self.published.append((msg, routing_key))
|
||||
|
||||
|
||||
class FakeChannel:
|
||||
def __init__(self):
|
||||
self.exchanges = {}
|
||||
self.is_closed = False
|
||||
|
||||
async def declare_exchange(self, name, typ, durable=True):
|
||||
self.exchanges[name] = FakeExchange(name)
|
||||
return self.exchanges[name]
|
||||
|
||||
async def declare_queue(self, name="", exclusive=True):
|
||||
return self
|
||||
|
||||
async def bind(self, exchange, routing_key):
|
||||
pass
|
||||
|
||||
|
||||
# ── 1. cluster.handle_image_generated resolves pending request ───────────
|
||||
|
||||
|
||||
def test_handle_image_generated_resolves_pending(monkeypatch):
|
||||
_reset()
|
||||
monkeypatch.setattr(cluster, "publish", _fake_publish)
|
||||
|
||||
cluster.CLUSTER_NODES["corsair"] = {
|
||||
"name": "corsair", "type": "worker", "status": "active",
|
||||
"capabilities": ["image_gen"],
|
||||
}
|
||||
|
||||
event = asyncio.Event()
|
||||
cluster._pending_image["req-123"] = ("", event)
|
||||
|
||||
asyncio.run(cluster.handle_image_generated(
|
||||
AMQP_EXCHANGE_SYSTEM, "node.corsair.image_generated",
|
||||
{"node_name": "corsair", "request_id": "req-123", "image_base64": "aW1hZ2U="},
|
||||
))
|
||||
|
||||
assert event.is_set()
|
||||
result = cluster._pending_image.get("req-123")
|
||||
assert result is not None
|
||||
assert result[0] == "aW1hZ2U="
|
||||
assert cluster.CLUSTER_NODES["corsair"]["last_seen"] is not None
|
||||
|
||||
|
||||
# ── 2. cluster.handle_image_failed resolves pending request ─────────────
|
||||
|
||||
|
||||
def test_handle_image_failed_resolves_pending(monkeypatch):
|
||||
_reset()
|
||||
monkeypatch.setattr(cluster, "publish", _fake_publish)
|
||||
|
||||
cluster.CLUSTER_NODES["corsair"] = {
|
||||
"name": "corsair", "type": "worker", "status": "active",
|
||||
}
|
||||
|
||||
event = asyncio.Event()
|
||||
cluster._pending_image["req-456"] = ("", event)
|
||||
|
||||
asyncio.run(cluster.handle_image_failed(
|
||||
AMQP_EXCHANGE_SYSTEM, "node.corsair.image_failed",
|
||||
{"node_name": "corsair", "request_id": "req-456", "error": "timeout"},
|
||||
))
|
||||
|
||||
assert event.is_set()
|
||||
result = cluster._pending_image.get("req-456")
|
||||
assert result is not None
|
||||
assert result[0] == ""
|
||||
|
||||
|
||||
# ── 3. cluster.handle_image_failed unknown node ─────────────────────────
|
||||
|
||||
|
||||
def test_handle_image_failed_unknown_node(caplog, monkeypatch):
|
||||
_reset()
|
||||
caplog.set_level("WARNING")
|
||||
monkeypatch.setattr(cluster, "publish", _fake_publish)
|
||||
|
||||
asyncio.run(cluster.handle_image_failed(
|
||||
AMQP_EXCHANGE_SYSTEM, "node.ghost.image_failed",
|
||||
{"node_name": "ghost", "request_id": "x", "error": "boom"},
|
||||
))
|
||||
|
||||
assert not any("unknown node" in rec.message for rec in caplog.records)
|
||||
|
||||
|
||||
# ── 4. cluster.request_image_generate publishes command ──────────────────
|
||||
|
||||
|
||||
def test_request_image_generate_publishes_command(monkeypatch):
|
||||
_reset()
|
||||
_published.clear()
|
||||
monkeypatch.setattr(cluster, "publish", _fake_publish)
|
||||
|
||||
cluster.CLUSTER_NODES["corsair"] = {
|
||||
"name": "corsair", "type": "worker", "status": "active",
|
||||
"capabilities": ["image_gen"],
|
||||
}
|
||||
|
||||
# Simulate immediate completion
|
||||
async def fake_wait():
|
||||
cluster._pending_image.clear()
|
||||
|
||||
original_wait_for = asyncio.wait_for
|
||||
|
||||
async def patched_wait_for(coro, timeout):
|
||||
cluster._pending_image["fake-id"] = ("aW1hZ2U=", asyncio.Event())
|
||||
cluster._pending_image["fake-id"][1].set()
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(asyncio, "wait_for", patched_wait_for)
|
||||
|
||||
result = asyncio.run(cluster.request_image_generate(
|
||||
"corsair", "a red dragon", width=512, height=512, steps=10,
|
||||
))
|
||||
|
||||
assert len(_published) == 1
|
||||
exchange, rk, payload = _published[0]
|
||||
assert exchange == AMQP_EXCHANGE_ADMIN
|
||||
assert rk == "node.corsair.cmd.image_generate"
|
||||
assert payload["prompt"] == "a red dragon"
|
||||
assert payload["width"] == 512
|
||||
assert payload["height"] == 512
|
||||
assert payload["steps"] == 10
|
||||
assert "request_id" in payload
|
||||
|
||||
|
||||
# ── 5. cluster.request_image_generate unknown node ──────────────────────
|
||||
|
||||
|
||||
def test_request_image_generate_unknown_node(monkeypatch):
|
||||
_reset()
|
||||
_published.clear()
|
||||
monkeypatch.setattr(cluster, "publish", _fake_publish)
|
||||
|
||||
result = asyncio.run(cluster.request_image_generate("ghost", "prompt"))
|
||||
assert result is None
|
||||
assert len(_published) == 0
|
||||
|
||||
|
||||
# ── 6. cluster.request_image_generate node lacks capability ─────────────
|
||||
|
||||
|
||||
def test_request_image_generate_no_capability(monkeypatch):
|
||||
_reset()
|
||||
_published.clear()
|
||||
monkeypatch.setattr(cluster, "publish", _fake_publish)
|
||||
|
||||
cluster.CLUSTER_NODES["jarvis"] = {
|
||||
"name": "jarvis", "type": "worker", "status": "active",
|
||||
"capabilities": ["llm"],
|
||||
}
|
||||
|
||||
result = asyncio.run(cluster.request_image_generate("jarvis", "prompt"))
|
||||
assert result is None
|
||||
assert len(_published) == 0
|
||||
|
||||
|
||||
# ── 7. cluster.request_image_generate timeout ───────────────────────────
|
||||
|
||||
|
||||
def test_request_image_generate_timeout(monkeypatch):
|
||||
_reset()
|
||||
_published.clear()
|
||||
monkeypatch.setattr(cluster, "publish", _fake_publish)
|
||||
|
||||
cluster.CLUSTER_NODES["corsair"] = {
|
||||
"name": "corsair", "type": "worker", "status": "active",
|
||||
"capabilities": ["image_gen"],
|
||||
}
|
||||
|
||||
async def timeout_wait(coro, timeout):
|
||||
raise asyncio.TimeoutError()
|
||||
|
||||
monkeypatch.setattr(asyncio, "wait_for", timeout_wait)
|
||||
|
||||
result = asyncio.run(cluster.request_image_generate("corsair", "prompt", timeout=1))
|
||||
assert result is None
|
||||
assert len(cluster._pending_image) == 0
|
||||
|
||||
|
||||
# ── 8. _find_image_node selects active image_gen node ───────────────────
|
||||
|
||||
|
||||
def test_find_image_node_selects_active():
|
||||
from routers.image import _find_image_node
|
||||
|
||||
_reset()
|
||||
cluster.CLUSTER_NODES["corsair"] = {
|
||||
"name": "corsair", "type": "worker", "status": "active",
|
||||
"capabilities": ["image_gen"],
|
||||
}
|
||||
cluster.CLUSTER_NODES["jarvis"] = {
|
||||
"name": "jarvis", "type": "worker", "status": "active",
|
||||
"capabilities": ["llm"],
|
||||
}
|
||||
|
||||
assert _find_image_node() == "corsair"
|
||||
|
||||
|
||||
def test_find_image_node_skips_inactive():
|
||||
from routers.image import _find_image_node
|
||||
|
||||
_reset()
|
||||
cluster.CLUSTER_NODES["corsair"] = {
|
||||
"name": "corsair", "type": "worker", "status": "error",
|
||||
"capabilities": ["image_gen"],
|
||||
}
|
||||
|
||||
assert _find_image_node() is None
|
||||
|
||||
|
||||
def test_find_image_node_no_image_gen():
|
||||
from routers.image import _find_image_node
|
||||
|
||||
_reset()
|
||||
cluster.CLUSTER_NODES["jarvis"] = {
|
||||
"name": "jarvis", "type": "worker", "status": "active",
|
||||
"capabilities": ["llm"],
|
||||
}
|
||||
|
||||
assert _find_image_node() is None
|
||||
|
||||
|
||||
# ── 9. POST /api/image/generate — no node available ─────────────────────
|
||||
|
||||
|
||||
def test_image_generate_no_node_503(tmp_path):
|
||||
_reset()
|
||||
with make_client(tmp_path) as client:
|
||||
headers = _admin_headers(client)
|
||||
resp = client.post("/api/image/generate", json={"prompt": "test"}, headers=headers)
|
||||
assert resp.status_code == 503
|
||||
assert "No image generation service" in resp.json()["detail"]
|
||||
|
||||
|
||||
# ── 10. POST /api/image/generate — empty prompt ─────────────────────────
|
||||
|
||||
|
||||
def test_image_generate_empty_prompt_400(tmp_path):
|
||||
_reset()
|
||||
with make_client(tmp_path) as client:
|
||||
headers = _admin_headers(client)
|
||||
resp = client.post("/api/image/generate", json={"prompt": ""}, headers=headers)
|
||||
assert resp.status_code == 400
|
||||
assert "Prompt is required" in resp.json()["detail"]
|
||||
|
||||
|
||||
# ── 11. POST /api/image/generate — happy path ──────────────────────────
|
||||
|
||||
|
||||
def test_image_generate_happy_path(tmp_path, monkeypatch):
|
||||
_reset()
|
||||
cluster.CLUSTER_NODES["corsair"] = {
|
||||
"name": "corsair", "type": "worker", "status": "active",
|
||||
"capabilities": ["image_gen"],
|
||||
}
|
||||
|
||||
fake_png = b"\x89PNG\r\n\x1a\n" + b"\x00" * 100
|
||||
fake_b64 = base64.b64encode(fake_png).decode()
|
||||
|
||||
async def fake_request_image_generate(**kwargs):
|
||||
return fake_b64
|
||||
|
||||
monkeypatch.setattr("routers.image.request_image_generate", fake_request_image_generate)
|
||||
|
||||
with make_client(tmp_path) as client:
|
||||
headers = _admin_headers(client)
|
||||
resp = client.post("/api/image/generate", json={
|
||||
"prompt": "a red dragon",
|
||||
"width": 512,
|
||||
"height": 512,
|
||||
}, headers=headers)
|
||||
assert resp.status_code == 200
|
||||
assert resp.headers["content-type"] == "image/png"
|
||||
assert resp.content == fake_png
|
||||
|
||||
|
||||
# ── 12. POST /api/image/generate — generation failed ───────────────────
|
||||
|
||||
|
||||
def test_image_generate_timeout_504(tmp_path, monkeypatch):
|
||||
_reset()
|
||||
cluster.CLUSTER_NODES["corsair"] = {
|
||||
"name": "corsair", "type": "worker", "status": "active",
|
||||
"capabilities": ["image_gen"],
|
||||
}
|
||||
|
||||
async def fake_request_image_generate(**kwargs):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr("routers.image.request_image_generate", fake_request_image_generate)
|
||||
|
||||
with make_client(tmp_path) as client:
|
||||
headers = _admin_headers(client)
|
||||
resp = client.post("/api/image/generate", json={"prompt": "test"}, headers=headers)
|
||||
assert resp.status_code == 504
|
||||
|
||||
|
||||
# ── 13. GET /api/image/status — available ──────────────────────────────
|
||||
|
||||
|
||||
def test_image_status_available(tmp_path):
|
||||
_reset()
|
||||
cluster.CLUSTER_NODES["corsair"] = {
|
||||
"name": "corsair", "type": "worker", "status": "active",
|
||||
"capabilities": ["image_gen"],
|
||||
"load": {"gpu_pct": 30},
|
||||
"last_seen": "2026-07-27T00:00:00Z",
|
||||
}
|
||||
|
||||
with make_client(tmp_path) as client:
|
||||
headers = _guest_headers(client)
|
||||
resp = client.get("/api/image/status", headers=headers)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["available"] is True
|
||||
assert len(data["nodes"]) == 1
|
||||
assert data["nodes"][0]["name"] == "corsair"
|
||||
|
||||
|
||||
# ── 14. GET /api/image/status — no nodes ───────────────────────────────
|
||||
|
||||
|
||||
def test_image_status_unavailable(tmp_path):
|
||||
_reset()
|
||||
with make_client(tmp_path) as client:
|
||||
headers = _guest_headers(client)
|
||||
resp = client.get("/api/image/status", headers=headers)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["available"] is False
|
||||
assert len(data["nodes"]) == 0
|
||||
|
||||
|
||||
# ── 15. node_agent.detect_capabilities — ComfyUI present ───────────────
|
||||
|
||||
|
||||
def test_detect_capabilities_comfyui_present(monkeypatch):
|
||||
cfg = agent.AgentConfig()
|
||||
cfg.comfyui_port = 8188
|
||||
|
||||
monkeypatch.setattr(agent, "HAS_HTTPX", True)
|
||||
|
||||
def fake_get(url, timeout=3):
|
||||
if "system_stats" in url:
|
||||
class R:
|
||||
status_code = 200
|
||||
return R()
|
||||
raise httpx.ConnectError("refused")
|
||||
|
||||
monkeypatch.setattr(httpx, "get", fake_get)
|
||||
|
||||
caps = agent.detect_capabilities(cfg)
|
||||
assert "image_gen" in caps
|
||||
assert "llm" in caps
|
||||
|
||||
|
||||
# ── 16. node_agent.detect_capabilities — ComfyUI absent ────────────────
|
||||
|
||||
|
||||
def test_detect_capabilities_comfyui_absent(monkeypatch):
|
||||
cfg = agent.AgentConfig()
|
||||
cfg.comfyui_port = 8188
|
||||
|
||||
monkeypatch.setattr(agent, "HAS_HTTPX", True)
|
||||
monkeypatch.setattr(httpx, "get", lambda url, timeout=3: (_ for _ in ()).throw(httpx.ConnectError("refused")))
|
||||
|
||||
caps = agent.detect_capabilities(cfg)
|
||||
assert "image_gen" not in caps
|
||||
assert "llm" in caps
|
||||
|
||||
|
||||
# ── 17. node_agent.detect_capabilities — httpx not installed ────────────
|
||||
|
||||
|
||||
def test_detect_capabilities_no_httpx(monkeypatch):
|
||||
cfg = agent.AgentConfig()
|
||||
monkeypatch.setattr(agent, "HAS_HTTPX", False)
|
||||
|
||||
caps = agent.detect_capabilities(cfg)
|
||||
assert "image_gen" not in caps
|
||||
|
||||
|
||||
# ── 18. node_agent.handle_image_generate — success ─────────────────────
|
||||
|
||||
|
||||
def test_node_agent_handle_image_generate_success(monkeypatch):
|
||||
monkeypatch.setattr(agent, "HAS_AIO_PIKA", True)
|
||||
monkeypatch.setattr(agent, "HAS_HTTPX", True)
|
||||
|
||||
cfg = agent.AgentConfig()
|
||||
cfg.node_name = "corsair"
|
||||
cfg.comfyui_port = 8188
|
||||
|
||||
fake_png = b"\x89PNG" + b"\x00" * 50
|
||||
fake_b64 = base64.b64encode(fake_png).decode()
|
||||
|
||||
async def fake_comfyui_generate(*a, **kw):
|
||||
return fake_b64
|
||||
|
||||
monkeypatch.setattr(agent, "_comfyui_generate", fake_comfyui_generate)
|
||||
|
||||
channel = FakeChannel()
|
||||
system_ex = FakeExchange("jc.system")
|
||||
channel.exchanges["jc.system"] = system_ex
|
||||
|
||||
asyncio.run(agent.handle_image_generate(
|
||||
cfg, channel, (FakeExchange(), system_ex),
|
||||
FakeMsg({
|
||||
"request_id": "req-789",
|
||||
"prompt": "a castle",
|
||||
"negative_prompt": "",
|
||||
"width": 1024,
|
||||
"height": 1024,
|
||||
"steps": 20,
|
||||
"seed": 42,
|
||||
"model": "",
|
||||
}),
|
||||
))
|
||||
|
||||
assert len(system_ex.published) == 1
|
||||
msg, rk = system_ex.published[0]
|
||||
assert rk == "node.corsair.image_generated"
|
||||
payload = json.loads(msg.body)
|
||||
assert payload["type"] == "image_generated"
|
||||
assert payload["request_id"] == "req-789"
|
||||
assert payload["image_base64"] == fake_b64
|
||||
|
||||
|
||||
# ── 19. node_agent.handle_image_generate — failure ─────────────────────
|
||||
|
||||
|
||||
def test_node_agent_handle_image_generate_failure(monkeypatch):
|
||||
monkeypatch.setattr(agent, "HAS_AIO_PIKA", True)
|
||||
monkeypatch.setattr(agent, "HAS_HTTPX", True)
|
||||
|
||||
cfg = agent.AgentConfig()
|
||||
cfg.node_name = "corsair"
|
||||
|
||||
async def fake_comfyui_generate(*a, **kw):
|
||||
raise RuntimeError("ComfyUI crashed")
|
||||
|
||||
monkeypatch.setattr(agent, "_comfyui_generate", fake_comfyui_generate)
|
||||
|
||||
channel = FakeChannel()
|
||||
system_ex = FakeExchange("jc.system")
|
||||
channel.exchanges["jc.system"] = system_ex
|
||||
|
||||
asyncio.run(agent.handle_image_generate(
|
||||
cfg, channel, (FakeExchange(), system_ex),
|
||||
FakeMsg({"request_id": "req-fail", "prompt": "test"}),
|
||||
))
|
||||
|
||||
assert len(system_ex.published) == 1
|
||||
msg, rk = system_ex.published[0]
|
||||
assert rk == "node.corsair.image_failed"
|
||||
payload = json.loads(msg.body)
|
||||
assert payload["type"] == "image_failed"
|
||||
assert "ComfyUI crashed" in payload["error"]
|
||||
|
||||
|
||||
# ── 20. node_agent.handle_image_generate — empty prompt ────────────────
|
||||
|
||||
|
||||
def test_node_agent_handle_image_generate_empty_prompt(monkeypatch):
|
||||
monkeypatch.setattr(agent, "HAS_AIO_PIKA", True)
|
||||
|
||||
cfg = agent.AgentConfig()
|
||||
cfg.node_name = "corsair"
|
||||
|
||||
channel = FakeChannel()
|
||||
system_ex = FakeExchange("jc.system")
|
||||
|
||||
asyncio.run(agent.handle_image_generate(
|
||||
cfg, channel, (FakeExchange(), system_ex),
|
||||
FakeMsg({"request_id": "req-x", "prompt": ""}),
|
||||
))
|
||||
|
||||
assert len(system_ex.published) == 0
|
||||
|
||||
|
||||
# ── 21. hardware.py — ComfyUI reachable ────────────────────────────────
|
||||
|
||||
|
||||
def test_assess_hardware_comfyui_reachable(tmp_path, monkeypatch):
|
||||
hardware.HARDWARE_STATE_PATH = tmp_path / "hw.json"
|
||||
monkeypatch.setattr(psutil, "virtual_memory", lambda: type("M", (), {"total": 16 * 1024 ** 3, "available": 8 * 1024 ** 3})())
|
||||
monkeypatch.setattr(psutil, "cpu_count", lambda: 8)
|
||||
|
||||
class MockProc:
|
||||
returncode = 1
|
||||
stdout = ""
|
||||
|
||||
monkeypatch.setattr(hardware.subprocess, "run", lambda cmd, **kw: MockProc())
|
||||
|
||||
async def mock_get(self, url, *args, **kwargs):
|
||||
class R:
|
||||
status_code = 200
|
||||
def json(self):
|
||||
if "CheckpointLoaderSimple" in url:
|
||||
return {"CheckpointLoaderSimple": {"input": {"required": {"ckpt_name": [["model.safetensors", "other.ckpt"]]}}}}
|
||||
return {}
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
if "8188" in url:
|
||||
return R()
|
||||
if "v1/models" in url:
|
||||
class R2:
|
||||
status_code = 200
|
||||
def json(self):
|
||||
return {"data": []}
|
||||
return R2()
|
||||
if "6333" in url:
|
||||
class R3:
|
||||
status_code = 200
|
||||
def json(self):
|
||||
return {"result": {"collections": []}}
|
||||
return R3()
|
||||
if "8888" in url:
|
||||
class R4:
|
||||
status_code = 200
|
||||
return R4()
|
||||
class R5:
|
||||
status_code = 200
|
||||
def json(self):
|
||||
return {}
|
||||
return R5()
|
||||
|
||||
monkeypatch.setattr(httpx.AsyncClient, "get", mock_get)
|
||||
|
||||
state = asyncio.run(hardware.assess_hardware())
|
||||
assert state["comfyui_reachable"] is True
|
||||
assert "model.safetensors" in state["comfyui_models"]
|
||||
|
||||
|
||||
# ── 22. hardware.py — ComfyUI unreachable ──────────────────────────────
|
||||
|
||||
|
||||
def test_assess_hardware_comfyui_unreachable(tmp_path, monkeypatch):
|
||||
hardware.HARDWARE_STATE_PATH = tmp_path / "hw.json"
|
||||
monkeypatch.setattr(psutil, "virtual_memory", lambda: type("M", (), {"total": 16 * 1024 ** 3, "available": 8 * 1024 ** 3})())
|
||||
monkeypatch.setattr(psutil, "cpu_count", lambda: 8)
|
||||
|
||||
class MockProc:
|
||||
returncode = 1
|
||||
stdout = ""
|
||||
|
||||
monkeypatch.setattr(hardware.subprocess, "run", lambda cmd, **kw: MockProc())
|
||||
|
||||
async def mock_get(self, url, *args, **kwargs):
|
||||
if "8188" in url:
|
||||
raise httpx.ConnectError("refused")
|
||||
if "v1/models" in url:
|
||||
class R2:
|
||||
status_code = 200
|
||||
def json(self):
|
||||
return {"data": []}
|
||||
return R2()
|
||||
if "6333" in url:
|
||||
class R3:
|
||||
status_code = 200
|
||||
def json(self):
|
||||
return {"result": {"collections": []}}
|
||||
return R3()
|
||||
if "8888" in url:
|
||||
class R4:
|
||||
status_code = 200
|
||||
return R4()
|
||||
class R5:
|
||||
status_code = 200
|
||||
def json(self):
|
||||
return {}
|
||||
return R5()
|
||||
|
||||
monkeypatch.setattr(httpx.AsyncClient, "get", mock_get)
|
||||
|
||||
state = asyncio.run(hardware.assess_hardware())
|
||||
assert state["comfyui_reachable"] is False
|
||||
assert state["comfyui_models"] == []
|
||||
|
||||
|
||||
# ── 23. node_agent config reads comfyui_port ───────────────────────────
|
||||
|
||||
|
||||
def test_config_from_ini_comfyui_port(tmp_path):
|
||||
ini = tmp_path / "caic-node-agent.conf"
|
||||
ini.write_text(
|
||||
"[agent]\n"
|
||||
"node_name = corsair\n"
|
||||
"capabilities = llm,image_gen\n"
|
||||
"comfyui_port = 8188\n"
|
||||
)
|
||||
cfg = agent.AgentConfig.from_ini(str(ini))
|
||||
assert cfg.comfyui_port == 8188
|
||||
assert "image_gen" in cfg.capabilities
|
||||
|
||||
|
||||
def test_config_from_ini_comfyui_port_default():
|
||||
cfg = agent.AgentConfig()
|
||||
assert cfg.comfyui_port == 8188
|
||||
|
||||
|
||||
# ── 24. SUBSCRIBE_TABLE includes image gen handlers ────────────────────
|
||||
|
||||
|
||||
def test_subscribe_table_includes_image_handlers():
|
||||
routing_keys = [rks for _, rks, _ in cluster.SUBSCRIBE_TABLE]
|
||||
all_keys = [rk for rks in routing_keys for rk in rks]
|
||||
assert "node.*.image_generated" in all_keys
|
||||
assert "node.*.image_failed" in all_keys
|
||||
@@ -2,7 +2,6 @@
|
||||
import asyncio
|
||||
|
||||
import cluster
|
||||
import triage
|
||||
from config import AMQP_EXCHANGE_ADMIN, AMQP_EXCHANGE_SYSTEM
|
||||
|
||||
|
||||
@@ -149,54 +148,3 @@ def test_handle_model_failed_unknown_node(caplog, monkeypatch):
|
||||
))
|
||||
|
||||
assert any("unknown node" in rec.message for rec in caplog.records)
|
||||
|
||||
|
||||
# ---------- 4. select_node() triggers swap when model mismatched ----------
|
||||
|
||||
|
||||
def test_select_node_code_triggers_swap(monkeypatch):
|
||||
_reset()
|
||||
_published.clear()
|
||||
monkeypatch.setattr(cluster, "publish", _fake_publish)
|
||||
|
||||
cluster.CLUSTER_NODES["jarvis"] = {
|
||||
"name": "jarvis", "type": "worker", "status": "active",
|
||||
"ip": "192.168.50.210",
|
||||
"active_model": {"name": "llama3.1", "port": 8081},
|
||||
"inventory": [
|
||||
{"filename": "qwen2.5-coder-14b-Q4_K_M.gguf", "name": "qwen2.5-coder", "version": "14b", "quant": "Q4_K_M"},
|
||||
],
|
||||
}
|
||||
|
||||
result = asyncio.run(triage.select_node("code"))
|
||||
|
||||
assert result is None
|
||||
# Swap should have been published
|
||||
assert any("cmd.swap_model" in rk for _, rk, _ in _published)
|
||||
# Node should now be swapping
|
||||
assert cluster.CLUSTER_NODES["jarvis"]["status"] == "swapping"
|
||||
|
||||
|
||||
# ---------- 5. select_node() returns None when node is already swapping ----------
|
||||
|
||||
|
||||
def test_select_node_swapping_returns_none(monkeypatch):
|
||||
_reset()
|
||||
_published.clear()
|
||||
monkeypatch.setattr(cluster, "publish", _fake_publish)
|
||||
|
||||
cluster.CLUSTER_NODES["jarvis"] = {
|
||||
"name": "jarvis", "type": "worker", "status": "swapping",
|
||||
"ip": "192.168.50.210",
|
||||
"active_model": {"name": "llama3.1", "port": 8081},
|
||||
"inventory": [
|
||||
{"filename": "qwen2.5-coder-14b-Q4_K_M.gguf", "name": "qwen2.5-coder"},
|
||||
],
|
||||
}
|
||||
|
||||
result = asyncio.run(triage.select_node("code"))
|
||||
|
||||
assert result is None
|
||||
# No swap command should be published while already swapping
|
||||
swap_published = any("cmd.swap_model" in rk for _, rk, _ in _published)
|
||||
assert not swap_published
|
||||
|
||||
@@ -1,141 +0,0 @@
|
||||
"""Tests for triage.py — query classification and node selection."""
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import httpx
|
||||
|
||||
import cluster
|
||||
import config
|
||||
import triage
|
||||
|
||||
|
||||
def _reset():
|
||||
cluster.CLUSTER_NODES.clear()
|
||||
cluster.CLUSTER_COORDINATOR = None
|
||||
|
||||
|
||||
_published = []
|
||||
|
||||
|
||||
async def _fake_publish(exchange, routing_key, payload):
|
||||
_published.append((exchange, routing_key, payload))
|
||||
|
||||
|
||||
class _MockPostResponse:
|
||||
def __init__(self, json_data: dict, status_code: int = 200):
|
||||
self._json_data = json_data
|
||||
self.status_code = status_code
|
||||
|
||||
def json(self):
|
||||
return self._json_data
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
|
||||
class _MockPostContext:
|
||||
def __init__(self, response: _MockPostResponse):
|
||||
self._response = response
|
||||
|
||||
async def __aenter__(self):
|
||||
return self._response
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
|
||||
# ---------- 1. classify_query returns valid classification ----------
|
||||
|
||||
|
||||
def test_classify_returns_valid(monkeypatch):
|
||||
async def post_stub(self, url, json=None, timeout=None):
|
||||
return _MockPostResponse({
|
||||
"choices": [{"message": {"content": "code"}}]
|
||||
})
|
||||
|
||||
monkeypatch.setattr(httpx.AsyncClient, "post", post_stub)
|
||||
|
||||
result = __import__("asyncio").run(triage.classify_query("write a python function"))
|
||||
assert result == "code"
|
||||
|
||||
|
||||
# ---------- 2. classify_query on error returns "general" ----------
|
||||
|
||||
|
||||
def test_classify_error_returns_general(monkeypatch):
|
||||
async def post_stub(self, url, json=None, timeout=None):
|
||||
raise httpx.ConnectError("connection refused")
|
||||
|
||||
monkeypatch.setattr(httpx.AsyncClient, "post", post_stub)
|
||||
|
||||
result = __import__("asyncio").run(triage.classify_query("any question"))
|
||||
assert result == "general"
|
||||
|
||||
|
||||
# ---------- 3. select_node("code") returns coder node ----------
|
||||
|
||||
|
||||
def test_select_node_code_returns_coder():
|
||||
_reset()
|
||||
cluster.CLUSTER_NODES["coder01"] = {
|
||||
"name": "coder01", "type": "worker", "status": "active",
|
||||
"ip": "192.168.50.210",
|
||||
"active_model": {"name": "qwen2.5-coder-14b", "port": 8082},
|
||||
}
|
||||
cluster.CLUSTER_NODES["general01"] = {
|
||||
"name": "general01", "type": "worker", "status": "active",
|
||||
"ip": "192.168.50.211",
|
||||
"active_model": {"name": "llama3.1", "port": 8081},
|
||||
}
|
||||
|
||||
node = asyncio.run(triage.select_node("code"))
|
||||
assert node is not None
|
||||
assert node["name"] == "coder01"
|
||||
|
||||
|
||||
# ---------- 4. select_node("general") with no matching node returns None ----------
|
||||
|
||||
|
||||
def test_select_node_general_no_match_returns_none():
|
||||
_reset()
|
||||
cluster.CLUSTER_NODES["coder01"] = {
|
||||
"name": "coder01", "type": "worker", "status": "active",
|
||||
"active_model": {"name": "qwen2.5-coder-14b", "port": 8082},
|
||||
}
|
||||
node = asyncio.run(triage.select_node("general"))
|
||||
assert node is None
|
||||
|
||||
|
||||
# ---------- 5. get_inference_url with coder node ----------
|
||||
|
||||
|
||||
def test_get_inference_url_with_coder_node(monkeypatch):
|
||||
_reset()
|
||||
async def fake_classify(query: str) -> str:
|
||||
return "code"
|
||||
monkeypatch.setattr(triage, "classify_query", fake_classify)
|
||||
|
||||
cluster.CLUSTER_NODES["coder01"] = {
|
||||
"name": "coder01", "type": "worker", "status": "active",
|
||||
"ip": "192.168.50.210",
|
||||
"active_model": {"name": "qwen2.5-coder-14b", "port": 8082},
|
||||
}
|
||||
|
||||
url = __import__("asyncio").run(triage.get_inference_url("write a loop in rust"))
|
||||
assert url == "http://192.168.50.210:8082/v1"
|
||||
|
||||
|
||||
# ---------- 6. get_inference_url with no nodes returns LLAMA_SERVER_BASE ----------
|
||||
|
||||
|
||||
def test_get_inference_url_no_nodes(monkeypatch):
|
||||
_reset()
|
||||
async def fake_classify(query: str) -> str:
|
||||
return "code"
|
||||
monkeypatch.setattr(triage, "classify_query", fake_classify)
|
||||
|
||||
url = __import__("asyncio").run(triage.get_inference_url("any question"))
|
||||
assert url == config.LLAMA_SERVER_BASE
|
||||
@@ -1,98 +0,0 @@
|
||||
"""cAIc — Query triage and cluster node selection."""
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
|
||||
from config import TRIAGE_BASE, TRIAGE_TIMEOUT, LLAMA_SERVER_BASE
|
||||
|
||||
log = logging.getLogger("caic")
|
||||
|
||||
_IDEAL_MODEL_MAP = {
|
||||
"code": {"name_contains": ["coder", "qwen"]},
|
||||
"general": {"name_contains": ["mistral", "llama"]},
|
||||
}
|
||||
|
||||
_CLASSIFICATION_PROMPT = """Classify the following user query into exactly one category. Respond with only the category name.
|
||||
|
||||
Categories:
|
||||
- general: everyday questions, chitchat, creative writing, advice, explanations
|
||||
- code: programming, debugging, code generation, technical questions about software
|
||||
- search: questions about current events, real-time information, weather, news, specific things that may have changed since training
|
||||
- rag: questions about specific documents, personal data, notes, memory, uploaded content
|
||||
|
||||
Query: {query}
|
||||
Category:"""
|
||||
|
||||
|
||||
async def classify_query(query: str) -> str:
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.post(
|
||||
f"{TRIAGE_BASE}/chat/completions",
|
||||
json={
|
||||
"model": "phi-4-mini",
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are a query classifier. Respond with exactly one word."},
|
||||
{"role": "user", "content": _CLASSIFICATION_PROMPT.format(query=query)},
|
||||
],
|
||||
"temperature": 0.0,
|
||||
"max_tokens": 10,
|
||||
},
|
||||
timeout=TRIAGE_TIMEOUT,
|
||||
)
|
||||
text = resp.json()["choices"][0]["message"]["content"].strip().lower()
|
||||
valid = {"general", "code", "search", "rag"}
|
||||
for v in valid:
|
||||
if v in text:
|
||||
return v
|
||||
except Exception:
|
||||
log.warning("triage classify_query failed, falling back to general", exc_info=True)
|
||||
return "general"
|
||||
|
||||
|
||||
async def select_node(classification: str) -> dict | None:
|
||||
from cluster import CLUSTER_NODES
|
||||
|
||||
if classification in ("search", "rag"):
|
||||
return None
|
||||
|
||||
ideal = _IDEAL_MODEL_MAP.get(classification, {})
|
||||
ideal_contains = ideal.get("name_contains", [])
|
||||
|
||||
# First pass: find an active node with the right model already loaded
|
||||
for node in CLUSTER_NODES.values():
|
||||
if node.get("status") != "active":
|
||||
continue
|
||||
am = node.get("active_model") or {}
|
||||
name = (am.get("name") or "").lower()
|
||||
if any(ideal in name for ideal in ideal_contains):
|
||||
return node
|
||||
|
||||
# Second pass: find an active node that can swap to the right model
|
||||
for node in CLUSTER_NODES.values():
|
||||
if node.get("status") != "active":
|
||||
continue
|
||||
inventory = node.get("inventory") or []
|
||||
for inv in inventory:
|
||||
inv_name = (inv.get("name") or "").lower()
|
||||
if any(ideal in inv_name for ideal in ideal_contains):
|
||||
from cluster import request_model_swap
|
||||
await request_model_swap(node["name"], inv["filename"])
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
|
||||
async def get_inference_url(query: str) -> str:
|
||||
if not query:
|
||||
return LLAMA_SERVER_BASE
|
||||
classification = await classify_query(query)
|
||||
if classification in ("search", "rag"):
|
||||
return LLAMA_SERVER_BASE
|
||||
node = await select_node(classification)
|
||||
if node:
|
||||
am = node.get("active_model") or {}
|
||||
port = am.get("port", 8081)
|
||||
ip = node.get("ip") or "127.0.0.1"
|
||||
return f"http://{ip}:{port}/v1"
|
||||
return LLAMA_SERVER_BASE
|
||||
Reference in New Issue
Block a user