gramps
9d1fd44d7f
- request_model_swap() publishes cmd.swap_model on jc.admin, sets node status to swapping - handle_model_ready() updates active_model from inventory, restores active status - handle_model_failed() sets node status to error with failure detail - select_node() made async with two-pass logic: find ready node or trigger swap - inventory field stored in node records for swap target lookup - Both handlers subscribed on jc.system via SUBSCRIBE_TABLE
248 lines
9.7 KiB
Python
248 lines
9.7 KiB
Python
"""
|
|
cAIc — Cluster protocol implementation.
|
|
Maintains node registry, event log, coordinator state, and ping-based health checks.
|
|
"""
|
|
import asyncio
|
|
import logging
|
|
import uuid
|
|
from collections import deque
|
|
from datetime import datetime, timezone
|
|
|
|
from amqp import publish, subscribe
|
|
from config import AMQP_EXCHANGE_ADMIN, AMQP_EXCHANGE_SYSTEM
|
|
|
|
log = logging.getLogger("caic")
|
|
|
|
CLUSTER_NODES: dict[str, dict] = {}
|
|
CLUSTER_EVENTS: deque = deque(maxlen=1000)
|
|
CLUSTER_COORDINATOR: str | None = None
|
|
_pending_pings: dict[str, tuple[str, asyncio.Event]] = {}
|
|
NODE_NAME: str = "ultron"
|
|
PING_TIMEOUT: float = 5.0
|
|
|
|
|
|
def _push_event(category: str, severity: str, node_name: str | None, message: str, details: dict | None = None) -> dict:
|
|
record = {
|
|
"category": category,
|
|
"severity": severity,
|
|
"node": node_name,
|
|
"message": message,
|
|
"details": details or {},
|
|
"timestamp": datetime.now(timezone.utc).isoformat() + "Z",
|
|
}
|
|
CLUSTER_EVENTS.append(record)
|
|
level = {"info": logging.INFO, "warn": logging.WARNING, "error": logging.ERROR, "critical": logging.CRITICAL}.get(severity, logging.INFO)
|
|
log.log(level, "[cluster] %s: %s", node_name or "system", message)
|
|
return record
|
|
|
|
|
|
async def handle_registration(exchange: str, routing_key: str, payload: dict) -> None:
|
|
global CLUSTER_COORDINATOR
|
|
node_name = payload.get("node_name", routing_key.split(".")[1] if "." in routing_key else "unknown")
|
|
node_type = payload.get("node_type", "worker")
|
|
|
|
if node_name in CLUSTER_NODES:
|
|
_push_event("cluster", "warn", node_name, "Duplicate registration rejected")
|
|
await publish(AMQP_EXCHANGE_ADMIN, f"node.{node_name}.rejected", {
|
|
"from": NODE_NAME, "node_name": node_name, "type": "rejected",
|
|
"reason": "duplicate_node_name",
|
|
"timestamp": datetime.now(timezone.utc).isoformat() + "Z",
|
|
})
|
|
return
|
|
|
|
if "node_type" not in payload or "capabilities" not in payload:
|
|
_push_event("cluster", "warn", node_name, "Malformed registration rejected")
|
|
await publish(AMQP_EXCHANGE_ADMIN, f"node.{node_name}.rejected", {
|
|
"from": NODE_NAME, "node_name": node_name, "type": "rejected",
|
|
"reason": "malformed_payload",
|
|
"timestamp": datetime.now(timezone.utc).isoformat() + "Z",
|
|
})
|
|
return
|
|
|
|
now = datetime.now(timezone.utc).isoformat() + "Z"
|
|
node = {
|
|
"name": node_name,
|
|
"type": node_type,
|
|
"status": "active",
|
|
"ip": payload.get("ip"),
|
|
"capabilities": payload.get("capabilities", []),
|
|
"active_model": payload.get("active_model"),
|
|
"inventory": payload.get("inventory", []),
|
|
"load": payload.get("load"),
|
|
"registered_at": now,
|
|
"last_seen": now,
|
|
}
|
|
CLUSTER_NODES[node_name] = node
|
|
_push_event("cluster", "info", node_name, f"Node registered (type={node_type})")
|
|
|
|
await publish(AMQP_EXCHANGE_ADMIN, f"node.{node_name}.admitted", {
|
|
"from": NODE_NAME, "node_name": node_name, "type": "admitted",
|
|
"coordinator": CLUSTER_COORDINATOR,
|
|
"timestamp": now,
|
|
})
|
|
|
|
if node_type == "coordinator" and CLUSTER_COORDINATOR is None:
|
|
CLUSTER_COORDINATOR = node_name
|
|
_push_event("cluster", "info", node_name, "Elected as coordinator")
|
|
await publish(AMQP_EXCHANGE_SYSTEM, "cluster.coordinator.response", {
|
|
"from": NODE_NAME, "type": "coord_response",
|
|
"coordinator": node_name, "nodes": list(CLUSTER_NODES.keys()),
|
|
"timestamp": now,
|
|
})
|
|
|
|
|
|
async def handle_deregistration(exchange: str, routing_key: str, payload: dict) -> None:
|
|
global CLUSTER_COORDINATOR
|
|
node_name = payload.get("node_name", routing_key.split(".")[1] if "." in routing_key else "unknown")
|
|
|
|
removed = CLUSTER_NODES.pop(node_name, None)
|
|
if not removed:
|
|
log.warning("[cluster] deregistration for unknown node %s", node_name)
|
|
return
|
|
|
|
_push_event("cluster", "info", node_name, "Node deregistered")
|
|
|
|
if CLUSTER_COORDINATOR == node_name:
|
|
CLUSTER_COORDINATOR = None
|
|
_push_event("cluster", "warn", node_name, "Coordinator deregistered — no coordinator active")
|
|
|
|
|
|
async def handle_pong(exchange: str, routing_key: str, payload: dict) -> None:
|
|
node_name = payload.get("node_name", routing_key.split(".")[1] if "." in routing_key else "unknown")
|
|
correlation_id = payload.get("correlation_id")
|
|
|
|
if correlation_id and correlation_id in _pending_pings:
|
|
_, event = _pending_pings.pop(correlation_id)
|
|
event.set()
|
|
|
|
if node_name in CLUSTER_NODES:
|
|
now = datetime.now(timezone.utc).isoformat() + "Z"
|
|
CLUSTER_NODES[node_name]["last_seen"] = now
|
|
if "status" in payload:
|
|
CLUSTER_NODES[node_name]["status"] = payload["status"]
|
|
if "active_model" in payload:
|
|
CLUSTER_NODES[node_name]["active_model"] = payload["active_model"]
|
|
if "load" in payload:
|
|
CLUSTER_NODES[node_name]["load"] = payload["load"]
|
|
else:
|
|
log.warning("[cluster] pong from unknown node %s", node_name)
|
|
|
|
|
|
async def handle_event(exchange: str, routing_key: str, payload: dict) -> None:
|
|
node_name = payload.get("node_name", routing_key.split(".")[1] if "." in routing_key else "unknown")
|
|
severity = payload.get("severity", "info")
|
|
message = payload.get("message", "")
|
|
details = payload.get("details")
|
|
_push_event("application", severity, node_name, message, details)
|
|
|
|
|
|
async def handle_coordinator_query(exchange: str, routing_key: str, payload: dict) -> None:
|
|
if CLUSTER_COORDINATOR is None:
|
|
return
|
|
now = datetime.now(timezone.utc).isoformat() + "Z"
|
|
await publish(AMQP_EXCHANGE_SYSTEM, "cluster.coordinator.response", {
|
|
"from": NODE_NAME, "type": "coord_response",
|
|
"coordinator": CLUSTER_COORDINATOR,
|
|
"nodes": list(CLUSTER_NODES.keys()),
|
|
"timestamp": now,
|
|
})
|
|
|
|
|
|
async def ping_node(node_name: str) -> bool:
|
|
global CLUSTER_COORDINATOR
|
|
if node_name not in CLUSTER_NODES:
|
|
return False
|
|
|
|
correlation_id = str(uuid.uuid4())
|
|
event = asyncio.Event()
|
|
_pending_pings[correlation_id] = (node_name, event)
|
|
|
|
now = datetime.now(timezone.utc).isoformat() + "Z"
|
|
await publish(AMQP_EXCHANGE_ADMIN, f"node.{node_name}.ping", {
|
|
"from": NODE_NAME, "node_name": node_name, "type": "ping",
|
|
"correlation_id": correlation_id, "timestamp": now,
|
|
})
|
|
|
|
try:
|
|
await asyncio.wait_for(event.wait(), timeout=PING_TIMEOUT)
|
|
return True
|
|
except asyncio.TimeoutError:
|
|
_pending_pings.pop(correlation_id, None)
|
|
CLUSTER_NODES.pop(node_name, None)
|
|
_push_event("cluster", "warn", node_name, "Node unresponsive — deregistered after ping timeout")
|
|
if CLUSTER_COORDINATOR == node_name:
|
|
CLUSTER_COORDINATOR = None
|
|
_push_event("cluster", "warn", node_name, "Coordinator unresponsive — no coordinator active")
|
|
return False
|
|
|
|
|
|
async def request_model_swap(node_name: str, model_filename: str) -> bool:
|
|
"""Request a worker node to swap its active model."""
|
|
if node_name not in CLUSTER_NODES:
|
|
log.warning("request_model_swap: unknown node %s", node_name)
|
|
return False
|
|
|
|
CLUSTER_NODES[node_name]["status"] = "swapping"
|
|
_push_event("cluster", "info", node_name, f"Model swap requested: {model_filename}")
|
|
|
|
now = datetime.now(timezone.utc).isoformat() + "Z"
|
|
await publish(AMQP_EXCHANGE_ADMIN, f"node.{node_name}.cmd.swap_model", {
|
|
"model_filename": model_filename,
|
|
"requested_at": now,
|
|
})
|
|
return True
|
|
|
|
|
|
async def handle_model_ready(exchange: str, routing_key: str, payload: dict) -> None:
|
|
node_name = payload.get("node_name", routing_key.split(".")[1] if "." in routing_key else "unknown")
|
|
model_filename = payload.get("active_model")
|
|
port = payload.get("port", 8081)
|
|
|
|
if node_name not in CLUSTER_NODES:
|
|
log.warning("handle_model_ready: unknown node %s", node_name)
|
|
return
|
|
|
|
inventory = CLUSTER_NODES[node_name].get("inventory") or []
|
|
model_info = None
|
|
for inv in inventory:
|
|
if inv.get("filename") == model_filename:
|
|
model_info = {**inv, "port": port}
|
|
break
|
|
if model_info is None:
|
|
model_info = {"filename": model_filename, "port": port}
|
|
|
|
now = datetime.now(timezone.utc).isoformat() + "Z"
|
|
CLUSTER_NODES[node_name]["active_model"] = model_info
|
|
CLUSTER_NODES[node_name]["status"] = "active"
|
|
CLUSTER_NODES[node_name]["last_seen"] = now
|
|
_push_event("cluster", "info", node_name, f"Model swap complete: {model_filename}")
|
|
|
|
|
|
async def handle_model_failed(exchange: str, routing_key: str, payload: dict) -> None:
|
|
node_name = payload.get("node_name", routing_key.split(".")[1] if "." in routing_key else "unknown")
|
|
error = payload.get("error", "unknown error")
|
|
|
|
if node_name not in CLUSTER_NODES:
|
|
log.warning("handle_model_failed: unknown node %s", node_name)
|
|
return
|
|
|
|
CLUSTER_NODES[node_name]["status"] = "error"
|
|
_push_event("cluster", "error", node_name, f"Model swap failed: {error}")
|
|
|
|
|
|
SUBSCRIBE_TABLE = [
|
|
(AMQP_EXCHANGE_ADMIN, ["node.*.register"], handle_registration),
|
|
(AMQP_EXCHANGE_ADMIN, ["node.*.deregister"], handle_deregistration),
|
|
(AMQP_EXCHANGE_ADMIN, ["node.*.pong"], handle_pong),
|
|
(AMQP_EXCHANGE_SYSTEM, ["node.*.event"], handle_event),
|
|
(AMQP_EXCHANGE_SYSTEM, ["cluster.coordinator.query"], handle_coordinator_query),
|
|
(AMQP_EXCHANGE_SYSTEM, ["node.*.model_ready"], handle_model_ready),
|
|
(AMQP_EXCHANGE_SYSTEM, ["node.*.model_failed"], handle_model_failed),
|
|
]
|
|
|
|
|
|
async def start_cluster_subscriptions() -> None:
|
|
for exchange, routing_keys, handler in SUBSCRIBE_TABLE:
|
|
await subscribe(exchange, routing_keys, handler)
|
|
log.info("cluster subscriptions started")
|