feat: resident runtime, shutdown command, observatory, and IPL logging hardening

- keep BEDS resident after IPL and coordinate clean shutdown
- propagate AMQP shutdown command across dispatcher pool
- add structured IPL milestone/event-chain logging with root GUID context
- add optional trace_on config for verbose method-entry diagnostics
- add dev purge-on-IPL controls for admin/logger collections
- add log level showcase events after IPL node-green
- add Mongo logger store helpers for chain/root lookup and purge
- add/modernize BEDS Observatory log_dumper utility UI and root record view
- refresh source headers and wiki docs for current architecture/runtime
- add architecture visual brief for leadership/image-generation workflows
This commit is contained in:
2026-04-10 13:42:39 -07:00
parent 0af80612bb
commit 14ec58318b
13 changed files with 1618 additions and 13 deletions

View File

@@ -49,6 +49,7 @@ use lapin::{
},
types::{AMQPValue, FieldTable, LongString, ShortString},
};
use tokio::sync::watch;
use crate::services::amqp::{DLX_EXCHANGE_NAME, EXCHANGE_NAME};
use crate::brokers::payload::{
@@ -98,6 +99,8 @@ pub async fn spawn(
queue_tag: String,
instance_id: u32,
template_registry: Arc<RuntimeTemplateRegistry>,
shutdown_tx: watch::Sender<bool>,
shutdown_rx: watch::Receiver<bool>,
) -> Result<tokio::task::JoinHandle<()>, BrokerError> {
let channel = conn.create_channel().await?;
@@ -163,7 +166,14 @@ pub async fn spawn(
tracing::info!("dispatcher[{}] queue '{}' declared and bound", instance_id, queue_name);
let handle = tokio::spawn(async move {
if let Err(e) = run(channel, queue_name, instance_id, template_registry).await {
if let Err(e) = run(
channel,
queue_name,
instance_id,
template_registry,
shutdown_tx,
shutdown_rx,
).await {
tracing::error!("dispatcher[{}] exited with error: {}", instance_id, e);
}
});
@@ -184,6 +194,8 @@ async fn run(
queue_name: String,
instance_id: u32,
template_registry: Arc<RuntimeTemplateRegistry>,
shutdown_tx: watch::Sender<bool>,
mut shutdown_rx: watch::Receiver<bool>,
) -> Result<(), BrokerError> {
let consumer_tag = format!("dispatcher-{}", instance_id);
@@ -198,7 +210,23 @@ async fn run(
tracing::info!("dispatcher[{}] consuming from '{}'", instance_id, queue_name);
while let Some(delivery) = consumer.next().await {
loop {
tokio::select! {
changed = shutdown_rx.changed() => {
if changed.is_ok() && *shutdown_rx.borrow() {
tracing::info!("dispatcher[{}] received global shutdown signal", instance_id);
break;
}
if changed.is_err() {
tracing::info!("dispatcher[{}] shutdown coordinator dropped", instance_id);
break;
}
}
maybe_delivery = consumer.next() => {
let Some(delivery) = maybe_delivery else {
break;
};
let delivery = match delivery {
Ok(d) => d,
Err(e) => {
@@ -274,6 +302,7 @@ async fn run(
"shutdown" => {
let _ = delivery.ack(BasicAckOptions::default()).await;
tracing::info!("dispatcher[{}] shutdown event received — exiting", instance_id);
let _ = shutdown_tx.send(true);
break;
}
_ => {
@@ -288,6 +317,8 @@ async fn run(
}
apply_ack_action(&channel, &delivery, &outcome.ack_action).await;
}
}
}
tracing::info!("dispatcher[{}] consume loop exited", instance_id);