feat: resident runtime, shutdown command, observatory, and IPL logging hardening
- keep BEDS resident after IPL and coordinate clean shutdown - propagate AMQP shutdown command across dispatcher pool - add structured IPL milestone/event-chain logging with root GUID context - add optional trace_on config for verbose method-entry diagnostics - add dev purge-on-IPL controls for admin/logger collections - add log level showcase events after IPL node-green - add Mongo logger store helpers for chain/root lookup and purge - add/modernize BEDS Observatory log_dumper utility UI and root record view - refresh source headers and wiki docs for current architecture/runtime - add architecture visual brief for leadership/image-generation workflows
This commit is contained in:
@@ -49,6 +49,7 @@ use lapin::{
|
||||
},
|
||||
types::{AMQPValue, FieldTable, LongString, ShortString},
|
||||
};
|
||||
use tokio::sync::watch;
|
||||
|
||||
use crate::services::amqp::{DLX_EXCHANGE_NAME, EXCHANGE_NAME};
|
||||
use crate::brokers::payload::{
|
||||
@@ -98,6 +99,8 @@ pub async fn spawn(
|
||||
queue_tag: String,
|
||||
instance_id: u32,
|
||||
template_registry: Arc<RuntimeTemplateRegistry>,
|
||||
shutdown_tx: watch::Sender<bool>,
|
||||
shutdown_rx: watch::Receiver<bool>,
|
||||
) -> Result<tokio::task::JoinHandle<()>, BrokerError> {
|
||||
let channel = conn.create_channel().await?;
|
||||
|
||||
@@ -163,7 +166,14 @@ pub async fn spawn(
|
||||
tracing::info!("dispatcher[{}] queue '{}' declared and bound", instance_id, queue_name);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
if let Err(e) = run(channel, queue_name, instance_id, template_registry).await {
|
||||
if let Err(e) = run(
|
||||
channel,
|
||||
queue_name,
|
||||
instance_id,
|
||||
template_registry,
|
||||
shutdown_tx,
|
||||
shutdown_rx,
|
||||
).await {
|
||||
tracing::error!("dispatcher[{}] exited with error: {}", instance_id, e);
|
||||
}
|
||||
});
|
||||
@@ -184,6 +194,8 @@ async fn run(
|
||||
queue_name: String,
|
||||
instance_id: u32,
|
||||
template_registry: Arc<RuntimeTemplateRegistry>,
|
||||
shutdown_tx: watch::Sender<bool>,
|
||||
mut shutdown_rx: watch::Receiver<bool>,
|
||||
) -> Result<(), BrokerError> {
|
||||
let consumer_tag = format!("dispatcher-{}", instance_id);
|
||||
|
||||
@@ -198,7 +210,23 @@ async fn run(
|
||||
|
||||
tracing::info!("dispatcher[{}] consuming from '{}'", instance_id, queue_name);
|
||||
|
||||
while let Some(delivery) = consumer.next().await {
|
||||
loop {
|
||||
tokio::select! {
|
||||
changed = shutdown_rx.changed() => {
|
||||
if changed.is_ok() && *shutdown_rx.borrow() {
|
||||
tracing::info!("dispatcher[{}] received global shutdown signal", instance_id);
|
||||
break;
|
||||
}
|
||||
if changed.is_err() {
|
||||
tracing::info!("dispatcher[{}] shutdown coordinator dropped", instance_id);
|
||||
break;
|
||||
}
|
||||
}
|
||||
maybe_delivery = consumer.next() => {
|
||||
let Some(delivery) = maybe_delivery else {
|
||||
break;
|
||||
};
|
||||
|
||||
let delivery = match delivery {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
@@ -274,6 +302,7 @@ async fn run(
|
||||
"shutdown" => {
|
||||
let _ = delivery.ack(BasicAckOptions::default()).await;
|
||||
tracing::info!("dispatcher[{}] shutdown event received — exiting", instance_id);
|
||||
let _ = shutdown_tx.send(true);
|
||||
break;
|
||||
}
|
||||
_ => {
|
||||
@@ -288,6 +317,8 @@ async fn run(
|
||||
}
|
||||
|
||||
apply_ack_action(&channel, &delivery, &outcome.ack_action).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!("dispatcher[{}] consume loop exited", instance_id);
|
||||
|
||||
Reference in New Issue
Block a user