diff --git a/README.md b/README.md index a016c06..d6e4432 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,7 @@ Full framework documentation lives in [`wiki/`](wiki/Home.md): - [IPL — Initial Program Load](wiki/04-ipl.md) — Bootstrap sequence, step by step - [Configuration System](wiki/05-configuration.md) — Layered TOML, env files, topology options - [Queue Topology](wiki/06-queue-topology.md) — AMQP exchanges, queues, routing keys +- [Design Notes & Discussions](wiki/07-notes.md) — Spew/no-nazi-twitter concept, open questions, architecture discussions in progress - [Template System](wiki/08-template-system.md) — REC and REL templates, TLA convention - [Event Lineage](wiki/09-event-lineage.md) — Compound event IDs, parent/child tracking - [Glossary](wiki/glossary.md) — Terms and abbreviations diff --git a/src/config/structs.rs b/src/config/structs.rs index 41aad8a..c4ca89f 100644 --- a/src/config/structs.rs +++ b/src/config/structs.rs @@ -1,78 +1,189 @@ +//! # config/structs.rs — BEDS Configuration Structs +//! +//! Typed configuration structs for all sections of `beds.toml`. Every struct +//! derives `serde::Deserialize` so the `config` crate can deserialise the +//! merged TOML directly into strongly-typed values. No runtime string lookups. +//! +//! ## Calling Agents +//! - `config/mod.rs` — deserialises into `BedsConfig` via `try_deserialize()` +//! - All service modules — receive sub-structs as `&cfg.broker_services`, etc. +//! +//! **Author:** mks +//! **Version:** 1.0 +//! +//! ## History +//! * `2026-04-02` - mks - original coding +//! * `2026-04-04` - mks - added RecNodeConfig, RelNodeConfig, RelInstanceConfig + use serde::Deserialize; use std::collections::HashMap; +/// Top-level BEDS configuration. Deserialised from the merged +/// `beds.toml` + `env_{name}.toml` at IPL step 1. #[derive(Debug, Deserialize)] pub struct BedsConfig { + /// Node identity — env name, version string, wbid pub id: IdConfig, + + /// Enables verbose debug output when true pub debug: bool, + + /// Enables journald syslog output when true pub syslog: bool, + + /// When true, log events also echo to stdout even when syslog is active pub syslog_mirror_console: bool, + + /// Enables audit trail logging for all data mutations pub audit_on: bool, + + /// Enables journal event recording pub journal_on: bool, + + /// RabbitMQ broker connection settings pub broker_services: BrokerServicesConfig, + + /// MongoDB (REC) node configs keyed by service name (e.g. "app_server") pub rec_services: HashMap, + + /// MariaDB (REL) node configs keyed by service name (e.g. "app_server") pub rel_services: HashMap, } +/// Node identity block. Identifies this node's role and environment. #[derive(Debug, Deserialize)] pub struct IdConfig { + /// Environment name — matches the env override file suffix (dev, qa, production) pub env_name: String, + + /// Application version string pub version: String, + + /// BEDS node identifier — unique name for this node in the cluster pub wbid: String, } +/// RabbitMQ broker connection settings and broker pool sizing. #[derive(Debug, Deserialize)] pub struct BrokerServicesConfig { - pub queue_tag : String, + /// Queue name prefix — prepended to all queue names to isolate environments. + /// Example: "prod_" produces queues like "prod_rec.read", "prod_log" + pub queue_tag: String, + + /// RabbitMQ virtual host — isolates traffic between environments pub vhost: String, + + /// Timer violation threshold in seconds — broker tasks exceeding this are logged pub timer_violation: u32, + + /// Maximum records transferred per AMQP message pub records_per_xfer: u32, + + /// Enables TCP keepalive on the broker connection pub keepalive: bool, + + /// AMQP heartbeat interval in seconds pub heartbeat: u32, + + /// Enables TLS on the broker connection pub use_ssl: bool, + + /// Path to TLS certificate file (only used when use_ssl is true) pub cert_path: String, + + /// Connection config for the primary broker node pub app_server: BrokerNodeConfig, } +/// Connection and pool sizing config for a single RabbitMQ node. #[derive(Debug, Deserialize)] pub struct BrokerNodeConfig { + /// Broker hostname or IP address pub host: String, + + /// AMQP port (default: 5672) pub port: u16, + + /// RabbitMQ management API port (optional — used for health checks and admin) pub api_port: Option, + + /// AMQP authentication username pub user: String, + + /// AMQP authentication password pub pass: String, + + /// Records per instance — used for per-broker throughput calculations pub rpi: u32, + + /// Broker instance counts for this node pub instances: BrokerInstancesConfig, } +/// Number of concurrent broker task instances per type on a single node. +/// +/// Each instance is a separate Tokio task. Increase these values when a node +/// has spare CPU/memory and a queue is saturating. #[derive(Debug, Deserialize)] pub struct BrokerInstancesConfig { + /// Number of read broker (rBroker) instances pub r_broker: u32, + + /// Number of write broker (wBroker) instances pub w_broker: u32, + + /// Number of migration/object broker (mBroker) instances pub m_broker: u32, } +/// Connection config for a single MongoDB (REC) node. #[derive(Debug, Deserialize)] pub struct RecNodeConfig { + /// MongoDB hostname or IP address pub host: String, + + /// MongoDB port (default: 27017) pub port: u16, + + /// Authentication username pub user: String, + + /// Authentication password pub pass: String, + + /// Default database name for this node pub database: String, + + /// Enables TLS on the MongoDB connection pub use_ssl: bool, } +/// Config for a MariaDB (REL) node. A node has a required master instance +/// and an optional secondary (read replica). #[derive(Debug, Deserialize)] pub struct RelNodeConfig { + /// Primary master instance — all writes go here pub master: RelInstanceConfig, + + /// Optional read replica — absence or unreachability is non-fatal pub secondary: Option, } +/// Connection config for a single MariaDB instance (master or secondary). #[derive(Debug, Deserialize, Clone)] pub struct RelInstanceConfig { + /// MariaDB hostname or IP address pub host: String, + + /// MariaDB port (default: 3306) pub port: u16, + + /// Authentication username pub user: String, + + /// Authentication password pub pass: String, + + /// Default database name pub database: String, } diff --git a/src/logging.rs b/src/logging.rs index f8a4aa5..1611944 100644 --- a/src/logging.rs +++ b/src/logging.rs @@ -1,26 +1,78 @@ +//! # logging.rs — BEDS Structured Logging Initialiser +//! +//! Initialises the `tracing` subscriber at IPL step 2. Supports two output +//! targets: journald (syslog) and console (stdout). Either or both may be +//! active depending on the config flags passed in. +//! +//! The subscriber must be initialised before any `tracing::info!` / +//! `tracing::warn!` / `tracing::error!` calls — the macros are no-ops until +//! a subscriber is registered. +//! +//! ## Calling Agents +//! - `ipl()` in main.rs — calls `init_from_config()` as the second IPL step +//! +//! ## Inputs +//! - `syslog: bool` — from `BedsConfig.syslog` +//! - `mirror_console: bool` — from `BedsConfig.syslog_mirror_console` +//! +//! ## Output Behaviour +//! | syslog | mirror_console | journald | console | +//! |--------|----------------|----------|---------| +//! | false | false | — | yes | +//! | false | true | — | yes | +//! | true | false | yes | — | +//! | true | true | yes | yes | +//! +//! **Author:** mks +//! **Version:** 1.0 +//! +//! ## History +//! * `2026-04-02` - mks - original coding + use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; +/// Initialises the `tracing` subscriber from BEDS config flags. +/// +/// Registers a global subscriber with up to two output layers — journald +/// and/or console — based on the `syslog` and `mirror_console` flags. The +/// log level is read from the `RUST_LOG` environment variable; defaults to +/// `info` if not set. +/// +/// Must be called exactly once, before any tracing macros are used. +/// Calling it a second time will panic (tracing enforces a single global +/// subscriber). +/// +/// # Arguments +/// +/// * `syslog` — enable journald output +/// * `mirror_console` — also write to console when journald is active +/// +/// # History +/// +/// * `2026-04-02` - mks - original coding pub fn init_from_config(syslog: bool, mirror_console: bool) { let registry = tracing_subscriber::registry() .with(EnvFilter::try_from_default_env() .unwrap_or_else(|_| EnvFilter::new("info"))); + // journald layer — active when syslog is enabled in config let journald_layer = if syslog { tracing_journald::layer().ok() } else { None }; - + + // console layer — active when syslog is off, or when mirroring is on let console_layer = if !syslog || mirror_console { Some(tracing_subscriber::fmt::layer() .with_target(false) .compact()) - }else { + } else { None }; - + registry .with(journald_layer) .with(console_layer) .init(); -} \ No newline at end of file +} diff --git a/wiki/07-notes.md b/wiki/07-notes.md new file mode 100644 index 0000000..1c6e8db --- /dev/null +++ b/wiki/07-notes.md @@ -0,0 +1,180 @@ +# Notes — Design Discussions and Open Questions + +This page is a scratch pad for design discussions, architecture decisions in progress, and ideas that don't fit neatly into the formal documentation yet. It is part of the permanent record. + +--- + +## Spew — A No-Nazi-Twitter Clone as the First BEDS Implementation + +### What It Is + +Spew is the working name for a social media platform built on top of BEDS. The concept emerged from a design discussion about what the "comsci flex" demonstration project for BEDS should be. The conclusion was: build a homelab-scale, decentralized, real-time social platform that solves the same engineering problems Twitter solved — but without the Nazis, and without the proprietary lock-in. + +The framing: **prove BEDS by building something hard on it.** A CRUD-over-REST todo app proves nothing. A real-time social graph at plausible production scale proves the broker architecture, the fan-out model, the moderation gate, and the IPFS storage tier all at once. + +BEDS goes in first, zero-state. Spew is the application layer on top. + +--- + +### The Justin Bieber Problem — Fan-Out at Scale + +The canonical interview question: *"Justin Bieber has 4 million followers. He posts a tweet. What happens?"* + +**The naive solution** (wrong at scale): + +On post, write to every follower's timeline. That's 4 million writes. It's O(n) on follower count, and it blocks the write path for a celebrity post. + +The wrong answer at an interview is to start talking about "objects" — it misses the operational reality entirely. + +**The BEDS solution:** + +O(1) write. **State change notification + client pull.** + +1. User posts. BEDS writes the post record to the REC store (MongoDB). One write. +2. BEDS publishes a notification event to `beds.events` on the `rec.write` routing key. One publish. +3. Subscribers receive the notification: *"user X has new content."* +4. Clients pull the timeline on demand — either on notification receipt or on next view. + +The timeline is assembled at read time, not at write time. The write path is flat regardless of follower count. + +**The implication for queue topology:** + +The `beds.events` topic exchange handles the fan-out routing. Routing keys allow fine-grained subscription: + +``` +post.{user_id} ← subscribe to a specific user's posts +post.# ← subscribe to all posts (firehose — admin only) +post.{tag}.{user_id} ← subscribe to posts from a user with a specific tag +``` + +A timeline service subscribes to `post.*` or a set of `post.{user_id}` keys for the users a given follower follows. The exchange does the fan-out. No 4 million writes. + +--- + +### Timeline Assembly + +Open question, not resolved: **absolute chronological order vs. subgrouped by author.** + +- **Absolute chronological** — the Twitter default. All posts from all followed users, sorted by timestamp. Easier to implement (sort by timestamp on read). Harder to skim when one user is prolific. +- **Subgrouped by author** — each followed user's posts are grouped together. Easier to skim. Requires a more complex read-time assembly query. Closer to what RSS readers do. + +The REC store (MongoDB) makes both easy — a compound index on `(author_id, timestamp)` supports both sort strategies without schema changes. Decision deferred until there is a UI prototype to react against. + +--- + +### Content Moderation — The KWIC Gate + +The problem: how do you stop hate speech, slurs, and known-bad content before it hits the exchange? + +**KWIC (Key Word In Context)** — a pre-commit moderation gate. + +Before a post is committed to the REC store and published to `beds.events`, it passes through a KWIC index validator: + +1. Post content is tokenized. +2. Each token is checked against a maintained KWIC index of prohibited terms, phrases, and patterns. +3. If the content fails the KWIC check, the write is rejected. The post never reaches the exchange. + +This is synchronous — the write is blocked until KWIC clears it. There is no async "we'll moderate it later" — the content either passes before it's published, or it doesn't publish. + +**Why KWIC?** + +KWIC was originally a bibliographic indexing technique — it presents each occurrence of a keyword in the context of surrounding text, making it easy for a human reviewer to scan for relevant usage. As a moderation gate, you maintain a KWIC index of prohibited terms and flag content that matches in context (not just keyword match — context matters for distinguishing slurs from citations). + +The index is separately maintainable, separately deployable, and can be updated without touching the application. A BEDS admin event (`adm` routing key) can trigger index refresh. + +**Implementation note:** + +KWIC validation is a natural broker task — a pre-write interceptor that runs before the wBroker dispatches to the MongoDB adapter. It does not require a separate service. The KWIC index lives in REC (MongoDB), loaded into memory at broker startup. + +--- + +### Decentralized Media Storage — IPFS + +The problem: media (images, video, audio) is expensive. Twitter's estimated storage footprint was roughly **1 exabyte** as of ~2022. You cannot store that on a homelab. You cannot store it on any single provider without vendor lock-in and ruinous cost. + +**The solution: IPFS (InterPlanetary File System).** + +IPFS is a content-addressed distributed storage system. Every piece of content is identified by its **CID (Content Identifier)** — a hash of the content itself. The content can be stored on any IPFS node in the network. As long as at least one node pins the content, it is retrievable. + +**How it integrates with BEDS:** + +1. User uploads media. The media goes to an IPFS node (local or public gateway). +2. IPFS returns a CID. +3. BEDS stores the CID in the post record (MongoDB). The media itself is never in BEDS. +4. On read, the client resolves the CID via any IPFS gateway. + +The BEDS post record looks something like: + +```json +{ + "post_id": "...", + "author_id": "...", + "text": "...", + "media_cid": "QmXyz...abc", + "timestamp": "..." +} +``` + +**Why this matters:** + +- Storage scales with the IPFS network, not with the BEDS node count +- Content is hash-verified — the CID proves the content hasn't been tampered with +- Multiple IPFS nodes can pin the same content — geographic redundancy without coordination +- No expiring posts (some competitors expire content to control storage costs) — once pinned, content persists as long as any node pins it + +**The moderation wrinkle:** + +IPFS content is immutable and content-addressed. You cannot delete a CID from the global network once published. The KWIC gate handles text. For media moderation, the approach is: BEDS can revoke the post record (remove the CID from the MongoDB document) without removing the content from IPFS. The content becomes unreachable through BEDS even if it technically still exists on IPFS. This is the operational reality of decentralized storage — handle it at the application layer. + +--- + +### Architecture Summary — How Spew Sits on BEDS + +``` +Spew Client (web / mobile) + │ + │ HTTP / WebSocket + ▼ + Spew API Layer + │ + │ AMQP via beds.events exchange + ▼ + BEDS Broker Pool (rBroker, wBroker, mBroker) + │ + ├─── KWIC Gate (pre-write, synchronous) + │ + ├─── REC Store (MongoDB) — posts, profiles, follows, notifications + │ + └─── REL Store (MariaDB) — user auth, sessions, account state + │ + │ AMQP fan-out on post.{user_id} routing keys + ▼ + Timeline Assembly Service (subscriber, on-demand pull) + │ + ▼ + IPFS — media storage (CIDs only stored in BEDS) +``` + +The application layer (Spew API) never touches MongoDB or MariaDB directly. Every data operation goes through `beds.events`. BEDS is infrastructure; Spew is the tenant. + +--- + +### Open Questions / Deferred Decisions + +| Question | Status | +|---|---| +| Timeline assembly: absolute chronological vs. subgrouped by author? | Deferred — needs UI prototype | +| IPFS pinning strategy — local node, public gateway, or both? | Deferred | +| KWIC index bootstrap — who maintains it, what format? | Deferred | +| Fan-out subscription model — per-user routing keys vs. wildcard + client filter? | Tentatively per-user keys | +| Notification delivery — WebSocket push vs. client poll? | Tentatively WebSocket on notification event | +| Rate limiting — per-user post rate enforced in KWIC gate or upstream? | Deferred | +| `post.#` firehose — admin-only or public opt-in? | Tentatively admin-only | + +--- + +### Why "Spew"? + +Because Twitter is a firehose and we're not pretending otherwise. + +Also: it's deliberately unglamorous. A name that takes itself seriously invites scope creep. "Spew" stays honest about what it is. diff --git a/wiki/Home.md b/wiki/Home.md index a08a596..99bf666 100644 --- a/wiki/Home.md +++ b/wiki/Home.md @@ -20,7 +20,9 @@ If you are reading this as a new contributor, start here and read in order. The ### Messaging - [Queue Topology](06-queue-topology.md) — AMQP exchanges, queues, routing keys, and the broker model -- [Broker Calls](07-broker-calls.md) — Every broker event type documented + +### Notes +- [Design Notes & Discussions](07-notes.md) — Spew/no-nazi-twitter concept, open questions, architecture discussions in progress ### Data - [Template System](08-template-system.md) — REC and REL templates, the TLA convention, schema-as-contract