using System.Diagnostics; using ytLive.Helpers; using ytLive.Services; namespace ytLive.Services.Audio; /// /// Owns the mic + desktop/game capture sources (TASK 4 ship step 4) and drives /// the footer meters; since TASK 9 it is also the live audio path into the /// encoder. Capture runs for the app's lifetime so both bars stay live in /// preview: mic samples are downmixed to mono, resampled to 48 kHz, put /// through the voice chain (bass → treble → noise gate → compressor), THEN /// level-metered and queued; loopback samples are resampled to 48 kHz stereo /// and queued. While live (), a 10 ms loop drains both /// queues, applies the honest gains (mic = MicVolume, loopback = unity /// × auto-duck when the mic is hot — the slider drives system volume via mixes them to stereo float, runs the /// master limiter (a -1 dBFS ceiling so the sum never clips the encoder), and /// writes the chunk to the encoder's audio pipe. Since TASK 47 the alert layer's /// clip audio joins the same mix (pre-scaled, unity-added, never ducked) through /// . /// public sealed class AudioMixer : IDisposable { public const int OutputSampleRate = 48000; private static readonly TimeSpan DefaultMixInterval = TimeSpan.FromMilliseconds(10); private readonly IAudioSource _mic; private readonly IAudioSource _loopback; private readonly AudioLevelMeter _meter; private readonly AudioLevelMeter _loopbackMeter; private readonly Action? _log; private readonly Func? _micGain; private readonly Func? _loopbackGain; private readonly TimeSpan _mixInterval; private readonly Func? _syncOffsetMs; private readonly AudioSyncDelay _syncDelay; private readonly AudioSyncDelay _micSyncDelay; private readonly Func? _micSyncOffsetMs; private float[]? _delayedMic; private int _advanceSamplesRemaining; private bool _started; // Live path (TASK 9): per-source resampling → voice chain on the mic → // thread-safe queues → the ducker + gain/mix loop → the encoder's pipe. private readonly AudioRingBuffer _micBuffer; private readonly AudioRingBuffer _loopbackBuffer; private readonly AudioRingBuffer _alertBuffer; private readonly VoiceFilterChain _voiceChain; private readonly AutoDucker _ducker; /// The ducker's CURRENT gain (1 = not ducking, 0.25 = the −12 dB duck). Read by the /// dock's badge: a duck indicator that is not the real value is decoration. public float DuckGain => _ducker.CurrentGain; /// Collapse the final mix to mono (creator ruling 2026-10-30: profoundly deaf on the /// right, single-sided audio is inaudible). Applied AFTER the limiter and sync delay, so what /// ships to the encoder — stream and recording alike — is what was folded. public bool MonoOutput { get; set; } /// Smoothed levels of the MIX (what the encoder receives), per channel. public float MasterLeft { get; private set; } public float MasterRight { get; private set; } /// Largest channel level — one ladder for the master, honest about the pair. public float MasterLevel => MathF.Max(MasterLeft, MasterRight); /// Channel gap in dB; a silent side counts at the −60 dBFS floor, so one-sided reads /// as the maximum imbalance rather than "nothing to compare". public float MasterImbalanceDb { get { const float floor = 0.001f; if (MasterLeft < floor && MasterRight < floor) return 0f; var l = MathF.Max(MasterLeft, floor); var r = MathF.Max(MasterRight, floor); return MathF.Abs(20f * MathF.Log10(l / r)); } } private readonly MasterLimiter _masterLimiter = new(); private TinyResampler? _micResampler; private TinyResampler? _loopbackResampler; private CancellationTokenSource? _liveCts; private NamedPipeAudioWriter _pipe = new(); private float[]? _micChunk; private float[]? _loopbackChunk; private float[]? _alertChunk; private float[]? _mixBuffer; private float[]? _delayedMix; private DateTime _lastLoopErrorLogged = DateTime.MinValue; private DateTime _nextStatsLog = DateTime.UtcNow + TimeSpan.FromSeconds(5); private long _micDrainedTotal; private long _loopDrainedTotal; private float _peakMix; public AudioMixer( IAudioSource mic, IAudioSource loopback, Action? log = null, Func? micGain = null, Func? loopbackGain = null, Func? syncOffsetMs = null, TimeSpan? mixInterval = null, Func? micSyncOffsetMs = null) { _mic = mic; _loopback = loopback; _meter = new AudioLevelMeter(); _loopbackMeter = new AudioLevelMeter(); _log = log; _micGain = micGain; _loopbackGain = loopbackGain; _syncOffsetMs = syncOffsetMs; _syncDelay = new AudioSyncDelay(OutputSampleRate); // The microphone alone, mono: a camera's frames reach the compositor later than the mic's // samples reach the pipe, so the voice has to wait for the picture. Loopback and alert audio // are not tied to a camera frame and must not move with it. _micSyncOffsetMs = micSyncOffsetMs; _micSyncDelay = new AudioSyncDelay(OutputSampleRate, channels: 1); _mixInterval = mixInterval ?? DefaultMixInterval; var seconds = _mixInterval.TotalSeconds; var framesPerTick = Math.Max(1, (int)Math.Round(OutputSampleRate * seconds)); // A tick normally owes ~10 ms; after a stall it may owe more, and the clock-based loop pays it // off in one write. Buffers therefore size to the largest chunk the planner can return. var maxChunkFrames = Math.Max(framesPerTick, OutputSampleRate / 5); _micBuffer = new AudioRingBuffer(OutputSampleRate * 2); _loopbackBuffer = new AudioRingBuffer(OutputSampleRate * 2 * 2); _alertBuffer = new AudioRingBuffer(OutputSampleRate * 2 * 8); _voiceChain = new VoiceFilterChain(OutputSampleRate); _ducker = new AutoDucker(); _micChunk = new float[maxChunkFrames]; _loopbackChunk = new float[maxChunkFrames * 2]; _alertChunk = new float[maxChunkFrames * 2]; _mixBuffer = new float[maxChunkFrames * 2]; _delayedMix = new float[maxChunkFrames * 2]; _mic.Started += OnMicStarted; _mic.SampleReady += OnMicSample; _loopback.SampleReady += OnLoopbackSample; _mic.Failed += OnMicFailed; _loopback.Failed += OnLoopbackFailed; } /// Current smoothed mic level (0..1), post voice chain. public float MicLevel => _meter.Level; /// Current smoothed desktop/game level (0..1). public float LoopbackLevel => _loopbackMeter.Level; /// Raised whenever the smoothed mic level changes. public event Action? MicLevelChanged; /// Raised whenever the smoothed desktop/game level changes. public event Action? LoopbackLevelChanged; /// Raised when the mic capture comes up (the status dot goes green). public event Action? MicConnected; /// Raised when the mic capture fails or dies (the status dot goes yellow). public event Action? MicFailed; public void Start() { if (_started) return; _started = true; _meter.Reset(); _loopback.Start(); _mic.Start(); } /// Swaps the mic source without touching loopback — used when the /// creator picks a different device mid-session. The level resets and the /// new source raises or . public void RestartMic() { _mic.Stop(); _meter.Reset(); _micBuffer.Clear(); _micResampler?.Reset(); _voiceChain.Reset(); MicLevelChanged?.Invoke(0); _mic.Start(); } public void Stop() { if (!_started) return; _started = false; StopLive(); _mic.Stop(); _loopback.Stop(); _meter.Reset(); _loopbackMeter.Reset(); MicLevelChanged?.Invoke(0); LoopbackLevelChanged?.Invoke(0); } /// Begins the live mix loop: drains the capture queues, applies the /// honest gains + auto-duck, and streams stereo float into the named audio /// pipe. Idempotent — safe to call once per go-live. public void StartLive(string pipeName) { if (_liveCts != null) return; // Capture runs from app STARTUP for the meters, so both rings carry up // to a full ring depth (~2s) of pre-live backlog by the time the user // hits record; draining it made every recorded event land ~2s late in // the audio track (2026-09-14, two confirmed takes). Discard it so the // recording starts at go-live. The writers keep running concurrently; // Clear under the ring's lock, so only the in-flight chunk is dropped. _micBuffer.Clear(); _loopbackBuffer.Clear(); _alertBuffer.Clear(); // Negative sync offset (audio runs BEHIND video): OBS's fix is to "eat // the head of the buffer" — discard the first |N| ms of the live stream // so every subsequent audio event lands |N| ms EARLIER in the recording. // Armed once at go-live (you can't eat the head mid-stream); positive // offsets stay live-reactive through the delay line in FillAndMix. var syncMs = _syncOffsetMs?.Invoke() ?? 0; _advanceSamplesRemaining = syncMs < 0 ? (int)Math.Round(OutputSampleRate / 1000.0 * -syncMs) * 2 : 0; if (_advanceSamplesRemaining > 0) _log?.Invoke($"Audio live: advance armed ({-syncMs} ms ahead)"); var cts = new CancellationTokenSource(); _liveCts = cts; _pipe = new NamedPipeAudioWriter(); _pipe.Start(pipeName); _log?.Invoke($"Audio live: started, pipe '{pipeName}'"); _ = Task.Run(() => LiveLoopAsync(_pipe, cts.Token)); } /// Ends the live mix loop and closes the audio pipe — ffmpeg sees /// EOF on the audio input. Safe when not live: the writer's Stop() is idempotent /// (StopStream now calls this on every rollback path). public void StopLive() { var cts = _liveCts; _liveCts = null; if (cts != null) { _log?.Invoke("Audio live: stopped"); cts.Cancel(); } _pipe.Stop(); } public void Dispose() { Stop(); _mic.Started -= OnMicStarted; _mic.SampleReady -= OnMicSample; _loopback.SampleReady -= OnLoopbackSample; _mic.Failed -= OnMicFailed; _loopback.Failed -= OnLoopbackFailed; _mic.Dispose(); _loopback.Dispose(); } private void OnMicStarted() { MicConnected?.Invoke(); } private void OnMicSample(AudioSample sample) { var mono = DownmixToMono(sample); mono = ResampleMic(mono, sample.SampleRate); for (var i = 0; i < mono.Length; i++) mono[i] = _voiceChain.Process(mono[i]); _micBuffer.Write(mono); // Push unconditionally: the ?. on the event would otherwise skip the // argument (and the meter update) when nothing is subscribed yet. The // level is the POST-filtered signal — the meter shows what the stream // will carry. var level = _meter.Push(new AudioSample(mono, OutputSampleRate, 1)); MicLevelChanged?.Invoke(level); } private void OnLoopbackSample(AudioSample sample) { var data = ResampleLoopback(sample.Samples, sample.SampleRate); _loopbackBuffer.Write(data); var level = _loopbackMeter.Push(new AudioSample(data, OutputSampleRate, sample.Channels)); LoopbackLevelChanged?.Invoke(level); } /// Drops a MEDIA clip's audio into the desktop/game ring (TASK 21's audio slice). /// It joins that group on purpose: the desktop fader and the mic auto-ducker then apply /// exactly as they do to game audio and TRAX, so a BRB bed ducks under the creator's /// voice without a second audio knob. The layer's own level/mute are applied upstream /// by . public void EnqueueDesktopMediaAudio(AudioSample sample) { var data = sample.Samples; if (data.Length == 0) return; if (sample.SampleRate != OutputSampleRate) data = ResampleLoopback(data, sample.SampleRate); if (sample.Channels == 1) data = UpmixToStereo(data); _loopbackBuffer.Write(data); var level = _loopbackMeter.Push(new AudioSample(data, OutputSampleRate, sample.Channels)); LoopbackLevelChanged?.Invoke(level); } /// Drops the alert clip's audio into its own ring (TASK 47 — the /// stream-alert video's soundtrack, pre-scaled by the alert layer to /// volume × fade). Mixed at unity into the live output — NO auto-duck: an /// alert must be heard over the mic, so the Creator's per-alert volume /// slider is the honest gain and nothing else is stacked on top. Monno and /// non-48k sources are normalized (the decoder already emits 48k stereo). public void EnqueueAlertAudio(AudioSample sample) { var data = sample.Samples; if (data.Length == 0) return; if (sample.SampleRate != OutputSampleRate) data = ResampleLoopback(data, sample.SampleRate); if (sample.Channels == 1) data = UpmixToStereo(data); _alertBuffer.Write(data); } private static float[] UpmixToStereo(float[] mono) { var stereo = new float[mono.Length * 2]; for (var i = 0; i < mono.Length; i++) { stereo[i * 2] = mono[i]; stereo[i * 2 + 1] = mono[i]; } return stereo; } private void OnMicFailed(Exception ex) { _log?.Invoke($"Mic capture failed: {ex.Message}"); MicLevelChanged?.Invoke(0); MicFailed?.Invoke(ex); } private void OnLoopbackFailed(Exception ex) { _log?.Invoke($"Desktop audio capture failed: {ex.Message}"); } private static float[] DownmixToMono(AudioSample sample) { if (sample.Channels <= 1) return sample.Samples; var frames = sample.Samples.Length / sample.Channels; var mono = new float[frames]; for (var i = 0; i < frames; i++) { var sum = 0f; for (var c = 0; c < sample.Channels; c++) sum += sample.Samples[i * sample.Channels + c]; mono[i] = sum / sample.Channels; } return mono; } private float[] ResampleMic(float[] mono, int inputRate) { if (inputRate == OutputSampleRate) return mono; _micResampler ??= new TinyResampler(inputRate, OutputSampleRate); if (_micResampler.NeedsResampling) return _micResampler.Process(mono); return mono; } private float[] ResampleLoopback(float[] interleaved, int inputRate) { if (inputRate == OutputSampleRate) return interleaved; _loopbackResampler ??= new TinyResampler(inputRate, OutputSampleRate); if (_loopbackResampler.NeedsResampling) return _loopbackResampler.Process(interleaved); return interleaved; } private async Task LiveLoopAsync(IAudioPipeWriter pipe, CancellationToken cancellationToken) { var mixBuffer = _mixBuffer!; var micChunk = _micChunk!; var loopbackChunk = _loopbackChunk!; // The audio clock is the monotonic clock, not the tick count. A fixed chunk per iteration turns // any scheduler error into a permanent A/V rate error (measured 2026-11-12: 2.1% fast, 224 ms // of drift in ten seconds); asking the clock how much is owed and producing exactly that keeps // the track the same length as the show. // The clock runs from the moment the session opens, NOT from the first successful write. Starting // it at connect made the track begin that much later than the picture: measured 2026-11-12, the // writes began 160 ms after ffmpeg started reading and the audio stream came out 137 ms shorter // than the video, so every word sat 160 ms late. Owed time is paid out of the capture ring when // the pipe connects, which is the audio that was missed — real audio, not padding. var clock = Stopwatch.StartNew(); long framesProduced = 0; var maxChunkFrames = micChunk.Length; while (!cancellationToken.IsCancellationRequested) { var nextTick = DateTime.UtcNow + _mixInterval; try { // Nowhere for audio to go until ffmpeg is reading: wait without producing, so the debt // stays owed and is paid from the ring the moment the pipe connects. if (!pipe.IsConnected) { await Task.Delay(1, cancellationToken).ConfigureAwait(false); continue; } var frames = AudioClock.FramesDue( clock.Elapsed.TotalSeconds, framesProduced, OutputSampleRate, maxChunkFrames); if (frames <= 0) { // Nothing owed yet — sooner than the tick interval, so a short yield, not a spin. await Task.Delay(1, cancellationToken).ConfigureAwait(false); continue; } var (_, micDrained, loopDrained) = FillAndMix(micChunk, loopbackChunk, mixBuffer, frames); var samples = frames * 2; // Negative offset: skip the armed |N| ms of stream head. The pipe // writer tolerates a partial chunk (any float length is valid); // dropping the head re-anchors the audio stream against the video // stream, making each event land earlier — OBS's eat-the-head fix. var writeFrom = 0; if (_advanceSamplesRemaining > 0) { var drop = Math.Min(_advanceSamplesRemaining, samples); _advanceSamplesRemaining -= drop; writeFrom = drop; } var wrote = writeFrom >= samples || await pipe.WriteAsync(mixBuffer.AsMemory(writeFrom, samples - writeFrom), cancellationToken).ConfigureAwait(false); framesProduced = AudioClock.Advance(framesProduced, frames, wrote); _micDrainedTotal += micDrained; _loopDrainedTotal += loopDrained; for (var i = writeFrom; i < samples; i++) { var abs = Math.Abs(mixBuffer[i]); if (abs > _peakMix) _peakMix = abs; } var now = DateTime.UtcNow; if (now >= _nextStatsLog && _log != null) { _nextStatsLog = now + TimeSpan.FromSeconds(5); _log($"Audio live: pipe connected={pipe.IsConnected} dropped writes={pipe.DroppedWrites} " + $"micLevel={_meter.Level:F3} loopLevel={_loopbackMeter.Level:F3} " + $"drained {_micDrainedTotal:N0}/{_loopDrainedTotal:N0} samples peakMix {_peakMix:F3} (per 5s)"); _micDrainedTotal = 0; _loopDrainedTotal = 0; _peakMix = 0; } } catch (OperationCanceledException) { break; } catch (Exception ex) { // Throttled + never on a dead session: the 2026-09-01 first-launch flood // was one logged NRE every 10ms tick (see _delayedMix fix). Log at most // once per 5s, and stop looping if the pipe writer is gone for good. if (cancellationToken.IsCancellationRequested) break; var now = DateTime.UtcNow; if (now - _lastLoopErrorLogged >= TimeSpan.FromSeconds(5)) { _lastLoopErrorLogged = now; AppLog.Write(ex, "Audio live loop error", AppLogErrorClass.Class2); _log?.Invoke($"Audio live loop error: {ex.Message}"); } } var delay = nextTick - DateTime.UtcNow; if (delay > TimeSpan.Zero) { try { await Task.Delay(delay, cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) { break; } } } } /// Drains one tick's worth of mic + loopback, silence-fills any /// underrun, applies the ducker and the honest gains, and mixes to stereo. /// Returns the pre-duck mic RMS and how many samples each buffer contributed /// (the 5s live-loop telemetry reads those to name whether capture or the /// pipe starved a silent recording). private (float MicRms, int MicDrained, int LoopDrained) FillAndMix( float[] micChunk, float[] loopbackChunk, float[] mix, int frames) { var micCount = _micBuffer.Read(micChunk, frames); var loopCount = _loopbackBuffer.Read(loopbackChunk, frames * 2); var alertChunk = _alertChunk!; var alertCount = _alertBuffer.Read(alertChunk, frames * 2); float micRms = 0; if (micCount == 0) { Array.Clear(micChunk, 0, frames); } else { double sumSquares = 0; for (var i = 0; i < micCount; i++) sumSquares += micChunk[i] * micChunk[i]; micRms = (float)Math.Sqrt(sumSquares / micCount); if (micCount < frames) Array.Clear(micChunk, micCount, frames - micCount); } if (loopCount == 0) { Array.Clear(loopbackChunk, 0, frames * 2); } else if (loopCount < frames * 2) { Array.Clear(loopbackChunk, loopCount, frames * 2 - loopCount); } if (alertCount == 0) { Array.Clear(alertChunk, 0, frames * 2); } else if (alertCount < frames * 2) { Array.Clear(alertChunk, alertCount, frames * 2 - alertCount); } // Mic-only delay: applied to the source, before the mix, so the desktop and alert audio keep // their timing. Reconfigured every tick like the mix line above (same 2026-09-12 trap: a // reallocating Configure would discard the buffered voice every 10 ms). var micDelayMs = _micSyncOffsetMs?.Invoke() ?? 0; if (micDelayMs > 0) { _micSyncDelay.Configure(micDelayMs); var delayedMic = _delayedMic; if (delayedMic is null || delayedMic.Length < frames) delayedMic = _delayedMic = new float[frames]; _micSyncDelay.Process(micChunk.AsSpan(0, frames), delayedMic.AsSpan(0, frames)); Array.Copy(delayedMic, micChunk, frames); } var duck = _ducker.Update(micRms); var micGain = (float)(_micGain?.Invoke() ?? 1.0); var loopGain = (float)(_loopbackGain?.Invoke() ?? 1.0) * duck; for (var i = 0; i < frames; i++) { var m = micChunk[i] * micGain; mix[i * 2] = m + loopbackChunk[i * 2] * loopGain + alertChunk[i * 2]; mix[i * 2 + 1] = m + loopbackChunk[i * 2 + 1] * loopGain + alertChunk[i * 2 + 1]; } var samples = frames * 2; _masterLimiter.Process(mix.AsSpan(0, samples)); _syncDelay.Configure(_syncOffsetMs?.Invoke() ?? 0); var delayed = _delayedMix; if (delayed is null || delayed.Length < samples) delayed = _delayedMix = new float[samples]; _syncDelay.Process(mix.AsSpan(0, samples), delayed.AsSpan(0, samples)); Array.Copy(delayed, mix, samples); if (MonoOutput) Helpers.AudioFold.ToMono(mix.AsSpan(0, samples), 2); MeasureMaster(mix.AsSpan(0, samples)); return (micRms, micCount, loopCount); } /// One pass over the finished mix: 1024 samples per channel, so the cost is noise /// beside the mixer it sits behind. private void MeasureMaster(ReadOnlySpan mix) { double left = 0, right = 0; var frames = mix.Length / 2; if (frames == 0) return; for (var i = 0; i < frames; i++) { left += mix[i * 2] * mix[i * 2]; right += mix[i * 2 + 1] * mix[i * 2 + 1]; } var l = (float)Math.Sqrt(left / frames); var r = (float)Math.Sqrt(right / frames); MasterLeft = MasterLeft * 0.8f + l * 0.2f; MasterRight = MasterRight * 0.8f + r * 0.2f; } }