diff --git a/Services/Audio/AudioClock.cs b/Services/Audio/AudioClock.cs new file mode 100644 index 0000000..50ee97e --- /dev/null +++ b/Services/Audio/AudioClock.cs @@ -0,0 +1,34 @@ +using System; + +namespace ytLive.Services.Audio; + +/// +/// How many frames the mix must produce now, derived from the monotonic clock rather than from a +/// tick count. +/// +/// Why this exists (2026-11-12): the live loop used to write a fixed 480 frames per +/// iteration and trust that every iteration was 10 ms. It is not: `Task.Delay` plus processing lands +/// near 9.8 ms under a raised timer resolution, and 102 ticks per second where 100 are expected is +/// 102% of an audio second per wall second. FFmpeg stamps audio by sample count, so the track comes out +/// longer than the show and the sound slides later against a correct picture — measured on a 9.9 s +/// recording: video 9.888 s (right), audio 10.112 s (+2.1%, 224 ms). +/// +/// The video side never had this bug because the pump paces off a stopwatch. This is the audio +/// equivalent: ask the clock what should exist, produce the difference, pad on underrun. Pure, so the +/// drift case is a test rather than a listening test. +/// +public static class AudioClock +{ + /// + /// Frames still owed at : expected-minus-produced, clamped to + /// 0... Zero means "not due yet"; the cap keeps a stalled loop from + /// dumping a huge burst in one write (the deficit stays owed and is paid off over the next ticks). + /// + public static int FramesDue(double elapsedSeconds, long framesProduced, int sampleRate, int maxFrames) + { + if (elapsedSeconds <= 0 || sampleRate <= 0 || maxFrames <= 0) return 0; + var expected = (long)Math.Round(elapsedSeconds * sampleRate); + var debt = expected - framesProduced; + return (int)Math.Clamp(debt, 0L, maxFrames); + } +} diff --git a/Services/Audio/AudioMixer.cs b/Services/Audio/AudioMixer.cs index 436048f..90aa2cc 100644 --- a/Services/Audio/AudioMixer.cs +++ b/Services/Audio/AudioMixer.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using ytLive.Helpers; using ytLive.Services; @@ -121,16 +122,19 @@ public sealed class AudioMixer : IDisposable var seconds = _mixInterval.TotalSeconds; var framesPerTick = Math.Max(1, (int)Math.Round(OutputSampleRate * seconds)); + // A tick normally owes ~10 ms; after a stall it may owe more, and the clock-based loop pays it + // off in one write. Buffers therefore size to the largest chunk the planner can return. + var maxChunkFrames = Math.Max(framesPerTick, OutputSampleRate / 5); _micBuffer = new AudioRingBuffer(OutputSampleRate * 2); _loopbackBuffer = new AudioRingBuffer(OutputSampleRate * 2 * 2); _alertBuffer = new AudioRingBuffer(OutputSampleRate * 2 * 8); _voiceChain = new VoiceFilterChain(OutputSampleRate); _ducker = new AutoDucker(); - _micChunk = new float[framesPerTick]; - _loopbackChunk = new float[framesPerTick * 2]; - _alertChunk = new float[framesPerTick * 2]; - _mixBuffer = new float[framesPerTick * 2]; - _delayedMix = new float[framesPerTick * 2]; + _micChunk = new float[maxChunkFrames]; + _loopbackChunk = new float[maxChunkFrames * 2]; + _alertChunk = new float[maxChunkFrames * 2]; + _mixBuffer = new float[maxChunkFrames * 2]; + _delayedMix = new float[maxChunkFrames * 2]; _mic.Started += OnMicStarted; _mic.SampleReady += OnMicSample; @@ -393,12 +397,32 @@ public sealed class AudioMixer : IDisposable var mixBuffer = _mixBuffer!; var micChunk = _micChunk!; var loopbackChunk = _loopbackChunk!; + + // The audio clock is the monotonic clock, not the tick count. A fixed chunk per iteration turns + // any scheduler error into a permanent A/V rate error (measured 2026-11-12: 2.1% fast, 224 ms + // of drift in ten seconds); asking the clock how much is owed and producing exactly that keeps + // the track the same length as the show. + var clock = Stopwatch.StartNew(); + long framesProduced = 0; + var maxChunkFrames = micChunk.Length; + while (!cancellationToken.IsCancellationRequested) { var nextTick = DateTime.UtcNow + _mixInterval; try { - var (_, micDrained, loopDrained) = FillAndMix(micChunk, loopbackChunk, mixBuffer); + var frames = AudioClock.FramesDue( + clock.Elapsed.TotalSeconds, framesProduced, OutputSampleRate, maxChunkFrames); + if (frames <= 0) + { + // Nothing owed yet — sooner than the tick interval, so a short yield, not a spin. + await Task.Delay(1, cancellationToken).ConfigureAwait(false); + continue; + } + + var (_, micDrained, loopDrained) = FillAndMix(micChunk, loopbackChunk, mixBuffer, frames); + framesProduced += frames; + var samples = frames * 2; // Negative offset: skip the armed |N| ms of stream head. The pipe // writer tolerates a partial chunk (any float length is valid); @@ -407,17 +431,18 @@ public sealed class AudioMixer : IDisposable var writeFrom = 0; if (_advanceSamplesRemaining > 0) { - var drop = Math.Min(_advanceSamplesRemaining, mixBuffer.Length); + var drop = Math.Min(_advanceSamplesRemaining, samples); _advanceSamplesRemaining -= drop; writeFrom = drop; } - if (writeFrom < mixBuffer.Length) - await pipe.WriteAsync(mixBuffer.AsMemory(writeFrom), cancellationToken).ConfigureAwait(false); + if (writeFrom < samples) + await pipe.WriteAsync(mixBuffer.AsMemory(writeFrom, samples - writeFrom), + cancellationToken).ConfigureAwait(false); _micDrainedTotal += micDrained; _loopDrainedTotal += loopDrained; - for (var i = writeFrom; i < mixBuffer.Length; i++) + for (var i = writeFrom; i < samples; i++) { var abs = Math.Abs(mixBuffer[i]); if (abs > _peakMix) _peakMix = abs; @@ -475,17 +500,18 @@ public sealed class AudioMixer : IDisposable /// Returns the pre-duck mic RMS and how many samples each buffer contributed /// (the 5s live-loop telemetry reads those to name whether capture or the /// pipe starved a silent recording). - private (float MicRms, int MicDrained, int LoopDrained) FillAndMix(float[] micChunk, float[] loopbackChunk, float[] mix) + private (float MicRms, int MicDrained, int LoopDrained) FillAndMix( + float[] micChunk, float[] loopbackChunk, float[] mix, int frames) { - var micCount = _micBuffer.Read(micChunk, micChunk.Length); - var loopCount = _loopbackBuffer.Read(loopbackChunk, loopbackChunk.Length); + var micCount = _micBuffer.Read(micChunk, frames); + var loopCount = _loopbackBuffer.Read(loopbackChunk, frames * 2); var alertChunk = _alertChunk!; - var alertCount = _alertBuffer.Read(alertChunk, alertChunk.Length); + var alertCount = _alertBuffer.Read(alertChunk, frames * 2); float micRms = 0; if (micCount == 0) { - Array.Clear(micChunk, 0, micChunk.Length); + Array.Clear(micChunk, 0, frames); } else { @@ -493,26 +519,26 @@ public sealed class AudioMixer : IDisposable for (var i = 0; i < micCount; i++) sumSquares += micChunk[i] * micChunk[i]; micRms = (float)Math.Sqrt(sumSquares / micCount); - if (micCount < micChunk.Length) - Array.Clear(micChunk, micCount, micChunk.Length - micCount); + if (micCount < frames) + Array.Clear(micChunk, micCount, frames - micCount); } if (loopCount == 0) { - Array.Clear(loopbackChunk, 0, loopbackChunk.Length); + Array.Clear(loopbackChunk, 0, frames * 2); } - else if (loopCount < loopbackChunk.Length) + else if (loopCount < frames * 2) { - Array.Clear(loopbackChunk, loopCount, loopbackChunk.Length - loopCount); + Array.Clear(loopbackChunk, loopCount, frames * 2 - loopCount); } if (alertCount == 0) { - Array.Clear(alertChunk, 0, alertChunk.Length); + Array.Clear(alertChunk, 0, frames * 2); } - else if (alertCount < alertChunk.Length) + else if (alertCount < frames * 2) { - Array.Clear(alertChunk, alertCount, alertChunk.Length - alertCount); + Array.Clear(alertChunk, alertCount, frames * 2 - alertCount); } // Mic-only delay: applied to the source, before the mix, so the desktop and alert audio keep @@ -523,40 +549,40 @@ public sealed class AudioMixer : IDisposable { _micSyncDelay.Configure(micDelayMs); var delayedMic = _delayedMic; - if (delayedMic is null || delayedMic.Length < micChunk.Length) - delayedMic = _delayedMic = new float[micChunk.Length]; - _micSyncDelay.Process(micChunk, delayedMic); - Array.Copy(delayedMic, micChunk, micChunk.Length); + if (delayedMic is null || delayedMic.Length < frames) + delayedMic = _delayedMic = new float[frames]; + _micSyncDelay.Process(micChunk.AsSpan(0, frames), delayedMic.AsSpan(0, frames)); + Array.Copy(delayedMic, micChunk, frames); } var duck = _ducker.Update(micRms); var micGain = (float)(_micGain?.Invoke() ?? 1.0); var loopGain = (float)(_loopbackGain?.Invoke() ?? 1.0) * duck; - var frames = micChunk.Length; for (var i = 0; i < frames; i++) { var m = micChunk[i] * micGain; mix[i * 2] = m + loopbackChunk[i * 2] * loopGain + alertChunk[i * 2]; mix[i * 2 + 1] = m + loopbackChunk[i * 2 + 1] * loopGain + alertChunk[i * 2 + 1]; } - _masterLimiter.Process(mix); + var samples = frames * 2; + _masterLimiter.Process(mix.AsSpan(0, samples)); _syncDelay.Configure(_syncOffsetMs?.Invoke() ?? 0); var delayed = _delayedMix; - if (delayed is null || delayed.Length < mix.Length) - delayed = _delayedMix = new float[mix.Length]; - _syncDelay.Process(mix, delayed); - Array.Copy(delayed, mix, mix.Length); + if (delayed is null || delayed.Length < samples) + delayed = _delayedMix = new float[samples]; + _syncDelay.Process(mix.AsSpan(0, samples), delayed.AsSpan(0, samples)); + Array.Copy(delayed, mix, samples); - if (MonoOutput) Helpers.AudioFold.ToMono(mix.AsSpan(0, mix.Length), 2); - MeasureMaster(mix); + if (MonoOutput) Helpers.AudioFold.ToMono(mix.AsSpan(0, samples), 2); + MeasureMaster(mix.AsSpan(0, samples)); return (micRms, micCount, loopCount); } /// One pass over the finished mix: 1024 samples per channel, so the cost is noise /// beside the mixer it sits behind. - private void MeasureMaster(float[] mix) + private void MeasureMaster(ReadOnlySpan mix) { double left = 0, right = 0; var frames = mix.Length / 2; diff --git a/Services/Audio/MasterLimiter.cs b/Services/Audio/MasterLimiter.cs index f786490..34d1b55 100644 --- a/Services/Audio/MasterLimiter.cs +++ b/Services/Audio/MasterLimiter.cs @@ -21,7 +21,7 @@ public sealed class MasterLimiter /// Scales the frame by the current gain. Gain drops instantly to /// ceiling/peak when the frame exceeds the ceiling and recovers /// toward unity slowly. - public void Process(float[] samples) + public void Process(Span samples) { var peak = 0f; foreach (var s in samples) diff --git a/docs/ai/output.md b/docs/ai/output.md index 2bec4a9..b1b5a29 100644 --- a/docs/ai/output.md +++ b/docs/ai/output.md @@ -299,6 +299,15 @@ devices, no timers). Presets `off / 150 / 200` sit in the LIP-SYNC bay under the whole-mix slider. Both lines are reconfigured every tick, so both carry the 2026-09-12 warning: a `Configure` that reallocates discards the buffered audio every 10 ms. +- **The audio clock is the wall clock, not the tick count (2026-11-12).** `LiveLoopAsync` asks + `Services/Audio/AudioClock.FramesDue(elapsed, produced, rate, maxChunk)` how many frames are owed and + produces exactly that, instead of writing a fixed 480 frames per iteration. The fixed version turned a + scheduler error into a permanent A/V rate error: iterations land near 9.8 ms rather than 10, which is + 102 ticks/s where 100 are expected — 2.1% more audio than wall time. Measured on a 9.9 s recording: + **video 9.888 s (correct, 594 frames at 60 fps), audio 10.112 s**, so the sound slid ~200 ms behind + the lips across ten seconds and no static delay could fix it. Buffers size to the largest chunk the + planner may return (`OutputSampleRate / 5`), and a stalled loop pays its debt over the following ticks + rather than dumping one huge write. Video never had this bug — the pump paces off a stopwatch. - **TRAX — free background music (TASK 8):** `MusicPlayer` = NAudio `MediaFoundationReader` (mp3/wav/m4a) → `VolumeWaveProvider16` at the hardcoded **0.20** bed (no slider) → `WaveOutEvent` on the default device, **looping on any clean natural end** (`PlaybackStopped` with `e.Exception == null` diff --git a/ytLive.Tests/AudioClockTests.cs b/ytLive.Tests/AudioClockTests.cs new file mode 100644 index 0000000..4eac890 --- /dev/null +++ b/ytLive.Tests/AudioClockTests.cs @@ -0,0 +1,50 @@ +using System; +using Xunit; +using ytLive.Services.Audio; + +namespace ytLive.Tests; + +/// +/// The mix's sample count comes from the clock, not from a tick count — the fix for the 2026-11-12 +/// drift (video 9.888 s against audio 10.112 s on a 9.9 s recording: the audio ran 2.1% long and the +/// sound slid ~200 ms behind the lips by the end of a ten-second clip). +/// +public class AudioClockTests +{ + [Fact] + public void The_mix_follows_the_wall_clock_even_when_the_tick_runs_short() + { + const int rate = 48000; + const int maxChunk = rate / 5; + + // The measured scheduler: iterations land every 9.8 ms, not the requested 10 ms. + var tickSeconds = 0.0098; + + long produced = 0; + long naive = 0; + var ticks = 0; + var elapsed = 0.0; + while (elapsed < 10.0) + { + var frames = AudioClock.FramesDue(elapsed, produced, rate, maxChunk); + produced += frames; + naive += (int)Math.Round(rate * 0.010); // what the old "480 per tick" loop wrote + ticks++; + elapsed += tickSeconds; + } + + var expected = (long)Math.Round(elapsed * rate); + + // The old loop drifted by the tick-rate error — about 2% of ten seconds — and kept drifting. + Assert.True(naive - expected > rate * 0.01, + $"the fixed-chunk loop should reproduce the drift (naive {naive} vs expected {expected})"); + + // Asking the clock keeps the produced count on the clock, within one chunk of payment. + Assert.True(Math.Abs(produced - expected) <= maxChunk, + $"clock-driven count must track the clock (produced {produced} vs expected {expected})"); + + // And a stalled loop pays its debt off gradually rather than dumping it in one write. + Assert.Equal(maxChunk, AudioClock.FramesDue(5.0, 0, rate, maxChunk)); + Assert.Equal(0, AudioClock.FramesDue(5.0, rate * 5, rate, maxChunk)); + } +}