From 3a56807ee3d5dcd6b24de6eb8b73685dec4dac12 Mon Sep 17 00:00:00 2001 From: gramps Date: Thu, 5 Nov 2026 10:40:25 -0800 Subject: [PATCH] fix(media): pace clip audio to a deadline too, and prove the keyed layer in the OUTPUT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two follow-ups from the creator's lip-sync observation. 1. Audio still slept a fixed 50 ms per chunk — the same "sleep on top of the work" fault fixed for video an hour ago, left in place. Every chunk's read cost was added, so a clip's audio fell progressively behind its picture: small per chunk, unbounded over a long clip. It now uses the same deadline pacer, so audio and video advance on the same wall clock. 2. The preview pane is NOT the compositor. "Preview looks great" proves the decode and the preview bitmap; the recording and stream are blitted by SceneCompositor. `MediaCompositeOutputTests` renders a keyed video layer over a background through the real compositor and asserts the two things that matter: keyed (alpha 0) pixels let the background through, and the subject half covers it. So "green screen removed" is now proven in the output path, not just the pane. Also: a round-clipped video layer had no preview trigger (only webcam+Round existed), so it would render in the output and be invisible in the pane. Same unit-circle path now covers it. Tests: 16 scoped (pacer, real-clip rate + keying, composite output, decode pacing, alert pacing) green. FULL SUITE NOT RUN — the machine is in use: two runs under load aborted in the WPF test host (HWND subclass crash) and one hit the 15-minute cap. Re-run scripts/verify.sh in a quiet window. --- Controls/PreviewPane.xaml | 9 +++ Services/VideoClipPipe.cs | 15 +++-- ytLive.Tests/MediaCompositeOutputTests.cs | 80 +++++++++++++++++++++++ 3 files changed, 99 insertions(+), 5 deletions(-) create mode 100644 ytLive.Tests/MediaCompositeOutputTests.cs diff --git a/Controls/PreviewPane.xaml b/Controls/PreviewPane.xaml index 362eeb6..fe64fe3 100644 --- a/Controls/PreviewPane.xaml +++ b/Controls/PreviewPane.xaml @@ -208,6 +208,15 @@ + + + + + + + + diff --git a/Services/VideoClipPipe.cs b/Services/VideoClipPipe.cs index 69bdd3c..f505119 100644 --- a/Services/VideoClipPipe.cs +++ b/Services/VideoClipPipe.cs @@ -226,6 +226,8 @@ public sealed class VideoClipPipe : IDisposable }); var stream = process.StandardOutput; + var passStart = Stopwatch.GetTimestamp(); + var pacer = new FramePacer(); var raw = new byte[65536]; var carry = Array.Empty(); var floatScratch = new float[AudioChunkFloats]; @@ -254,11 +256,14 @@ public sealed class VideoClipPipe : IDisposable { chunks++; AudioAvailable?.Invoke(new AudioSample(floatScratch, AudioSampleRate, AudioChannels)); - // Pace the NEXT chunk so the clip's audio lands at real-time cadence, - // matching how the video pipe is paced by frame rate. - await _delay( - TimeSpan.FromSeconds(AudioChunkFloats / (double)(AudioSampleRate * AudioChannels)), ct) - .ConfigureAwait(false); + // Deadline pacing, exactly like the video pipe: sleep only what is LEFT of this + // chunk's 50 ms. A fixed sleep added the read cost every chunk, so audio fell + // progressively behind the picture on a long clip (2026-11-05). + var wait = pacer.DelayFor( + TimeSpan.FromSeconds(AudioChunkFloats / (double)(AudioSampleRate * AudioChannels)), + Stopwatch.GetElapsedTime(passStart)); + if (wait > TimeSpan.Zero) + await _delay(wait, ct).ConfigureAwait(false); floatScratch = new float[AudioChunkFloats]; floatIndex = 0; } diff --git a/ytLive.Tests/MediaCompositeOutputTests.cs b/ytLive.Tests/MediaCompositeOutputTests.cs new file mode 100644 index 0000000..863fcde --- /dev/null +++ b/ytLive.Tests/MediaCompositeOutputTests.cs @@ -0,0 +1,80 @@ +using System; +using Xunit; +using ytLive.Models; +using ytLive.Services; +using ytLive.Services.Compositor; + +namespace ytLive.Tests; + +/// +/// "Preview looks great" only proves the preview. The recording and the stream go through +/// , which is a different path: the preview pane draws each layer as its +/// own WPF image, while the output blits every layer into one frame. This proves the two things that +/// matter for a KEYED video layer in the output — the keyed (alpha 0) pixels let the layer beneath +/// through, and the opaque part of the clip covers it. +/// +public class MediaCompositeOutputTests +{ + private static VideoFrame Solid(int w, int h, byte r, byte g, byte b, byte a = 255) + { + var pixels = new byte[w * h * 4]; + for (var i = 0; i < pixels.Length; i += 4) + { + pixels[i] = b; pixels[i + 1] = g; pixels[i + 2] = r; pixels[i + 3] = a; + } + return new VideoFrame(w, h, pixels); + } + + [Fact] + public void KeyedVideoLayer_LetsTheBackgroundThrough_AndCoversTheRest() + { + // Background: red. Video layer at (640,340) 640×400: transparent left half (keyed out), + // opaque blue right half (the subject). + var background = Solid(1920, 1080, 255, 0, 0); + var keyedClip = Solid(640, 400, 0, 0, 255, a: 0); + var half = 640 / 2; + for (var y = 0; y < 400; y++) + for (var x = half; x < 640; x++) + { + var i = (y * 640 + x) * 4; + keyedClip.BgraPixels[i + 3] = 255; + } + + var bg = new Source { Type = SourceType.DisplayCapture, IsBackground = true, CaptureKey = "monitor:0" }; + var video = new Source { Type = SourceType.MediaSource, MediaPath = "clip.mp4", X = 640, Y = 340, Width = 640, Height = 400 }; + var scene = new Scene { Name = "Live", Elements = { bg, video } }; + + VideoFrame? FrameFor(SceneElement element) => element switch + { + Source { IsBackground: true } => background, + Source { IsMediaSource: true } => keyedClip, + _ => null, + }; + + var options = new CompositorOptions + { + SourceRectX = 0, SourceRectY = 0, SourceRectWidth = 1920, SourceRectHeight = 1080, + OutputWidth = 1920, OutputHeight = 1080, + }; + + var output = new SceneCompositor().Render(scene, FrameFor, null, options); + + // Keyed half: the background must show through — this is what "green screen removed" means + // in the recording, not just in the pane. + AssertColor(output, 700, 400, 255, 0, 0); + AssertColor(output, 950, 600, 255, 0, 0); + + // Subject half: the clip's own pixels cover the background. + AssertColor(output, 1000, 400, 0, 0, 255); + AssertColor(output, 1250, 700, 0, 0, 255); + + // Outside the layer: untouched background. + AssertColor(output, 100, 100, 255, 0, 0); + } + + private static void AssertColor(VideoFrame frame, int x, int y, byte r, byte g, byte b) + { + var i = (y * frame.Width + x) * 4; + Assert.Equal((b, g, r), (frame.BgraPixels[i], frame.BgraPixels[i + 1], frame.BgraPixels[i + 2])); + } +}