From a2219be198dcf9dd4de39c1781fa8d0c5194a668 Mon Sep 17 00:00:00 2001 From: gramps Date: Mon, 31 Aug 2026 19:47:38 -0700 Subject: [PATCH] TASK 21 slice 2b: native-FPS pacing in MediaVideoSource (probe -> delay seam) - MediaVideoSource takes optional IFrameRateProbe? + Func? delay seams (default Task.Delay); probes FPS once in RunAsync, delays by 1/fps after each emitted frame. No probe/unknown fps -> no pacing (ffmpeg pipe backpressure throttles). - Test: MediaVideoSource_PacesFramesByProbedFps (fake probe returns 1000fps + recording delay; one delay per frame ~= 1ms). Media tests 5/5, build 0 warnings. Derivative reference: per-frame delay pacing of decoded output, standard in media playback. --- HANDOFF.md | 15 +++++++----- Services/MediaVideoSource.cs | 17 ++++++++++++- TASKS.md | 2 +- ai.md | 11 ++++++--- ytLive.Tests/MediaVideoSourceTests.cs | 35 +++++++++++++++++++++++++++ 5 files changed, 68 insertions(+), 12 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 9096e5a..269e656 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -56,11 +56,14 @@ subprocess text, null if ffprobe absent), and `FfmpegLocator.ProbeFileName` (`ffprobe.exe`) + conditional extraction alongside ffmpeg.exe. Tests: `ytLive.Tests/FfmpegFrameRateParserTests.cs` (6 pure units + 1 probe integration via fake locator/process); `FfmpegLocatorTests` still green — 15/15 pass, build -0 warnings. **Next wiring step (slice 2b):** inject the probe + a pacing delay -seam into `MediaVideoSource` and pace `FrameAvailable` by 1/fps. +0 warnings. Slice 2b shipped too: `MediaVideoSource` now takes optional +`IFrameRateProbe?` + `Func? delay` seams, +probes FPS once in `RunAsync`, and paces each emitted frame by 1/fps (unknown/ +absent probe → no pacing). New test `MediaVideoSource_PacesFramesByProbedFps` +(fake probe + recording delay, one delay per frame ≈1ms). Media tests 5/5, +build 0 warnings. -Next (slice 2b): wire the probe into `MediaVideoSource` for native-FPS pacing. -Slice 3: loop control (restart decode on `Completed` when `MediaIsLooping`). +Next (slice 3): loop control (restart decode on `Completed` when `MediaIsLooping`). **Landmine:** a stale `testhost` can lock `ytLive.Tests`'s `ytLive.dll` and break `dotnet build` of the test project — kill it first (`cmd.exe /c "taskkill /PID N /F"`) @@ -70,8 +73,8 @@ pure tests run in WSL; user verifies the GUI suites on native Windows PowerShell ## Next -1. Commit slice 2a (this unit): ffprobe extraction + parser + probe seam + tests + docs. -2. Slice 2b: wire the probe into `MediaVideoSource` for native-FPS pacing (delay seam + tests). +1. Commit slice 2b (this unit): native-FPS pacing in `MediaVideoSource` + pacing test + docs. +2. Slice 3: loop control (restart decode on `Completed` when `MediaIsLooping`). 3. Slice 3: loop control (restart decode on `Completed` when `MediaIsLooping`). 4. UI picker slice: acquire/release media sessions on add/remove (the remaining slice-1-step-4 open end). 5. Await user push checkpoint for the pending TASK 22/21 commits. diff --git a/Services/MediaVideoSource.cs b/Services/MediaVideoSource.cs index e02182a..750805f 100644 --- a/Services/MediaVideoSource.cs +++ b/Services/MediaVideoSource.cs @@ -25,6 +25,8 @@ public sealed class MediaVideoSource : IMediaFrameSource private readonly int _height; private readonly IFfmpegLocator _locator; private readonly IDecodeProcess _process; + private readonly IFrameRateProbe? _probe; + private readonly Func _delay; private readonly CancellationTokenSource _cts = new(); private IDisposable? _run; @@ -33,13 +35,17 @@ public sealed class MediaVideoSource : IMediaFrameSource int width, int height, IFfmpegLocator locator, - IDecodeProcess process) + IDecodeProcess process, + IFrameRateProbe? frameRateProbe = null, + Func? delay = null) { _path = path; _width = width; _height = height; _locator = locator; _process = process; + _probe = frameRateProbe; + _delay = delay ?? ((t, ct) => Task.Delay(t, ct)); } public Task StartAsync(CancellationToken cancellationToken = default) @@ -70,6 +76,11 @@ public sealed class MediaVideoSource : IMediaFrameSource { try { + double? frameRate = null; + if (_probe != null) + frameRate = await _probe.ProbeAsync(_path, _cts.Token).ConfigureAwait(false); + var frameDuration = frameRate is > 0 ? TimeSpan.FromSeconds(1.0 / frameRate.Value) : TimeSpan.Zero; + var ffmpegPath = await _locator.LocateAsync(_cts.Token).ConfigureAwait(false); var args = $"-hide_banner -loglevel error -i \"{_path}\" " + @@ -93,7 +104,11 @@ public sealed class MediaVideoSource : IMediaFrameSource var read = await stream.ReadAsync(buffer, 0, buffer.Length, _cts.Token).ConfigureAwait(false); if (read <= 0) break; foreach (var frame in reader.Feed(buffer.AsSpan(0, read).ToArray())) + { FrameAvailable?.Invoke(frame); + if (frameDuration > TimeSpan.Zero) + await _delay(frameDuration, _cts.Token).ConfigureAwait(false); + } } } catch (OperationCanceledException) diff --git a/TASKS.md b/TASKS.md index 2fc602f..9757030 100644 --- a/TASKS.md +++ b/TASKS.md @@ -1044,7 +1044,7 @@ The tasks below are ordered by dependency and risk. Each task builds on the prev 4. ✅ `MediaVideoSource` implements `IMediaFrameSource`: FFmpeg raw video decoder → `VideoFrame` pipeline — spawns ffmpeg `-f rawvideo -pix_fmt bgra`, drains the pipe via pure `RawVideoFrameReader` (`Services/RawVideoFrameReader.cs`), raises `FrameAvailable`/`Completed`; lifecycle is `StartAsync`/`StopAsync`; decode process behind `IDecodeProcess`/`FfmpegDecodeProcess` seam (binary-stdout mirror of `IEncoderProcess`). Shipped 2026-08-31; refactored onto `IMediaFrameSource` with this slice. 4b. ✅ Slice 2a — native-FPS probe seam: `FfmpegFrameRateParser` (pure, prefers `avg_frame_rate=` then `r_frame_rate=`, rational N/N/M, unknown→null) + `IFrameRateProbe`/`FfmpegFrameRateProbe` (derives sibling `ffprobe.exe` from the located ffmpeg dir, reuses the `IDecodeProcess` seam; null if ffprobe absent) + `FfmpegLocator.ProbeFileName` now also extracts `ffprobe.exe` from the pinned archive (conditional). Tests: `FfmpegFrameRateParserTests` (6 pure units + 1 probe integration via fake locator/process) + `FfmpegLocatorTests` still green — 15/15. 5. ✅ `IMediaFrameSource` + `MediaVideoSourceManager` (`Services/IMediaFrameSource.cs`, `Services/MediaVideoSourceManager.cs`): app-wide decode-session ownership refcounted by `MediaPath` with a `Func` factory seam; `AcquireAsync`/`ReleaseAsync`/`ReleaseAllAsync`/`GetLatestFrame`; coalesces frames onto the UI dispatcher onto a single shared `WriteableBitmap` per file (mirror of `ScreenCaptureManager`); `MediaFailed` + `PreviewBitmapChanged`. Unit tests + one integration test (bitmap share/coalesce) — `MediaVideoSourceManagerTests`, 5/5 pass. -6. 🔶 Native-FPS pacing — frames currently emit as fast as the pipe produces them; ffprobe FPS probe + per-frame pacing seam still open (slice 2). +6. ✅ Native-FPS pacing (slice 2b): `MediaVideoSource` takes optional `IFrameRateProbe?` + `Func? delay` seams, probes FPS once in `RunAsync`, and delays by 1/fps after each emitted frame; unknown/absent probe → no pacing. Test: `MediaVideoSource_PacesFramesByProbedFps` (fake probe + recording delay, one delay per frame ≈ 1ms). 7. ☐ Wire into `FramePump` resolver — `Source { Type: MediaSource }` → latest video frame — **resolver + preview routing + manager wired; session acquisition (start/stop on add/remove) still open (comes with the UI picker)** 8. ☐ Wire into `SceneCompositor` — render media source as an image element at its position/size 9. ☐ Loop control — `IsLooping` property, restart on end diff --git a/ai.md b/ai.md index 8e3dcd4..ba7ab37 100644 --- a/ai.md +++ b/ai.md @@ -683,12 +683,15 @@ seam:** `Func`, `Func` resolver, `Func? delay` (default `Task.Delay`) + seams, probes FPS once in `RunAsync`, and delays by 1/fps after each emitted frame; no probe/unknown → no + pacing (ffmpeg's own pipe backpressure already throttles the decode). + Still open: session acquisition on add/remove (UI picker), loop control. - **Social bar on the output (bar bug-fix branch):** the `FramePump` takes an optional `socialBar: Func<(VideoFrame? Frame, SocialBarPosition Position)>?` seam, re-read **every frame** (so a mid-stream position flip applies immediately). The strip is pre-rasterized by `Compositor/SocialBarRenderer.cs` diff --git a/ytLive.Tests/MediaVideoSourceTests.cs b/ytLive.Tests/MediaVideoSourceTests.cs index 8738ccc..a5588ec 100644 --- a/ytLive.Tests/MediaVideoSourceTests.cs +++ b/ytLive.Tests/MediaVideoSourceTests.cs @@ -90,6 +90,41 @@ public class MediaVideoSourceTests public void Dispose() => _stream.Dispose(); } + [Fact] + public async Task MediaVideoSource_PacesFramesByProbedFps() + { + const int w = 2, h = 1; // frame bytes = 8 + var payload = new byte[16]; // two frames + var delays = new List(); + + using var source = new MediaVideoSource( + "clip.mp4", w, h, + new FakeLocator(), + new FakeDecodeProcess(payload), + frameRateProbe: new FakeFrameRateProbe(1000.0), + delay: (_d, _) => { delays.Add(_d); return Task.CompletedTask; }); + + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + source.Completed += () => completed.TrySetResult(true); + + await source.StartAsync(); + await completed.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + // One pacing delay per emitted frame, each = 1/1000s. + Assert.Equal(2, delays.Count); + foreach (var d in delays) + Assert.True(Math.Abs((d - TimeSpan.FromMilliseconds(1)).TotalMilliseconds) < 0.001, + $"expected ~1ms, got {d.TotalMilliseconds}ms"); + } + + private sealed class FakeFrameRateProbe : IFrameRateProbe + { + private readonly double? _fps; + public FakeFrameRateProbe(double? fps) => _fps = fps; + public Task ProbeAsync(string path, CancellationToken ct = default) + => Task.FromResult(_fps); + } + private sealed class FakeLocator : IFfmpegLocator { public Task LocateAsync(CancellationToken ct = default) => Task.FromResult("ffmpeg.exe");