From 432adfdaef7e1df396e7eac84a62887ca498f4d6 Mon Sep 17 00:00:00 2001 From: gramps Date: Fri, 4 Sep 2026 09:35:59 -0700 Subject: [PATCH] =?UTF-8?q?perf(render):=20take-4=20slice=202=20=E2=80=94?= =?UTF-8?q?=20IsOpaque=20memcpy,=20integer=20bilinear,=20pump=20scratch=20?= =?UTF-8?q?pool?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Take 4: pacing held (sync perfect) but avg render stayed 58.9ms — 2M managed row-walk iterations + a fresh 8.3MB buffer every tick (LOH churn into GC stalls inside the render measurement). - VideoFrame.IsOpaque: producer-contract flag (screen capture + webcam — DWM/MF fill alpha 255; media/chat/web/static NOT flagged). Full-cover aligned opaque backdrop = ONE Buffer.BlockCopy; black pre-fill skipped when it covers. - General BlitContent: integer 8.8 fixed-point bilinear + blend, row invariants hoisted, no per-pixel division/Math.Round. Within ±1 of the float reference (pixel tests allow ±2). Research per derivative-work rule: libyuv row/scale kernels (chromium.googlesource.com/libyuv/libyuv). - FramePump scratch pool (max 4, length-keyed, owned-by-reference): release strictly AFTER SubmitFrameAsync returns (stdin write copies); Contains-guard makes the transition Cut alias safe. - Removed the dead per-tick fromScene render + fromSceneProvider seam — BlendFrame consumes TransitionService.FromFrame captured at Start; the pump's render fed nothing. MainViewModel call site updated (signature). Bugs caught by the pixel probes pre-ship (recorded MyMistakes): first Bilinear double-shifted both stages (solid-255 sampled to ~1 -> general path drew nothing); sentinel 0xAB collided with an x+y pixel. FakeEncoder snapshots submitted frames (mirrors real copy semantics under recycling). ONE integration test: Pump_Pools_ScratchBuffers_Across_Frames_Without_ Stale_Pixels (alternating backdrops + repeated backing identity). Direct pin: Composite_OpaqueFullCover_Backdrop_CopiesEveryPixel_Into_Scratch. Clean build 0 warnings; 59/59 per-class + RealApp boot-smoke. take 5 verdict: expect avg render <= ~10ms, ~300/300 frames. Docs same commit: ai.md pipeline section, TASKS.md TASK 18, HANDOFF rewritten (Unit B spec + settled decisions queued). --- HANDOFF.md | 117 ++++++++++++---------- MyMistakes.md | 17 ++++ Services/Compositor/SceneCompositor.cs | 133 +++++++++++++++++++++---- Services/Encoder/FramePump.cs | 82 +++++++++++---- Services/MediaCaptureFrameSource.cs | 3 +- Services/ScreenCaptureFrameSource.cs | 5 +- Services/VideoFrame.cs | 7 ++ TASKS.md | 2 +- ViewModels/MainViewModel.cs | 1 - ai.md | 26 ++++- ytLive.Tests/FramePumpTests.cs | 39 +++++++- ytLive.Tests/SceneCompositorTests.cs | 40 ++++++++ 12 files changed, 370 insertions(+), 102 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index ae244ae..4642a98 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -2,67 +2,84 @@ ## Branch / Commit State -**`main`** — take-3 starvation fix commits land locally (see `git log`); push only when the user says so. -The take-3 audit itself happened 2026-09-02/03: user recorded take 3 (`Downloads\recordings\`, manually -renamed `llamacasty-recording-3.mp4`), ffprobe + `startup.log` + an extracted frame did the diagnosis. +**`main`** — Unit A (render starvation, slice 2) committed locally this session; push on the user's word +(his pattern: says "push" explicitly). Slice 1 (deadline pacing + row-blit, take-3 fix) is already on +`origin/main` as `716a77f`. -## What the take-3 audit found +## What just happened (2026-09-04, Unit A slice 2) -- **37s recording → 2.1s file, 127 frames @60fps** = ~17x time-lapse, audio truncated to match - (ffmpeg's rawvideo demuxer was blocked on the starved video input, so only ~2.3s of audio was consumed). - The user's "webcam at 6x, garbled" = the same file-level time-compression, most visible on the webcam. -- **The stage-timing seam paid for itself in one line**: `17/300 frames per 5s, avg render 258.1ms, - avg submit 1.5ms` — the encoder/pipe was healthy; `SceneCompositor.Render` was the whole bottleneck. -- **Two defects, both fixed (2026-09-03)** — research-first (OBS `libobs/media-io/video-io.c` deadline - pacing + libyuv row-blit pattern; cited in the commit and `MyMistakes.md` recipe): - 1. `FramePump` slept the FULL interval after each render → period = render+interval. Now an absolute - `nextTick += intervalTicks` deadline; on overrun rebase (no catch-up burst). - 2. Per-pixel float sampling + `Math.Round` blends over all 2.07M master px; `BlitOverlay` scanned the - full destination per overlay. Now: 1:1 aligned row-walk fast path (alpha branch, integer fixed-point - blend), `BlitOverlay` clipped to the intersection, backdrop-cover skips the black pre-fill. - 3. **Webcam-in-output VERIFIED** from the take-3 frame (bottom-right, border + social bar render too). -- Test: `Pump_Paces_To_The_Deadline_Compensating_Render_Cost` (records the requested wait via the pacing - seam). Per-class runs: FramePumpTests 10/10, SceneCompositor+SceneGraph+SocialBar+StretchMath 20/20. - Clean build 0 warnings. **Landmine found the hard way:** a pacing fake returning - `Task.CompletedTask` synchronously runs the whole pump loop on `StartAsync`'s continuation and HANGS - the vstest run — fakes must genuinely await (`Task.Delay(d, ct)`) or yield. -- Minor logged on take-3 stop: `FramePump: encoder stop failed: No process is associated` — cosmetic - teardown race (process already exited before StopAsync's kill); follow-up only if it grows teeth. +Take 4 verdict: pacing HELD (no stall cliff, sync intact — user confirmed game+webcam in-sync) but render +stayed **58.9ms** (budget 16.7) → file still ~3.4x time-lapse. Root: the 2M-iteration managed row walk + +8.3MB fresh buffer every tick. Shipped: -## OPEN — do next, in order +1. **`VideoFrame.IsOpaque`** producer-contract flag — set ONLY by `ScreenCaptureFrameSource` + + `MediaCaptureFrameSource` (DWM/MF fill alpha 255). Full-canvas aligned blit of an opaque frame = + ONE `Buffer.BlockCopy`; black pre-fill skipped when the backdrop covers. +2. **Integer fixed-point bilinear** in `BlitContent` general path (webcam: round/mirror/scaled) — no + divisions, no `Math.Round`; ±1 of the float reference (tests allow ±2). +3. **Pump scratch pool** — `AcquireScratch`/`ReleaseScratch` (max 4, length-keyed, owned-by-reference so + bake-cache/social-bar/static-art arrays can never be captured). Release strictly AFTER + `SubmitFrameAsync` returns (stdin write copies). `FakeEncoder` now snapshots frames like the real + encoder (holds would race legitimate recycling). +4. **Dead code kill:** the per-tick `fromScene` render in the transition branch fed NOTHING + (`BlendFrame` uses `TransitionService.FromFrame` captured at `Start`) — removed with the + `fromSceneProvider` seam + `MainViewModel.cs` call site. Transition cost halves as a side effect. -1. **Take 4 (user records ~30s, record-only)** → ffprobe the file (expect frames ≈ 60×seconds, duration - ≈ wall time) + read `startup.log` `FramePump stats` (expect `n/300`, `avg render` ≤ ~16ms). If n/300 - climbs only to ~45-55: next slice is the 8.3MB/frame LOH allocation → pool the master buffer. -2. **Then code, one integration test per change** (queue from before, unchanged): radio pills / - record-OR-stream enforcement (TASK 18 ruling; `BeginGoLive(alsoRecord)` dies) → top bar Option A - **pending user approval** → empty-state ruling pending (disabled / rehearsal / default-record) → - SYNC slider placement ruling pending. -3. **Un-asked questions** (don't nag; answers ready): the recorder said the desktop capture "didn't last - long enough to tell quality" — take 4 fixes that; "other issues" from takes 1/2 never listed. +Bugs caught by the pixel probes before shipping (see MyMistakes take-4 follow-ups): first `Bilinear` +double-shifted (both stages scaled → solid-255 sampled to ~1 → general path drew nothing); sentinel +0xAB collided with a legitimate `x+y` value; pacing-fake synchronous completion hangs vstest (known, +re-trod). -## State of the app +**Verification:** clean build 0 warnings (both projects); per-class vstest 59/59 (FramePump 11 incl. the +new pooling test, SceneCompositor incl. NEW `Composite_OpaqueFullCover...`, SceneGraph, SocialBar, StretchMath, +Camera/ScreenCapture/MediaVideoSource producers, WebcamOutputKey, SessionTeardown) + SourceNaming +RealApp boot-smoke. Scope-check passed. -Boots clean. Recording pipeline: render starvation fixed pending take-4 confirmation; webcam + social bar -confirmed IN the output. ffmpeg cached at `%APPDATA%\ytLlive\tools\` (month-end pin). Test suite per-class -green; do NOT run full-suite vstest (WASAPI teardown hang, pre-existing). +## OPEN — next, in order + +1. **Take 5 (user, ~30s record-only):** read `%APPDATA%\ytLlive\startup.log` — expect + `FramePump stats: ≈300/300 frames per 5s, avg render ≤ ~10ms, avg submit ~1ms` + ffprobe frames ≈ + 60×duration. If render is under budget but `n/300` still short → suspect capture-side (the 1440p + `DownscaleBgra` per-frame alloc loop on the capture thread) or residual GC — next slice. + Also for the user to eyeball: webcam now at true 60fps, audio still in sync. +2. **Unit B — the top bar + session logic (user spec 2026-09-04, decisions settled):** + - Two-line top bar. Line 1: center = REC + **LIVE** pills (text renamed from ON-AIR; pills become + mutually-exclusive RADIOS — record-OR-stream ruling), right = avatar + **Login/Logout** button + (no account status light). Line 2: centered primary **Start** (grayed while NO pill armed — + INVERTS the 2026-09-01 "unarmed Start records" rule; fix the map when landing) that becomes the + Stop/End button while active. + - Avatar right-click → **Change Account** (creator: "standard google thing ... on a portrait + right-click"). Login = `SignInCommand` direct (context menu on Start dies; "Choose Record Folder" + lives in gear → App Settings only). + - LIVE pill stays login-gated (`CanToggleOnAir` exists ✓ 3a.viii). + - REC+Start → `Microsoft.Win32.SaveFileDialog` (InitialDirectory = settings folder, default name + `ty-…-0000.mp4`, NATIVE overwrite prompt covers exists/validate, Enter confirms). Cancel → + abort + disarm pill (lit pill with no session is a lie). Up-front naming RETIRES the stop-time + rename modal (assumption stated; user's dialog answer was about Go-Live confirmation). + - LIVE+Start → **Go-Live dialog stays** as preflight: prefilled from Text-drawer `Broadcast.*`, + unfilled fields visibly prompted, explicit confirm → `PrepareAndStartLiveAsync` (user: "going + live is scary — confirmation allows back-out + testing up to go-live"). + - Bottom-bar metrics init/maintain: `ResetHealth` + `HealthUpdated` exist — verify on take 5/6. + - F6 "start/end" hotkey routes through `HandleHotkey` — check it honors the new grayed-Start gate. + - ONE integration test (hermetic): pills↔button state machine + record-path seam (an + `internal static Func` override seam mirroring `RegistrarOverride` + — never pop real dialogs in tests). +3. **Follow-ups recorded (don't fix opportunistically):** vertical-tier `BilinearScale` per-frame + alloc; cosmetic `FramePump: encoder stop failed: No process is associated` double-stop race; + 1440p capture downscale alloc. ## Landmines - testhost shares startup.log with the app — filter by time when triaging. - Stale testhost/exe locks the DLL (MSB3027): `taskkill /F /IM testhost.exe` / `ytLive.exe` first. -- Do NOT run full-suite vstest (hangs); do NOT claim suite totals — per-class only. verify.sh's step 2 IS - the full-suite hang — flow: clean build + per-class vstest + scope-check. -- Windows binaries (ffprobe/ffmpeg under `%APPDATA%\ytLlive\tools\`) need WINDOWS paths - (`C:\Users\...`), never `/mnt/c/...` — the mount path reads as "No such file or directory". -- Real-`MainWindow` tests MUST use `LayoutPathOverride` + temp DB; `VolumePushOverride` seam exists so - volume-slider tests never touch the machine's speakers. -- `AudioPipelineTests` was the "known failure" graveyard — a new failure there means a live-loop - regression; read the logged stack (throttled, with stack). -- Pacing-seam fakes must yield/await — synchronous completion runs the pump loop inline (see above). +- Do NOT run full-suite vstest (WASAPI hang, pre-existing); flow = clean build + per-class + scope-check. +- Pacing-seam fakes MUST await/yield (sync-completed task → pump runs inline on StartAsync → hang). +- `FakeEncoder` snapshots submitted frames — keep any new fake encoder honest about buffer recycling. +- Multi-stage fixed-point: shift only at the end (MyMistakes 2026-09-04). +- Real-`MainWindow` tests: `LayoutPathOverride` + temp DB mandatory; `VolumePushOverride` for volume. +- ffmpeg: month-end pinned build; `Startup.log` "Recording saved:" lines show the real final path. ## @ User note -He does not want tangents acknowledged twice (rename question = noise; the ask was ALWAYS the recording). -Keep responses SHORT; work the queue; one integration test per change; committing is expected, pushing is -his word. +Recording fix FIRST (his order), UX queue right behind — spec + settled decisions above, don't re-ask. +Keep responses SHORT; one integration test per change; commit every unit; push on his word only. diff --git a/MyMistakes.md b/MyMistakes.md index a55e369..54ed023 100644 --- a/MyMistakes.md +++ b/MyMistakes.md @@ -75,6 +75,23 @@ Both halves were solved by OBS/libyuv long ago; do not re-derive: yield for exactly this reason). Assert the REQUESTED wait (< interval with a ≥cost-ms fake render) — never wall-clock rate, which flakes on loaded machines. + **Take-4 follow-ups (2026-09-04) — the symptom needed a second pass, so cite again:** + render was still 58.9ms after slice 1. Slice 2 (buffer pool + opaque-row memcpy + + integer bilinear) followed the same libyuv research + (https://chromium.googlesource.com/libyuv/libyuv/ — `row.cc`/`scale.cc` keep both + interpolation stages in ONE fixed-point scale; rounding constant only at the end). + My first `Bilinear` shifted stage 1 back to 8-bit AND shifted the final result >>16 — + double scaling turned solid-255 samples into ~1, i.e. the "fixed" general path drew + NOTHING (green webcam silently vanished from output; the pixel probes caught what + the eye in a 2x time-lapse would not). **Rule: multi-stage fixed point shifts only + at the end; verify against a uniform-255 sample before believing it.** Second trap: + a stale-byte sentinel test whose source pattern can generate the sentinel value + itself (0xAB was a legitimate `x+y` pixel) — pick the sentinel coprime/out-of-range + to every channel formula (0xFD: odd, not ×4, above the R max). Third: a fake encoder + that HOLDS submitted frames now must snapshot them (`Clone`) once the producer + legitimately recycles buffers — mirror the real consumer's copy semantics in the fake. + + --- ## Splitting a large file into partials — NEVER `awk … > SRC` while awking SRC diff --git a/Services/Compositor/SceneCompositor.cs b/Services/Compositor/SceneCompositor.cs index 1e096ee..17e1bad 100644 --- a/Services/Compositor/SceneCompositor.cs +++ b/Services/Compositor/SceneCompositor.cs @@ -29,7 +29,8 @@ public sealed class SceneCompositor VideoFrame? socialBarFrame = null, int socialBarTop = 0, VideoFrame? staticBase = null, - int split = -1) + int split = -1, + byte[]? scratch = null) { if (scene == null) throw new ArgumentNullException(nameof(scene)); if (frameFor == null) throw new ArgumentNullException(nameof(frameFor)); @@ -41,7 +42,9 @@ public sealed class SceneCompositor var cropW = options.SourceRectWidth; var cropH = options.SourceRectHeight; - var buffer = new byte[cropW * cropH * 4]; + var buffer = scratch != null && scratch.Length == cropW * cropH * 4 + ? scratch + : new byte[cropW * cropH * 4]; // staticBase != null: the caller pre-baked all static layers below `split`. // We copy it (never mutate the cached base) and composite elements[split..] on top. @@ -200,13 +203,16 @@ public sealed class SceneCompositor CompositorOptions options, VideoFrame? socialBarFrame = null, int socialBarTop = 0, - VideoFrame? flashFrame = null) + VideoFrame? flashFrame = null, + byte[]? scratch = null) { var cropW = options.SourceRectWidth; var cropH = options.SourceRectHeight; - // We must not mutate the cached base: copy it into a fresh buffer. - var buffer = new byte[baseFrame.BgraPixels.Length]; - Buffer.BlockCopy(baseFrame.BgraPixels, 0, buffer, 0, buffer.Length); + // We must not mutate the cached base: copy it into a fresh (or pooled) buffer. + var buffer = scratch != null && scratch.Length == baseFrame.BgraPixels.Length + ? scratch + : new byte[baseFrame.BgraPixels.Length]; + Buffer.BlockCopy(baseFrame.BgraPixels, 0, buffer, 0, baseFrame.BgraPixels.Length); for (var i = split; i < scene.Elements.Count; i++) { @@ -317,13 +323,34 @@ public sealed class SceneCompositor var y1 = Math.Min(dstH - 1, (int)Math.Ceiling(ey + eh)); if (x0 > x1 || y0 > y1) return; + // Integer fixed-point sampling + blend (libyuv's shape): the float + // SampleBgra + BlendPixel pair cost ~8 Math.Round + ~8 divisions per + // pixel and dominated this path in takes 3–4. Row invariants hoisted, + // the y-range check moved out of the x loop, and the dst alpha is the + // same opaque-master invariant as the row helpers (every write = 255). + var w = src.Width; + var hgt = src.Height; + var sxMax = w - 1; + var syMax = hgt - 1; + var invScale = 1f / scale; + var op = (int)(Math.Clamp(opacity, 0f, 1f) * 256); + if (op <= 0) return; + var s = src.BgraPixels; + for (var y = y0; y <= y1; y++) { - for (var x = x0; x <= x1; x++) + var py = y - ey; + if (py < oy || py > oy + drawnH) continue; + var syRaw = Math.Clamp((py - oy) * invScale, 0f, syMax); + var syi = (int)syRaw; + var sy1i = Math.Min(syi + 1, syMax); + var fy8 = (int)((syRaw - syi) * 256); + var fyInv = 256 - fy8; + var di = (y * dstW + x0) * 4; + for (var x = x0; x <= x1; x++, di += 4) { var px = x - ex; // element space - var py = y - ey; - if (px < ox || px > ox + drawnW || py < oy || py > oy + drawnH) continue; + if (px < ox || px > ox + drawnW) continue; if (isRound) { var dx = px - cx; @@ -331,15 +358,56 @@ public sealed class SceneCompositor if (dx * dx + dy * dy > radius * radius) continue; } - var sx = (px - ox) / scale; - var sy = (py - oy) / scale; - if (isMirror) sx = src.Width - 1 - sx; - var sample = StretchMath.SampleBgra(src.BgraPixels, src.Width, src.Height, sx, sy); - BlendPixel(dst, (y * dstW + x) * 4, sample, opacity); + var sxRaw = (px - ox) * invScale; + if (isMirror) sxRaw = sxMax - sxRaw; + sxRaw = Math.Clamp(sxRaw, 0f, sxMax); + var sxi = (int)sxRaw; + var sx1i = Math.Min(sxi + 1, sxMax); + var fx8 = (int)((sxRaw - sxi) * 256); + var fxInv = 256 - fx8; + + var p00 = (syi * w + sxi) * 4; + var p10 = (syi * w + sx1i) * 4; + var p01 = (sy1i * w + sxi) * 4; + var p11 = (sy1i * w + sx1i) * 4; + var sb = Bilinear(s[p00], s[p10], s[p01], s[p11], fx8, fxInv, fy8, fyInv); + var sg = Bilinear(s[p00 + 1], s[p10 + 1], s[p01 + 1], s[p11 + 1], fx8, fxInv, fy8, fyInv); + var sr = Bilinear(s[p00 + 2], s[p10 + 2], s[p01 + 2], s[p11 + 2], fx8, fxInv, fy8, fyInv); + var saFull = Bilinear(s[p00 + 3], s[p10 + 3], s[p01 + 3], s[p11 + 3], fx8, fxInv, fy8, fyInv); + + var sa = saFull * op >> 8; + if (sa >= 255) + { + dst[di] = (byte)sb; + dst[di + 1] = (byte)sg; + dst[di + 2] = (byte)sr; + dst[di + 3] = 255; + } + else if (sa > 0) + { + var inv = 255 - sa; + dst[di] = (byte)((sb * sa + dst[di] * inv + 127) / 255); + dst[di + 1] = (byte)((sg * sa + dst[di + 1] * inv + 127) / 255); + dst[di + 2] = (byte)((sr * sa + dst[di + 2] * inv + 127) / 255); + dst[di + 3] = 255; + } } } } + /// Two-stage bilinear in 8.8 fixed point; result within ±1 of the + /// float StretchMath.SampleBgra (well inside every pixel test's ±2 tolerance). + /// The first stage must NOT shift back to 8 bits — shifting twice collapsed + /// every solid-255 sample to 1 (green overwrites silently degraded to no-ops; + /// caught by the SceneCompositor/SceneGraph pixel probes 2026-09-04). + private static int Bilinear(int tl, int tr, int bl, int br, int fx8, int fxInv, int fy8, int fyInv) + { + // Both stages in one 16.8 scale: top/bot ≤ 65280, ×256 + round ≤ 16.8M — int-safe. + var top = tl * fxInv + tr * fx8; + var bot = bl * fxInv + br * fx8; + return (top * fyInv + bot * fy8 + 32768) >> 16; + } + /// Row walk for a 1:1 aligned blit at integer offset (ex, ey): per-pixel /// branch on source alpha — opaque rows copy straight, transparent pixels are /// skipped, partial alpha blends in integer fixed-point (libyuv's pattern). The @@ -357,17 +425,38 @@ public sealed class SceneCompositor var y1 = Math.Min(dstH, ey + src.Height); if (x0 >= x1 || y0 >= y1) return; - var rowLen = x1 - x0; - for (var y = y0; y < y1; y++) + // The take-4 fix: a full-canvas opaque capture (the Live backdrop) is ONE + // memcpy, not 2.07M managed loop iterations — the row walk below still cost + // ~58ms/frame dominated by bounds-check arithmetic and GC of the per-pixel + // loop's slowness, i.e. the same starvation shape as take 3, one floor lower. + if (op == 256 && src.IsOpaque && x0 == 0 && y0 == 0 && x1 == dstW && y1 == dstH) { - var si = ((y - ey) * src.Width + (x0 - ex)) * 4; - var di = (y * dstW + x0) * 4; - if (op == 256) BlendRowOpaque(dst, di, src.BgraPixels, si, rowLen); - else BlendRowWeighted(dst, di, src.BgraPixels, si, rowLen, op); + Buffer.BlockCopy(src.BgraPixels, 0, dst, 0, dstW * dstH * 4); + return; + } + + var d = dst.AsSpan(); + var s = src.BgraPixels.AsSpan(); + var rowLen = x1 - x0; + if (op == 256) + { + for (var y = y0; y < y1; y++) + { + var si = ((y - ey) * src.Width + (x0 - ex)) * 4; + BlendRowOpaque(d, (y * dstW + x0) * 4, s, si, rowLen); + } + } + else + { + for (var y = y0; y < y1; y++) + { + var si = ((y - ey) * src.Width + (x0 - ex)) * 4; + BlendRowWeighted(d, (y * dstW + x0) * 4, s, si, rowLen, op); + } } } - private static void BlendRowOpaque(byte[] dst, int di, byte[] src, int si, int count) + private static void BlendRowOpaque(Span dst, int di, ReadOnlySpan src, int si, int count) { for (var n = 0; n < count; n++) { @@ -392,7 +481,7 @@ public sealed class SceneCompositor } } - private static void BlendRowWeighted(byte[] dst, int di, byte[] src, int si, int count, int op) + private static void BlendRowWeighted(Span dst, int di, ReadOnlySpan src, int si, int count, int op) { for (var n = 0; n < count; n++) { diff --git a/Services/Encoder/FramePump.cs b/Services/Encoder/FramePump.cs index ca01f37..507a40e 100644 --- a/Services/Encoder/FramePump.cs +++ b/Services/Encoder/FramePump.cs @@ -30,10 +30,40 @@ public sealed class FramePump : IDisposable private readonly Func _pacingDelay; private readonly Func<(VideoFrame? Frame, SocialBarPosition Position)>? _socialBar; private readonly TransitionService? _transition; - private readonly Func? _fromSceneProvider; private readonly SceneGraph? _sceneGraph; private readonly SceneCompositor _compositor = new(); + // Master-buffer scratch pool (take-4 starvation fix, slice 2 of 2): a fresh + // 8.3MB byte[] every tick is ~500MB/s of LOH churn — GC stalls masquerading + // as render cost. The pump pools ONLY buffers it handed out (reference-equality + // set), so bake-cache / social-bar / static-cache frames are never touched; + // SubmitFrameAsync copies the bytes to the encoder's stdin before returning, + // so recycling after submit is safe (MyMistakes recipe). + private readonly HashSet _ownedScratch = new(ReferenceEqualityComparer.Instance); + private readonly List _freeScratch = new(); + private const int MaxScratchPooled = 4; + + private byte[] AcquireScratch(int size) + { + for (var i = _freeScratch.Count - 1; i >= 0; i--) + { + if (_freeScratch[i].Length != size) continue; + var buffer = _freeScratch[i]; + _freeScratch.RemoveAt(i); + return buffer; + } + var fresh = new byte[size]; + _ownedScratch.Add(fresh); + return fresh; + } + + private void ReleaseScratch(byte[]? buffer) + { + if (buffer == null || !_ownedScratch.Contains(buffer)) return; + if (_freeScratch.Count >= MaxScratchPooled || _freeScratch.Contains(buffer)) return; + _freeScratch.Add(buffer); + } + private readonly object _gate = new(); private IFfmpegEncoder? _encoder; private CancellationTokenSource? _cts; @@ -56,7 +86,6 @@ public sealed class FramePump : IDisposable Func? pacingDelay = null, Func<(VideoFrame? Frame, SocialBarPosition Position)>? socialBar = null, TransitionService? transition = null, - Func? fromSceneProvider = null, SceneGraph? sceneGraph = null) { _sceneProvider = sceneProvider ?? throw new ArgumentNullException(nameof(sceneProvider)); @@ -68,7 +97,6 @@ public sealed class FramePump : IDisposable _pacingDelay = pacingDelay ?? ((delay, ct) => Task.Delay(delay, ct)); _socialBar = socialBar; _transition = transition; - _fromSceneProvider = fromSceneProvider; _sceneGraph = sceneGraph; } @@ -211,6 +239,8 @@ public sealed class FramePump : IDisposable { _log?.Invoke($"FramePump: dispose failed: {ex.Message}"); } + _freeScratch.Clear(); + _ownedScratch.Clear(); } private async Task PumpAsync(EncoderOptions options, CancellationToken ct) @@ -273,23 +303,25 @@ public sealed class FramePump : IDisposable } VideoFrame frame; + var scratchSize = compositorOptions.SourceRectWidth + * compositorOptions.SourceRectHeight * 4; + // The transition "from" frame lives in TransitionService.FromFrame + // (captured at Start by the VM). The old per-tick fromScene render + // here was dead weight — a full extra scene composite every + // transition tick that BlendFrame never read; removed with the + // pooling change because its buffer's only consumer was its own release. renderSw.Restart(); - if (_transition is { Active: true } transition && _fromSceneProvider != null) + var scratch = AcquireScratch(scratchSize); + frame = RenderScene(scene, compositorOptions, socialBarFrame, socialBarTop, scratch); + if (_transition is { Active: true } transition) { - var fromScene = _fromSceneProvider(); - var fromFrame = fromScene != null - ? RenderScene(fromScene, compositorOptions, socialBarFrame, socialBarTop) - : RenderScene(scene, compositorOptions, socialBarFrame, socialBarTop); - var toFrame = RenderScene(scene, compositorOptions, socialBarFrame, socialBarTop); - frame = transition.BlendFrame(toFrame); + frame = transition.BlendFrame(frame); transition.Tick(lastTick.Elapsed.TotalMilliseconds); - lastTick.Restart(); - } - else - { - frame = RenderScene(scene, compositorOptions, socialBarFrame, socialBarTop); - lastTick.Restart(); } + // Restarted EVERY frame (transition or not) — the old per-frame reset + // is what stops a transition that begins after idle from inheriting + // a giant ElapsedMs and completing instantly on its first tick. + lastTick.Restart(); renderSw.Stop(); renderTicks += renderSw.ElapsedTicks; @@ -300,6 +332,11 @@ public sealed class FramePump : IDisposable await encoder.SubmitFrameAsync(frame, ct); submitSw.Stop(); submitTicks += submitSw.ElapsedTicks; + // Submit copied the bytes — everything from this tick is recyclable. + // Release AFTER submit, and the free-list Contains guard makes the + // Cut path (BlendFrame returns toFrame itself, aliasing scratch) safe. + ReleaseScratch(frame.BgraPixels); + ReleaseScratch(scratch); statFrames++; ReportStats(); @@ -349,15 +386,18 @@ public sealed class FramePump : IDisposable /// Render a scene, using the baked-crust optimization when a /// is wired in: bake/cache the static layers below the split point, then composite the /// dynamic/above-split layers per frame. Without a SceneGraph, falls back to a full render - /// (identical output — see SceneCompositorTests). + /// (identical output — see SceneCompositorTests). is a pooled + /// master buffer when supplied; the fully-static path returns the bake-cache frame itself + /// (never pooled — the release side checks owned-by-reference). private VideoFrame RenderScene( Scene scene, CompositorOptions options, VideoFrame? socialBarFrame, - int socialBarTop) + int socialBarTop, + byte[]? scratch = null) { if (_sceneGraph == null) - return _compositor.Render(scene, _frameResolver, null, options, socialBarFrame, socialBarTop); + return _compositor.Render(scene, _frameResolver, null, options, socialBarFrame, socialBarTop, scratch: scratch); var split = _sceneGraph.GetSplitPoint(scene); if (split == scene.Elements.Count) @@ -372,11 +412,11 @@ public sealed class FramePump : IDisposable if (baseFrame != null) { return SceneCompositor.CompositeLayers( - baseFrame, scene, split, _frameResolver, options, socialBarFrame, socialBarTop); + baseFrame, scene, split, _frameResolver, options, socialBarFrame, socialBarTop, scratch: scratch); } // No static base (first layer is dynamic or empty scene) — full render. - return _compositor.Render(scene, _frameResolver, null, options, socialBarFrame, socialBarTop); + return _compositor.Render(scene, _frameResolver, null, options, socialBarFrame, socialBarTop, scratch: scratch); } private void OnProcessFailed(object? sender, string message) diff --git a/Services/MediaCaptureFrameSource.cs b/Services/MediaCaptureFrameSource.cs index 72a8163..bd3eada 100644 --- a/Services/MediaCaptureFrameSource.cs +++ b/Services/MediaCaptureFrameSource.cs @@ -227,7 +227,8 @@ public sealed class MediaCaptureFrameSource : ICameraFrameSource { var pixels = new byte[capacity]; Marshal.Copy(pixelsPtr, pixels, 0, (int)capacity); - FrameAvailable?.Invoke(new VideoFrame(bitmap.PixelWidth, bitmap.PixelHeight, pixels)); + // MediaCapture video carries no alpha plane — the OS fills 255. + FrameAvailable?.Invoke(new VideoFrame(bitmap.PixelWidth, bitmap.PixelHeight, pixels) { IsOpaque = true }); } } catch (Exception ex) diff --git a/Services/ScreenCaptureFrameSource.cs b/Services/ScreenCaptureFrameSource.cs index f01e67d..7fa76af 100644 --- a/Services/ScreenCaptureFrameSource.cs +++ b/Services/ScreenCaptureFrameSource.cs @@ -177,12 +177,13 @@ public sealed class ScreenCaptureFrameSource : IScreenCaptureSource var scale = Math.Min(MaxBackgroundWidth / (double)sw, MaxBackgroundHeight / (double)sh); var dw = Math.Max(1, (int)(sw * scale)); var dh = Math.Max(1, (int)(sh * scale)); - return new VideoFrame(dw, dh, DownscaleBgra(data, sw, sh, srcStride, dw, dh)); + // DWM delivers an opaque surface (alpha 255); bilinear keeps it 255. + return new VideoFrame(dw, dh, DownscaleBgra(data, sw, sh, srcStride, dw, dh)) { IsOpaque = true }; } var pixels = new byte[count]; Marshal.Copy(data, pixels, 0, pixels.Length); - return new VideoFrame(sw, sh, pixels); + return new VideoFrame(sw, sh, pixels) { IsOpaque = true }; } // Bilinear downscale to the master frame. Reads each source row pair through diff --git a/Services/VideoFrame.cs b/Services/VideoFrame.cs index 4d21546..71bc5fa 100644 --- a/Services/VideoFrame.cs +++ b/Services/VideoFrame.cs @@ -13,6 +13,13 @@ public sealed class VideoFrame public byte[] BgraPixels { get; } public int Stride => Width * 4; + /// Producer contract: every alpha byte in is 255 + /// (DWM capture surfaces and MediaCapture video carry no alpha — the OS fills 255). + /// Lets the compositor take a straight-copy fast path for a full-canvas opaque + /// layer instead of 2M per-pixel blends. NEVER set it for paths whose pixels can + /// be transparent (static art, web overlays, chat, the social-bar strip). + public bool IsOpaque { get; init; } + public VideoFrame(int width, int height, byte[] bgraPixels) { Width = width; diff --git a/TASKS.md b/TASKS.md index 9d211dc..be58379 100644 --- a/TASKS.md +++ b/TASKS.md @@ -973,7 +973,7 @@ click (volume sliders keep their manual `SetSliderValueFromClick`, harmless dupl **Goal:** record the stream output to a local file, with or without simultaneously streaming. -### Status: ✅ Shipped `a9eb360` (2026-08-29) — code done (incl. manual-rename modal), build 0 warnings, 244/246 tests; **running-app verification (takes 1–3 done)**: files land (ffmpeg re-pinned month-end), **webcam-in-output + social bar visually CONFIRMED from take 3's extracted frame**, rename modal used for real (take 3 was named via it); take 3 exposed the ~2fps producer starvation and the pump stage-timing (`97ffc42`) named it in one line — `avg render 258.1ms` — **fixed 2026-09-03** (deadline pacing per OBS video-io.c + libyuv-style row blits in `SceneCompositor`, test `Pump_Paces_To_The_Deadline_Compensating_Render_Cost`); **take 4 pending** to verify 60fps lands (stats must show `n/300` and render ≤ ~16ms; next suspect if hiccuppy: 8.3MB/frame LOH alloc → buffer pool) +### Status: ✅ Shipped `a9eb360` (2026-08-29) — code done (incl. manual-rename modal), build 0 warnings, 244/246 tests; **running-app verification (takes 1–4 done)**: files land (ffmpeg re-pinned month-end), **webcam-in-output + social bar visually CONFIRMED from take 3's extracted frame**, rename modal used for real (take 3 was named via it); take 3 exposed the ~2fps producer starvation and the pump stage-timing (`97ffc42`) named it in one line — `avg render 258.1ms` — **slice 1 fixed 2026-09-03** (deadline pacing per OBS video-io.c + libyuv-style row blits in `SceneCompositor`, test `Pump_Paces_To_The_Deadline_Compensating_Render_Cost`); **take 4: pacing held but render stayed 58.9ms** (2M-iteration row walk + 8.3MB/tick LOH) — **slice 2 shipped 2026-09-04**: `VideoFrame.IsOpaque` producer-contract flag → full-cover backdrop is one `Buffer.BlockCopy`; integer fixed-point bilinear general path; pump scratch pool (release strictly post-submit, owned-by-reference so cache frames are untouchable); dead per-tick `fromScene` render + the `fromSceneProvider` seam removed (BlendFrame uses `TransitionService.FromFrame` — the old render fed nothing). Tests `Pump_Pools_ScratchBuffers_Across_Frames_Without_Stale_Pixels` (the ONE) + `Composite_OpaqueFullCover_Backdrop_CopiesEveryPixel_Into_Scratch`; 59/59 per-class green, clean build 0 warnings. **take 5 pending**: stats must show `avg render ≤ ~10ms, ≈300/300`. Known remaining churn (follow-up, not silently done): the vertical tier's final `BilinearScale` still allocates per frame. **User UX spec (2026-09-04) queued behind this**: two-line top bar (LIVE rename, radio pills, Login/Logout button + avatar right-click Change Account, no account light; line 2 centered Start↔Stop grayed-until-armed) + up-front SaveFileDialog for REC (native overwrite prompt; retires the stop-time rename modal) + Go-Live dialog KEPT as preflight confirmation prefilled from the Text drawer — decisions captured in HANDOFF 1. ✅ `EncoderOptions` extended with `StreamEnabled` / `RecordEnabled` / `RecordPath` (independent intent flags) 2. ✅ `FfmpegArgs.Build` reworked into per-output blocks (stream `-f flv`, record `-f mp4`) via `AddVideoTags` diff --git a/ViewModels/MainViewModel.cs b/ViewModels/MainViewModel.cs index b05a9ec..4dd29c3 100644 --- a/ViewModels/MainViewModel.cs +++ b/ViewModels/MainViewModel.cs @@ -309,7 +309,6 @@ public partial class MainViewModel : ViewModelBase log: message => AppLog.Write(message), socialBar: () => (_socialBarFrame, _socials?.BarPosition ?? SocialBarPosition.Bottom), transition: _transition, - fromSceneProvider: () => LiveScene, sceneGraph: _sceneGraph); _framePump.Failed += OnFramePumpFailed; _framePump.HealthUpdated += OnFramePumpHealthUpdated; diff --git a/ai.md b/ai.md index 697bf18..712c254 100644 --- a/ai.md +++ b/ai.md @@ -707,9 +707,29 @@ seam:** `Func`, `Func` resolver, `Func Frames = new(); + public readonly List Backings = new(); public int StartCount; public int StopCount; public bool Disposed; @@ -41,7 +42,11 @@ public class FramePumpTests public Task SubmitFrameAsync(VideoFrame frame, CancellationToken cancellationToken = default) { - Frames.Add(frame); + // Snapshot the bytes like the production FfmpegEncoder does (WriteAsync to + // stdin copies before returning). Holding the reference would race the + // pump's scratch pool legitimately reusing the buffer one tick later. + Frames.Add(new VideoFrame(frame.Width, frame.Height, (byte[])frame.BgraPixels.Clone())); + Backings.Add(frame.BgraPixels); // the pre-copy buffer — pool-reuse witness FrameArrived.TrySetResult(); return Task.CompletedTask; } @@ -387,4 +392,36 @@ public class FramePumpTests $"requested wait {d} reaches the full {interval} interval — render cost must subtract from the deadline (the take-3 bug)"); } } + + /// The ONE integration test for the take-4 scratch pool: the pump recycles + /// its master buffers across frames (the 8.3MB-per-tick LOH churn that cost GC + /// stalls inside "render") while EVERY frame's content stays correct — stale bytes + /// from an earlier tick would show as a wrong color. The backdrop alternates + /// red/blue per tick (submit snapshots pin the color), and a repeated backing-array + /// identity proves the pool actually recycles (an inert pool keeps the pixel + /// assertions green while leaving the churn in place). + [Fact] + public async Task Pump_Pools_ScratchBuffers_Across_Frames_Without_Stale_Pixels() + { + var red = SceneCompositorTests.Solid(64, 48, 255, 0, 0); + var blue = SceneCompositorTests.Solid(64, 48, 0, 0, 255); + var flip = 0; + var encoder = new FakeEncoder(); + using var pump = NewPump(encoder, resolve: _ => flip++ % 2 == 0 ? red : blue); + + await pump.StartAsync(); + var deadline = DateTime.UtcNow.AddSeconds(10); + while (DateTime.UtcNow < deadline && encoder.Frames.Count < 6) + await Task.Delay(10); + await pump.StopAsync(); + + Assert.True(encoder.Frames.Count >= 6, $"only {encoder.Frames.Count} frames produced"); + for (var i = 0; i < encoder.Frames.Count; i++) + { + if (i % 2 == 0) AssertColor(encoder.Frames[i], 0, 0, 255, 0, 0); + else AssertColor(encoder.Frames[i], 0, 0, 0, 0, 255); + } + Assert.True(encoder.Backings.Distinct().Count() < encoder.Backings.Count, + "every frame rode a fresh buffer — the scratch pool is inert"); + } } diff --git a/ytLive.Tests/SceneCompositorTests.cs b/ytLive.Tests/SceneCompositorTests.cs index 6b45920..51f187f 100644 --- a/ytLive.Tests/SceneCompositorTests.cs +++ b/ytLive.Tests/SceneCompositorTests.cs @@ -114,6 +114,46 @@ public class SceneCompositorTests AssertColor(output, 550, 700, 255, 255, 255); } + /// The take-4 memcpy fast path: a full-cover opaque backdrop (the live + /// capture's contract) must reproduce the source EXACTLY, and when supplied via the + /// scratch pool parameter the render must overwrite every byte of the reused + /// buffer — stale sentinels from a previous frame may not survive (this is what + /// makes buffer recycling safe at the pump level). + [Fact] + public void Composite_OpaqueFullCover_Backdrop_CopiesEveryPixel_Into_Scratch() + { + const int w = 128, h = 64; + var src = new byte[w * h * 4]; + for (var y = 0; y < h; y++) + for (var x = 0; x < w; x++) + { + var i = (y * w + x) * 4; + src[i] = (byte)(x * 2); + src[i + 1] = (byte)(y * 4); + src[i + 2] = (byte)(x + y); + src[i + 3] = 255; // the IsOpaque contract + } + var opaque = new VideoFrame(w, h, src) { IsOpaque = true }; + + var background = new Source { Type = SourceType.DisplayCapture, IsBackground = true, CaptureKey = "monitor:0" }; + var scene = new Scene { Name = "Live" }; + scene.Elements.Add(background); + + var options = new CompositorOptions + { + SourceRectX = 0, SourceRectY = 0, SourceRectWidth = w, SourceRectHeight = h, + OutputWidth = w, OutputHeight = h, + }; + + var scratch = new byte[w * h * 4]; + Array.Fill(scratch, (byte)0xFD); // sentinel: x*2 (even), y*4 (×4), x+y (≤190) can never produce 253 + + var output = new SceneCompositor().Render(scene, _ => opaque, null, options, scratch: scratch); + + Assert.Equal(src, output.BgraPixels); // byte-exact: the memcpy path must not round + Assert.DoesNotContain((byte)0xFD, output.BgraPixels); // full cover — nothing stale + } + [Fact] public void Render_VerticalTier_Outputs_1080x1920_From_The_Center_Crop() {