perf(render): take-4 slice 2 — IsOpaque memcpy, integer bilinear, pump scratch pool

Take 4: pacing held (sync perfect) but avg render stayed 58.9ms — 2M managed row-walk iterations + a fresh 8.3MB buffer every tick (LOH churn into GC stalls inside the render measurement). - VideoFrame.IsOpaque: producer-contract flag (screen capture + webcam — DWM/MF fill alpha 255; media/chat/web/static NOT flagged). Full-cover aligned opaque backdrop = ONE Buffer.BlockCopy; black pre-fill skipped when it covers. - General BlitContent: integer 8.8 fixed-point bilinear + blend, row invariants hoisted, no per-pixel division/Math.Round. Within ±1 of the float reference (pixel tests allow ±2). Research per derivative-work rule: libyuv row/scale kernels (chromium.googlesource.com/libyuv/libyuv). - FramePump scratch pool (max 4, length-keyed, owned-by-reference): release strictly AFTER SubmitFrameAsync returns (stdin write copies); Contains-guard makes the transition Cut alias safe. - Removed the dead per-tick fromScene render + fromSceneProvider seam — BlendFrame consumes TransitionService.FromFrame captured at Start; the pump's render fed nothing. MainViewModel call site updated (signature). Bugs caught by the pixel probes pre-ship (recorded MyMistakes): first Bilinear double-shifted both stages (solid-255 sampled to ~1 -> general path drew nothing); sentinel 0xAB collided with an x+y pixel. FakeEncoder snapshots submitted frames (mirrors real copy semantics under recycling). ONE integration test: Pump_Pools_ScratchBuffers_Across_Frames_Without_ Stale_Pixels (alternating backdrops + repeated backing identity). Direct pin: Composite_OpaqueFullCover_Backdrop_CopiesEveryPixel_Into_Scratch. Clean build 0 warnings; 59/59 per-class + RealApp boot-smoke. take 5 verdict: expect avg render <= ~10ms, ~300/300 frames. Docs same commit: ai.md pipeline section, TASKS.md TASK 18, HANDOFF rewritten (Unit B spec + settled decisions queued).
2026-09-04 09:35:59 -07:00
parent 716a77f61a
commit 432adfdaef
12 changed files with 370 additions and 102 deletions
@@ -29,7 +29,8 @@ public sealed class SceneCompositor
        VideoFrame? socialBarFrame = null,
        int socialBarTop = 0,
        VideoFrame? staticBase = null,
-        int split = -1)
+        int split = -1,
+        byte[]? scratch = null)
    {
        if (scene == null) throw new ArgumentNullException(nameof(scene));
        if (frameFor == null) throw new ArgumentNullException(nameof(frameFor));
@@ -41,7 +42,9 @@ public sealed class SceneCompositor

        var cropW = options.SourceRectWidth;
        var cropH = options.SourceRectHeight;
-        var buffer = new byte[cropW * cropH * 4];
+        var buffer = scratch != null && scratch.Length == cropW * cropH * 4
+            ? scratch
+            : new byte[cropW * cropH * 4];

        // staticBase != null: the caller pre-baked all static layers below `split`.
        // We copy it (never mutate the cached base) and composite elements[split..] on top.
@@ -200,13 +203,16 @@ public sealed class SceneCompositor
        CompositorOptions options,
        VideoFrame? socialBarFrame = null,
        int socialBarTop = 0,
-        VideoFrame? flashFrame = null)
+        VideoFrame? flashFrame = null,
+        byte[]? scratch = null)
    {
        var cropW = options.SourceRectWidth;
        var cropH = options.SourceRectHeight;
-        // We must not mutate the cached base: copy it into a fresh buffer.
-        var buffer = new byte[baseFrame.BgraPixels.Length];
-        Buffer.BlockCopy(baseFrame.BgraPixels, 0, buffer, 0, buffer.Length);
+        // We must not mutate the cached base: copy it into a fresh (or pooled) buffer.
+        var buffer = scratch != null && scratch.Length == baseFrame.BgraPixels.Length
+            ? scratch
+            : new byte[baseFrame.BgraPixels.Length];
+        Buffer.BlockCopy(baseFrame.BgraPixels, 0, buffer, 0, baseFrame.BgraPixels.Length);

        for (var i = split; i < scene.Elements.Count; i++)
        {
@@ -317,13 +323,34 @@ public sealed class SceneCompositor
        var y1 = Math.Min(dstH - 1, (int)Math.Ceiling(ey + eh));
        if (x0 > x1 || y0 > y1) return;

+        // Integer fixed-point sampling + blend (libyuv's shape): the float
+        // SampleBgra + BlendPixel pair cost ~8 Math.Round + ~8 divisions per
+        // pixel and dominated this path in takes 3–4. Row invariants hoisted,
+        // the y-range check moved out of the x loop, and the dst alpha is the
+        // same opaque-master invariant as the row helpers (every write = 255).
+        var w = src.Width;
+        var hgt = src.Height;
+        var sxMax = w - 1;
+        var syMax = hgt - 1;
+        var invScale = 1f / scale;
+        var op = (int)(Math.Clamp(opacity, 0f, 1f) * 256);
+        if (op <= 0) return;
+        var s = src.BgraPixels;
+
        for (var y = y0; y <= y1; y++)
        {
-            for (var x = x0; x <= x1; x++)
+            var py = y - ey;
+            if (py < oy || py > oy + drawnH) continue;
+            var syRaw = Math.Clamp((py - oy) * invScale, 0f, syMax);
+            var syi = (int)syRaw;
+            var sy1i = Math.Min(syi + 1, syMax);
+            var fy8 = (int)((syRaw - syi) * 256);
+            var fyInv = 256 - fy8;
+            var di = (y * dstW + x0) * 4;
+            for (var x = x0; x <= x1; x++, di += 4)
            {
                var px = x - ex; // element space
-                var py = y - ey;
-                if (px < ox || px > ox + drawnW || py < oy || py > oy + drawnH) continue;
+                if (px < ox || px > ox + drawnW) continue;
                if (isRound)
                {
                    var dx = px - cx;
@@ -331,15 +358,56 @@ public sealed class SceneCompositor
                    if (dx * dx + dy * dy > radius * radius) continue;
                }

-                var sx = (px - ox) / scale;
-                var sy = (py - oy) / scale;
-                if (isMirror) sx = src.Width - 1 - sx;
-                var sample = StretchMath.SampleBgra(src.BgraPixels, src.Width, src.Height, sx, sy);
-                BlendPixel(dst, (y * dstW + x) * 4, sample, opacity);
+                var sxRaw = (px - ox) * invScale;
+                if (isMirror) sxRaw = sxMax - sxRaw;
+                sxRaw = Math.Clamp(sxRaw, 0f, sxMax);
+                var sxi = (int)sxRaw;
+                var sx1i = Math.Min(sxi + 1, sxMax);
+                var fx8 = (int)((sxRaw - sxi) * 256);
+                var fxInv = 256 - fx8;
+
+                var p00 = (syi * w + sxi) * 4;
+                var p10 = (syi * w + sx1i) * 4;
+                var p01 = (sy1i * w + sxi) * 4;
+                var p11 = (sy1i * w + sx1i) * 4;
+                var sb = Bilinear(s[p00], s[p10], s[p01], s[p11], fx8, fxInv, fy8, fyInv);
+                var sg = Bilinear(s[p00 + 1], s[p10 + 1], s[p01 + 1], s[p11 + 1], fx8, fxInv, fy8, fyInv);
+                var sr = Bilinear(s[p00 + 2], s[p10 + 2], s[p01 + 2], s[p11 + 2], fx8, fxInv, fy8, fyInv);
+                var saFull = Bilinear(s[p00 + 3], s[p10 + 3], s[p01 + 3], s[p11 + 3], fx8, fxInv, fy8, fyInv);
+
+                var sa = saFull * op >> 8;
+                if (sa >= 255)
+                {
+                    dst[di] = (byte)sb;
+                    dst[di + 1] = (byte)sg;
+                    dst[di + 2] = (byte)sr;
+                    dst[di + 3] = 255;
+                }
+                else if (sa > 0)
+                {
+                    var inv = 255 - sa;
+                    dst[di] = (byte)((sb * sa + dst[di] * inv + 127) / 255);
+                    dst[di + 1] = (byte)((sg * sa + dst[di + 1] * inv + 127) / 255);
+                    dst[di + 2] = (byte)((sr * sa + dst[di + 2] * inv + 127) / 255);
+                    dst[di + 3] = 255;
+                }
            }
        }
    }

+    /// <summary>Two-stage bilinear in 8.8 fixed point; result within ±1 of the
+    /// float StretchMath.SampleBgra (well inside every pixel test's ±2 tolerance).
+    /// The first stage must NOT shift back to 8 bits — shifting twice collapsed
+    /// every solid-255 sample to 1 (green overwrites silently degraded to no-ops;
+    /// caught by the SceneCompositor/SceneGraph pixel probes 2026-09-04).</summary>
+    private static int Bilinear(int tl, int tr, int bl, int br, int fx8, int fxInv, int fy8, int fyInv)
+    {
+        // Both stages in one 16.8 scale: top/bot ≤ 65280, ×256 + round ≤ 16.8M — int-safe.
+        var top = tl * fxInv + tr * fx8;
+        var bot = bl * fxInv + br * fx8;
+        return (top * fyInv + bot * fy8 + 32768) >> 16;
+    }
+
    /// <summary>Row walk for a 1:1 aligned blit at integer offset (ex, ey): per-pixel
    /// branch on source alpha — opaque rows copy straight, transparent pixels are
    /// skipped, partial alpha blends in integer fixed-point (libyuv's pattern). The
@@ -357,17 +425,38 @@ public sealed class SceneCompositor
        var y1 = Math.Min(dstH, ey + src.Height);
        if (x0 >= x1 || y0 >= y1) return;

-        var rowLen = x1 - x0;
-        for (var y = y0; y < y1; y++)
+        // The take-4 fix: a full-canvas opaque capture (the Live backdrop) is ONE
+        // memcpy, not 2.07M managed loop iterations — the row walk below still cost
+        // ~58ms/frame dominated by bounds-check arithmetic and GC of the per-pixel
+        // loop's slowness, i.e. the same starvation shape as take 3, one floor lower.
+        if (op == 256 && src.IsOpaque && x0 == 0 && y0 == 0 && x1 == dstW && y1 == dstH)
        {
-            var si = ((y - ey) * src.Width + (x0 - ex)) * 4;
-            var di = (y * dstW + x0) * 4;
-            if (op == 256) BlendRowOpaque(dst, di, src.BgraPixels, si, rowLen);
-            else BlendRowWeighted(dst, di, src.BgraPixels, si, rowLen, op);
+            Buffer.BlockCopy(src.BgraPixels, 0, dst, 0, dstW * dstH * 4);
+            return;
+        }
+
+        var d = dst.AsSpan();
+        var s = src.BgraPixels.AsSpan();
+        var rowLen = x1 - x0;
+        if (op == 256)
+        {
+            for (var y = y0; y < y1; y++)
+            {
+                var si = ((y - ey) * src.Width + (x0 - ex)) * 4;
+                BlendRowOpaque(d, (y * dstW + x0) * 4, s, si, rowLen);
+            }
+        }
+        else
+        {
+            for (var y = y0; y < y1; y++)
+            {
+                var si = ((y - ey) * src.Width + (x0 - ex)) * 4;
+                BlendRowWeighted(d, (y * dstW + x0) * 4, s, si, rowLen, op);
+            }
        }
    }

-    private static void BlendRowOpaque(byte[] dst, int di, byte[] src, int si, int count)
+    private static void BlendRowOpaque(Span<byte> dst, int di, ReadOnlySpan<byte> src, int si, int count)
    {
        for (var n = 0; n < count; n++)
        {
@@ -392,7 +481,7 @@ public sealed class SceneCompositor
        }
    }

-    private static void BlendRowWeighted(byte[] dst, int di, byte[] src, int si, int count, int op)
+    private static void BlendRowWeighted(Span<byte> dst, int di, ReadOnlySpan<byte> src, int si, int count, int op)
    {
        for (var n = 0; n < count; n++)
        {
@@ -30,10 +30,40 @@ public sealed class FramePump : IDisposable
    private readonly Func<TimeSpan, CancellationToken, Task> _pacingDelay;
    private readonly Func<(VideoFrame? Frame, SocialBarPosition Position)>? _socialBar;
    private readonly TransitionService? _transition;
-    private readonly Func<Scene?>? _fromSceneProvider;
    private readonly SceneGraph? _sceneGraph;
    private readonly SceneCompositor _compositor = new();

+    // Master-buffer scratch pool (take-4 starvation fix, slice 2 of 2): a fresh
+    // 8.3MB byte[] every tick is ~500MB/s of LOH churn — GC stalls masquerading
+    // as render cost. The pump pools ONLY buffers it handed out (reference-equality
+    // set), so bake-cache / social-bar / static-cache frames are never touched;
+    // SubmitFrameAsync copies the bytes to the encoder's stdin before returning,
+    // so recycling after submit is safe (MyMistakes recipe).
+    private readonly HashSet<byte[]> _ownedScratch = new(ReferenceEqualityComparer.Instance);
+    private readonly List<byte[]> _freeScratch = new();
+    private const int MaxScratchPooled = 4;
+
+    private byte[] AcquireScratch(int size)
+    {
+        for (var i = _freeScratch.Count - 1; i >= 0; i--)
+        {
+            if (_freeScratch[i].Length != size) continue;
+            var buffer = _freeScratch[i];
+            _freeScratch.RemoveAt(i);
+            return buffer;
+        }
+        var fresh = new byte[size];
+        _ownedScratch.Add(fresh);
+        return fresh;
+    }
+
+    private void ReleaseScratch(byte[]? buffer)
+    {
+        if (buffer == null || !_ownedScratch.Contains(buffer)) return;
+        if (_freeScratch.Count >= MaxScratchPooled || _freeScratch.Contains(buffer)) return;
+        _freeScratch.Add(buffer);
+    }
+
    private readonly object _gate = new();
    private IFfmpegEncoder? _encoder;
    private CancellationTokenSource? _cts;
@@ -56,7 +86,6 @@ public sealed class FramePump : IDisposable
        Func<TimeSpan, CancellationToken, Task>? pacingDelay = null,
        Func<(VideoFrame? Frame, SocialBarPosition Position)>? socialBar = null,
        TransitionService? transition = null,
-        Func<Scene?>? fromSceneProvider = null,
        SceneGraph? sceneGraph = null)
    {
        _sceneProvider = sceneProvider ?? throw new ArgumentNullException(nameof(sceneProvider));
@@ -68,7 +97,6 @@ public sealed class FramePump : IDisposable
        _pacingDelay = pacingDelay ?? ((delay, ct) => Task.Delay(delay, ct));
        _socialBar = socialBar;
        _transition = transition;
-        _fromSceneProvider = fromSceneProvider;
        _sceneGraph = sceneGraph;
    }

@@ -211,6 +239,8 @@ public sealed class FramePump : IDisposable
        {
            _log?.Invoke($"FramePump: dispose failed: {ex.Message}");
        }
+        _freeScratch.Clear();
+        _ownedScratch.Clear();
    }

    private async Task PumpAsync(EncoderOptions options, CancellationToken ct)
@@ -273,23 +303,25 @@ public sealed class FramePump : IDisposable
                    }

                    VideoFrame frame;
+                    var scratchSize = compositorOptions.SourceRectWidth
+                        * compositorOptions.SourceRectHeight * 4;
+                    // The transition "from" frame lives in TransitionService.FromFrame
+                    // (captured at Start by the VM). The old per-tick fromScene render
+                    // here was dead weight — a full extra scene composite every
+                    // transition tick that BlendFrame never read; removed with the
+                    // pooling change because its buffer's only consumer was its own release.
                    renderSw.Restart();
-                    if (_transition is { Active: true } transition && _fromSceneProvider != null)
+                    var scratch = AcquireScratch(scratchSize);
+                    frame = RenderScene(scene, compositorOptions, socialBarFrame, socialBarTop, scratch);
+                    if (_transition is { Active: true } transition)
                    {
-                        var fromScene = _fromSceneProvider();
-                        var fromFrame = fromScene != null
-                            ? RenderScene(fromScene, compositorOptions, socialBarFrame, socialBarTop)
-                            : RenderScene(scene, compositorOptions, socialBarFrame, socialBarTop);
-                        var toFrame = RenderScene(scene, compositorOptions, socialBarFrame, socialBarTop);
-                        frame = transition.BlendFrame(toFrame);
+                        frame = transition.BlendFrame(frame);
                        transition.Tick(lastTick.Elapsed.TotalMilliseconds);
-                        lastTick.Restart();
-                    }
-                    else
-                    {
-                        frame = RenderScene(scene, compositorOptions, socialBarFrame, socialBarTop);
-                        lastTick.Restart();
                    }
+                    // Restarted EVERY frame (transition or not) — the old per-frame reset
+                    // is what stops a transition that begins after idle from inheriting
+                    // a giant ElapsedMs and completing instantly on its first tick.
+                    lastTick.Restart();
                    renderSw.Stop();
                    renderTicks += renderSw.ElapsedTicks;

@@ -300,6 +332,11 @@ public sealed class FramePump : IDisposable
                    await encoder.SubmitFrameAsync(frame, ct);
                    submitSw.Stop();
                    submitTicks += submitSw.ElapsedTicks;
+                    // Submit copied the bytes — everything from this tick is recyclable.
+                    // Release AFTER submit, and the free-list Contains guard makes the
+                    // Cut path (BlendFrame returns toFrame itself, aliasing scratch) safe.
+                    ReleaseScratch(frame.BgraPixels);
+                    ReleaseScratch(scratch);
                    statFrames++;
                    ReportStats();

@@ -349,15 +386,18 @@ public sealed class FramePump : IDisposable
    /// <summary>Render a scene, using the baked-crust optimization when a <see cref="SceneGraph"/>
    /// is wired in: bake/cache the static layers below the split point, then composite the
    /// dynamic/above-split layers per frame. Without a SceneGraph, falls back to a full render
-    /// (identical output — see SceneCompositorTests).</summary>
+    /// (identical output — see SceneCompositorTests). <paramref name="scratch"/> is a pooled
+    /// master buffer when supplied; the fully-static path returns the bake-cache frame itself
+    /// (never pooled — the release side checks owned-by-reference).</summary>
    private VideoFrame RenderScene(
        Scene scene,
        CompositorOptions options,
        VideoFrame? socialBarFrame,
-        int socialBarTop)
+        int socialBarTop,
+        byte[]? scratch = null)
    {
        if (_sceneGraph == null)
-            return _compositor.Render(scene, _frameResolver, null, options, socialBarFrame, socialBarTop);
+            return _compositor.Render(scene, _frameResolver, null, options, socialBarFrame, socialBarTop, scratch: scratch);

        var split = _sceneGraph.GetSplitPoint(scene);
        if (split == scene.Elements.Count)
@@ -372,11 +412,11 @@ public sealed class FramePump : IDisposable
        if (baseFrame != null)
        {
            return SceneCompositor.CompositeLayers(
-                baseFrame, scene, split, _frameResolver, options, socialBarFrame, socialBarTop);
+                baseFrame, scene, split, _frameResolver, options, socialBarFrame, socialBarTop, scratch: scratch);
        }

        // No static base (first layer is dynamic or empty scene) — full render.
-        return _compositor.Render(scene, _frameResolver, null, options, socialBarFrame, socialBarTop);
+        return _compositor.Render(scene, _frameResolver, null, options, socialBarFrame, socialBarTop, scratch: scratch);
    }

    private void OnProcessFailed(object? sender, string message)
@@ -227,7 +227,8 @@ public sealed class MediaCaptureFrameSource : ICameraFrameSource
            {
                var pixels = new byte[capacity];
                Marshal.Copy(pixelsPtr, pixels, 0, (int)capacity);
-                FrameAvailable?.Invoke(new VideoFrame(bitmap.PixelWidth, bitmap.PixelHeight, pixels));
+                // MediaCapture video carries no alpha plane — the OS fills 255.
+                FrameAvailable?.Invoke(new VideoFrame(bitmap.PixelWidth, bitmap.PixelHeight, pixels) { IsOpaque = true });
            }
        }
        catch (Exception ex)
@@ -177,12 +177,13 @@ public sealed class ScreenCaptureFrameSource : IScreenCaptureSource
            var scale = Math.Min(MaxBackgroundWidth / (double)sw, MaxBackgroundHeight / (double)sh);
            var dw = Math.Max(1, (int)(sw * scale));
            var dh = Math.Max(1, (int)(sh * scale));
-            return new VideoFrame(dw, dh, DownscaleBgra(data, sw, sh, srcStride, dw, dh));
+            // DWM delivers an opaque surface (alpha 255); bilinear keeps it 255.
+            return new VideoFrame(dw, dh, DownscaleBgra(data, sw, sh, srcStride, dw, dh)) { IsOpaque = true };
        }

        var pixels = new byte[count];
        Marshal.Copy(data, pixels, 0, pixels.Length);
-        return new VideoFrame(sw, sh, pixels);
+        return new VideoFrame(sw, sh, pixels) { IsOpaque = true };
    }

    // Bilinear downscale to the master frame. Reads each source row pair through
@@ -13,6 +13,13 @@ public sealed class VideoFrame
    public byte[] BgraPixels { get; }
    public int Stride => Width * 4;

+    /// <summary>Producer contract: every alpha byte in <see cref="BgraPixels"/> is 255
+    /// (DWM capture surfaces and MediaCapture video carry no alpha — the OS fills 255).
+    /// Lets the compositor take a straight-copy fast path for a full-canvas opaque
+    /// layer instead of 2M per-pixel blends. NEVER set it for paths whose pixels can
+    /// be transparent (static art, web overlays, chat, the social-bar strip).</summary>
+    public bool IsOpaque { get; init; }
+
    public VideoFrame(int width, int height, byte[] bgraPixels)
    {
        Width = width;