Files
LlamaCasty/ytLive.Tests/SceneCompositorTests.cs
T
gramps efe88b8a0b fix(compositor): keep straight alpha in the paste-cache raster — the web widget's recording black box
The element-space raster builds on a TRANSPARENT base but BlitContentRaw's
partial-alpha branch applied the opaque-dst source-over blend: color premultiplied
by the sampled alpha, then alpha forced to 255. Pasting that raster saw a==255 and
straight-copied darkened ink over the scene — a recording box that the raw-bitmap
preview (correct alpha) never showed. Transparent margins and opaque content were
unaffected, which is why every take looped on the page/CSS while the capture was
transparent all along (15:51 dumps: alpha max 255, mean ~19, zero 57%).

BlitContentRaw now takes transparentDst; the raster call passes true and writes
straight color + straight alpha so the paste rows (BlendRowOpaque/Weighted) do the
real source-over onto the opaque master. Master paths byte-identical.

ONE integration test PasteCache_SemiTransparentLayer_RevealsBackdrop_NotOpaqueInk:
50%-blue over red reads (127,0,128) fixed vs (0,0,128) buggy — proven both ways
(verified by stashing the fix: fails before, passes after). Clean build, 0 warnings;
22/23 compositor-class tests pass, the sole failure the documented pre-existing
Composite_FullScene_MasterPixels pixel (1380,700).

Alpha-compositing model: standard source-over with producer-cached surfaces, the
OBS/libyuv paste model already cited in ai.md/MyMistakes (rawvideo recipe, row-blit
BLEND_NONE / straight-alpha branches); full story in MyMistakes (RESOLVED entry).

Per the good-dog rule: one integration test, memory updates (MyMistakes/ai.md/
HANDOFF) in the same commit.
2026-09-12 08:57:58 -07:00

512 lines
22 KiB
C#
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
using Xunit;
using ytLive.Models;
using ytLive.Services;
using ytLive.Services.Compositor;
namespace ytLive.Tests;
/// <summary>
/// The output compositor (TASK 4 ship step 1): renders a scene into the encoder's
/// master frame, mirroring the XAML preview minus the editing chrome. The integration
/// test composites a full scene (background + round webcam + images + mirror + border)
/// and asserts per-layer probe pixels; the vertical-tier test asserts the 9:16 crop
/// and upscale. Frames are pushed by the test — no capture managers involved.
/// </summary>
public class SceneCompositorTests
{
public static VideoFrame Solid(int w, int h, byte r, byte g, byte b)
{
var pixels = new byte[w * h * 4];
for (var i = 0; i < pixels.Length; i += 4)
{
pixels[i] = b;
pixels[i + 1] = g;
pixels[i + 2] = r;
pixels[i + 3] = 255;
}
return new VideoFrame(w, h, pixels);
}
/// <summary>Left half = leftColor, right half = rightColor.</summary>
private static VideoFrame Split(int w, int h, byte lr, byte lg, byte lb, byte rr, byte rg, byte rb)
{
var pixels = new byte[w * h * 4];
var half = w / 2;
for (var y = 0; y < h; y++)
{
for (var x = 0; x < w; x++)
{
var i = (y * w + x) * 4;
pixels[i] = x < half ? lb : rb;
pixels[i + 1] = x < half ? lg : rg;
pixels[i + 2] = x < half ? lr : rr;
pixels[i + 3] = 255;
}
}
return new VideoFrame(w, h, pixels);
}
private static void AssertColor(VideoFrame frame, int x, int y, byte r, byte g, byte b)
{
var i = (y * frame.Width + x) * 4;
var br = frame.BgraPixels[i + 2];
var bg = frame.BgraPixels[i + 1];
var bb = frame.BgraPixels[i];
Assert.True(
Math.Abs(br - r) <= 2 && Math.Abs(bg - g) <= 2 && Math.Abs(bb - b) <= 2,
$"pixel ({x},{y}): expected rgb({r},{g},{b}), got rgb({br},{bg},{bb})");
}
[Fact]
public void Composite_FullScene_MasterPixels()
{
var red = Solid(1920, 1080, 255, 0, 0); // background
var green = Solid(1280, 720, 0, 255, 0); // webcam
var split = Split(100, 100, 0, 255, 255, 255, 0, 255); // image: left cyan, right magenta
var background = new Source { Type = SourceType.DisplayCapture, IsBackground = true, CaptureKey = "monitor:0" };
var roundWebcam = new WebcamSceneConfig { X = 100, Y = 100, Width = 300, Height = 300, ClipShape = ClipShape.Round };
var imageA = new Source { Type = SourceType.Image, X = 1200, Y = 600, Width = 200, Height = 200 };
var imageB = new Source
{
Type = SourceType.Image, X = 500, Y = 700, Width = 100, Height = 100, IsMirrored = true,
BorderColor = "#ffffff", BorderOpacity = 1, BorderWidth = 4,
};
var scene = new Scene { Name = "Live" };
scene.Elements.Add(background);
scene.Elements.Add(roundWebcam);
scene.Elements.Add(imageA);
scene.Elements.Add(imageB);
VideoFrame? FrameFor(SceneElement e) => e switch
{
WebcamSceneConfig => green,
Source { IsBackground: true } => red,
Source { Type: SourceType.Image } => split,
_ => null,
};
var options = new CompositorOptions
{
SourceRectX = 0, SourceRectY = 0, SourceRectWidth = 1920, SourceRectHeight = 1080,
OutputWidth = 1920, OutputHeight = 1080,
};
var output = new SceneCompositor().Render(scene, FrameFor, null, options);
Assert.Equal(1920, output.Width);
Assert.Equal(1080, output.Height);
// background at the frame corner
AssertColor(output, 0, 0, 255, 0, 0);
// round webcam: center is the (square-cropped) webcam feed
AssertColor(output, 250, 250, 0, 255, 0);
// round webcam: element-square corner is OUTSIDE the circle -> background shows
AssertColor(output, 101, 101, 255, 0, 0);
// imageA (unmirrored): left half cyan, right half magenta
AssertColor(output, 1220, 700, 0, 255, 255);
AssertColor(output, 1380, 700, 255, 0, 255);
// imageB (mirrored): halves swap — element left shows the source's right (magenta)
AssertColor(output, 520, 750, 255, 0, 255);
AssertColor(output, 580, 750, 0, 255, 255);
// imageB border: white ring at the top edge (drawn over the content)
AssertColor(output, 550, 700, 255, 255, 255);
}
/// <summary>The take-4 memcpy fast path: a full-cover opaque backdrop (the live
/// capture's contract) must reproduce the source EXACTLY, and when supplied via the
/// <c>scratch</c> pool parameter the render must overwrite every byte of the reused
/// buffer — stale sentinels from a previous frame may not survive (this is what
/// makes buffer recycling safe at the pump level).</summary>
[Fact]
public void Composite_OpaqueFullCover_Backdrop_CopiesEveryPixel_Into_Scratch()
{
const int w = 128, h = 64;
var src = new byte[w * h * 4];
for (var y = 0; y < h; y++)
for (var x = 0; x < w; x++)
{
var i = (y * w + x) * 4;
src[i] = (byte)(x * 2);
src[i + 1] = (byte)(y * 4);
src[i + 2] = (byte)(x + y);
src[i + 3] = 255; // the IsOpaque contract
}
var opaque = new VideoFrame(w, h, src) { IsOpaque = true };
var background = new Source { Type = SourceType.DisplayCapture, IsBackground = true, CaptureKey = "monitor:0" };
var scene = new Scene { Name = "Live" };
scene.Elements.Add(background);
var options = new CompositorOptions
{
SourceRectX = 0, SourceRectY = 0, SourceRectWidth = w, SourceRectHeight = h,
OutputWidth = w, OutputHeight = h,
};
var scratch = new byte[w * h * 4];
Array.Fill(scratch, (byte)0xFD); // sentinel: x*2 (even), y*4 (×4), x+y (≤190) can never produce 253
var output = new SceneCompositor().Render(scene, _ => opaque, null, options, scratch: scratch);
Assert.Equal(src, output.BgraPixels); // byte-exact: the memcpy path must not round
Assert.DoesNotContain((byte)0xFD, output.BgraPixels); // full cover — nothing stale
}
/// <summary>The ONE integration test for the paste cache (take-8): a second render of
/// the SAME frames must produce byte-identical output (the hit path pastes a cached
/// element-space raster — position/opacity/margins/round-clip must survive it), and a
/// NEW source array must propagate (the cache keys on frame identity, so a stale
/// raster can never outlive its content). Covers the margins the FullScene probe
/// pixels don't: outside-the-circle alpha and the black pre-fill interaction.</summary>
[Fact]
public void PasteCache_RepeatRender_IsByteIdentical_And_ContentChangePropagates()
{
var red = Solid(1920, 1080, 255, 0, 0);
var green = Solid(1280, 720, 0, 255, 0);
var blue = Solid(1280, 720, 0, 0, 255);
var background = new Source { Type = SourceType.DisplayCapture, IsBackground = true, CaptureKey = "monitor:0" };
var roundWebcam = new WebcamSceneConfig { X = 100, Y = 100, Width = 300, Height = 300, ClipShape = ClipShape.Round };
var scene = new Scene { Name = "Live" };
scene.Elements.Add(background);
scene.Elements.Add(roundWebcam);
VideoFrame? FrameFor(SceneElement e) => e switch
{
WebcamSceneConfig => green,
Source { IsBackground: true } => red,
_ => null,
};
var options = new CompositorOptions
{
SourceRectX = 0, SourceRectY = 0, SourceRectWidth = 1920, SourceRectHeight = 1080,
OutputWidth = 1920, OutputHeight = 1080,
};
var compositor = new SceneCompositor();
var first = compositor.Render(scene, FrameFor, null, options);
var second = compositor.Render(scene, FrameFor, null, options); // paste-cache hit
Assert.Equal(first.BgraPixels, second.BgraPixels);
// margins: inside the circle = webcam, outside = backdrop (both renders)
foreach (var f in new[] { first, second })
{
AssertColor(f, 250, 250, 0, 255, 0);
AssertColor(f, 101, 101, 255, 0, 0);
Assert.Equal((byte)255, f.BgraPixels[(250 * 1920 + 250) * 4 + 3]); // alpha stays opaque
}
// Content change (new array, same geometry) must show through the cache.
VideoFrame? FrameForBlue(SceneElement e) => e switch
{
WebcamSceneConfig => blue,
Source { IsBackground: true } => red,
_ => null,
};
var third = compositor.Render(scene, FrameForBlue, null, options);
AssertColor(third, 250, 250, 0, 0, 255);
AssertColor(third, 101, 101, 255, 0, 0);
}
/// <summary>The ONE integration test for the capture buffer ring + paste-cache epoch
/// (take-11 spike fix): a recycled capture buffer hands out the SAME byte[] with
/// different content; only VideoFrame.Epoch can keep the paste cache honest. Without
/// the epoch in the key, frame 2 false-hits frame 1's raster and the recording shows
/// stale pixels for as long as the array keeps cycling — the exact bug this pins.</summary>
[Fact]
public void PasteCache_RecycledArrayWithNewEpoch_ReRasterizes_NotStaleHits()
{
var pixels = new byte[64 * 32 * 4];
for (var i = 0; i < pixels.Length; i += 4)
{
pixels[i] = 0; pixels[i + 1] = 0; pixels[i + 2] = 255; pixels[i + 3] = 255; // red
}
var gen1 = new VideoFrame(64, 32, pixels) { IsOpaque = true, Epoch = 1 };
var webcam = new WebcamSceneConfig { X = 4, Y = 4, Width = 32, Height = 16 };
var scene = new Scene { Name = "Live" };
scene.Elements.Add(webcam);
var options = new CompositorOptions
{
SourceRectX = 0, SourceRectY = 0, SourceRectWidth = 128, SourceRectHeight = 64,
OutputWidth = 128, OutputHeight = 64,
};
var compositor = new SceneCompositor();
var out1 = compositor.Render(scene, _ => gen1, null, options);
AssertColor(out1, 20, 12, 255, 0, 0);
// Generation 2: the SAME array, new content, bumped epoch (what the capture
// ring does 60 times a second).
for (var i = 0; i < pixels.Length; i += 4)
{
pixels[i] = 0; pixels[i + 1] = 255; pixels[i + 2] = 0; pixels[i + 3] = 255; // green
}
var gen2 = new VideoFrame(64, 32, pixels) { IsOpaque = true, Epoch = 2 };
var out2 = compositor.Render(scene, _ => gen2, null, options);
AssertColor(out2, 20, 12, 0, 255, 0); // a stale hit would still be red
// And the same epoch twice still hits the cache (idempotent paste).
var out3 = compositor.Render(scene, _ => gen2, null, options);
Assert.Equal(out2.BgraPixels, out3.BgraPixels);
}
/// <summary>THE page-transparency fix (the web widget's recording "black box"): a
/// semi-transparent layer flowing through the paste-cache raster must reveal the
/// backdrop through it (true source-over), never an opaque darkened block. The raster
/// is built on a TRANSPARENT base, but the old partial-alpha branch applied the
/// opaque-dst blend (premultiplied color + forced alpha 255), so every translucent
/// widget pixel pasted as opaque ink — the box in recordings while the raw-bitmap
/// preview stayed correct. A half-alpha blue widget over a red backdrop must read
/// (127,0,128) here; the buggy raster pastes (0,0,128).</summary>
[Fact]
public void PasteCache_SemiTransparentLayer_RevealsBackdrop_NotOpaqueInk()
{
var red = Solid(128, 64, 255, 0, 0); // live backdrop
var web = new byte[32 * 16 * 4]; // 2:1 widget canvas, half-alpha blue, uniform
for (var i = 0; i < web.Length; i += 4)
{
web[i] = 255; web[i + 1] = 0; web[i + 2] = 0; web[i + 3] = 128;
}
var semitransparent = new VideoFrame(32, 16, web); // NOT IsOpaque
var background = new Source { Type = SourceType.DisplayCapture, IsBackground = true, CaptureKey = "monitor:0" };
var widget = new Source { Type = SourceType.WebSource, X = 8, Y = 8, Width = 16, Height = 16 };
var scene = new Scene { Name = "Live" };
scene.Elements.Add(background);
scene.Elements.Add(widget);
VideoFrame? FrameFor(SceneElement e) => e switch
{
Source { IsBackground: true } => red,
Source { Type: SourceType.WebSource } => semitransparent,
_ => null,
};
var options = new CompositorOptions
{
SourceRectX = 0, SourceRectY = 0, SourceRectWidth = 128, SourceRectHeight = 64,
OutputWidth = 128, OutputHeight = 64,
};
var compositor = new SceneCompositor();
var build = compositor.Render(scene, FrameFor, null, options); // raster build + paste
var hit = compositor.Render(scene, FrameFor, null, options); // paste-cache hit
foreach (var output in new[] { build, hit })
{
AssertColor(output, 16, 16, 127, 0, 128); // backdrop shows through the 50% blue
AssertColor(output, 4, 4, 255, 0, 0); // outside the element: untouched backdrop
}
Assert.Equal(build.BgraPixels, hit.BgraPixels);
}
/// <summary>take-16 regression gate — GC churn convicted mechanically: a ring-style
/// producer hands out a small set of rotating arrays with a NEW EPOCH every frame
/// (exactly what the capture/camera rings do). The compositor must re-rasterize INTO
/// its reused per-element raster, so steady-state allocation per render is near zero.
/// Before the fix every new cache key minted a fresh raster array (~625KB for the
/// webcam at 30fps ≈ 18MB/s) — gen2 pauses followed, and they ate camera frames in
/// the real app. The content assertion rides along: the pasted pixels must match the
/// CURRENT ring generation, never a stale one.</summary>
[Fact]
public void PasteCache_RingRecycledSources_ReRasterInPlace_WithoutAllocationChurn()
{
const int SrcW = 640, SrcH = 360;
var ring = new byte[4][];
for (var s = 0; s < ring.Length; s++)
{
ring[s] = new byte[SrcW * SrcH * 4];
for (var i = 0; i < ring[s].Length; i += 4)
{
ring[s][i] = 0; ring[s][i + 1] = 0;
ring[s][i + 2] = (byte)(40 + s * 50);
ring[s][i + 3] = 255;
}
}
var background = new Source { Type = SourceType.DisplayCapture, IsBackground = true, CaptureKey = "monitor:0" };
var webcam = new WebcamSceneConfig { X = 100, Y = 100, Width = 320, Height = 180 };
var scene = new Scene { Name = "Live" };
scene.Elements.Add(background);
scene.Elements.Add(webcam);
var options = new CompositorOptions
{
SourceRectX = 0, SourceRectY = 0, SourceRectWidth = 1280, SourceRectHeight = 720,
OutputWidth = 1280, OutputHeight = 720,
};
var bg = Solid(1280, 720, 0, 255, 0);
var compositor = new SceneCompositor();
long epoch = 0;
VideoFrame? FrameFor(SceneElement e) => e switch
{
WebcamSceneConfig => new VideoFrame(SrcW, SrcH, ring[epoch % ring.Length])
{ IsOpaque = true, Epoch = ++epoch },
_ => bg,
};
var scratch = new byte[1280 * 720 * 4];
for (var i = 0; i < 8; i++) compositor.Render(scene, FrameFor, null, options, scratch: scratch);
var before = GC.GetAllocatedBytesForCurrentThread();
const int iterations = 200;
VideoFrame? last = null;
for (var i = 0; i < iterations; i++) last = compositor.Render(scene, FrameFor, null, options, scratch: scratch);
var perFrame = (GC.GetAllocatedBytesForCurrentThread() - before) / (double)iterations;
Assert.True(perFrame < 16 * 1024,
$"compositor allocated {perFrame:F0} bytes/frame under a ring producer — raster reuse regressed");
Assert.NotNull(last);
AssertColor(last!, 260, 190, (byte)(40 + 3 * 50), 0, 0);
AssertColor(last!, 5, 700, 0, 255, 0);
}
[Fact]
public void Render_VerticalTier_Outputs_1080x1920_From_The_Center_Crop()
{
var red = Solid(1920, 1080, 255, 0, 0);
var green = Solid(1280, 720, 0, 255, 0);
var background = new Source { Type = SourceType.DisplayCapture, IsBackground = true, CaptureKey = "monitor:0" };
var roundWebcam = new WebcamSceneConfig { X = 656, Y = 100, Width = 300, Height = 300, ClipShape = ClipShape.Round };
var scene = new Scene { Name = "Live" };
scene.Elements.Add(background);
scene.Elements.Add(roundWebcam);
VideoFrame? FrameFor(SceneElement e) => e switch
{
WebcamSceneConfig => green,
Source { IsBackground: true } => red,
_ => null,
};
var options = new CompositorOptions
{
SourceRectX = 656, SourceRectY = 0, SourceRectWidth = 607, SourceRectHeight = 1080,
OutputWidth = 1080, OutputHeight = 1920,
};
var output = new SceneCompositor().Render(scene, FrameFor, null, options);
Assert.Equal(1080, output.Width);
Assert.Equal(1920, output.Height);
// top-left of the crop is pure background (webcam starts at crop y=100, inset from the corner)
AssertColor(output, 0, 0, 255, 0, 0);
// the webcam center (crop 150,250) scales to output ~(267,444) and stays green
AssertColor(output, 267, 444, 0, 255, 0);
// far from the webcam, still background
AssertColor(output, 978, 1688, 255, 0, 0);
}
[Fact]
public void Composite_WithFlash_BlendsOverContent()
{
var red = Solid(1920, 1080, 255, 0, 0);
var background = new Source { Type = SourceType.DisplayCapture, IsBackground = true, CaptureKey = "monitor:0" };
var scene = new Scene { Name = "Live" };
scene.Elements.Add(background);
// flash: a semi-transparent white pixel at the center of a master-sized frame
var flash = new VideoFrame(1920, 1080, new byte[1920 * 1080 * 4]);
var ci = (1080 / 2 * 1920 + 1920 / 2) * 4;
flash.BgraPixels[ci] = 255;
flash.BgraPixels[ci + 1] = 255;
flash.BgraPixels[ci + 2] = 255;
flash.BgraPixels[ci + 3] = 64; // ~25% alpha
var options = new CompositorOptions
{
SourceRectX = 0, SourceRectY = 0, SourceRectWidth = 1920, SourceRectHeight = 1080,
OutputWidth = 1920, OutputHeight = 1080,
};
var output = new SceneCompositor().Render(scene, _ => red, flash, options);
// red lightened by 25% white: ~(255, 63, 63)
AssertColor(output, 960, 540, 255, 64, 64);
// untouched corner stays pure red
AssertColor(output, 0, 0, 255, 0, 0);
}
[Fact]
public void Composite_WithSocialBar_OverlaysAboveFlash_AtTopOrBottom()
{
var red = Solid(1920, 1080, 255, 0, 0);
var background = new Source { Type = SourceType.DisplayCapture, IsBackground = true, CaptureKey = "monitor:0" };
var scene = new Scene { Name = "Live" };
scene.Elements.Add(background);
var options = new CompositorOptions
{
SourceRectX = 0, SourceRectY = 0, SourceRectWidth = 1920, SourceRectHeight = 1080,
OutputWidth = 1920, OutputHeight = 1080,
};
var compositor = new SceneCompositor();
// master-sized overlay: opaque green at (0,0), 50%-white at the center
var bar = new VideoFrame(1920, 1080, new byte[1920 * 1080 * 4]);
var g = 0;
bar.BgraPixels[g] = 0; bar.BgraPixels[g + 1] = 255; bar.BgraPixels[g + 2] = 0; bar.BgraPixels[g + 3] = 255;
var w = (540 * 1920 + 960) * 4;
bar.BgraPixels[w] = 255; bar.BgraPixels[w + 1] = 255; bar.BgraPixels[w + 2] = 255; bar.BgraPixels[w + 3] = 128;
// flash: opaque magenta at (0,0) — the bar must cover it
var flash = new VideoFrame(1920, 1080, new byte[1920 * 1080 * 4]);
flash.BgraPixels[0] = 255; flash.BgraPixels[1] = 0; flash.BgraPixels[2] = 255; flash.BgraPixels[3] = 255;
var top = compositor.Render(scene, _ => red, flash, options, bar, 0);
AssertColor(top, 0, 0, 0, 255, 0); // bar above flash at the top-left
AssertColor(top, 960, 540, 255, 127, 127); // 50% white over red
AssertColor(top, 100, 100, 255, 0, 0); // empty overlay area: background
var bottom = compositor.Render(scene, _ => red, flash, options, bar, 1040);
AssertColor(bottom, 0, 1040, 0, 255, 0); // bar drawn at the bottom edge
AssertColor(bottom, 0, 1039, 255, 0, 0); // background just above the bar
AssertColor(bottom, 960, 540, 255, 0, 0); // bar region moved away from center
}
}
public class StretchMathTests
{
[Fact]
public void UniformToFill_SameAspect_Is_Exact_Fit_With_No_Offset()
{
var (scale, ox, oy) = StretchMath.UniformToFill(400, 300, 800, 600);
Assert.Equal(0.5f, scale, 3);
Assert.Equal(0f, ox, 3);
Assert.Equal(0f, oy, 3);
}
[Fact]
public void UniformToFill_WiderSource_Crops_And_Centers_Vertically()
{
var (scale, ox, oy) = StretchMath.UniformToFill(400, 200, 800, 600);
Assert.Equal(0.5f, scale, 3);
Assert.Equal(0f, ox, 3);
Assert.Equal(-50f, oy, 3); // drawn 400x300 into 400x200 -> 50px crop top and bottom
}
[Fact]
public void BilinearScale_SameSize_ReturnsTheInput()
{
var src = SceneCompositorTests.Solid(4, 4, 10, 20, 30);
Assert.Same(src, StretchMath.BilinearScale(src, 4, 4));
}
[Fact]
public void BilinearScale_Downscales_To_Target_Size()
{
var src = SceneCompositorTests.Solid(16, 16, 0, 255, 0);
var scaled = StretchMath.BilinearScale(src, 8, 8);
Assert.Equal(8, scaled.Width);
Assert.Equal(8, scaled.Height);
var i = 0;
Assert.Equal(255, scaled.BgraPixels[i + 1]); // solid green survives the scale
}
}