diff --git a/MyMistakes.md b/MyMistakes.md index 4d1f467..40f5bf7 100644 --- a/MyMistakes.md +++ b/MyMistakes.md @@ -15,7 +15,40 @@ ## 🔬 Recipes registry -### Shrink / re-encode an image for the README (screenshots → small hero image) +### Web-overlay transparency: measure with the alpha crop, COMPOSE with the full canvas + +(2026-09-05, take-19-2/take-20 — "the webcam is a black square under the web widget". + +**Symptom:** WebView2-based web overlays (StreamElements alert hosts etc.) rendered +with hard opaque boxes over the webcam in the RECORDING but the preview looked fine, +and the FIRST take after a rebuild was good but later takes went opaque — a +stateful-looking bug. + +**Root cause:** the capture was alpha-CROPPED to the widget's content bounding box +(`FindContentBounds` — every non-zero-alpha pixel) and that CROP was handed to the +compositor, which `UniformToFill`-scales a source into the element rect. A crop has +NO transparent margins by construction, so the composite zoomed the opaque content +to cover the ENTIRE element rect (hard box!) whenever the widget drew anything. It +looked stateful because an idle/transparent page yields a full-frame crop ≈ the +correct full-canvas viewport (good), while any real content yields a tight crop + +zoom (bad). + +**The rule (OBS browser-source model):** the page is a FIXED canvas (1920×1080 here); +the element rect is a VIEWPORT onto it. Measure/select with the alpha crop; hand the +composite the FULL canvas with its transparent background intact — `UniformToFill` +then maps the page region 1:1 into the element rect and the page's transparent +margins reveal the layers beneath (the webcam). When feeding a compositor that +scales sources to fill a rect, NEVER feed it a crop of your source unless the rect +is meant to re-frame it. + +Recipe: +1. Keep `FindContentBounds` for the preview bitmap / selection bounding box. +2. Feed the compositor a full-canvas frame whose background is alpha 0, from an + 8-deep ring (same identity/Epoch discipline as every producer — a single reused + scratch array tears under the consumer). +3. Pin the reveal contract with a compositor test: full-canvas transparent-margin + frame over a webcam rect → margin pixel = webcam color, opaque badge pixel = + widget color. Worked out 2026-08-29 (the recipe was NEVER recorded the first time it was done, so it had to be re-derived from scratch — that's the incident this entry exists to end). diff --git a/Services/WebView2Manager.cs b/Services/WebView2Manager.cs index 409670f..5595174 100644 --- a/Services/WebView2Manager.cs +++ b/Services/WebView2Manager.cs @@ -33,16 +33,36 @@ public sealed class WebView2Manager : IDisposable // Buffer reuse (2026-09-04, same lesson as the capture/camera rings): every // capture tick used to mint a fresh 8.3MB canvas array + a fresh crop array // (~83MB/s of LOH at 10Hz → gen2 pauses surfaced as the pump's 35-40ms - // "worst render" spikes). Canvas scratch is reused while the size holds; - // the frame handed to the compositor rotates an 8-deep ring with a monotonic - // Epoch (identity-keyed consumers must see each generation). The preview - // WriteableBitmap is recreated only when the cropped size changes. - public byte[]? CanvasScratch; + // "worst render" spikes). The full-canvas slots rotate an 8-deep ring with a + // monotonic Epoch (identity-keyed consumers — the compositor's paste cache — + // must see each generation); the crop slots feed the preview WriteableBitmap, + // recreated only when the cropped size changes. + public readonly byte[]?[] CanvasRing = new byte[8][]; + public int CanvasRingNext; + public long Epoch; public readonly byte[]?[] OutRing = new byte[8][]; public int OutRingNext; - public long Epoch; public WriteableBitmap? PreviewBitmap; + public byte[] RentCanvasBuffer(int size) + { + for (var tries = 0; tries < CanvasRing.Length; tries++) + { + var idx = (CanvasRingNext + tries) % CanvasRing.Length; + var buf = CanvasRing[idx]; + if (buf != null && buf.Length == size) + { + CanvasRingNext = (idx + 1) % CanvasRing.Length; + return buf; + } + } + var slot = CanvasRingNext; + CanvasRingNext = (slot + 1) % CanvasRing.Length; + var fresh = new byte[size]; + CanvasRing[slot] = fresh; + return fresh; + } + public byte[] RentOutBuffer(int size) { for (var tries = 0; tries < OutRing.Length; tries++) @@ -76,11 +96,10 @@ public sealed class WebView2Manager : IDisposable // OBS model: the page renders at the MASTER CANVAS size (1920×1080), // stable, never tracked. Full-bleed widgets (designed for the canvas) // render fully instead of being clipped to a small box-sized viewport; - // dragging the box never reflows the page. The capture is tightly - // cropped to the widget's ALPHA bounding box (FindContentBounds) and - // the display (Stretch=Fill) maps it flush under the box, anchored at - // (0,0) — no DOM query, so no timing fragility; worst case is the - // full frame, which is the confirmed-good rendering. + // dragging the box never reflows the page. The alpha bounding box + // (FindContentBounds) measures the widget's extent for the UI/selection; + // the COMPOSITE consumes the FULL canvas — the element rect is a viewport + // onto the page, whose transparent margins reveal the layers beneath. var webView = new WebView2 { Width = 1920, @@ -190,11 +209,15 @@ public sealed class WebView2Manager : IDisposable // The widget's extent is measured from the captured frame itself: the // bounding box of non-transparent pixels (the background is injected - // transparent, so the alpha channel IS the widget). This is immune to - // DOM timing (fonts/images/animations), needs no JS round-trip, and - // CANNOT break rendering: if the widget fills the canvas the box equals - // the full frame (identical to the confirmed-good uncropped display); if - // it floats in transparent margin, the crop maps it flush under the box. + // transparent, so the alpha channel IS the widget). Immune to DOM timing + // (fonts/images/animations), needs no JS round-trip, and CANNOT break the + // render: if the widget fills the canvas the box equals the full frame; if + // it floats in transparent margin the box is the measured content extent — + // used ONLY for the preview bitmap / selection. The frame handed to the + // compositor is always the FULL canvas, so the page's transparent margins + // keep revealing the layers beneath (the take-19-2/take-20 "black square + // over the webcam" was the CROP fed to the compositor: UniformToFill then + // zoomed the opaque content box to cover the whole element rect). internal static (int X, int Y, int W, int H) FindContentBounds(byte[] pixels, int pixW, int pixH, int stride) { int minX = pixW, minY = pixH, maxX = -1, maxY = -1; @@ -245,9 +268,7 @@ public sealed class WebView2Manager : IDisposable if (pixW < 1 || pixH < 1) return; var need = pixW * pixH * 4; - var pixels = session.CanvasScratch is { } scratch && scratch.Length >= need - ? scratch - : (session.CanvasScratch = new byte[need]); + var pixels = session.RentCanvasBuffer(need); formatted.CopyPixels(pixels, pixW * 4, 0); var stride = pixW * 4; @@ -266,7 +287,10 @@ public sealed class WebView2Manager : IDisposable session.PreviewBitmap = new WriteableBitmap(cropW, cropH, 96, 96, PixelFormats.Bgra32, null); session.PreviewBitmap.WritePixels(new Int32Rect(0, 0, cropW, cropH), outPixels, cropW * 4, 0); - session.LatestFrame = new VideoFrame(cropW, cropH, outPixels) { Epoch = ++session.Epoch }; + // The compositor consumes the FULL canvas (alpha intact): the element rect + // scales the page region onto itself and the transparent margins reveal what + // is beneath. The alpha crop above exists only for the preview/selection. + session.LatestFrame = new VideoFrame(pixW, pixH, pixels) { Epoch = ++session.Epoch }; // The preview bitmap is now MUTABLE and reused (CameraManager precedent): // WPF re-renders it after WritePixels; handlers must not Freeze it. diff --git a/ai.md b/ai.md index 1c5d539..a9267bc 100644 --- a/ai.md +++ b/ai.md @@ -27,6 +27,15 @@ This project's hard lesson: the web-source bounding saga (2026-08) burned 9 commits rediscovering that a browser source is a *fixed canvas* — OBS keeps it at a stable page size, crops/hugs content via the box, and clips at the edge. The apparent "gap" that ended the saga was a widget's own CSS glow effect, not a bug. +Refined 2026-09-05 (take-19-2/take-20 "webcam black square"): **measure with the +alpha crop, but CONSUME the full canvas.** The alpha bounding box (`FindContentBounds`) +is the widget's true extent for the UI/selection box; the frame handed to the +compositor must be the FULL transparent-background canvas, because the element rect +is a viewport onto the page — `UniformToFill` on the CROP zoomed the opaque content +box to cover the whole element rect (a hard box over the webcam) the moment the +widget drew any content. Stateful take-1-good/take-2-bad: an idle transparent page +produces a full-frame crop (≈ the correct full-canvas viewport), any real content +produced a tight crop + zoom. **Derived-solution rule (2026-08-29, driven by the image-shrink incident):** when you work out any one-off, reusable solution (recipe / workaround / how-to), write diff --git a/ytLive.Tests/SceneCompositorTests.cs b/ytLive.Tests/SceneCompositorTests.cs index d160e34..00375b9 100644 --- a/ytLive.Tests/SceneCompositorTests.cs +++ b/ytLive.Tests/SceneCompositorTests.cs @@ -114,6 +114,70 @@ public class SceneCompositorTests AssertColor(output, 550, 700, 255, 255, 255); } + /// The ONE integration test for the web-overlay transparency regression + /// (take-19-2/take-20 "black square over the webcam"): a FULL-canvas web frame + /// (the page background is transparent; the widget content lives at its canvas + /// position) is composited into a webcam-covered scene. The element rect is a + /// viewport onto the page, so the page's transparent margins must REVEAL the + /// layers beneath (the webcam), while the widget's opaque content still draws. + /// Feeding the ALPHA CROP instead of the full canvas zoomed the opaque content + /// box to cover the whole element rect — this test pins the reveal contract. + [Fact] + public void Composite_FullCanvasWebSource_TransparentMarginsRevealWebcam() + { + var red = Solid(1920, 1080, 255, 0, 0); + var green = Solid(1280, 720, 0, 255, 0); + var blue = Solid(1920, 1080, 0, 0, 255); + + // FULL-canvas web frame (what WebView2Manager hands to the compositor): + // transparent everywhere except an opaque badge at its canvas position. + var webPixels = new byte[1920 * 1080 * 4]; + for (var y = 800; y < 880; y++) + for (var x = 1400; x < 1520; x++) + { + var i = (y * 1920 + x) * 4; + webPixels[i] = blue.BgraPixels[i]; + webPixels[i + 1] = blue.BgraPixels[i + 1]; + webPixels[i + 2] = blue.BgraPixels[i + 2]; + webPixels[i + 3] = 255; + } + var webCanvas = new VideoFrame(1920, 1080, webPixels); + + var background = new Source { Type = SourceType.DisplayCapture, IsBackground = true, CaptureKey = "monitor:0" }; + var webcam = new WebcamSceneConfig { X = 1360, Y = 760, Width = 536, Height = 294, ClipShape = ClipShape.Traditional }; + var webSource = new Source { Type = SourceType.WebSource, X = 1279, Y = 709, Width = 703, Height = 389 }; + + var scene = new Scene { Name = "Live" }; + scene.Elements.Add(background); + scene.Elements.Add(webcam); + scene.Elements.Add(webSource); + + VideoFrame? FrameFor(SceneElement e) => e switch + { + WebcamSceneConfig => green, + Source { IsBackground: true } => red, + Source { Type: SourceType.WebSource } => webCanvas, + _ => null, + }; + + var options = new CompositorOptions + { + SourceRectX = 0, SourceRectY = 0, SourceRectWidth = 1920, SourceRectHeight = 1080, + OutputWidth = 1920, OutputHeight = 1080, + }; + + var output = new SceneCompositor().Render(scene, FrameFor, null, options); + + // (1777,984) ≈ canvas (1360,760): inside the web element rect AND inside the + // webcam rect, but in the page's transparent margin -> the WEBCAM must show. + AssertColor(output, 1777, 984, 0, 255, 0); + // (1810,1010) ≈ canvas (1450,825): inside the widget's opaque badge -> the + // widget content draws over the webcam. + AssertColor(output, 1810, 1010, 0, 0, 255); + // deep margin north of the webcam: the backdrop shows through untouched. + AssertColor(output, 1300, 720, 255, 0, 0); + } + /// The take-4 memcpy fast path: a full-cover opaque backdrop (the live /// capture's contract) must reproduce the source EXACTLY, and when supplied via the /// scratch pool parameter the render must overwrite every byte of the reused