diff --git a/MyMistakes.md b/MyMistakes.md index 40f5bf7..4d1f467 100644 --- a/MyMistakes.md +++ b/MyMistakes.md @@ -15,40 +15,7 @@ ## 🔬 Recipes registry -### Web-overlay transparency: measure with the alpha crop, COMPOSE with the full canvas - -(2026-09-05, take-19-2/take-20 — "the webcam is a black square under the web widget". - -**Symptom:** WebView2-based web overlays (StreamElements alert hosts etc.) rendered -with hard opaque boxes over the webcam in the RECORDING but the preview looked fine, -and the FIRST take after a rebuild was good but later takes went opaque — a -stateful-looking bug. - -**Root cause:** the capture was alpha-CROPPED to the widget's content bounding box -(`FindContentBounds` — every non-zero-alpha pixel) and that CROP was handed to the -compositor, which `UniformToFill`-scales a source into the element rect. A crop has -NO transparent margins by construction, so the composite zoomed the opaque content -to cover the ENTIRE element rect (hard box!) whenever the widget drew anything. It -looked stateful because an idle/transparent page yields a full-frame crop ≈ the -correct full-canvas viewport (good), while any real content yields a tight crop + -zoom (bad). - -**The rule (OBS browser-source model):** the page is a FIXED canvas (1920×1080 here); -the element rect is a VIEWPORT onto it. Measure/select with the alpha crop; hand the -composite the FULL canvas with its transparent background intact — `UniformToFill` -then maps the page region 1:1 into the element rect and the page's transparent -margins reveal the layers beneath (the webcam). When feeding a compositor that -scales sources to fill a rect, NEVER feed it a crop of your source unless the rect -is meant to re-frame it. - -Recipe: -1. Keep `FindContentBounds` for the preview bitmap / selection bounding box. -2. Feed the compositor a full-canvas frame whose background is alpha 0, from an - 8-deep ring (same identity/Epoch discipline as every producer — a single reused - scratch array tears under the consumer). -3. Pin the reveal contract with a compositor test: full-canvas transparent-margin - frame over a webcam rect → margin pixel = webcam color, opaque badge pixel = - widget color. +### Shrink / re-encode an image for the README (screenshots → small hero image) Worked out 2026-08-29 (the recipe was NEVER recorded the first time it was done, so it had to be re-derived from scratch — that's the incident this entry exists to end). diff --git a/Services/WebView2Manager.cs b/Services/WebView2Manager.cs index 5595174..409670f 100644 --- a/Services/WebView2Manager.cs +++ b/Services/WebView2Manager.cs @@ -33,36 +33,16 @@ public sealed class WebView2Manager : IDisposable // Buffer reuse (2026-09-04, same lesson as the capture/camera rings): every // capture tick used to mint a fresh 8.3MB canvas array + a fresh crop array // (~83MB/s of LOH at 10Hz → gen2 pauses surfaced as the pump's 35-40ms - // "worst render" spikes). The full-canvas slots rotate an 8-deep ring with a - // monotonic Epoch (identity-keyed consumers — the compositor's paste cache — - // must see each generation); the crop slots feed the preview WriteableBitmap, - // recreated only when the cropped size changes. - public readonly byte[]?[] CanvasRing = new byte[8][]; - public int CanvasRingNext; - public long Epoch; + // "worst render" spikes). Canvas scratch is reused while the size holds; + // the frame handed to the compositor rotates an 8-deep ring with a monotonic + // Epoch (identity-keyed consumers must see each generation). The preview + // WriteableBitmap is recreated only when the cropped size changes. + public byte[]? CanvasScratch; public readonly byte[]?[] OutRing = new byte[8][]; public int OutRingNext; + public long Epoch; public WriteableBitmap? PreviewBitmap; - public byte[] RentCanvasBuffer(int size) - { - for (var tries = 0; tries < CanvasRing.Length; tries++) - { - var idx = (CanvasRingNext + tries) % CanvasRing.Length; - var buf = CanvasRing[idx]; - if (buf != null && buf.Length == size) - { - CanvasRingNext = (idx + 1) % CanvasRing.Length; - return buf; - } - } - var slot = CanvasRingNext; - CanvasRingNext = (slot + 1) % CanvasRing.Length; - var fresh = new byte[size]; - CanvasRing[slot] = fresh; - return fresh; - } - public byte[] RentOutBuffer(int size) { for (var tries = 0; tries < OutRing.Length; tries++) @@ -96,10 +76,11 @@ public sealed class WebView2Manager : IDisposable // OBS model: the page renders at the MASTER CANVAS size (1920×1080), // stable, never tracked. Full-bleed widgets (designed for the canvas) // render fully instead of being clipped to a small box-sized viewport; - // dragging the box never reflows the page. The alpha bounding box - // (FindContentBounds) measures the widget's extent for the UI/selection; - // the COMPOSITE consumes the FULL canvas — the element rect is a viewport - // onto the page, whose transparent margins reveal the layers beneath. + // dragging the box never reflows the page. The capture is tightly + // cropped to the widget's ALPHA bounding box (FindContentBounds) and + // the display (Stretch=Fill) maps it flush under the box, anchored at + // (0,0) — no DOM query, so no timing fragility; worst case is the + // full frame, which is the confirmed-good rendering. var webView = new WebView2 { Width = 1920, @@ -209,15 +190,11 @@ public sealed class WebView2Manager : IDisposable // The widget's extent is measured from the captured frame itself: the // bounding box of non-transparent pixels (the background is injected - // transparent, so the alpha channel IS the widget). Immune to DOM timing - // (fonts/images/animations), needs no JS round-trip, and CANNOT break the - // render: if the widget fills the canvas the box equals the full frame; if - // it floats in transparent margin the box is the measured content extent — - // used ONLY for the preview bitmap / selection. The frame handed to the - // compositor is always the FULL canvas, so the page's transparent margins - // keep revealing the layers beneath (the take-19-2/take-20 "black square - // over the webcam" was the CROP fed to the compositor: UniformToFill then - // zoomed the opaque content box to cover the whole element rect). + // transparent, so the alpha channel IS the widget). This is immune to + // DOM timing (fonts/images/animations), needs no JS round-trip, and + // CANNOT break rendering: if the widget fills the canvas the box equals + // the full frame (identical to the confirmed-good uncropped display); if + // it floats in transparent margin, the crop maps it flush under the box. internal static (int X, int Y, int W, int H) FindContentBounds(byte[] pixels, int pixW, int pixH, int stride) { int minX = pixW, minY = pixH, maxX = -1, maxY = -1; @@ -268,7 +245,9 @@ public sealed class WebView2Manager : IDisposable if (pixW < 1 || pixH < 1) return; var need = pixW * pixH * 4; - var pixels = session.RentCanvasBuffer(need); + var pixels = session.CanvasScratch is { } scratch && scratch.Length >= need + ? scratch + : (session.CanvasScratch = new byte[need]); formatted.CopyPixels(pixels, pixW * 4, 0); var stride = pixW * 4; @@ -287,10 +266,7 @@ public sealed class WebView2Manager : IDisposable session.PreviewBitmap = new WriteableBitmap(cropW, cropH, 96, 96, PixelFormats.Bgra32, null); session.PreviewBitmap.WritePixels(new Int32Rect(0, 0, cropW, cropH), outPixels, cropW * 4, 0); - // The compositor consumes the FULL canvas (alpha intact): the element rect - // scales the page region onto itself and the transparent margins reveal what - // is beneath. The alpha crop above exists only for the preview/selection. - session.LatestFrame = new VideoFrame(pixW, pixH, pixels) { Epoch = ++session.Epoch }; + session.LatestFrame = new VideoFrame(cropW, cropH, outPixels) { Epoch = ++session.Epoch }; // The preview bitmap is now MUTABLE and reused (CameraManager precedent): // WPF re-renders it after WritePixels; handlers must not Freeze it. diff --git a/ai.md b/ai.md index a9267bc..1c5d539 100644 --- a/ai.md +++ b/ai.md @@ -27,15 +27,6 @@ This project's hard lesson: the web-source bounding saga (2026-08) burned 9 commits rediscovering that a browser source is a *fixed canvas* — OBS keeps it at a stable page size, crops/hugs content via the box, and clips at the edge. The apparent "gap" that ended the saga was a widget's own CSS glow effect, not a bug. -Refined 2026-09-05 (take-19-2/take-20 "webcam black square"): **measure with the -alpha crop, but CONSUME the full canvas.** The alpha bounding box (`FindContentBounds`) -is the widget's true extent for the UI/selection box; the frame handed to the -compositor must be the FULL transparent-background canvas, because the element rect -is a viewport onto the page — `UniformToFill` on the CROP zoomed the opaque content -box to cover the whole element rect (a hard box over the webcam) the moment the -widget drew any content. Stateful take-1-good/take-2-bad: an idle transparent page -produces a full-frame crop (≈ the correct full-canvas viewport), any real content -produced a tight crop + zoom. **Derived-solution rule (2026-08-29, driven by the image-shrink incident):** when you work out any one-off, reusable solution (recipe / workaround / how-to), write diff --git a/ytLive.Tests/SceneCompositorTests.cs b/ytLive.Tests/SceneCompositorTests.cs index 00375b9..d160e34 100644 --- a/ytLive.Tests/SceneCompositorTests.cs +++ b/ytLive.Tests/SceneCompositorTests.cs @@ -114,70 +114,6 @@ public class SceneCompositorTests AssertColor(output, 550, 700, 255, 255, 255); } - /// The ONE integration test for the web-overlay transparency regression - /// (take-19-2/take-20 "black square over the webcam"): a FULL-canvas web frame - /// (the page background is transparent; the widget content lives at its canvas - /// position) is composited into a webcam-covered scene. The element rect is a - /// viewport onto the page, so the page's transparent margins must REVEAL the - /// layers beneath (the webcam), while the widget's opaque content still draws. - /// Feeding the ALPHA CROP instead of the full canvas zoomed the opaque content - /// box to cover the whole element rect — this test pins the reveal contract. - [Fact] - public void Composite_FullCanvasWebSource_TransparentMarginsRevealWebcam() - { - var red = Solid(1920, 1080, 255, 0, 0); - var green = Solid(1280, 720, 0, 255, 0); - var blue = Solid(1920, 1080, 0, 0, 255); - - // FULL-canvas web frame (what WebView2Manager hands to the compositor): - // transparent everywhere except an opaque badge at its canvas position. - var webPixels = new byte[1920 * 1080 * 4]; - for (var y = 800; y < 880; y++) - for (var x = 1400; x < 1520; x++) - { - var i = (y * 1920 + x) * 4; - webPixels[i] = blue.BgraPixels[i]; - webPixels[i + 1] = blue.BgraPixels[i + 1]; - webPixels[i + 2] = blue.BgraPixels[i + 2]; - webPixels[i + 3] = 255; - } - var webCanvas = new VideoFrame(1920, 1080, webPixels); - - var background = new Source { Type = SourceType.DisplayCapture, IsBackground = true, CaptureKey = "monitor:0" }; - var webcam = new WebcamSceneConfig { X = 1360, Y = 760, Width = 536, Height = 294, ClipShape = ClipShape.Traditional }; - var webSource = new Source { Type = SourceType.WebSource, X = 1279, Y = 709, Width = 703, Height = 389 }; - - var scene = new Scene { Name = "Live" }; - scene.Elements.Add(background); - scene.Elements.Add(webcam); - scene.Elements.Add(webSource); - - VideoFrame? FrameFor(SceneElement e) => e switch - { - WebcamSceneConfig => green, - Source { IsBackground: true } => red, - Source { Type: SourceType.WebSource } => webCanvas, - _ => null, - }; - - var options = new CompositorOptions - { - SourceRectX = 0, SourceRectY = 0, SourceRectWidth = 1920, SourceRectHeight = 1080, - OutputWidth = 1920, OutputHeight = 1080, - }; - - var output = new SceneCompositor().Render(scene, FrameFor, null, options); - - // (1777,984) ≈ canvas (1360,760): inside the web element rect AND inside the - // webcam rect, but in the page's transparent margin -> the WEBCAM must show. - AssertColor(output, 1777, 984, 0, 255, 0); - // (1810,1010) ≈ canvas (1450,825): inside the widget's opaque badge -> the - // widget content draws over the webcam. - AssertColor(output, 1810, 1010, 0, 0, 255); - // deep margin north of the webcam: the backdrop shows through untouched. - AssertColor(output, 1300, 720, 255, 0, 0); - } - /// The take-4 memcpy fast path: a full-cover opaque backdrop (the live /// capture's contract) must reproduce the source EXACTLY, and when supplied via the /// scratch pool parameter the render must overwrite every byte of the reused