gramps
a11b15e444
TASK 43's alert box grows a real video celebration. Per-alert IAlertClipDecoder
(ffmpeg bgra + f32le pipes, real-time paced, disposed at drain) plays the shipped
Assets/alert-default.mp4 (stamped into the Asset table at startup) unless the
creator picks their own file — path reference only, never stored in the DB; the
six AlertRenderer animations stay the fallback. ~0.3s fade rides the alpha
envelope on straight-source copies (EOF freeze-frames then fades out); audio
forwards to a new AudioMixer alert ring (8s, 48k stereo) drained at unity — no
duck, creator ruling — scaled by volume × fade. An auto-composed marquee ticker
('Funder — Super Chat · $10.00', 140px/s) scrolls top-of-frame via a
FramePump._alertTicker seam through Render/CompositeLayers, mixed into the cache
signature (dynamic overlay, never baked). New Stream Alerts section in LeftPanel.
Derivative-work references (how OBS/Streamlabs alert boxes do per-alert video):
- https://support.streamlabs.com/hc/en-us/articles/217741147-Setting-Up-Your-Streamlabs-Alerts (custom image/video per alert type + variations)
- https://obsproject.com/kb/stream-tutorial-2-alerts (alert overlay as an on-screen zone)
- https://streamlabs.com/content-hub/widgets/alert-box (per-event alert playback)
Good Dog: AlertLayerVideoTests drives a fake IAlertClipDecoder through the whole
lifecycle in one pass (custom path wins, decoder spawns/disposes, fade envelope
0→127→255, audio volume×fade, ticker scrolls, EOF fade-drain to idle). It caught
the clip branch of Advance not clearing _current before AdvanceToNext — the layer
stayed IsPlaying after drain (MyMistakes post-mortem).
Full vstest 319/319; clean build 0 warnings; scope check green.
758 lines
34 KiB
C#
758 lines
34 KiB
C#
using ytLive.Helpers;
|
||
using ytLive.Models;
|
||
|
||
namespace ytLive.Services.Compositor;
|
||
|
||
/// <summary>
|
||
/// The output compositor: renders a scene into the encoder's master frame (tightly-packed
|
||
/// BGRA8 <see cref="VideoFrame"/>) exactly as the XAML preview renders it, minus the
|
||
/// editing chrome (SelectionOverlay, DimRects, badge, placeholder). The preview stays the
|
||
/// editing view; this is the output view — see TASKS.md "Ship step 1 — Scene compositor".
|
||
///
|
||
/// Frame sources are supplied by a <see cref="Func{SceneElement, VideoFrame}"/> resolver
|
||
/// (the caller maps webcam → DeviceId, images → AssetId, background → CaptureKey), keeping
|
||
/// the compositor pure and free of WPF and of the capture managers.
|
||
/// </summary>
|
||
public sealed class SceneCompositor
|
||
{
|
||
// Paste cache (take-8 fix, 2026-09-04): the Live scene re-sampled ~680k bilinear
|
||
// pixels EVERY tick even though only the camera actually changes at that rate —
|
||
// chat/web/image layers are identical frame after frame. Same shape as OBS (the
|
||
// source surface is cached, the compositor blits): a scaled/round-clipped/mirrored
|
||
// layer rasterizes once and is pasted (row-copy/blend) on every later tick.
|
||
// Keyed by the SOURCE FRAME IDENTITY (array + Epoch + CropBounds) — producers hand
|
||
// out fresh immutable byte[], or recycled-ring arrays whose monotonic Epoch distinguishes
|
||
// each generation (screen capture, take-11 fix). CropBounds must be in the key because
|
||
// the same frame (same Pixels/Epoch/Width/Height) can have different CropBounds as
|
||
// the element is resized — a stale cached raster with wrong crop pixels would corrupt
|
||
// the output. Bounds-checked eviction clears wholesale.
|
||
private sealed record PasteKey(byte[] Pixels, long Epoch, int SrcW, int SrcH, int DstW, int DstH, bool Round, bool Mirror, (int X, int Y, int W, int H)? CropBounds);
|
||
private static readonly Dictionary<PasteKey, VideoFrame> _pasteCache = new();
|
||
private static readonly Dictionary<SceneElement, VideoFrame> _elementRasters = new(ReferenceEqualityComparer.Instance);
|
||
private static readonly object _pasteGate = new();
|
||
private const int MaxPasteEntries = 256;
|
||
|
||
/// <summary>
|
||
/// Composite <paramref name="scene"/> into the tier's output frame. Layer order (back →
|
||
/// front): live background (the scene's <c>IsBackground</c> source) → background image →
|
||
/// visible elements (z-order = <c>Elements</c> order, mirroring the XAML DataTemplate) →
|
||
/// branding flash → social bar (a global overlay; spans the master width at
|
||
/// <paramref name="socialBarTop"/>) → stream-alert ticker (a global overlay pinned to
|
||
/// the very top, TASK 47). Transparent regions read opaque black.
|
||
/// </summary>
|
||
public VideoFrame Render(
|
||
Scene scene,
|
||
Func<SceneElement, VideoFrame?> frameFor,
|
||
VideoFrame? flashFrame,
|
||
CompositorOptions options,
|
||
VideoFrame? socialBarFrame = null,
|
||
int socialBarTop = 0,
|
||
VideoFrame? staticBase = null,
|
||
int split = -1,
|
||
byte[]? scratch = null,
|
||
VideoFrame? tickerFrame = null)
|
||
{
|
||
if (scene == null) throw new ArgumentNullException(nameof(scene));
|
||
if (frameFor == null) throw new ArgumentNullException(nameof(frameFor));
|
||
if (options == null) throw new ArgumentNullException(nameof(options));
|
||
if (options.SourceRectWidth <= 0 || options.SourceRectHeight <= 0)
|
||
throw new ArgumentException("The source rect must be positive.", nameof(options));
|
||
if (options.OutputWidth <= 0 || options.OutputHeight <= 0)
|
||
throw new ArgumentException("The output size must be positive.", nameof(options));
|
||
|
||
var cropW = options.SourceRectWidth;
|
||
var cropH = options.SourceRectHeight;
|
||
var buffer = scratch != null && scratch.Length == cropW * cropH * 4
|
||
? scratch
|
||
: new byte[cropW * cropH * 4];
|
||
|
||
// staticBase != null: the caller pre-baked all static layers below `split`.
|
||
// We copy it (never mutate the cached base) and composite elements[split..] on top.
|
||
if (staticBase != null)
|
||
{
|
||
if (staticBase.Width != cropW || staticBase.Height != cropH)
|
||
throw new ArgumentException("The static base must match the source rect.", nameof(staticBase));
|
||
Buffer.BlockCopy(staticBase.BgraPixels, 0, buffer, 0, buffer.Length);
|
||
}
|
||
else
|
||
{
|
||
if (split < 0) split = scene.Elements.Count;
|
||
|
||
var elements = scene.Elements;
|
||
var liveCapture = elements.OfType<Source>().FirstOrDefault(s => s.IsBackground);
|
||
var liveCaptureFrame = liveCapture != null && liveCapture.IsVisible ? frameFor(liveCapture) : null;
|
||
|
||
// A master-sized live backdrop paints every pixel of the base — the
|
||
// opaque-black pre-fill (2M byte writes) would be dead work. Capture
|
||
// frames are opaque BGRA by contract (the WinRT path ignores alpha),
|
||
// so the 1:1 row walk covers the full rect.
|
||
var covered = liveCaptureFrame != null
|
||
&& liveCaptureFrame.Width == cropW && liveCaptureFrame.Height == cropH;
|
||
if (!covered)
|
||
for (var i = 3; i < buffer.Length; i += 4)
|
||
buffer[i] = 255; // opaque black base
|
||
|
||
if (liveCaptureFrame != null)
|
||
BlitContent(buffer, cropW, cropH, 0, 0, cropW, cropH, liveCaptureFrame, 1f, false, false);
|
||
|
||
var background = elements.OfType<Source>().FirstOrDefault(s => s.Type == SourceType.Background);
|
||
var backgroundFrame = background != null ? frameFor(background) : null;
|
||
if (backgroundFrame != null)
|
||
BlitContent(buffer, cropW, cropH, 0, 0, cropW, cropH, backgroundFrame, 1f, false, false);
|
||
|
||
foreach (var element in elements)
|
||
{
|
||
if (!element.IsVisible) continue;
|
||
switch (element)
|
||
{
|
||
case Source { IsBackground: true }:
|
||
case Source { Type: SourceType.Background }:
|
||
case Source { Type: SourceType.TextOverlay }:
|
||
continue;
|
||
}
|
||
var fullFrame = frameFor(element);
|
||
if (fullFrame == null) continue;
|
||
BlitBakedElement(buffer, cropW, cropH, options, element, fullFrame);
|
||
}
|
||
}
|
||
|
||
// Composite the above-split layers (dynamic + any static above them) per frame.
|
||
for (var i = Math.Max(0, split); i < scene.Elements.Count; i++)
|
||
{
|
||
var element = scene.Elements[i];
|
||
if (!element.IsVisible) continue;
|
||
switch (element)
|
||
{
|
||
case Source { IsBackground: true }:
|
||
case Source { Type: SourceType.Background }:
|
||
case Source { Type: SourceType.TextOverlay }:
|
||
continue;
|
||
}
|
||
var frame = frameFor(element);
|
||
if (frame == null) continue;
|
||
BlitBakedElement(buffer, cropW, cropH, options, element, frame);
|
||
}
|
||
|
||
if (flashFrame != null)
|
||
BlitOverlay(buffer, cropW, cropH, options, flashFrame, 0, 0);
|
||
|
||
if (socialBarFrame != null && socialBarFrame.Width > 0 && socialBarFrame.Height > 0)
|
||
BlitOverlay(buffer, cropW, cropH, options, socialBarFrame, 0, socialBarTop);
|
||
|
||
if (tickerFrame != null && tickerFrame.Width > 0 && tickerFrame.Height > 0)
|
||
BlitOverlay(buffer, cropW, cropH, options, tickerFrame, 0, 0);
|
||
|
||
return StretchMath.BilinearScale(
|
||
new VideoFrame(cropW, cropH, buffer), options.OutputWidth, options.OutputHeight);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Render the static base of a scene: the opaque black layer, the live-capture
|
||
/// background, the background image, and every <paramref name="split"/> z-layers
|
||
/// below the split point (all of which are static by construction — see
|
||
/// <see cref="SceneElement.Kind"/>). The result is a source-rect sized frame that
|
||
/// <see cref="Render"/> can start from and composite the dynamic/above-split layers
|
||
/// onto each tick. Returns null when there is no static layer at all.
|
||
/// </summary>
|
||
public static VideoFrame? BakeStaticBase(
|
||
Scene scene,
|
||
int split,
|
||
Func<SceneElement, VideoFrame?> frameFor,
|
||
Func<CompositorOptions> options,
|
||
VideoFrame? socialBarFrame = null,
|
||
int socialBarTop = 0,
|
||
VideoFrame? flashFrame = null)
|
||
{
|
||
if (split <= 0) return null;
|
||
var opts = options();
|
||
if (opts.SourceRectWidth <= 0 || opts.SourceRectHeight <= 0) return null;
|
||
|
||
var cropW = opts.SourceRectWidth;
|
||
var cropH = opts.SourceRectHeight;
|
||
var buffer = new byte[cropW * cropH * 4];
|
||
for (var i = 3; i < buffer.Length; i += 4)
|
||
buffer[i] = 255;
|
||
|
||
var elements = scene.Elements;
|
||
|
||
var liveCapture = elements.OfType<Source>().FirstOrDefault(s => s.IsBackground);
|
||
if (liveCapture != null)
|
||
{
|
||
var liveCaptureFrame = liveCapture.IsVisible ? frameFor(liveCapture) : null;
|
||
if (liveCaptureFrame != null && split > elements.IndexOf(liveCapture))
|
||
BlitContent(buffer, cropW, cropH, 0, 0, cropW, cropH, liveCaptureFrame, 1f, false, false);
|
||
}
|
||
|
||
var background = elements.OfType<Source>().FirstOrDefault(s => s.Type == SourceType.Background);
|
||
if (background != null)
|
||
{
|
||
var backgroundFrame = frameFor(background);
|
||
if (backgroundFrame != null && split > elements.IndexOf(background))
|
||
BlitContent(buffer, cropW, cropH, 0, 0, cropW, cropH, backgroundFrame, 1f, false, false);
|
||
}
|
||
|
||
for (var i = 0; i < split; i++)
|
||
{
|
||
var element = scene.Elements[i];
|
||
if (!element.IsVisible) continue;
|
||
switch (element)
|
||
{
|
||
case Source { IsBackground: true }:
|
||
case Source { Type: SourceType.Background }:
|
||
case Source { Type: SourceType.TextOverlay }:
|
||
continue;
|
||
}
|
||
var frame = frameFor(element);
|
||
if (frame == null) continue;
|
||
BlitBakedElement(buffer, cropW, cropH, opts, element, frame);
|
||
}
|
||
|
||
var baked = new VideoFrame(cropW, cropH, buffer);
|
||
if (flashFrame != null)
|
||
baked = Overlay(baked, opts, flashFrame, 0, 0);
|
||
if (socialBarFrame != null && socialBarFrame.Width > 0 && socialBarFrame.Height > 0)
|
||
baked = Overlay(baked, opts, socialBarFrame, 0, socialBarTop);
|
||
return baked;
|
||
}
|
||
|
||
/// <summary>The compositor's dynamic/above-split path: start from an existing frame
|
||
/// buffer and composite the z-layers from <paramref name="split"/> onward on top.
|
||
/// When <paramref name="baseFrame"/> is null the caller must fall back to full render.
|
||
/// The social bar never reaches here (it IS baked into the static base) but the alert
|
||
/// ticker must: it is dynamic by construction — only ever present while an alert plays —
|
||
/// so it overlays LAST, pinned to the very top (TASK 47).</summary>
|
||
public static VideoFrame CompositeLayers(
|
||
VideoFrame baseFrame,
|
||
Scene scene,
|
||
int split,
|
||
Func<SceneElement, VideoFrame?> frameFor,
|
||
CompositorOptions options,
|
||
VideoFrame? socialBarFrame = null,
|
||
int socialBarTop = 0,
|
||
VideoFrame? flashFrame = null,
|
||
byte[]? scratch = null,
|
||
VideoFrame? tickerFrame = null)
|
||
{
|
||
var cropW = options.SourceRectWidth;
|
||
var cropH = options.SourceRectHeight;
|
||
// We must not mutate the cached base: copy it into a fresh (or pooled) buffer.
|
||
var buffer = scratch != null && scratch.Length == baseFrame.BgraPixels.Length
|
||
? scratch
|
||
: new byte[baseFrame.BgraPixels.Length];
|
||
Buffer.BlockCopy(baseFrame.BgraPixels, 0, buffer, 0, baseFrame.BgraPixels.Length);
|
||
|
||
for (var i = split; i < scene.Elements.Count; i++)
|
||
{
|
||
var element = scene.Elements[i];
|
||
if (!element.IsVisible) continue;
|
||
switch (element)
|
||
{
|
||
case Source { IsBackground: true }:
|
||
case Source { Type: SourceType.Background }:
|
||
case Source { Type: SourceType.TextOverlay }:
|
||
continue;
|
||
}
|
||
var frame = frameFor(element);
|
||
if (frame == null) continue;
|
||
BlitBakedElement(buffer, cropW, cropH, options, element, frame);
|
||
}
|
||
|
||
if (tickerFrame != null && tickerFrame.Width > 0 && tickerFrame.Height > 0)
|
||
BlitOverlay(buffer, cropW, cropH, options, tickerFrame, 0, 0);
|
||
|
||
return StretchMath.BilinearScale(new VideoFrame(cropW, cropH, buffer), options.OutputWidth, options.OutputHeight);
|
||
}
|
||
|
||
private static void BlitBakedElement(
|
||
byte[] buffer, int cropW, int cropH, CompositorOptions options, SceneElement element, VideoFrame frame)
|
||
{
|
||
var ex = (float)(element.X - options.SourceRectX);
|
||
var ey = (float)(element.Y - options.SourceRectY);
|
||
var ew = (float)element.Width;
|
||
var eh = (float)element.Height;
|
||
var isRound = element.ClipShape == ClipShape.Round;
|
||
|
||
// All rect-placed elements route through the paste cache (take-9 lesson): the
|
||
// IsOpaque bypass kept re-sampling the webcam (~156k px) EVERY tick even when
|
||
// the device frame had not changed; an opaque source rasterizes to an opaque
|
||
// raster whose paste is a per-pixel copy branch, so hits beat the sampler and
|
||
// misses cost exactly what the bypass cost.
|
||
BlitCachedLayer(buffer, cropW, cropH, ex, ey, ew, eh, frame, element, isRound);
|
||
if (element.HasBorder)
|
||
DrawBorder(buffer, cropW, cropH, ex, ey, ew, eh, element, isRound);
|
||
}
|
||
|
||
/// <summary>Non-opaque layer blit through the paste cache: the element-space raster
|
||
/// (scaled to the element rect, round-clipped, mirrored) depends only on the source
|
||
/// frame + rect + shape/mirror — NOT on position or opacity. Static content (chat,
|
||
/// web overlays, images, the media frame between updates) is sampled ONCE and pasted
|
||
/// at integer position with per-tixel blend; only layers whose source frame actually
|
||
/// changed (webcam at device fps) pay resampling.</summary>
|
||
private static void BlitCachedLayer(
|
||
byte[] buffer, int cropW, int cropH,
|
||
float ex, float ey, float ew, float eh,
|
||
VideoFrame frame, SceneElement element, bool isRound)
|
||
{
|
||
if (ew <= 0 || eh <= 0) return;
|
||
var dw = (int)System.Math.Ceiling(ew);
|
||
var dh = (int)System.Math.Ceiling(eh);
|
||
if (dw <= 0 || dh <= 0) return;
|
||
var mirror = element.IsMirrored;
|
||
|
||
PasteKey? key = null;
|
||
VideoFrame? raster = null;
|
||
lock (_pasteGate)
|
||
{
|
||
key = new PasteKey(frame.BgraPixels, frame.Epoch, frame.Width, frame.Height, dw, dh, isRound, mirror, frame.CropBounds);
|
||
_pasteCache.TryGetValue(key, out raster);
|
||
}
|
||
|
||
if (raster == null)
|
||
{
|
||
// Rasterize in ELEMENT space on a TRANSPARENT base: the sampler writes 255
|
||
// alpha only where it actually draws, so untouched margins (outside the
|
||
// round clip, past the UniformToFill edges) stay alpha 0 and the paste
|
||
// reveals the master beneath instead of black. The ARRAY is reused per
|
||
// element (see _elementRasters note) — new source generations re-raster
|
||
// in place, so the steady state allocates nothing.
|
||
lock (_pasteGate)
|
||
{
|
||
if (_elementRasters.TryGetValue(element, out var existing)
|
||
&& existing.Width == dw && existing.Height == dh)
|
||
raster = existing;
|
||
}
|
||
if (raster == null) raster = new VideoFrame(dw, dh, new byte[dw * dh * 4]);
|
||
Array.Clear(raster.BgraPixels);
|
||
BlitContentRaw(raster.BgraPixels, dw, dh, 0, 0, ew, eh, frame, 1f, isRound, mirror, transparentDst: true);
|
||
lock (_pasteGate)
|
||
{
|
||
// Only the SOURCE frame's identity is a real key owner; drop any older
|
||
// entry that shared this element's reused raster so a stale key can never
|
||
// paste a mid-update array.
|
||
if (_pasteCache.Count >= MaxPasteEntries) _pasteCache.Clear();
|
||
_pasteCache[key!] = raster;
|
||
_elementRasters[element] = raster;
|
||
}
|
||
}
|
||
|
||
// Paste at integer position; opacity blends. The raster is never flagged
|
||
// IsOpaque — round clips/margins carry alpha 0 — so the row blend always runs.
|
||
var op = (int)(System.Math.Clamp(element.Opacity, 0f, 1f) * 256);
|
||
if (op <= 0) return;
|
||
var px = (int)System.Math.Floor(ex);
|
||
var py = (int)System.Math.Floor(ey);
|
||
var x0 = System.Math.Max(0, px);
|
||
var y0 = System.Math.Max(0, py);
|
||
var x1 = System.Math.Min(cropW, px + dw);
|
||
var y1 = System.Math.Min(cropH, py + dh);
|
||
if (x0 >= x1 || y0 >= y1) return;
|
||
var d = buffer.AsSpan();
|
||
var s = raster.BgraPixels;
|
||
for (var y = y0; y < y1; y++)
|
||
{
|
||
var si = ((y - py) * dw + (x0 - px)) * 4;
|
||
var len = x1 - x0;
|
||
if (op == 256)
|
||
BlendRowOpaque(d, (y * cropW + x0) * 4, s.AsSpan(si, len * 4), 0, len);
|
||
else
|
||
BlendRowWeighted(d, (y * cropW + x0) * 4, s.AsSpan(si, len * 4), 0, len, op);
|
||
}
|
||
}
|
||
|
||
private static VideoFrame Overlay(VideoFrame frame, CompositorOptions options, VideoFrame overlay, int sx0, int sy0)
|
||
{
|
||
var cropW = options.SourceRectWidth;
|
||
var cropH = options.SourceRectHeight;
|
||
var buffer = new byte[frame.BgraPixels.Length];
|
||
Buffer.BlockCopy(frame.BgraPixels, 0, buffer, 0, buffer.Length);
|
||
BlitOverlay(buffer, cropW, cropH, options, overlay, sx0, sy0);
|
||
return new VideoFrame(cropW, cropH, buffer);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Render a cross-fade blend between two scenes. Used by <see cref="TransitionService"/>
|
||
/// during fade/move transitions: the caller renders each scene individually, then blends
|
||
/// the two results at the given alpha (0 = fully from, 1 = fully to).
|
||
/// </summary>
|
||
public static VideoFrame BlendFrames(VideoFrame from, VideoFrame to, float alpha)
|
||
{
|
||
if (from.Width != to.Width || from.Height != to.Height)
|
||
{
|
||
if (alpha <= 0.5f) return from;
|
||
return to;
|
||
}
|
||
|
||
var count = from.BgraPixels.Length;
|
||
var result = new byte[count];
|
||
var inv = 1f - alpha;
|
||
|
||
for (var i = 0; i < count; i += 4)
|
||
{
|
||
result[i] = (byte)(from.BgraPixels[i] * inv + to.BgraPixels[i] * alpha);
|
||
result[i + 1] = (byte)(from.BgraPixels[i + 1] * inv + to.BgraPixels[i + 1] * alpha);
|
||
result[i + 2] = (byte)(from.BgraPixels[i + 2] * inv + to.BgraPixels[i + 2] * alpha);
|
||
result[i + 3] = 255;
|
||
}
|
||
|
||
return new VideoFrame(from.Width, from.Height, result);
|
||
}
|
||
|
||
/// <summary>
|
||
/// UniformToFill blit of a source frame into element rect (ex, ey, ew, eh), with
|
||
/// straight-alpha source-over, optional round clip (true circle, hard edge) and
|
||
/// horizontal mirror around the element center.
|
||
/// </summary>
|
||
private static void BlitContent(
|
||
byte[] dst, int dstW, int dstH,
|
||
float ex, float ey, float ew, float eh,
|
||
VideoFrame src, float opacity, bool isRound, bool isMirror)
|
||
{
|
||
if (ew <= 0 || eh <= 0) return;
|
||
|
||
var (scale, ox, oy) = StretchMath.UniformToFill(ew, eh, src.Width, src.Height);
|
||
|
||
// 1:1 aligned fast path (the full-frame backdrop, master-sized overlays):
|
||
// bilinear sampling at scale 1 with integer offsets is the identity, so the
|
||
// blit is a row walk, not 2M float samples + rounds. Research-first (2026-09-03,
|
||
// take-3 fix; libyuv's row-based BLEND_NONE pattern —
|
||
// https://chromium.googlesource.com/libyuv/libyuv/): the 258ms/frame render
|
||
// that time-lapsed take 3 lived in the general loop below.
|
||
if (!isRound && !isMirror && scale == 1f && ox == 0f && oy == 0f
|
||
&& ex == MathF.Floor(ex) && ey == MathF.Floor(ey))
|
||
{
|
||
Blit1To1Rows(dst, dstW, dstH, (int)ex, (int)ey, src, opacity);
|
||
return;
|
||
}
|
||
|
||
BlitContentRaw(dst, dstW, dstH, ex, ey, ew, eh, src, opacity, isRound, isMirror);
|
||
}
|
||
|
||
/// <summary>The general scaled/clipped/mirrored sampler (integer fixed-point
|
||
/// bilinear + source-over). Also rasterizes paste-cache entries in ELEMENT space
|
||
/// onto a TRANSPARENT base (alpha 0) so pasting over the master keeps the round
|
||
/// clip and transparency of the untouched margins.</summary>
|
||
private static void BlitContentRaw(
|
||
byte[] dst, int dstW, int dstH,
|
||
float ex, float ey, float ew, float eh,
|
||
VideoFrame src, float opacity, bool isRound, bool isMirror,
|
||
bool transparentDst = false)
|
||
{
|
||
if (ew <= 0 || eh <= 0) return;
|
||
|
||
float scale, ox, oy, invScale;
|
||
float drawnW, drawnH;
|
||
// No CropBounds means "the whole source canvas is the crop region" — cbW/cbH
|
||
// MUST default to the full source dims, not 0. A zero cbW/cbH makes every
|
||
// sxCrop/syCrop below evaluate to 0 regardless of position, collapsing the
|
||
// entire sampled rect to source pixel (0,0) (the webcam gray-block regression,
|
||
// 2026-09-12: introduced by the CropBounds Fill-style scaling work the same
|
||
// day — CropBounds-bearing sources, i.e. the web widget, were never affected).
|
||
var cbX = 0;
|
||
var cbY = 0;
|
||
var cbW = src.Width;
|
||
var cbH = src.Height;
|
||
|
||
if (src.CropBounds is { } cb)
|
||
{
|
||
cbX = cb.X;
|
||
cbY = cb.Y;
|
||
cbW = cb.W;
|
||
cbH = cb.H;
|
||
scale = Math.Max(ew / cbW, eh / cbH);
|
||
invScale = 1f / scale;
|
||
drawnW = cbW * scale;
|
||
drawnH = cbH * scale;
|
||
ox = (ew - drawnW) / 2f;
|
||
oy = (eh - drawnH) / 2f;
|
||
}
|
||
else
|
||
{
|
||
var (uScale, uOx, uOy) = StretchMath.UniformToFill(ew, eh, src.Width, src.Height);
|
||
scale = uScale;
|
||
invScale = 1f / uScale;
|
||
ox = uOx;
|
||
oy = uOy;
|
||
drawnW = src.Width * uScale;
|
||
drawnH = src.Height * uScale;
|
||
}
|
||
|
||
var radius = Math.Min(ew, eh) / 2f;
|
||
var cx = ew / 2f;
|
||
var cy = eh / 2f;
|
||
|
||
var x0 = Math.Max(0, (int)Math.Floor(ex));
|
||
var y0 = Math.Max(0, (int)Math.Floor(ey));
|
||
var x1 = Math.Min(dstW - 1, (int)Math.Ceiling(ex + ew));
|
||
var y1 = Math.Min(dstH - 1, (int)Math.Ceiling(ey + eh));
|
||
if (x0 > x1 || y0 > y1) return;
|
||
|
||
var op = (int)(Math.Clamp(opacity, 0f, 1f) * 256);
|
||
if (op <= 0) return;
|
||
var s = src.BgraPixels;
|
||
var srcStride = src.Width; // canvas stride in pixels (not bytes)
|
||
|
||
for (var y = y0; y <= y1; y++)
|
||
{
|
||
var py = y - ey;
|
||
if (py < oy || py > oy + drawnH) continue;
|
||
|
||
// syNorm: normalized [0,1] position within the drawn content (vertically)
|
||
var syNorm = Math.Clamp((py - oy) / drawnH, 0f, 1f);
|
||
// syCrop: pixel position within the crop region
|
||
var syCrop = syNorm * cbH;
|
||
// syCanvas: canvas pixel row (for buffer indexing)
|
||
var syCanvas = Math.Clamp(cbY + syCrop, 0f, src.Height - 1f);
|
||
var syi = (int)syCanvas;
|
||
var sy1i = Math.Min(syi + 1, src.Height - 1);
|
||
var fy8 = (int)((syCanvas - syi) * 256);
|
||
var fyInv = 256 - fy8;
|
||
|
||
var di = (y * dstW + x0) * 4;
|
||
for (var x = x0; x <= x1; x++, di += 4)
|
||
{
|
||
var px = x - ex;
|
||
if (px < ox || px > ox + drawnW) continue;
|
||
if (isRound)
|
||
{
|
||
var dx = px - cx;
|
||
var dy = py - cy;
|
||
if (dx * dx + dy * dy > radius * radius) continue;
|
||
}
|
||
|
||
// sxNorm: normalized [0,1] position within the drawn content (horizontally)
|
||
var sxNorm = Math.Clamp((px - ox) / drawnW, 0f, 1f);
|
||
// sxCrop: pixel position within the crop region
|
||
var sxCrop = sxNorm * cbW;
|
||
// sxCanvas: canvas pixel column (for buffer indexing)
|
||
var sxCanvas = Math.Clamp(cbX + sxCrop, 0f, src.Width - 1f);
|
||
if (isMirror) sxCanvas = src.Width - 1 - sxCanvas;
|
||
var sxi = (int)sxCanvas;
|
||
var sx1i = Math.Min(sxi + 1, src.Width - 1);
|
||
var fx8 = (int)((sxCanvas - sxi) * 256);
|
||
var fxInv = 256 - fx8;
|
||
|
||
// ALL pixel indexing uses srcStride (canvas stride), NOT crop width
|
||
var p00 = (syi * srcStride + sxi) * 4;
|
||
var p10 = (syi * srcStride + sx1i) * 4;
|
||
var p01 = (sy1i * srcStride + sxi) * 4;
|
||
var p11 = (sy1i * srcStride + sx1i) * 4;
|
||
var sb = Bilinear(s[p00], s[p10], s[p01], s[p11], fx8, fxInv, fy8, fyInv);
|
||
var sg = Bilinear(s[p00 + 1], s[p10 + 1], s[p01 + 1], s[p11 + 1], fx8, fxInv, fy8, fyInv);
|
||
var sr = Bilinear(s[p00 + 2], s[p10 + 2], s[p01 + 2], s[p11 + 2], fx8, fxInv, fy8, fyInv);
|
||
var saFull = Bilinear(s[p00 + 3], s[p10 + 3], s[p01 + 3], s[p11 + 3], fx8, fxInv, fy8, fyInv);
|
||
|
||
var sa = saFull * op >> 8;
|
||
if (sa >= 255)
|
||
{
|
||
dst[di] = (byte)sb;
|
||
dst[di + 1] = (byte)sg;
|
||
dst[di + 2] = (byte)sr;
|
||
dst[di + 3] = 255;
|
||
}
|
||
else if (sa > 0)
|
||
{
|
||
if (transparentDst)
|
||
{
|
||
// Element-raster pass on a TRANSPARENT base (the paste cache):
|
||
// keep straight color + straight alpha so the paste rows can
|
||
// source-over onto the opaque master. Premultiplying here (the
|
||
// opaque-dst blend) stamped alpha 255 onto already-alpha-scaled
|
||
// color — the paste over-wrote the scene with darkened ink, the
|
||
// web widget's "black box" in recordings.
|
||
dst[di] = (byte)sb;
|
||
dst[di + 1] = (byte)sg;
|
||
dst[di + 2] = (byte)sr;
|
||
dst[di + 3] = (byte)sa;
|
||
}
|
||
else
|
||
{
|
||
var inv = 255 - sa;
|
||
dst[di] = (byte)((sb * sa + dst[di] * inv + 127) / 255);
|
||
dst[di + 1] = (byte)((sg * sa + dst[di + 1] * inv + 127) / 255);
|
||
dst[di + 2] = (byte)((sr * sa + dst[di + 2] * inv + 127) / 255);
|
||
dst[di + 3] = 255;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>Two-stage bilinear in 8.8 fixed point; result within ±1 of the
|
||
/// float StretchMath.SampleBgra (well inside every pixel test's ±2 tolerance).
|
||
/// The first stage must NOT shift back to 8 bits — shifting twice collapsed
|
||
/// every solid-255 sample to 1 (green overwrites silently degraded to no-ops;
|
||
/// caught by the SceneCompositor/SceneGraph pixel probes 2026-09-04).</summary>
|
||
private static int Bilinear(int tl, int tr, int bl, int br, int fx8, int fxInv, int fy8, int fyInv)
|
||
{
|
||
// Both stages in one 16.8 scale: top/bot ≤ 65280, ×256 + round ≤ 16.8M — int-safe.
|
||
var top = tl * fxInv + tr * fx8;
|
||
var bot = bl * fxInv + br * fx8;
|
||
return (top * fyInv + bot * fy8 + 32768) >> 16;
|
||
}
|
||
|
||
/// <summary>Row walk for a 1:1 aligned blit at integer offset (ex, ey): per-pixel
|
||
/// branch on source alpha — opaque rows copy straight, transparent pixels are
|
||
/// skipped, partial alpha blends in integer fixed-point (libyuv's pattern). The
|
||
/// master buffer's destination alpha is always opaque (black base paints 255 and
|
||
/// every blend writes 255), so dst alpha is never read — matching BlendPixel's
|
||
/// source-over result for opaque dst within ±1.</summary>
|
||
private static void Blit1To1Rows(byte[] dst, int dstW, int dstH, int ex, int ey, VideoFrame src, float opacity)
|
||
{
|
||
var op = (int)(Math.Clamp(opacity, 0f, 1f) * 256);
|
||
if (op <= 0) return;
|
||
|
||
var x0 = Math.Max(0, ex);
|
||
var y0 = Math.Max(0, ey);
|
||
var x1 = Math.Min(dstW, ex + src.Width);
|
||
var y1 = Math.Min(dstH, ey + src.Height);
|
||
if (x0 >= x1 || y0 >= y1) return;
|
||
|
||
// The take-4 fix: a full-canvas opaque capture (the Live backdrop) is ONE
|
||
// memcpy, not 2.07M managed loop iterations — the row walk below still cost
|
||
// ~58ms/frame dominated by bounds-check arithmetic and GC of the per-pixel
|
||
// loop's slowness, i.e. the same starvation shape as take 3, one floor lower.
|
||
if (op == 256 && src.IsOpaque && x0 == 0 && y0 == 0 && x1 == dstW && y1 == dstH)
|
||
{
|
||
Buffer.BlockCopy(src.BgraPixels, 0, dst, 0, dstW * dstH * 4);
|
||
return;
|
||
}
|
||
|
||
var d = dst.AsSpan();
|
||
var s = src.BgraPixels.AsSpan();
|
||
var rowLen = x1 - x0;
|
||
if (op == 256)
|
||
{
|
||
for (var y = y0; y < y1; y++)
|
||
{
|
||
var si = ((y - ey) * src.Width + (x0 - ex)) * 4;
|
||
BlendRowOpaque(d, (y * dstW + x0) * 4, s, si, rowLen);
|
||
}
|
||
}
|
||
else
|
||
{
|
||
for (var y = y0; y < y1; y++)
|
||
{
|
||
var si = ((y - ey) * src.Width + (x0 - ex)) * 4;
|
||
BlendRowWeighted(d, (y * dstW + x0) * 4, s, si, rowLen, op);
|
||
}
|
||
}
|
||
}
|
||
|
||
private static void BlendRowOpaque(Span<byte> dst, int di, ReadOnlySpan<byte> src, int si, int count)
|
||
{
|
||
for (var n = 0; n < count; n++)
|
||
{
|
||
var a = src[si + 3];
|
||
if (a == 255)
|
||
{
|
||
dst[di] = src[si];
|
||
dst[di + 1] = src[si + 1];
|
||
dst[di + 2] = src[si + 2];
|
||
dst[di + 3] = 255;
|
||
}
|
||
else if (a != 0)
|
||
{
|
||
var inv = 255 - a;
|
||
dst[di] = (byte)((src[si] * a + dst[di] * inv + 127) / 255);
|
||
dst[di + 1] = (byte)((src[si + 1] * a + dst[di + 1] * inv + 127) / 255);
|
||
dst[di + 2] = (byte)((src[si + 2] * a + dst[di + 2] * inv + 127) / 255);
|
||
dst[di + 3] = 255;
|
||
}
|
||
si += 4;
|
||
di += 4;
|
||
}
|
||
}
|
||
|
||
private static void BlendRowWeighted(Span<byte> dst, int di, ReadOnlySpan<byte> src, int si, int count, int op)
|
||
{
|
||
for (var n = 0; n < count; n++)
|
||
{
|
||
var a = src[si + 3] * op >> 8;
|
||
if (a >= 255)
|
||
{
|
||
dst[di] = src[si];
|
||
dst[di + 1] = src[si + 1];
|
||
dst[di + 2] = src[si + 2];
|
||
dst[di + 3] = 255;
|
||
}
|
||
else if (a > 0)
|
||
{
|
||
var inv = 255 - a;
|
||
dst[di] = (byte)((src[si] * a + dst[di] * inv + 127) / 255);
|
||
dst[di + 1] = (byte)((src[si + 1] * a + dst[di + 1] * inv + 127) / 255);
|
||
dst[di + 2] = (byte)((src[si + 2] * a + dst[di + 2] * inv + 127) / 255);
|
||
dst[di + 3] = 255;
|
||
}
|
||
si += 4;
|
||
di += 4;
|
||
}
|
||
}
|
||
|
||
/// <summary>Centered OBS-style border stroke: rect ring (Traditional) or circle ring (Round).</summary>
|
||
private static void DrawBorder(
|
||
byte[] dst, int dstW, int dstH,
|
||
float ex, float ey, float ew, float eh,
|
||
SceneElement element, bool isRound)
|
||
{
|
||
if (!element.TryGetBorderColor(out var r, out var g, out var b)) return;
|
||
var half = element.BorderWidth / 2f;
|
||
var alpha = (float)(element.Opacity * element.BorderOpacity);
|
||
if (half <= 0 || alpha <= 0) return;
|
||
|
||
var cx = ex + ew / 2f;
|
||
var cy = ey + eh / 2f;
|
||
var radius = Math.Min(ew, eh) / 2f;
|
||
|
||
var x0 = Math.Max(0, (int)Math.Floor(ex - half));
|
||
var y0 = Math.Max(0, (int)Math.Floor(ey - half));
|
||
var x1 = Math.Min(dstW - 1, (int)Math.Ceiling(ex + ew + half));
|
||
var y1 = Math.Min(dstH - 1, (int)Math.Ceiling(ey + eh + half));
|
||
|
||
for (var y = y0; y <= y1; y++)
|
||
{
|
||
for (var x = x0; x <= x1; x++)
|
||
{
|
||
float d = isRound
|
||
? MathF.Sqrt((x - cx) * (x - cx) + (y - cy) * (y - cy)) - radius
|
||
: MathF.Max(MathF.Max(ex - x, x - (ex + ew)), MathF.Max(ey - y, y - (ey + eh)));
|
||
if (MathF.Abs(d) <= half)
|
||
BlendPixel(dst, (y * dstW + x) * 4, (b, g, r, (byte)255), alpha);
|
||
}
|
||
}
|
||
}
|
||
|
||
/// <summary>1:1 copy of a master-sized overlay (flash, social bar) cropped to the
|
||
/// active source rect. The overlay is positioned in master space by (sx0, sy0).
|
||
/// Iterates only the intersection rect — the old loop walked every destination
|
||
/// pixel to blit a ~64px strip (2M iterations of bounds checks per overlay).</summary>
|
||
private static void BlitOverlay(
|
||
byte[] dst, int dstW, int dstH, CompositorOptions options,
|
||
VideoFrame overlay, int sx0, int sy0)
|
||
{
|
||
var dx0 = Math.Max(0, sx0 - options.SourceRectX);
|
||
var dy0 = Math.Max(0, sy0 - options.SourceRectY);
|
||
var dx1 = Math.Min(dstW, sx0 - options.SourceRectX + overlay.Width);
|
||
var dy1 = Math.Min(dstH, sy0 - options.SourceRectY + overlay.Height);
|
||
for (var y = dy0; y < dy1; y++)
|
||
{
|
||
var si = ((y + options.SourceRectY - sy0) * overlay.Width
|
||
+ dx0 + options.SourceRectX - sx0) * 4;
|
||
BlendRowOpaque(dst, (y * dstW + dx0) * 4, overlay.BgraPixels, si, dx1 - dx0);
|
||
}
|
||
}
|
||
|
||
/// <summary>Straight-alpha source-over blend; the frame's alpha (and the layer opacity) drives coverage.</summary>
|
||
private static void BlendPixel(byte[] dst, int di, (byte B, byte G, byte R, byte A) src, float opacity)
|
||
{
|
||
var sa = src.A / 255f * opacity;
|
||
if (sa <= 0) return;
|
||
var da = dst[di + 3] / 255f;
|
||
var outA = sa + da * (1 - sa);
|
||
if (outA <= 0) return;
|
||
dst[di] = (byte)Math.Round((src.B * sa + dst[di] * da * (1 - sa)) / outA);
|
||
dst[di + 1] = (byte)Math.Round((src.G * sa + dst[di + 1] * da * (1 - sa)) / outA);
|
||
dst[di + 2] = (byte)Math.Round((src.R * sa + dst[di + 2] * da * (1 - sa)) / outA);
|
||
dst[di + 3] = (byte)Math.Round(outA * 255);
|
||
}
|
||
}
|