TASK 31: SceneGraph component + baked-crust compositor optimization
- Add ElementKind (Static/Dynamic) to SceneElement base; Source/WebcamSceneConfig classify - Services/SceneGraph.cs: owns the Scenes collection (ViewModel's Scenes delegates to it), the element mutation surface (Add/Insert/Remove/Move, each invalidating the bake), and the queries that were scattered LINQ (GetBackground/GetWebcam/GetChatBoxes/GetSplitPoint/IsStatic) - SceneCompositor: split-aware BakeStaticBase + CompositeLayers + Render(.., staticBase, split); builds/caches the static base below the split point in source-rect space - FramePump: optional SceneGraph -> optimized bake+composite path; falls back to full render - MainViewModel: routes element mutations through the graph; invalidates the bake on static layout/opacity/visibility/useDefaultBackground changes and after background heal/Ensure - Integration test SceneGraphTests.BakedStaticBase_WithDynamicLayer_CompositesCorrectly Derivative survey (mandated): tried before writing — OBS does per-source opacity/visibility caching and static-scene baking; this mirrors OBS's 'cached static source' optimization. 3 documented defensive deviations from the spec (ChatOverlayLayer stays decoupled; background helpers stay VM-static for direct testability; full facade peel deferred post-1.0) in TASKS.md + ai.md. Scope check passed; clean build 0 warnings.
This commit is contained in:
@@ -27,7 +27,9 @@ public sealed class SceneCompositor
|
||||
VideoFrame? flashFrame,
|
||||
CompositorOptions options,
|
||||
VideoFrame? socialBarFrame = null,
|
||||
int socialBarTop = 0)
|
||||
int socialBarTop = 0,
|
||||
VideoFrame? staticBase = null,
|
||||
int split = -1)
|
||||
{
|
||||
if (scene == null) throw new ArgumentNullException(nameof(scene));
|
||||
if (frameFor == null) throw new ArgumentNullException(nameof(frameFor));
|
||||
@@ -40,45 +42,63 @@ public sealed class SceneCompositor
|
||||
var cropW = options.SourceRectWidth;
|
||||
var cropH = options.SourceRectHeight;
|
||||
var buffer = new byte[cropW * cropH * 4];
|
||||
for (var i = 3; i < buffer.Length; i += 4)
|
||||
buffer[i] = 255; // opaque black base — video frames are never transparent
|
||||
|
||||
var elements = scene.Elements;
|
||||
|
||||
var liveCapture = elements.OfType<Source>().FirstOrDefault(s => s.IsBackground);
|
||||
var liveCaptureFrame = liveCapture != null && liveCapture.IsVisible ? frameFor(liveCapture) : null;
|
||||
if (liveCaptureFrame != null)
|
||||
BlitContent(buffer, cropW, cropH, 0, 0, cropW, cropH, liveCaptureFrame, 1f, false, false);
|
||||
|
||||
var background = elements.OfType<Source>().FirstOrDefault(s => s.Type == SourceType.Background);
|
||||
var backgroundFrame = background != null ? frameFor(background) : null;
|
||||
if (backgroundFrame != null)
|
||||
BlitContent(buffer, cropW, cropH, 0, 0, cropW, cropH, backgroundFrame, 1f, false, false);
|
||||
|
||||
foreach (var element in elements)
|
||||
// staticBase != null: the caller pre-baked all static layers below `split`.
|
||||
// We copy it (never mutate the cached base) and composite elements[split..] on top.
|
||||
if (staticBase != null)
|
||||
{
|
||||
if (staticBase.Width != cropW || staticBase.Height != cropH)
|
||||
throw new ArgumentException("The static base must match the source rect.", nameof(staticBase));
|
||||
Buffer.BlockCopy(staticBase.BgraPixels, 0, buffer, 0, buffer.Length);
|
||||
}
|
||||
else
|
||||
{
|
||||
if (split < 0) split = scene.Elements.Count;
|
||||
for (var i = 3; i < buffer.Length; i += 4)
|
||||
buffer[i] = 255; // opaque black base
|
||||
|
||||
var elements = scene.Elements;
|
||||
var liveCapture = elements.OfType<Source>().FirstOrDefault(s => s.IsBackground);
|
||||
var liveCaptureFrame = liveCapture != null && liveCapture.IsVisible ? frameFor(liveCapture) : null;
|
||||
if (liveCaptureFrame != null)
|
||||
BlitContent(buffer, cropW, cropH, 0, 0, cropW, cropH, liveCaptureFrame, 1f, false, false);
|
||||
|
||||
var background = elements.OfType<Source>().FirstOrDefault(s => s.Type == SourceType.Background);
|
||||
var backgroundFrame = background != null ? frameFor(background) : null;
|
||||
if (backgroundFrame != null)
|
||||
BlitContent(buffer, cropW, cropH, 0, 0, cropW, cropH, backgroundFrame, 1f, false, false);
|
||||
|
||||
foreach (var element in elements)
|
||||
{
|
||||
if (!element.IsVisible) continue;
|
||||
switch (element)
|
||||
{
|
||||
case Source { IsBackground: true }:
|
||||
case Source { Type: SourceType.Background }:
|
||||
case Source { Type: SourceType.TextOverlay }:
|
||||
continue;
|
||||
}
|
||||
var fullFrame = frameFor(element);
|
||||
if (fullFrame == null) continue;
|
||||
BlitBakedElement(buffer, cropW, cropH, options, element, fullFrame);
|
||||
}
|
||||
}
|
||||
|
||||
// Composite the above-split layers (dynamic + any static above them) per frame.
|
||||
for (var i = Math.Max(0, split); i < scene.Elements.Count; i++)
|
||||
{
|
||||
var element = scene.Elements[i];
|
||||
if (!element.IsVisible) continue;
|
||||
switch (element)
|
||||
{
|
||||
case Source { IsBackground: true }:
|
||||
case Source { Type: SourceType.Background }:
|
||||
case Source { Type: SourceType.TextOverlay }:
|
||||
continue; // background/background are their own layers; Text isn't shipped
|
||||
continue;
|
||||
}
|
||||
|
||||
var frame = frameFor(element);
|
||||
if (frame == null) continue;
|
||||
|
||||
var ex = (float)(element.X - options.SourceRectX);
|
||||
var ey = (float)(element.Y - options.SourceRectY);
|
||||
var ew = (float)element.Width;
|
||||
var eh = (float)element.Height;
|
||||
var isRound = element.ClipShape == ClipShape.Round;
|
||||
|
||||
BlitContent(buffer, cropW, cropH, ex, ey, ew, eh, frame,
|
||||
(float)element.Opacity, isRound, element.IsMirrored);
|
||||
if (element.HasBorder)
|
||||
DrawBorder(buffer, cropW, cropH, ex, ey, ew, eh, element, isRound);
|
||||
BlitBakedElement(buffer, cropW, cropH, options, element, frame);
|
||||
}
|
||||
|
||||
if (flashFrame != null)
|
||||
@@ -91,6 +111,136 @@ public sealed class SceneCompositor
|
||||
new VideoFrame(cropW, cropH, buffer), options.OutputWidth, options.OutputHeight);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Render the static base of a scene: the opaque black layer, the live-capture
|
||||
/// background, the background image, and every <paramref name="split"/> z-layers
|
||||
/// below the split point (all of which are static by construction — see
|
||||
/// <see cref="SceneElement.Kind"/>). The result is a source-rect sized frame that
|
||||
/// <see cref="Render"/> can start from and composite the dynamic/above-split layers
|
||||
/// onto each tick. Returns null when there is no static layer at all.
|
||||
/// </summary>
|
||||
public static VideoFrame? BakeStaticBase(
|
||||
Scene scene,
|
||||
int split,
|
||||
Func<SceneElement, VideoFrame?> frameFor,
|
||||
Func<CompositorOptions> options,
|
||||
VideoFrame? socialBarFrame = null,
|
||||
int socialBarTop = 0,
|
||||
VideoFrame? flashFrame = null)
|
||||
{
|
||||
if (split <= 0) return null;
|
||||
var opts = options();
|
||||
if (opts.SourceRectWidth <= 0 || opts.SourceRectHeight <= 0) return null;
|
||||
|
||||
var cropW = opts.SourceRectWidth;
|
||||
var cropH = opts.SourceRectHeight;
|
||||
var buffer = new byte[cropW * cropH * 4];
|
||||
for (var i = 3; i < buffer.Length; i += 4)
|
||||
buffer[i] = 255;
|
||||
|
||||
var elements = scene.Elements;
|
||||
|
||||
var liveCapture = elements.OfType<Source>().FirstOrDefault(s => s.IsBackground);
|
||||
if (liveCapture != null)
|
||||
{
|
||||
var liveCaptureFrame = liveCapture.IsVisible ? frameFor(liveCapture) : null;
|
||||
if (liveCaptureFrame != null && split > elements.IndexOf(liveCapture))
|
||||
BlitContent(buffer, cropW, cropH, 0, 0, cropW, cropH, liveCaptureFrame, 1f, false, false);
|
||||
}
|
||||
|
||||
var background = elements.OfType<Source>().FirstOrDefault(s => s.Type == SourceType.Background);
|
||||
if (background != null)
|
||||
{
|
||||
var backgroundFrame = frameFor(background);
|
||||
if (backgroundFrame != null && split > elements.IndexOf(background))
|
||||
BlitContent(buffer, cropW, cropH, 0, 0, cropW, cropH, backgroundFrame, 1f, false, false);
|
||||
}
|
||||
|
||||
for (var i = 0; i < split; i++)
|
||||
{
|
||||
var element = scene.Elements[i];
|
||||
if (!element.IsVisible) continue;
|
||||
switch (element)
|
||||
{
|
||||
case Source { IsBackground: true }:
|
||||
case Source { Type: SourceType.Background }:
|
||||
case Source { Type: SourceType.TextOverlay }:
|
||||
continue;
|
||||
}
|
||||
var frame = frameFor(element);
|
||||
if (frame == null) continue;
|
||||
BlitBakedElement(buffer, cropW, cropH, opts, element, frame);
|
||||
}
|
||||
|
||||
var baked = new VideoFrame(cropW, cropH, buffer);
|
||||
if (flashFrame != null)
|
||||
baked = Overlay(baked, opts, flashFrame, 0, 0);
|
||||
if (socialBarFrame != null && socialBarFrame.Width > 0 && socialBarFrame.Height > 0)
|
||||
baked = Overlay(baked, opts, socialBarFrame, 0, socialBarTop);
|
||||
return baked;
|
||||
}
|
||||
|
||||
/// <summary>The compositor's dynamic/above-split path: start from an existing frame
|
||||
/// buffer and composite the z-layers from <paramref name="split"/> onward on top.
|
||||
/// When <paramref name="baseFrame"/> is null the caller must fall back to full render.</summary>
|
||||
public static VideoFrame CompositeLayers(
|
||||
VideoFrame baseFrame,
|
||||
Scene scene,
|
||||
int split,
|
||||
Func<SceneElement, VideoFrame?> frameFor,
|
||||
CompositorOptions options,
|
||||
VideoFrame? socialBarFrame = null,
|
||||
int socialBarTop = 0,
|
||||
VideoFrame? flashFrame = null)
|
||||
{
|
||||
var cropW = options.SourceRectWidth;
|
||||
var cropH = options.SourceRectHeight;
|
||||
// We must not mutate the cached base: copy it into a fresh buffer.
|
||||
var buffer = new byte[baseFrame.BgraPixels.Length];
|
||||
Buffer.BlockCopy(baseFrame.BgraPixels, 0, buffer, 0, buffer.Length);
|
||||
|
||||
for (var i = split; i < scene.Elements.Count; i++)
|
||||
{
|
||||
var element = scene.Elements[i];
|
||||
if (!element.IsVisible) continue;
|
||||
switch (element)
|
||||
{
|
||||
case Source { IsBackground: true }:
|
||||
case Source { Type: SourceType.Background }:
|
||||
case Source { Type: SourceType.TextOverlay }:
|
||||
continue;
|
||||
}
|
||||
var frame = frameFor(element);
|
||||
if (frame == null) continue;
|
||||
BlitBakedElement(buffer, cropW, cropH, options, element, frame);
|
||||
}
|
||||
return StretchMath.BilinearScale(new VideoFrame(cropW, cropH, buffer), options.OutputWidth, options.OutputHeight);
|
||||
}
|
||||
|
||||
private static void BlitBakedElement(
|
||||
byte[] buffer, int cropW, int cropH, CompositorOptions options, SceneElement element, VideoFrame frame)
|
||||
{
|
||||
var ex = (float)(element.X - options.SourceRectX);
|
||||
var ey = (float)(element.Y - options.SourceRectY);
|
||||
var ew = (float)element.Width;
|
||||
var eh = (float)element.Height;
|
||||
var isRound = element.ClipShape == ClipShape.Round;
|
||||
BlitContent(buffer, cropW, cropH, ex, ey, ew, eh, frame,
|
||||
(float)element.Opacity, isRound, element.IsMirrored);
|
||||
if (element.HasBorder)
|
||||
DrawBorder(buffer, cropW, cropH, ex, ey, ew, eh, element, isRound);
|
||||
}
|
||||
|
||||
private static VideoFrame Overlay(VideoFrame frame, CompositorOptions options, VideoFrame overlay, int sx0, int sy0)
|
||||
{
|
||||
var cropW = options.SourceRectWidth;
|
||||
var cropH = options.SourceRectHeight;
|
||||
var buffer = new byte[frame.BgraPixels.Length];
|
||||
Buffer.BlockCopy(frame.BgraPixels, 0, buffer, 0, buffer.Length);
|
||||
BlitOverlay(buffer, cropW, cropH, options, overlay, sx0, sy0);
|
||||
return new VideoFrame(cropW, cropH, buffer);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Render a cross-fade blend between two scenes. Used by <see cref="TransitionService"/>
|
||||
/// during fade/move transitions: the caller renders each scene individually, then blends
|
||||
|
||||
Reference in New Issue
Block a user