gramps/LlamaCasty
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perf(video): move tier scaling to ffmpeg

Browse Source 
The frame pump now submits crop-sized BGRA frames; FFmpeg scales them to final tier dimensions. Bundled scaler benchmark: 300 608x1080 to 1080x1920 frames in 0.435s (scaler-only).

Reference: https://ffmpeg.org/ffmpeg-filters.html#scale-1
This commit is contained in:
gramps
2026-11-04 10:52:35 -08:00
parent 76e8c22815
commit 6310f50c27
11 changed files with 93 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ytLive.Tests/SceneCompositorTests.cs
+39 -16
View File
@@ -2,15 +2,16 @@
using ytLive.Models;
using ytLive.Services;
using ytLive.Services.Compositor;
using ytLive.Services.Encoder;
namespace ytLive.Tests;
/// <summary>
/// The output compositor (TASK 4 ship step 1): renders a scene into the encoder's
/// master frame, mirroring the XAML preview minus the editing chrome. The integration
/// raw input frame, mirroring the XAML preview minus the editing chrome. The integration
/// test composites a full scene (background + round webcam + images + mirror + border)
/// and asserts per-layer probe pixels; the vertical-tier test asserts the 9:16 crop
/// and upscale. Frames are pushed by the test — no capture managers involved.
/// and asserts per-layer probe pixels; the vertical-tier test checks the crop frame and
/// FFmpeg's final 9:16 resize. Frames are pushed by the test — no capture managers involved.
/// </summary>
public class SceneCompositorTests
{
@@ -407,9 +408,18 @@ public class SceneCompositorTests
}
[Fact]
public void Render_VerticalTier_Outputs_1080x1920_From_The_Center_Crop()
public void Render_VerticalTier_Crops_And_Configures_Ffmpeg_1080x1920_Scale()
{
var red = Solid(1920, 1080, 255, 0, 0);
var backgroundPixels = new byte[1920 * 1080 * 4];
for (var y = 0; y < 1080; y++)
for (var x = 0; x < 1920; x++)
{
var i = (y * 1920 + x) * 4;
backgroundPixels[i] = (byte)(x * 255 / 1919);
backgroundPixels[i + 1] = (byte)(y * 255 / 1079);
backgroundPixels[i + 3] = 255;
}
var backgroundFrame = new VideoFrame(1920, 1080, backgroundPixels);
var green = Solid(1280, 720, 0, 255, 0);
var background = new Source { Type = SourceType.DisplayCapture, IsBackground = true, CaptureKey = "monitor:0" };
@@ -421,26 +431,40 @@ public class SceneCompositorTests
VideoFrame? FrameFor(SceneElement e) => e switch
{
WebcamSceneConfig => green,
Source { IsBackground: true } => red,
Source { IsBackground: true } => backgroundFrame,
_ => null,
};
var options = new CompositorOptions
{
SourceRectX = 656, SourceRectY = 0, SourceRectWidth = 607, SourceRectHeight = 1080,
OutputWidth = 1080, OutputHeight = 1920,
SourceRectX = 656, SourceRectY = 0, SourceRectWidth = 608, SourceRectHeight = 1080,
OutputWidth = 608, OutputHeight = 1080,
};
var output = new SceneCompositor().Render(scene, FrameFor, null, options);
Assert.Equal(1080, output.Width);
Assert.Equal(1920, output.Height);
// top-left of the crop is pure background (webcam starts at crop y=100, inset from the corner)
AssertColor(output, 0, 0, 255, 0, 0);
// the webcam center (crop 150,250) scales to output ~(267,444) and stays green
AssertColor(output, 267, 444, 0, 255, 0);
Assert.Equal(608, output.Width);
Assert.Equal(1080, output.Height);
// The centered crop and webcam are composed at source size.
AssertColor(output, 0, 0, 0, 0, 87);
AssertColor(output, 150, 250, 0, 255, 0);
// far from the webcam, still background
AssertColor(output, 978, 1688, 255, 0, 0);
AssertColor(output, 550, 950, 0, 224, 160);
// FFmpeg receives the crop, then scales it to the tier's encoded dimensions.
var args = FfmpegArgs.Build(new EncoderOptions
{
RtmpUrl = "rtmp://a.rtmp.youtube.com/live2/key",
InputWidth = output.Width,
InputHeight = output.Height,
Width = 1080,
Height = 1920,
Fps = 60,
}, "libopenh264").ToArray();
var inputSize = Array.IndexOf(args, "-video_size");
var scaleFilter = Array.IndexOf(args, "-vf");
Assert.Equal("608x1080", args[inputSize + 1]);
Assert.Equal("scale=1080:1920:flags=bilinear,setsar=1", args[scaleFilter + 1]);
}
[Fact]
@@ -616,4 +640,3 @@ public class StretchMathTests
Assert.Equal(255, scaled.BgraPixels[i + 1]); // solid green survives the scale
}
}