perf(video): move tier scaling to ffmpeg
The frame pump now submits crop-sized BGRA frames; FFmpeg scales them to final tier dimensions. Bundled scaler benchmark: 300 608x1080 to 1080x1920 frames in 0.435s (scaler-only). Reference: https://ffmpeg.org/ffmpeg-filters.html#scale-1
This commit is contained in:
@@ -1,11 +1,11 @@
|
||||
namespace ytLive.Services.Compositor;
|
||||
|
||||
/// <summary>
|
||||
/// Where the compositor draws from (the active quality tier's output rect over the
|
||||
/// 1920×1080 master) and at what size. 16:9 tiers = the full master 1:1; the vertical
|
||||
/// 9:16 tier = the centered 607×1080 crop scaled up to 1080×1920. The source rect is
|
||||
/// integer-aligned — MainViewModel's <c>OutputRectX</c> can be 656.5, so the caller
|
||||
/// rounds before building these options.
|
||||
/// Where the compositor draws from (the active quality tier's crop over the
|
||||
/// 1920×1080 master) and the size of the BGRA frame it sends to the encoder. The
|
||||
/// compositor frame is the crop itself; FFmpeg scales it to the selected output
|
||||
/// dimensions. The source rect is integer-aligned — MainViewModel's
|
||||
/// <c>OutputRectX</c> can be 656.5, so the caller rounds before building these options.
|
||||
/// </summary>
|
||||
public sealed class CompositorOptions
|
||||
{
|
||||
|
||||
@@ -6,10 +6,11 @@ using ytLive.Models;
|
||||
namespace ytLive.Services.Compositor;
|
||||
|
||||
/// <summary>
|
||||
/// The output compositor: renders a scene into the encoder's master frame (tightly-packed
|
||||
/// The output compositor: renders a scene into the encoder's raw input frame (tightly-packed
|
||||
/// BGRA8 <see cref="VideoFrame"/>) exactly as the XAML preview renders it, minus the
|
||||
/// editing chrome (SelectionOverlay, DimRects, badge, placeholder). The preview stays the
|
||||
/// editing view; this is the output view — see TASKS.md "Ship step 1 — Scene compositor".
|
||||
/// editing chrome (SelectionOverlay, DimRects, badge, placeholder). FFmpeg scales that frame
|
||||
/// to the selected output dimensions. The preview stays the editing view; this is the output
|
||||
/// view — see TASKS.md "Ship step 1 — Scene compositor".
|
||||
///
|
||||
/// Frame sources are supplied by a <see cref="Func{SceneElement, VideoFrame}"/> resolver
|
||||
/// (the caller maps webcam → DeviceId, images → AssetId, background → CaptureKey), keeping
|
||||
|
||||
@@ -14,8 +14,16 @@ public sealed class EncoderOptions
|
||||
public const string DefaultAudioPipeName = "ytllive_audio";
|
||||
|
||||
public string RtmpUrl { get; init; } = string.Empty;
|
||||
/// <summary>Final encoded output width and height.</summary>
|
||||
public int Width { get; init; } = 1920;
|
||||
public int Height { get; init; } = 1080;
|
||||
|
||||
/// <summary>Raw BGRA frame size from the compositor. Zero uses the final output size.</summary>
|
||||
public int InputWidth { get; init; }
|
||||
public int InputHeight { get; init; }
|
||||
|
||||
public int RawWidth => InputWidth > 0 ? InputWidth : Width;
|
||||
public int RawHeight => InputHeight > 0 ? InputHeight : Height;
|
||||
public int Fps { get; init; } = 60;
|
||||
public int BitrateKbps { get; init; } = 8000;
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ public static class FfmpegArgs
|
||||
"-hide_banner", "-loglevel", "info",
|
||||
"-stats", "-stats_period", "0.5",
|
||||
"-f", "rawvideo", "-pix_fmt", "bgra",
|
||||
"-video_size", $"{options.Width}x{options.Height}",
|
||||
"-video_size", $"{options.RawWidth}x{options.RawHeight}",
|
||||
"-framerate", options.Fps.ToString(),
|
||||
"-i", "pipe:0",
|
||||
"-f", "f32le", "-ar", options.AudioSampleRate.ToString(),
|
||||
@@ -53,6 +53,10 @@ public static class FfmpegArgs
|
||||
{
|
||||
var gop = options.GopSize;
|
||||
args.AddRange(["-map", "0:v", "-map", "1:a"]);
|
||||
if (options.RawWidth != options.Width || options.RawHeight != options.Height)
|
||||
{
|
||||
args.AddRange(["-vf", $"scale={options.Width}:{options.Height}:flags=bilinear,setsar=1"]);
|
||||
}
|
||||
args.AddRange(["-c:v", videoEncoder]);
|
||||
args.AddRange(["-b:v", $"{options.BitrateKbps}k", "-maxrate", $"{options.BitrateKbps}k",
|
||||
"-bufsize", $"{options.BitrateKbps * 2}k"]);
|
||||
|
||||
@@ -10,7 +10,7 @@ namespace ytLive.Services.Encoder;
|
||||
/// The live frame producer (TASK 4 ship step 5): the bridge between the capture
|
||||
/// managers + compositor and the encoder. While live it snapshots the active
|
||||
/// scene each tick, resolves every element to its latest frame, composites it
|
||||
/// into the tier's output frame, and paces frames into the encoder at the tier's
|
||||
/// into the tier's cropped input frame, and paces frames into the encoder at the tier's
|
||||
/// FPS. All collaborators are constructor-injected seams (scene, resolver,
|
||||
/// options, encoder factory, pacing delay) so the pump stays free of WPF and of
|
||||
/// the capture managers and is fully hermetic in tests.
|
||||
@@ -722,11 +722,9 @@ public sealed class FramePump : IDisposable
|
||||
byte[]? scratch, Func<SceneElement, VideoFrame?> resolver,
|
||||
CompositeTimings? timings = null)
|
||||
{
|
||||
// 1:1 guard: the cache stores the OUTPUT bytes and scratch is source-sized —
|
||||
// at 1:1 they are the same buffer size. Off-size tiers (a vertical 1080×1920
|
||||
// tier, an H264-1080p tier) fall back to a per-tick full render; the deployed
|
||||
// config is master-sized and caching a per-tick fresh scale buffer is the known
|
||||
// vertical-tier follow-up (see ai.md), not silently baked here.
|
||||
// The cache stores the compositor frame at crop size; FFmpeg scales that frame
|
||||
// to the final output dimensions. With product options, scratch and rendered
|
||||
// frames are therefore the same size, including for vertical output.
|
||||
var useCache = scratch != null && options.SourceRectWidth == options.OutputWidth
|
||||
&& options.SourceRectHeight == options.OutputHeight;
|
||||
var signature = useCache
|
||||
|
||||
+1
-1
@@ -50,7 +50,7 @@ External-facing logic: YouTube API, persistence. See
|
||||
| `SocialValidator.cs` | `ISocialValidator` seam + `HttpSocialValidator` default: validates a social handle/URL by constructing the canonical profile URL and issuing an HTTP GET — the "act of validation" before adding to the bar. 200/redirect = exists; 404/connection failure = rejected. Fediverse `@user@domain` additionally does a best-effort nodeinfo lookup (`/.well-known/nodeinfo` → `software.name`) so the entry can show the instance's real logo; nodeinfo failure still validates (generic fediverse glyph). If the identity domain's nodeinfo is blocked (SSO) but the bare root 302s to the real instance (YunoHost default-app subdomains), the lookup follows the redirect and asks that host. Since the bar bug-fix branch the resolution **probes well-known subdomains** (`mastodon.`, `social.`, ... order in `FediverseSubdomainCandidates`) when both the identity domain and its redirect come up empty, bounded by a ~15s linked-CTS budget; `ResolveFediverseSoftwareAsync(domain, ct)` on the seam powers the load-time heal. Constructor takes optional `HttpClient` + `Action<string>` log for tests |
|
||||
| `Compositor/SocialBarRenderer.cs` | **Social bar output strip** (bar bug-fix branch): rasterizes the global social bar into a transparent straight-alpha BGRA8 `VideoFrame` strip (1920 wide, 40px content + 24px glow pad) via `RenderTargetBitmap` — one Path (logo) + one TextBlock (handle) per entry, white on transparent with the preview's green `#2ecc71` glow baked in (Pbgra32 → straight-alpha unpremultiply). UI thread only; the frame is immutable afterwards so the `FramePump` reads it from its own thread |
|
||||
| `Compositor/AlertTickerRenderer.cs` | **Shared ticker renderer (TASK 47):** marquee frame rendered per call by pure byte-math; the text pill is rasterized once per text on the UI thread (cached, unpremultiplied straight alpha). Supports AlertBox active-event tickers and ordinary scene-local ticker sources, with per-source `Ticker` settings. **Dynamic output — never baked** (unlike the social bar) |
|
||||
| `Encoder/EncoderOptions.cs` | One go-live's encoder config: full `RtmpUrl` (ingest + stream key), W×H/FPS/bitrate from the quality tier, `VideoEncoder` (null = probe), audio sample rate/channels, `GopSize` = FPS×4 (the ≤4s keyframe bound) |
|
||||
| `Encoder/EncoderOptions.cs` | One go-live's encoder config: full `RtmpUrl` (ingest + stream key), raw input W×H from the compositor crop, final output W×H/FPS/bitrate from the quality tier, `VideoEncoder` (null = probe), audio sample rate/channels, `GopSize` = FPS×4 (the ≤4s keyframe bound) |
|
||||
| `Encoder/IFfmpegEncoder.cs` | **TASK 4 ship step 3 seam**: start/feed/stop the live encoder; `HealthUpdated` (parsed `StreamHealth`), `ProcessFailed` on unexpected non-zero exit. Constructor-injected locator + process factory (tests fake both) |
|
||||
| `Encoder/FfmpegEncoder.cs` | The default encoder: spawn `ffmpeg.exe` (probe `-encoders` first → hardware NVENC/QSV/AMF, OpenH264 fallback — never libx264), feed raw BGRA frames into stdin, parse `-stats` lines into `StreamHealth`, graceful stop via stdin EOF + 10s kill watchdog. Constructed by `MainViewModel` since ship step 5 (`new FfmpegEncoder(new FfmpegLocator())`), driven by the `FramePump` |
|
||||
| `Encoder/IEncoderProcess.cs` | Seam around the spawned subprocess (probe + encoder): redirected stdin/stdout/stderr + exit control — the encoder never touches `Process` directly |
|
||||
|
||||
Reference in New Issue
Block a user