diff --git a/Services/Compositor/CompositorOptions.cs b/Services/Compositor/CompositorOptions.cs index b20df8c..2f840ec 100644 --- a/Services/Compositor/CompositorOptions.cs +++ b/Services/Compositor/CompositorOptions.cs @@ -1,11 +1,11 @@ namespace ytLive.Services.Compositor; /// -/// Where the compositor draws from (the active quality tier's output rect over the -/// 1920×1080 master) and at what size. 16:9 tiers = the full master 1:1; the vertical -/// 9:16 tier = the centered 607×1080 crop scaled up to 1080×1920. The source rect is -/// integer-aligned — MainViewModel's OutputRectX can be 656.5, so the caller -/// rounds before building these options. +/// Where the compositor draws from (the active quality tier's crop over the +/// 1920×1080 master) and the size of the BGRA frame it sends to the encoder. The +/// compositor frame is the crop itself; FFmpeg scales it to the selected output +/// dimensions. The source rect is integer-aligned — MainViewModel's +/// OutputRectX can be 656.5, so the caller rounds before building these options. /// public sealed class CompositorOptions { diff --git a/Services/Compositor/SceneCompositor.cs b/Services/Compositor/SceneCompositor.cs index fa039a7..9f08a70 100644 --- a/Services/Compositor/SceneCompositor.cs +++ b/Services/Compositor/SceneCompositor.cs @@ -6,10 +6,11 @@ using ytLive.Models; namespace ytLive.Services.Compositor; /// -/// The output compositor: renders a scene into the encoder's master frame (tightly-packed +/// The output compositor: renders a scene into the encoder's raw input frame (tightly-packed /// BGRA8 ) exactly as the XAML preview renders it, minus the -/// editing chrome (SelectionOverlay, DimRects, badge, placeholder). The preview stays the -/// editing view; this is the output view — see TASKS.md "Ship step 1 — Scene compositor". +/// editing chrome (SelectionOverlay, DimRects, badge, placeholder). FFmpeg scales that frame +/// to the selected output dimensions. The preview stays the editing view; this is the output +/// view — see TASKS.md "Ship step 1 — Scene compositor". /// /// Frame sources are supplied by a resolver /// (the caller maps webcam → DeviceId, images → AssetId, background → CaptureKey), keeping diff --git a/Services/Encoder/EncoderOptions.cs b/Services/Encoder/EncoderOptions.cs index 5427109..1b08334 100644 --- a/Services/Encoder/EncoderOptions.cs +++ b/Services/Encoder/EncoderOptions.cs @@ -14,8 +14,16 @@ public sealed class EncoderOptions public const string DefaultAudioPipeName = "ytllive_audio"; public string RtmpUrl { get; init; } = string.Empty; + /// Final encoded output width and height. public int Width { get; init; } = 1920; public int Height { get; init; } = 1080; + + /// Raw BGRA frame size from the compositor. Zero uses the final output size. + public int InputWidth { get; init; } + public int InputHeight { get; init; } + + public int RawWidth => InputWidth > 0 ? InputWidth : Width; + public int RawHeight => InputHeight > 0 ? InputHeight : Height; public int Fps { get; init; } = 60; public int BitrateKbps { get; init; } = 8000; diff --git a/Services/Encoder/FfmpegArgs.cs b/Services/Encoder/FfmpegArgs.cs index 51760cd..15108e7 100644 --- a/Services/Encoder/FfmpegArgs.cs +++ b/Services/Encoder/FfmpegArgs.cs @@ -19,7 +19,7 @@ public static class FfmpegArgs "-hide_banner", "-loglevel", "info", "-stats", "-stats_period", "0.5", "-f", "rawvideo", "-pix_fmt", "bgra", - "-video_size", $"{options.Width}x{options.Height}", + "-video_size", $"{options.RawWidth}x{options.RawHeight}", "-framerate", options.Fps.ToString(), "-i", "pipe:0", "-f", "f32le", "-ar", options.AudioSampleRate.ToString(), @@ -53,6 +53,10 @@ public static class FfmpegArgs { var gop = options.GopSize; args.AddRange(["-map", "0:v", "-map", "1:a"]); + if (options.RawWidth != options.Width || options.RawHeight != options.Height) + { + args.AddRange(["-vf", $"scale={options.Width}:{options.Height}:flags=bilinear,setsar=1"]); + } args.AddRange(["-c:v", videoEncoder]); args.AddRange(["-b:v", $"{options.BitrateKbps}k", "-maxrate", $"{options.BitrateKbps}k", "-bufsize", $"{options.BitrateKbps * 2}k"]); diff --git a/Services/Encoder/FramePump.cs b/Services/Encoder/FramePump.cs index af58824..c8cde7c 100644 --- a/Services/Encoder/FramePump.cs +++ b/Services/Encoder/FramePump.cs @@ -10,7 +10,7 @@ namespace ytLive.Services.Encoder; /// The live frame producer (TASK 4 ship step 5): the bridge between the capture /// managers + compositor and the encoder. While live it snapshots the active /// scene each tick, resolves every element to its latest frame, composites it -/// into the tier's output frame, and paces frames into the encoder at the tier's +/// into the tier's cropped input frame, and paces frames into the encoder at the tier's /// FPS. All collaborators are constructor-injected seams (scene, resolver, /// options, encoder factory, pacing delay) so the pump stays free of WPF and of /// the capture managers and is fully hermetic in tests. @@ -722,11 +722,9 @@ public sealed class FramePump : IDisposable byte[]? scratch, Func resolver, CompositeTimings? timings = null) { - // 1:1 guard: the cache stores the OUTPUT bytes and scratch is source-sized — - // at 1:1 they are the same buffer size. Off-size tiers (a vertical 1080×1920 - // tier, an H264-1080p tier) fall back to a per-tick full render; the deployed - // config is master-sized and caching a per-tick fresh scale buffer is the known - // vertical-tier follow-up (see ai.md), not silently baked here. + // The cache stores the compositor frame at crop size; FFmpeg scales that frame + // to the final output dimensions. With product options, scratch and rendered + // frames are therefore the same size, including for vertical output. var useCache = scratch != null && options.SourceRectWidth == options.OutputWidth && options.SourceRectHeight == options.OutputHeight; var signature = useCache diff --git a/Services/index.md b/Services/index.md index ac56886..ca13b04 100644 --- a/Services/index.md +++ b/Services/index.md @@ -50,7 +50,7 @@ External-facing logic: YouTube API, persistence. See | `SocialValidator.cs` | `ISocialValidator` seam + `HttpSocialValidator` default: validates a social handle/URL by constructing the canonical profile URL and issuing an HTTP GET — the "act of validation" before adding to the bar. 200/redirect = exists; 404/connection failure = rejected. Fediverse `@user@domain` additionally does a best-effort nodeinfo lookup (`/.well-known/nodeinfo` → `software.name`) so the entry can show the instance's real logo; nodeinfo failure still validates (generic fediverse glyph). If the identity domain's nodeinfo is blocked (SSO) but the bare root 302s to the real instance (YunoHost default-app subdomains), the lookup follows the redirect and asks that host. Since the bar bug-fix branch the resolution **probes well-known subdomains** (`mastodon.`, `social.`, ... order in `FediverseSubdomainCandidates`) when both the identity domain and its redirect come up empty, bounded by a ~15s linked-CTS budget; `ResolveFediverseSoftwareAsync(domain, ct)` on the seam powers the load-time heal. Constructor takes optional `HttpClient` + `Action` log for tests | | `Compositor/SocialBarRenderer.cs` | **Social bar output strip** (bar bug-fix branch): rasterizes the global social bar into a transparent straight-alpha BGRA8 `VideoFrame` strip (1920 wide, 40px content + 24px glow pad) via `RenderTargetBitmap` — one Path (logo) + one TextBlock (handle) per entry, white on transparent with the preview's green `#2ecc71` glow baked in (Pbgra32 → straight-alpha unpremultiply). UI thread only; the frame is immutable afterwards so the `FramePump` reads it from its own thread | | `Compositor/AlertTickerRenderer.cs` | **Shared ticker renderer (TASK 47):** marquee frame rendered per call by pure byte-math; the text pill is rasterized once per text on the UI thread (cached, unpremultiplied straight alpha). Supports AlertBox active-event tickers and ordinary scene-local ticker sources, with per-source `Ticker` settings. **Dynamic output — never baked** (unlike the social bar) | -| `Encoder/EncoderOptions.cs` | One go-live's encoder config: full `RtmpUrl` (ingest + stream key), W×H/FPS/bitrate from the quality tier, `VideoEncoder` (null = probe), audio sample rate/channels, `GopSize` = FPS×4 (the ≤4s keyframe bound) | +| `Encoder/EncoderOptions.cs` | One go-live's encoder config: full `RtmpUrl` (ingest + stream key), raw input W×H from the compositor crop, final output W×H/FPS/bitrate from the quality tier, `VideoEncoder` (null = probe), audio sample rate/channels, `GopSize` = FPS×4 (the ≤4s keyframe bound) | | `Encoder/IFfmpegEncoder.cs` | **TASK 4 ship step 3 seam**: start/feed/stop the live encoder; `HealthUpdated` (parsed `StreamHealth`), `ProcessFailed` on unexpected non-zero exit. Constructor-injected locator + process factory (tests fake both) | | `Encoder/FfmpegEncoder.cs` | The default encoder: spawn `ffmpeg.exe` (probe `-encoders` first → hardware NVENC/QSV/AMF, OpenH264 fallback — never libx264), feed raw BGRA frames into stdin, parse `-stats` lines into `StreamHealth`, graceful stop via stdin EOF + 10s kill watchdog. Constructed by `MainViewModel` since ship step 5 (`new FfmpegEncoder(new FfmpegLocator())`), driven by the `FramePump` | | `Encoder/IEncoderProcess.cs` | Seam around the spawned subprocess (probe + encoder): redirected stdin/stdout/stderr + exit control — the encoder never touches `Process` directly | diff --git a/ViewModels/MainViewModel.Streaming.Operations.cs b/ViewModels/MainViewModel.Streaming.Operations.cs index 41a0d06..5b4886c 100644 --- a/ViewModels/MainViewModel.Streaming.Operations.cs +++ b/ViewModels/MainViewModel.Streaming.Operations.cs @@ -422,6 +422,7 @@ public partial class MainViewModel : ViewModelBase { var url = _rtmpUrlProvider(); var quality = OutputProfile.Resolve(SelectedQuality, QualityOptions, HasProOutput); + var compositor = BuildCompositorOptions(); return new EncoderOptions { RtmpUrl = url ?? string.Empty, @@ -430,6 +431,8 @@ public partial class MainViewModel : ViewModelBase RecordPath = _activeRecordPath, Width = quality.Width, Height = quality.Height, + InputWidth = compositor.OutputWidth, + InputHeight = compositor.OutputHeight, Fps = quality.Fps, BitrateKbps = (int)Math.Round(quality.Bitrate * 1000), }; diff --git a/ViewModels/MainViewModel.cs b/ViewModels/MainViewModel.cs index 9d63af4..96d5888 100644 --- a/ViewModels/MainViewModel.cs +++ b/ViewModels/MainViewModel.cs @@ -654,19 +654,21 @@ public partial class MainViewModel : ViewModelBase // The tier's crop rect over the 1920x1080 master, integer-aligned (the VM's // OutputRect* are doubles — the vertical 607.5 half-pixel crop rounds to 608). + // FFmpeg, not the compositor, scales this crop to the tier's final dimensions. // internal = test seam (the zoom layer must NOT change the output crop — the // box is a composited layer, never the crop source). internal CompositorOptions BuildCompositorOptions() { - var quality = SelectedQuality; + var sourceRectWidth = (int)Math.Round(OutputRectWidth); + var sourceRectHeight = (int)Math.Round(OutputRectHeight); return new CompositorOptions { SourceRectX = (int)Math.Round(OutputRectX), SourceRectY = (int)Math.Round(OutputRectY), - SourceRectWidth = (int)Math.Round(OutputRectWidth), - SourceRectHeight = (int)Math.Round(OutputRectHeight), - OutputWidth = quality.Width, - OutputHeight = quality.Height, + SourceRectWidth = sourceRectWidth, + SourceRectHeight = sourceRectHeight, + OutputWidth = sourceRectWidth, + OutputHeight = sourceRectHeight, }; } // The credit's cadence, timing, placement and licence gate all live in // BrandFlashPresenter; MainViewModel only routes it to the frame pump and the diff --git a/docs/ai/frame-pipeline.md b/docs/ai/frame-pipeline.md index b1ce807..e2705a4 100644 --- a/docs/ai/frame-pipeline.md +++ b/docs/ai/frame-pipeline.md @@ -254,9 +254,10 @@ stays XAML (editing view); the compositor is the output view. `VisualBrush` samples all of `CanvasGrid`, and if the dims were inside it the box would show the dim shading — especially after the box is dragged away and the fixed capture slides under the dims. -- **Render the active tier's output rect directly** (`CompositorOptions {SourceRectX/Y/W/H, - OutputWidth, OutputHeight}`, fed from `MainViewModel.OutputRect*`): 16:9 = full 1920×1080 1:1; the - vertical 9:16 tier = composite the centered 607×1080 crop then bilinear-upscale to 1080×1920. +- **Composite the active tier's crop at crop size** (`CompositorOptions {SourceRectX/Y/W/H, + OutputWidth, OutputHeight}`, fed from `MainViewModel.OutputRect*`): 16:9 = full 1920×1080; the + vertical 9:16 tier = centered 608×1080 crop. FFmpeg's `scale` filter resizes the raw BGRA frame + to the selected output tier after it leaves the frame pump; `setsar=1` keeps square pixels. - **CPU posture:** with FFmpeg as a subprocess the master crosses a CPU readback to the pipe every frame anyway, so GPU compositing buys little at this layer count (2-3 live layers; static layers pre-composite once into a cached base). GPU effort belongs to **NVENC** (the encoder), not composition; @@ -363,9 +364,11 @@ seam:** `Func`, `Func` resolver, `Func /// The output compositor (TASK 4 ship step 1): renders a scene into the encoder's -/// master frame, mirroring the XAML preview minus the editing chrome. The integration +/// raw input frame, mirroring the XAML preview minus the editing chrome. The integration /// test composites a full scene (background + round webcam + images + mirror + border) -/// and asserts per-layer probe pixels; the vertical-tier test asserts the 9:16 crop -/// and upscale. Frames are pushed by the test — no capture managers involved. +/// and asserts per-layer probe pixels; the vertical-tier test checks the crop frame and +/// FFmpeg's final 9:16 resize. Frames are pushed by the test — no capture managers involved. /// public class SceneCompositorTests { @@ -407,9 +408,18 @@ public class SceneCompositorTests } [Fact] - public void Render_VerticalTier_Outputs_1080x1920_From_The_Center_Crop() + public void Render_VerticalTier_Crops_And_Configures_Ffmpeg_1080x1920_Scale() { - var red = Solid(1920, 1080, 255, 0, 0); + var backgroundPixels = new byte[1920 * 1080 * 4]; + for (var y = 0; y < 1080; y++) + for (var x = 0; x < 1920; x++) + { + var i = (y * 1920 + x) * 4; + backgroundPixels[i] = (byte)(x * 255 / 1919); + backgroundPixels[i + 1] = (byte)(y * 255 / 1079); + backgroundPixels[i + 3] = 255; + } + var backgroundFrame = new VideoFrame(1920, 1080, backgroundPixels); var green = Solid(1280, 720, 0, 255, 0); var background = new Source { Type = SourceType.DisplayCapture, IsBackground = true, CaptureKey = "monitor:0" }; @@ -421,26 +431,40 @@ public class SceneCompositorTests VideoFrame? FrameFor(SceneElement e) => e switch { WebcamSceneConfig => green, - Source { IsBackground: true } => red, + Source { IsBackground: true } => backgroundFrame, _ => null, }; var options = new CompositorOptions { - SourceRectX = 656, SourceRectY = 0, SourceRectWidth = 607, SourceRectHeight = 1080, - OutputWidth = 1080, OutputHeight = 1920, + SourceRectX = 656, SourceRectY = 0, SourceRectWidth = 608, SourceRectHeight = 1080, + OutputWidth = 608, OutputHeight = 1080, }; var output = new SceneCompositor().Render(scene, FrameFor, null, options); - Assert.Equal(1080, output.Width); - Assert.Equal(1920, output.Height); - // top-left of the crop is pure background (webcam starts at crop y=100, inset from the corner) - AssertColor(output, 0, 0, 255, 0, 0); - // the webcam center (crop 150,250) scales to output ~(267,444) and stays green - AssertColor(output, 267, 444, 0, 255, 0); + Assert.Equal(608, output.Width); + Assert.Equal(1080, output.Height); + // The centered crop and webcam are composed at source size. + AssertColor(output, 0, 0, 0, 0, 87); + AssertColor(output, 150, 250, 0, 255, 0); // far from the webcam, still background - AssertColor(output, 978, 1688, 255, 0, 0); + AssertColor(output, 550, 950, 0, 224, 160); + + // FFmpeg receives the crop, then scales it to the tier's encoded dimensions. + var args = FfmpegArgs.Build(new EncoderOptions + { + RtmpUrl = "rtmp://a.rtmp.youtube.com/live2/key", + InputWidth = output.Width, + InputHeight = output.Height, + Width = 1080, + Height = 1920, + Fps = 60, + }, "libopenh264").ToArray(); + var inputSize = Array.IndexOf(args, "-video_size"); + var scaleFilter = Array.IndexOf(args, "-vf"); + Assert.Equal("608x1080", args[inputSize + 1]); + Assert.Equal("scale=1080:1920:flags=bilinear,setsar=1", args[scaleFilter + 1]); } [Fact] @@ -616,4 +640,3 @@ public class StretchMathTests Assert.Equal(255, scaled.BgraPixels[i + 1]); // solid green survives the scale } } -