From ad9b3e1e48a1cf038f37be6f97c0bce4870a4285 Mon Sep 17 00:00:00 2001 From: gramps Date: Fri, 7 Aug 2026 14:15:50 -0700 Subject: [PATCH] Screen backdrop (schema v5/v6) + five-scene catalog + webcam polish: live desktop/game capture as a permanent non-deletable bottom layer (Source.IsBackdrop), auto full-screen game detection (Win32FullScreenDetector) else primary display, GraphicsCapturePicker re-designation, refcounted shared ScreenCaptureManager, Live-only backdrop by policy (HasBackdrop + v5-v6 backfill + EnforceBackdropPolicy, checkbox gone), SceneCatalog (Starting/Live/BRB/Chat/Ending) with + button re-adding missing scenes, webcam mid-session transparent-container fix (GetPreviewBitmap propagation), Chat half-screen-area cap, 'Add Webcam' always opens the picker (SwapWebcamIdentityAsync, no silent resurrect), WindowsRuntimeMarshal frame-read + 5s-throttled errors, docs updated, tests (65 passing) --- MainWindow.xaml | 90 ++++- MainWindow.xaml.cs | 40 +- Models/Scene.cs | 11 + Models/SceneCatalog.cs | 32 ++ Models/Source.cs | 28 +- Models/index.md | 5 +- Services/CameraManager.cs | 12 + Services/CaptureInterop.cs | 52 +++ Services/Direct3D11Helper.cs | 83 +++++ Services/IFullScreenDetector.cs | 21 ++ Services/IScreenCaptureSource.cs | 15 + Services/LayoutStore.cs | 83 ++++- Services/ScreenCaptureFrameSource.cs | 224 +++++++++++ Services/ScreenCaptureManager.cs | 246 +++++++++++++ Services/ScreenCaptureSourceFactory.cs | 74 ++++ Services/Win32FullScreenDetector.cs | 194 ++++++++++ Services/index.md | 12 +- TASKS.md | 22 +- ViewModels/MainViewModel.cs | 387 +++++++++++++++++--- ViewModels/index.md | 2 +- ai.md | 125 ++++++- ytLive.Tests/BackdropTests.cs | 118 ++++++ ytLive.Tests/LayoutStorePersistenceTests.cs | 145 ++++++++ ytLive.Tests/RoundClipInteractionTests.cs | 15 +- ytLive.Tests/SceneCatalogTests.cs | 77 ++++ ytLive.Tests/ScreenCaptureManagerTests.cs | 189 ++++++++++ ytLive.Tests/WebcamSafeguardTests.cs | 46 ++- 27 files changed, 2232 insertions(+), 116 deletions(-) create mode 100644 Models/SceneCatalog.cs create mode 100644 Services/CaptureInterop.cs create mode 100644 Services/Direct3D11Helper.cs create mode 100644 Services/IFullScreenDetector.cs create mode 100644 Services/IScreenCaptureSource.cs create mode 100644 Services/ScreenCaptureFrameSource.cs create mode 100644 Services/ScreenCaptureManager.cs create mode 100644 Services/ScreenCaptureSourceFactory.cs create mode 100644 Services/Win32FullScreenDetector.cs create mode 100644 ytLive.Tests/BackdropTests.cs create mode 100644 ytLive.Tests/SceneCatalogTests.cs create mode 100644 ytLive.Tests/ScreenCaptureManagerTests.cs diff --git a/MainWindow.xaml b/MainWindow.xaml index 815e55b..e5231f7 100644 --- a/MainWindow.xaml +++ b/MainWindow.xaml @@ -13,6 +13,8 @@ WindowStartupLocation="CenterScreen" Closing="MainWindow_Closing" Loaded="MainWindow_Loaded" + Activated="MainWindow_Activated" + Deactivated="MainWindow_Deactivated" PreviewMouseLeftButtonDown="Window_PreviewMouseLeftButtonDown"> @@ -130,11 +132,27 @@ - + - @@ -265,12 +283,25 @@ + + + + + + + + + @@ -305,10 +336,31 @@ - + + + + + + + + + + @@ -339,6 +391,12 @@ + + + + @@ -368,9 +426,15 @@ - - - - - - - - + + + + + diff --git a/MainWindow.xaml.cs b/MainWindow.xaml.cs index e251594..5dced42 100644 --- a/MainWindow.xaml.cs +++ b/MainWindow.xaml.cs @@ -46,6 +46,16 @@ public partial class MainWindow : Window _viewModel.Shutdown(); } + private void MainWindow_Activated(object? sender, EventArgs e) + { + _viewModel.RefreshBackdropAutoCapture(); + } + + private void MainWindow_Deactivated(object? sender, EventArgs e) + { + _viewModel.NoteBackgroundWindow(); + } + private void UpdateTaskbarOverlay() { TaskbarInfo.Overlay = _viewModel.IsLive @@ -91,6 +101,16 @@ public partial class MainWindow : Window } } + private void AddSceneButton_Click(object sender, RoutedEventArgs e) + { + if (sender is Button { ContextMenu: { } menu } button) + { + menu.PlacementTarget = button; + menu.Placement = System.Windows.Controls.Primitives.PlacementMode.Bottom; + menu.IsOpen = true; + } + } + // ─── Preview image selection / move / resize ─── private bool _isDraggingOverlay; private bool _isResizing; @@ -166,6 +186,22 @@ public partial class MainWindow : Window _viewModel.RemoveSourceCommand.Execute(element); } + private void BackdropMenu_ChangeCapture(object sender, RoutedEventArgs e) + { + _ = _viewModel.ChangeBackdropCaptureAsync(); + } + + private void BackdropMenu_RefreshCapture(object sender, RoutedEventArgs e) + { + _viewModel.RefreshBackdropAutoCapture(); + } + + private void SourceMenu_Remove(object sender, RoutedEventArgs e) + { + if (sender is MenuItem { DataContext: SceneElement element }) + _viewModel.RemoveSourceCommand.Execute(element); + } + private void OpacitySlider_ValueChanged(object sender, RoutedPropertyChangedEventArgs e) => OpacityValueText.Text = $"{Math.Round(e.NewValue * 100)}%"; @@ -227,8 +263,8 @@ public partial class MainWindow : Window else if (_isResizing) { var isWebcam = selected is WebcamSceneConfig; - var maxW = isWebcam ? MainViewModel.WebcamMaxWidth : 1920; - var maxH = isWebcam ? MainViewModel.WebcamMaxHeight : 1080; + var maxW = isWebcam ? MainViewModel.MaxWebcamWidthFor(_viewModel.ActiveScene?.Name) : 1920; + var maxH = isWebcam ? MainViewModel.MaxWebcamHeightFor(_viewModel.ActiveScene?.Name) : 1080; var newW = Math.Clamp(p.X - selected.X, 32, maxW); var newH = newW / _resizeAspect; if (newH > maxH) diff --git a/Models/Scene.cs b/Models/Scene.cs index e063fcb..a16a5c5 100644 --- a/Models/Scene.cs +++ b/Models/Scene.cs @@ -11,6 +11,7 @@ public class Scene : INotifyPropertyChanged private string _name = string.Empty; private bool _isEditing; private bool _isHidden; + private bool _hasBackdrop; public string Name { @@ -30,6 +31,16 @@ public class Scene : INotifyPropertyChanged set => Set(ref _isHidden, value); } + /// + /// Whether the scene has a live desktop/game backdrop layer. Enforced by + /// policy: only the canonical Live scene ever has one (see SceneCatalog). + /// + public bool HasBackdrop + { + get => _hasBackdrop; + set => Set(ref _hasBackdrop, value); + } + /// /// The scene's rendered elements in z-order (back to front): multi-instance /// Sources plus this scene's webcam usage (), if any. diff --git a/Models/SceneCatalog.cs b/Models/SceneCatalog.cs new file mode 100644 index 0000000..6e72eb2 --- /dev/null +++ b/Models/SceneCatalog.cs @@ -0,0 +1,32 @@ +using System; +using System.Linq; + +namespace ytLive.Models; + +/// +/// The app's five canonical scenes. Scenes are worked on by name; you can delete +/// them ("work with less"), but the app only ever adds back one of these five — +/// anything beyond them is OBS territory. The live desktop/game backdrop exists +/// only in the Live scene. +/// +public static class SceneCatalog +{ + public const string Starting = "Starting"; + public const string Live = "Live"; + public const string Brb = "BRB"; + public const string Chat = "Chat"; + public const string Ending = "Ending"; + + public static readonly string[] All = { Starting, Live, Brb, Chat, Ending }; + + public static bool IsCanonical(string? name) + => name != null && All.Contains(name.Trim(), StringComparer.OrdinalIgnoreCase); + + public static bool Is(string? name, string canonical) + => name != null && string.Equals(name.Trim(), canonical, StringComparison.OrdinalIgnoreCase); + + public static bool IsChat(string? name) => Is(name, Chat); + + /// Only the Live scene carries the live desktop/game backdrop. + public static bool HasBackdrop(string? name) => Is(name, Live); +} diff --git a/Models/Source.cs b/Models/Source.cs index e6ec022..c54f6c8 100644 --- a/Models/Source.cs +++ b/Models/Source.cs @@ -27,7 +27,22 @@ public enum ClipShape public class Source : SceneElement { private SourceType _type; - public SourceType Type { get => _type; set => Set(ref _type, value); } + public SourceType Type + { + get => _type; + set + { + if (Set(ref _type, value)) + { + Raise(nameof(IsLiveCapture)); + Raise(nameof(DisplaySource)); + } + } + } + + /// The permanent scene backdrop (live desktop/game capture). Never removable/reorderable. + private bool _isBackdrop; + public bool IsBackdrop { get => _isBackdrop; set => Set(ref _isBackdrop, value); } private bool _isEnabled = true; public bool IsEnabled { get => _isEnabled; set => Set(ref _isEnabled, value); } @@ -39,6 +54,15 @@ public class Source : SceneElement private IntPtr? _windowHandle; public IntPtr? WindowHandle { get => _windowHandle; set => Set(ref _windowHandle, value); } + /// + /// Identifies what this live-capture source points at: "monitor:<index>" or + /// "window:<hwnd>". Persisted so a re-designated backdrop survives a reload. + /// + private string? _captureKey; + public string? CaptureKey { get => _captureKey; set => Set(ref _captureKey, value); } + + public bool IsLiveCapture => Type is SourceType.DisplayCapture or SourceType.WindowCapture; + // Image (asset stored in the layout database) private string? _assetId; private ImageSource? _imageSource; @@ -59,7 +83,7 @@ public class Source : SceneElement public ImageSource? ImageSource => _imageSource; - public override ImageSource? DisplaySource => _imageSource; + public override ImageSource? DisplaySource => IsLiveCapture ? VideoImageSource : _imageSource; public override bool IsImageSource => Type == SourceType.Image; } diff --git a/Models/index.md b/Models/index.md index cf76703..a98e08d 100644 --- a/Models/index.md +++ b/Models/index.md @@ -5,8 +5,9 @@ Plain data types. No logic beyond what a property can carry. See | File | Purpose | |------|---------| -| `Scene.cs` | A scene: `Name`, `IsHidden`, `IsChatScene`, `IsEditing`, `Elements` collection (images + webcam config), `WebcamConfig` accessor | -| `Source.cs` | An image source: `SourceType` enum (Image/Screen/Background/Text — **Webcam removed**), `X`/`Y`/`Width`/`Height`, `Opacity`, `IsEnabled`, `ImageSource` (static) / `VideoImageSource` (live frames), `DisplaySource`, `ClipShape` enum (Traditional/Round), `IsMirrored` + `MirrorScale`, `MirrorButtonText`/`ShapeButtonText`, asset identity, `IsImageSource` (XAML binds this, never `Type`) | +| `Scene.cs` | A scene: `Name`, `IsHidden`, `IsChatScene`, `IsEditing`, **`HasBackdrop`** (Live-only policy flag, default off — only the canonical Live scene has one, see `SceneCatalog`), `Elements` collection (images + webcam config), `WebcamConfig` accessor | +| `SceneCatalog.cs` | The five canonical scenes (Starting/Live/BRB/Chat/Ending) — the product, by name: `All`, `IsCanonical`, `Is(name, canonical)` (case-insensitive trim), `IsChat`, `HasBackdrop(name)` (true only for Live). Drives the empty-DB seed, the "+" missing-scenes menu, and `MainViewModel.EnforceBackdropPolicy` | +| `Source.cs` | A source: `SourceType` enum (DisplayCapture/WindowCapture/Background/Image/TextOverlay — **Webcam removed**), `X`/`Y`/`Width`/`Height`, `Opacity`, `IsEnabled`, `ImageSource` (static) / `VideoImageSource` (live frames), `DisplaySource` (`IsLiveCapture` → video, else static), `ClipShape` enum (Traditional/Round), `IsMirrored` + `MirrorScale`, `MirrorButtonText`/`ShapeButtonText`, asset identity, `IsImageSource` (XAML binds this, never `Type`), **backdrop surface**: `IsBackdrop` (permanent bottom layer, never removable), `CaptureKey` (persisted `monitor:`/`window:`/`picker:`), `IsLiveCapture` (`Type` is Display/WindowCapture) | | `SceneElement.cs` | Base for anything placeable in a scene: shared layout/clip/mirror/border surface, `virtual IsWebcam`/`virtual IsImageSource`; `ToggleClipShape` + persisted `RectWidth`/`RectHeight` (pre-Round rect so round→rect restores after reload) | | `Webcam.cs` | Singleton webcam identity: `Id`, `DeviceId`, `Name` (one row app-wide) | | `WebcamSceneConfig.cs` | Per-scene webcam placement (subclass of `SceneElement`): geometry + `IsVisible` + border (`BorderColor`/`BorderOpacity`/`BorderWidth`/`BorderAnimation`) + `VideoImageSource`; `WebcamId` links to `Webcam` | diff --git a/Services/CameraManager.cs b/Services/CameraManager.cs index d8088ed..0b5dedb 100644 --- a/Services/CameraManager.cs +++ b/Services/CameraManager.cs @@ -150,6 +150,18 @@ public sealed class CameraManager : IDisposable return _sessions.TryGetValue(deviceId, out var session) ? session.LatestFrame : null; } + /// + /// The shared WriteableBitmap for a running session, or null when the camera + /// hasn't produced its first frame yet. only + /// fires once (first frame), so configs created after the session started must + /// pick the bitmap up from here. + /// + public WriteableBitmap? GetPreviewBitmap(string deviceId) + { + lock (_gate) + return _sessions.TryGetValue(deviceId, out var session) ? session.PreviewBitmap : null; + } + public void Dispose() { List sessions; diff --git a/Services/CaptureInterop.cs b/Services/CaptureInterop.cs new file mode 100644 index 0000000..7941ae3 --- /dev/null +++ b/Services/CaptureInterop.cs @@ -0,0 +1,52 @@ +using System; +using System.Runtime.InteropServices; +using Windows.Graphics.Capture; + +namespace ytLive.Services; + +/// +/// Desktop-interop bridges for Graphics Capture that CsWinRT can't project: +/// creating a from an HMONITOR/HWND via the +/// activation factory's IGraphicsCaptureItemInterop, and wiring the +/// GraphicsCapturePicker to an owner window via IInitializeWithWindow. +/// GUIDs and signatures come from the official Windows GraphicsCapture samples. +/// +internal static class CaptureInterop +{ + private static readonly Guid GraphicsCaptureItemGuid = new("79C3F95B-31F7-4EC2-A464-632EF5D30760"); + + public static GraphicsCaptureItem CreateForMonitor(IntPtr hmonitor) + { + var interop = GraphicsCaptureItem.As(); + return GraphicsCaptureItem.FromAbi(interop.CreateForMonitor(hmonitor, GraphicsCaptureItemGuid)); + } + + public static GraphicsCaptureItem CreateForWindow(IntPtr hwnd) + { + var interop = GraphicsCaptureItem.As(); + return GraphicsCaptureItem.FromAbi(interop.CreateForWindow(hwnd, GraphicsCaptureItemGuid)); + } + + public static void SetWindowOwner(GraphicsCapturePicker picker, IntPtr hwnd) + { + var interop = (IInitializeWithWindow)(object)picker; + interop.Initialize(hwnd); + } + + [ComImport] + [Guid("3628E81B-3CAC-4C60-B7F4-23CE0E0C3356")] + [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + private interface IGraphicsCaptureItemInterop + { + IntPtr CreateForWindow([In] IntPtr window, [In] ref Guid iid); + IntPtr CreateForMonitor([In] IntPtr monitor, [In] ref Guid iid); + } + + [ComImport] + [Guid("3E68D4BD-7135-4D10-8018-9FB6D9F33FA1")] + [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + private interface IInitializeWithWindow + { + void Initialize(IntPtr hwnd); + } +} diff --git a/Services/Direct3D11Helper.cs b/Services/Direct3D11Helper.cs new file mode 100644 index 0000000..2bd55a5 --- /dev/null +++ b/Services/Direct3D11Helper.cs @@ -0,0 +1,83 @@ +using System; +using System.Runtime.InteropServices; +using Windows.Graphics.DirectX.Direct3D11; +using WinRT; + +namespace ytLive.Services; + +/// +/// Bridges a native D3D11 device into the WinRT +/// the capture API needs. CsWinRT doesn't project the Direct3D11Helper static, +/// so this creates the device with D3D11CreateDevice and converts it via the +/// WinRT interop export CreateDirect3D11DeviceFromDXGIDevice (d3d11.dll). +/// +/// This deliberately does NOT use the older QI-for-IDirect3DDxgiInterfaceAccess +/// trick: the raw D3D11 device stopped exposing that interface on newer Windows +/// (verified E_NOINTERFACE on build 26200, hardware and WARP alike), while +/// CreateDirect3D11DeviceFromDXGIDevice keeps working. One shared device per +/// process. +/// +internal static class Direct3D11Helper +{ + private const uint D3D11CreateDeviceBgraSupport = 0x20; + private const uint D3D11SdkVersion = 7; + private const int DriverTypeHardware = 1; + + // IDirect3DDxgiInterfaceAccess (the legacy bridge) is only exposed on an + // 11.1 device AND requires the 11.1 runtime to be in the requested set — + // with pFeatureLevels = NULL D3D11CreateDevice never creates 11.1, so the + // array must be explicit (11.1 first, then descending). + private static readonly int[] FeatureLevels = { 0xB100, 0xB000, 0xA100, 0xA000, 0x9300, 0x9200, 0x9100 }; + + private static IDirect3DDevice? _sharedDevice; + private static readonly object Gate = new(); + + public static IDirect3DDevice CreateDevice() + { + if (_sharedDevice != null) return _sharedDevice; + lock (Gate) + { + return _sharedDevice ??= CreateDeviceCore(); + } + } + + private static IDirect3DDevice CreateDeviceCore() + { + var hr = D3D11CreateDevice(IntPtr.Zero, DriverTypeHardware, IntPtr.Zero, D3D11CreateDeviceBgraSupport, + FeatureLevels, (uint)FeatureLevels.Length, D3D11SdkVersion, out var devicePtr, out _, out var contextPtr); + if (hr != 0) + throw Marshal.GetExceptionForHR(hr)!; + + try + { + var dxgiGuid = new Guid("54EC77FA-1377-44E6-8C32-88FD5F44C84C"); + var qiHr = Marshal.QueryInterface(devicePtr, ref dxgiGuid, out var dxgiDevice); + if (qiHr != 0) + throw Marshal.GetExceptionForHR(qiHr)!; + try + { + hr = CreateDirect3D11DeviceFromDXGIDevice(dxgiDevice, out var winrtDevice); + if (hr != 0) + throw Marshal.GetExceptionForHR(hr)!; + return MarshalInterface.FromAbi(winrtDevice); + } + finally + { + Marshal.Release(dxgiDevice); + } + } + finally + { + Marshal.Release(devicePtr); + Marshal.Release(contextPtr); + } + } + + [DllImport("d3d11.dll")] + private static extern int D3D11CreateDevice(IntPtr pAdapter, int driverType, IntPtr software, + uint flags, int[]? featureLevels, uint featureLevelsCount, uint sdkVersion, + out IntPtr device, out int featureLevel, out IntPtr immediateContext); + + [DllImport("d3d11.dll", ExactSpelling = true)] + private static extern int CreateDirect3D11DeviceFromDXGIDevice(IntPtr dxgiDevice, out IntPtr graphicsDevice); +} diff --git a/Services/IFullScreenDetector.cs b/Services/IFullScreenDetector.cs new file mode 100644 index 0000000..45e1800 --- /dev/null +++ b/Services/IFullScreenDetector.cs @@ -0,0 +1,21 @@ +namespace ytLive.Services; + +/// +/// Identifies which monitor holds a full-screen window (a game/app running +/// borderless or exclusive fullscreen). Seam so ScreenCaptureManager is +/// testable without Win32 interop. +/// +public interface IFullScreenDetector +{ + /// + /// The index of the monitor a full-screen foreground window covers, or null + /// when the foreground window is windowed, missing, or one of our own. + /// + int? GetForegroundFullScreenMonitorIndex(); + + /// Index of the primary (taskbar-hosting) monitor. + int PrimaryMonitorIndex(); + + /// All monitors in capture-key order (enumeration order). + IReadOnlyList GetDisplays(); +} diff --git a/Services/IScreenCaptureSource.cs b/Services/IScreenCaptureSource.cs new file mode 100644 index 0000000..d297674 --- /dev/null +++ b/Services/IScreenCaptureSource.cs @@ -0,0 +1,15 @@ +namespace ytLive.Services; + +/// +/// A running screen-capture source for one target key ("monitor:<n>" or +/// "window:<hwnd>"). Raises normalized BGRA frames from a worker thread; +/// callers must marshal to the UI thread. Seam so ScreenCaptureManager is +/// testable without WinRT. +/// +public interface IScreenCaptureSource +{ + string Key { get; } + event Action? FrameAvailable; + Task StartAsync(); + Task StopAsync(); +} diff --git a/Services/LayoutStore.cs b/Services/LayoutStore.cs index a3b1bd8..356fa85 100644 --- a/Services/LayoutStore.cs +++ b/Services/LayoutStore.cs @@ -40,6 +40,7 @@ public class LayoutStore : IDisposable Name TEXT NOT NULL, IsHidden INTEGER NOT NULL DEFAULT 0, IsChatScene INTEGER NOT NULL DEFAULT 0, + HasBackdrop INTEGER NOT NULL DEFAULT 1, SortOrder INTEGER NOT NULL DEFAULT 0 ); """, @@ -69,6 +70,8 @@ public class LayoutStore : IDisposable DeviceId TEXT, ClipShape TEXT NOT NULL DEFAULT 'Traditional', IsMirrored INTEGER NOT NULL DEFAULT 0, + IsBackdrop INTEGER NOT NULL DEFAULT 0, + CaptureKey TEXT, SortOrder INTEGER NOT NULL DEFAULT 0 ); """, @@ -110,11 +113,12 @@ public class LayoutStore : IDisposable } MigrateSourceTable(); MigrateWebcamConfigTable(); + MigrateSceneTable(); if (GetUserVersion() < 3) MigrateToV3(); using (var cmd = _connection.CreateCommand()) { - cmd.CommandText = "PRAGMA user_version = 4;"; + cmd.CommandText = "PRAGMA user_version = 6;"; cmd.ExecuteNonQuery(); } } @@ -153,6 +157,20 @@ public class LayoutStore : IDisposable cmd.CommandText = "ALTER TABLE Source ADD COLUMN IsMirrored INTEGER NOT NULL DEFAULT 0;"; cmd.ExecuteNonQuery(); } + + if (!columns.Contains("IsBackdrop")) + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = "ALTER TABLE Source ADD COLUMN IsBackdrop INTEGER NOT NULL DEFAULT 0;"; + cmd.ExecuteNonQuery(); + } + + if (!columns.Contains("CaptureKey")) + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = "ALTER TABLE Source ADD COLUMN CaptureKey TEXT;"; + cmd.ExecuteNonQuery(); + } } // v3 → v4: WebcamSceneConfig gains RectWidth/RectHeight (the pre-Round rect, @@ -260,6 +278,47 @@ public class LayoutStore : IDisposable tx.Commit(); } + // v5 → v6: Scene gains HasBackdrop. Added columns default to true; the + // ONE-TIME backfill (runs only when the column is first added) turns every + // non-Live canonical scene off and drops their backdrop sources — the fix for + // DBs saved before the Live-only policy. MainViewModel.EnforceBackdropPolicy + // re-runs the same normalization on every load for DBs that miss the backfill. + private void MigrateSceneTable() + { + var columns = new HashSet(StringComparer.OrdinalIgnoreCase); + using (var cmd = _connection.CreateCommand()) + { + cmd.CommandText = "PRAGMA table_info(Scene);"; + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + columns.Add(reader.GetString(1)); + } + + if (columns.Contains("HasBackdrop")) return; + + using (var cmd = _connection.CreateCommand()) + { + cmd.CommandText = "ALTER TABLE Scene ADD COLUMN HasBackdrop INTEGER NOT NULL DEFAULT 1;"; + cmd.ExecuteNonQuery(); + } + + const string noBackdrop = "('starting', 'brb', 'chat', 'ending')"; + using (var cmd = _connection.CreateCommand()) + { + cmd.CommandText = $"UPDATE Scene SET HasBackdrop = 0 WHERE LOWER(TRIM(Name)) IN {noBackdrop};"; + cmd.ExecuteNonQuery(); + } + using (var cmd = _connection.CreateCommand()) + { + cmd.CommandText = $""" + DELETE FROM Source + WHERE IsBackdrop = 1 AND SceneId IN + (SELECT Id FROM Scene WHERE LOWER(TRIM(Name)) IN {noBackdrop}); + """; + cmd.ExecuteNonQuery(); + } + } + public List Load() { Webcam = null; @@ -269,7 +328,7 @@ public class LayoutStore : IDisposable using (var cmd = _connection.CreateCommand()) { - cmd.CommandText = "SELECT Id, Name, IsHidden, IsChatScene FROM Scene ORDER BY SortOrder"; + cmd.CommandText = "SELECT Id, Name, IsHidden, IsChatScene, HasBackdrop FROM Scene ORDER BY SortOrder"; using var reader = cmd.ExecuteReader(); while (reader.Read()) { @@ -279,6 +338,7 @@ public class LayoutStore : IDisposable Name = reader.GetString(1), IsHidden = reader.GetInt32(2) != 0, IsChatScene = reader.GetInt32(3) != 0, + HasBackdrop = reader.GetInt32(4) != 0, }); } } @@ -302,7 +362,8 @@ public class LayoutStore : IDisposable { cmd.CommandText = """ SELECT Id, SceneId, AssetId, Type, Name, IsEnabled, - X, Y, Width, Height, Opacity, MonitorIndex, ClipShape, IsMirrored + X, Y, Width, Height, Opacity, MonitorIndex, ClipShape, IsMirrored, + IsBackdrop, CaptureKey FROM Source ORDER BY SortOrder """; using var reader = cmd.ExecuteReader(); @@ -324,6 +385,8 @@ public class LayoutStore : IDisposable MonitorIndex = reader.IsDBNull(11) ? null : reader.GetInt32(11), ClipShape = Enum.TryParse(reader.GetString(12), out var clip) ? clip : ClipShape.Traditional, IsMirrored = reader.GetInt32(13) != 0, + IsBackdrop = reader.GetInt32(14) != 0, + CaptureKey = reader.IsDBNull(15) ? null : reader.GetString(15), }; if (!sourcesByScene.TryGetValue(sceneId, out var list)) sourcesByScene[sceneId] = list = new List(); @@ -412,14 +475,15 @@ public class LayoutStore : IDisposable using (var cmd = _connection.CreateCommand()) { cmd.CommandText = """ - INSERT INTO Scene (Id, Name, IsHidden, IsChatScene, SortOrder) - VALUES ($id, $name, $isHidden, $isChat, $sort) + INSERT INTO Scene (Id, Name, IsHidden, IsChatScene, HasBackdrop, SortOrder) + VALUES ($id, $name, $isHidden, $isChat, $hasBackdrop, $sort) """; cmd.Transaction = tx; var idP = cmd.Parameters.Add("$id", SqliteType.Text); var nameP = cmd.Parameters.Add("$name", SqliteType.Text); var hiddenP = cmd.Parameters.Add("$isHidden", SqliteType.Integer); var chatP = cmd.Parameters.Add("$isChat", SqliteType.Integer); + var backdropP = cmd.Parameters.Add("$hasBackdrop", SqliteType.Integer); var sortP = cmd.Parameters.Add("$sort", SqliteType.Integer); var sort = 0; @@ -429,6 +493,7 @@ public class LayoutStore : IDisposable nameP.Value = scene.Name; hiddenP.Value = scene.IsHidden ? 1 : 0; chatP.Value = scene.IsChatScene ? 1 : 0; + backdropP.Value = scene.HasBackdrop ? 1 : 0; sortP.Value = sort++; cmd.ExecuteNonQuery(); } @@ -439,10 +504,10 @@ public class LayoutStore : IDisposable cmd.CommandText = """ INSERT INTO Source (Id, SceneId, AssetId, Type, Name, IsEnabled, X, Y, Width, Height, Opacity, MonitorIndex, - ClipShape, IsMirrored, SortOrder) + ClipShape, IsMirrored, IsBackdrop, CaptureKey, SortOrder) VALUES ($id, $sceneId, $assetId, $type, $name, $isEnabled, $x, $y, $w, $h, $opacity, $monitor, - $clip, $mirrored, $sort) + $clip, $mirrored, $isBackdrop, $captureKey, $sort) """; cmd.Transaction = tx; var idP = cmd.Parameters.Add("$id", SqliteType.Text); @@ -459,6 +524,8 @@ public class LayoutStore : IDisposable var monitorP = cmd.Parameters.Add("$monitor", SqliteType.Integer); var clipP = cmd.Parameters.Add("$clip", SqliteType.Text); var mirroredP = cmd.Parameters.Add("$mirrored", SqliteType.Integer); + var isBackdropP = cmd.Parameters.Add("$isBackdrop", SqliteType.Integer); + var captureKeyP = cmd.Parameters.Add("$captureKey", SqliteType.Text); var sortP = cmd.Parameters.Add("$sort", SqliteType.Integer); foreach (var scene in scenes) @@ -481,6 +548,8 @@ public class LayoutStore : IDisposable monitorP.Value = (object?)source.MonitorIndex ?? DBNull.Value; clipP.Value = source.ClipShape.ToString(); mirroredP.Value = source.IsMirrored ? 1 : 0; + isBackdropP.Value = source.IsBackdrop ? 1 : 0; + captureKeyP.Value = (object?)source.CaptureKey ?? DBNull.Value; sortP.Value = sort++; cmd.ExecuteNonQuery(); } diff --git a/Services/ScreenCaptureFrameSource.cs b/Services/ScreenCaptureFrameSource.cs new file mode 100644 index 0000000..834bf4d --- /dev/null +++ b/Services/ScreenCaptureFrameSource.cs @@ -0,0 +1,224 @@ +using System; +using System.Runtime.InteropServices; +using System.Runtime.InteropServices.WindowsRuntime; +using System.Threading.Tasks; +using Windows.Graphics; +using Windows.Graphics.Capture; +using Windows.Graphics.DirectX; +using Windows.Graphics.DirectX.Direct3D11; +using Windows.Graphics.Imaging; +using ytLive.Helpers; + +namespace ytLive.Services; + +/// +/// A live screen capture for one target key ("monitor:<n>" or +/// "window:<hwnd>"). Owns the GraphicsCaptureItem, a free-threaded +/// Direct3D11CaptureFramePool and the capture session; frames are converted on +/// the capture worker thread from the GPU surface to a CPU BGRA VideoFrame. +/// Known OS limits: DRM content captures as black frames; capture pauses while +/// the app is minimized (the frame pool simply stops delivering frames). +/// +public sealed class ScreenCaptureFrameSource : IScreenCaptureSource +{ + private readonly object _gate = new(); + private GraphicsCaptureItem _item; + private Direct3D11CaptureFramePool? _framePool; + private GraphicsCaptureSession? _session; + private SizeInt32 _poolSize; + private bool _started; + private bool _framePending; + private DateTime _lastErrorLog = DateTime.MinValue; + + // The composition master frame (see ai.md "Resolution tiers"): the backdrop + // is an input layer, so we never hold a CPU frame bigger than the master. + private const int MaxBackdropWidth = 1920; + private const int MaxBackdropHeight = 1080; + + // A failing conversion must not re-flood the log at frame rate. + private static readonly TimeSpan ErrorLogThrottle = TimeSpan.FromSeconds(5); + + public string Key { get; } + public event Action? FrameAvailable; + + internal ScreenCaptureFrameSource(string key, GraphicsCaptureItem item) + { + Key = key; + _item = item; + } + + /// Wraps an item chosen through the OS GraphicsCapturePicker. + public static ScreenCaptureFrameSource CreateForPicker(string key, GraphicsCaptureItem item) + => new(key, item); + + public static ScreenCaptureFrameSource CreateForMonitor(int monitorIndex) + { + var hmonitor = Win32FullScreenDetector.GetMonitorHandle(monitorIndex); + if (hmonitor == IntPtr.Zero) + throw new InvalidOperationException($"Monitor {monitorIndex} is not connected"); + var item = CaptureInterop.CreateForMonitor(hmonitor); + return new ScreenCaptureFrameSource($"monitor:{monitorIndex}", item); + } + + public static ScreenCaptureFrameSource CreateForWindow(IntPtr hwnd) + { + var item = CaptureInterop.CreateForWindow(hwnd); + return new ScreenCaptureFrameSource($"window:0x{hwnd.ToInt64():X}", item); + } + + public Task StartAsync() + { + lock (_gate) + { + if (_started) return Task.CompletedTask; + var item = _item; + var device = Direct3D11Helper.CreateDevice(); + var framePool = Direct3D11CaptureFramePool.CreateFreeThreaded( + device, DirectXPixelFormat.B8G8R8A8UIntNormalized, 2, item.Size); + _poolSize = item.Size; + var session = framePool.CreateCaptureSession(item); + framePool.FrameArrived += OnFrameArrived; + session.StartCapture(); + _framePool = framePool; + _session = session; + _started = true; + } + return Task.CompletedTask; + } + + public Task StopAsync() + { + lock (_gate) + { + _started = false; + _framePending = false; + if (_framePool != null) + _framePool.FrameArrived -= OnFrameArrived; + _session?.Dispose(); + _session = null; + _framePool?.Dispose(); + _framePool = null; + } + return Task.CompletedTask; + } + + private void OnFrameArrived(Direct3D11CaptureFramePool sender, object args) + { + var frame = sender.TryGetNextFrame(); + if (frame == null) return; + lock (_gate) + { + if (!_started) + { + frame.Dispose(); + return; + } + } + + if (frame.ContentSize.Width != _poolSize.Width || frame.ContentSize.Height != _poolSize.Height) + { + sender.Recreate(Direct3D11Helper.CreateDevice(), + DirectXPixelFormat.B8G8R8A8UIntNormalized, 2, frame.ContentSize); + _poolSize = frame.ContentSize; + } + + if (_framePending) + { + frame.Dispose(); + return; + } + _framePending = true; + _ = ProcessFrameAsync(frame); + } + + private async Task ProcessFrameAsync(Direct3D11CaptureFrame frame) + { + try + { + using (frame) + using (var softwareBitmap = await SoftwareBitmap.CreateCopyFromSurfaceAsync( + frame.Surface, BitmapAlphaMode.Ignore)) + { + FrameAvailable?.Invoke(CopyToVideoFrame(softwareBitmap)); + } + } + catch (Exception ex) + { + var now = DateTime.UtcNow; + if (now - _lastErrorLog >= ErrorLogThrottle) + { + _lastErrorLog = now; + AppLog.Write($"ScreenCaptureFrameSource: frame conversion failed: {ex.Message}"); + } + } + finally + { + _framePending = false; + } + } + + private static VideoFrame CopyToVideoFrame(SoftwareBitmap bitmap) + { + var sw = bitmap.PixelWidth; + var sh = bitmap.PixelHeight; + using var buffer = bitmap.LockBuffer(BitmapBufferAccessMode.Read); + using var reference = buffer.CreateReference(); + if (!WindowsRuntimeMarshal.TryGetDataUnsafe(reference, out var data, out var capacity)) + throw new InvalidOperationException("Could not access the frame buffer."); + + var srcStride = sw * 4; + var count = (int)Math.Min(capacity, (uint)(sh * srcStride)); + + // The master frame is 1920×1080; a larger monitor is scaled down here so + // the CPU never holds a frame above the master (the rendered layer fills + // the frame with UniformToFill regardless). + if (sw > MaxBackdropWidth || sh > MaxBackdropHeight) + { + var scale = Math.Min(MaxBackdropWidth / (double)sw, MaxBackdropHeight / (double)sh); + var dw = Math.Max(1, (int)(sw * scale)); + var dh = Math.Max(1, (int)(sh * scale)); + return new VideoFrame(dw, dh, DownscaleBgra(data, sw, sh, srcStride, dw, dh)); + } + + var pixels = new byte[count]; + Marshal.Copy(data, pixels, 0, pixels.Length); + return new VideoFrame(sw, sh, pixels); + } + + // Bilinear downscale to the master frame. Reads each source row pair through + // Marshal.Copy (no unsafe), writing tightly packed BGRA output. + private static byte[] DownscaleBgra(IntPtr src, int sw, int sh, int srcStride, int dw, int dh) + { + var row0 = new byte[srcStride]; + var row1 = new byte[srcStride]; + var dst = new byte[dw * dh * 4]; + var xs = sw / (double)dw; + var ys = sh / (double)dh; + + for (var y = 0; y < dh; y++) + { + var sy = Math.Min(sh - 1, (int)(y * ys)); + var sy1 = Math.Min(sh - 1, sy + 1); + var fy = (y * ys) - sy; + Marshal.Copy(IntPtr.Add(src, sy * srcStride), row0, 0, srcStride); + Marshal.Copy(IntPtr.Add(src, sy1 * srcStride), row1, 0, srcStride); + + var dRow = y * dw * 4; + for (var x = 0; x < dw; x++) + { + var sx = Math.Min(sw - 1, (int)(x * xs)); + var sx1 = Math.Min(sw - 1, sx + 1); + var fx = (x * xs) - sx; + for (var c = 0; c < 4; c++) + { + var i0 = sx * 4 + c; + var i1 = sx1 * 4 + c; + var top = row0[i0] + (row0[i1] - row0[i0]) * fx; + var bottom = row1[i0] + (row1[i1] - row1[i0]) * fx; + dst[dRow + x * 4 + c] = (byte)(top + (bottom - top) * fy); + } + } + } + return dst; + } +} diff --git a/Services/ScreenCaptureManager.cs b/Services/ScreenCaptureManager.cs new file mode 100644 index 0000000..2acff60 --- /dev/null +++ b/Services/ScreenCaptureManager.cs @@ -0,0 +1,246 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using System.Windows; +using System.Windows.Media; +using System.Windows.Media.Imaging; +using System.Windows.Threading; +using ytLive.Helpers; + +namespace ytLive.Services; + +/// +/// Owns screen-capture sessions app-wide, refcounted by target key +/// ("monitor:<n>", "window:<hwnd>", "picker:<...>"). One target = +/// at most one capture, one shared WriteableBitmap; the last release stops and +/// disposes the source. Frames arrive on a worker thread and are coalesced onto +/// the UI dispatcher (at most one pending copy per session, using the latest +/// frame). Mirrors CameraManager. +/// +public sealed class ScreenCaptureManager : IDisposable +{ + private sealed class CaptureSession + { + public string Key { get; } + public IScreenCaptureSource Source { get; } + public Action? FrameHandler; + public int RefCount; + public bool Started; + public WriteableBitmap? PreviewBitmap; + public VideoFrame? LatestFrame; + public bool FramePending; + + public CaptureSession(string key, IScreenCaptureSource source) + { + Key = key; + Source = source; + RefCount = 1; + } + } + + private readonly Func _sourceFactory; + private readonly Dispatcher? _uiDispatcher; + private readonly Dictionary _sessions = new(); + private readonly object _gate = new(); + + /// Raised on the UI thread when a capture's shared preview bitmap is first created. + public event Action? PreviewBitmapChanged; + + /// Raised when a capture cannot be created or started (monitor gone, access denied, no target). + public event Action? CaptureFailed; + + public ScreenCaptureManager(Func sourceFactory, Dispatcher? uiDispatcher = null) + { + _sourceFactory = sourceFactory; + _uiDispatcher = uiDispatcher; + } + + /// Increments the refcount for a target key, starting capture the first time. + public async Task AcquireAsync(string key) + { + if (string.IsNullOrWhiteSpace(key)) return false; + + CaptureSession session; + bool shouldStart; + lock (_gate) + { + if (_sessions.TryGetValue(key, out var existing)) + { + existing.RefCount++; + shouldStart = false; + session = existing; + } + else + { + IScreenCaptureSource? source; + try + { + source = _sourceFactory(key); + } + catch (Exception ex) + { + AppLog.Write($"ScreenCaptureManager: creating capture '{key}' failed: {ex.Message}"); + CaptureFailed?.Invoke(key, ex.Message); + return false; + } + if (source == null) + { + CaptureFailed?.Invoke(key, "No capture target for this key"); + return false; + } + session = new CaptureSession(key, source); + session.FrameHandler = frame => OnFrameAvailable(session, frame); + session.Source.FrameAvailable += session.FrameHandler; + _sessions[key] = session; + shouldStart = true; + } + } + + if (!shouldStart) return session.Started; + + try + { + await session.Source.StartAsync(); + session.Started = true; + return true; + } + catch (Exception ex) + { + lock (_gate) + _sessions.Remove(key); + session.Source.FrameAvailable -= session.FrameHandler; + AppLog.Write($"ScreenCaptureManager: failed to start capture '{key}': {ex.Message}"); + CaptureFailed?.Invoke(key, ex.Message); + await SafeStopAsync(session.Source); + return false; + } + } + + /// Decrements the refcount; stops and disposes the source at zero. + public async Task ReleaseAsync(string key) + { + CaptureSession? toStop = null; + lock (_gate) + { + if (!_sessions.TryGetValue(key, out var session)) return; + if (--session.RefCount > 0) return; + _sessions.Remove(key); + toStop = session; + } + + if (toStop == null) return; + toStop.Source.FrameAvailable -= toStop.FrameHandler; + await SafeStopAsync(toStop.Source); + toStop.PreviewBitmap = null; + } + + /// + /// Releases a target unconditionally (every ref), regardless of how many + /// scenes hold it. Used on capture re-designation and layout reloads where + /// the old key's refcount isn't known after the scenes are replaced. + /// + public async Task ReleaseAllAsync(string key) + { + CaptureSession? toStop = null; + lock (_gate) + { + if (!_sessions.TryGetValue(key, out var session)) return; + session.RefCount = 0; + _sessions.Remove(key); + toStop = session; + } + + if (toStop == null) return; + toStop.Source.FrameAvailable -= toStop.FrameHandler; + await SafeStopAsync(toStop.Source); + toStop.PreviewBitmap = null; + } + + public void Dispose() + { + List sessions; + lock (_gate) + { + sessions = new List(_sessions.Values); + _sessions.Clear(); + } + + foreach (var session in sessions) + { + session.Source.FrameAvailable -= session.FrameHandler; + _ = SafeStopAsync(session.Source); + } + } + + private static async Task SafeStopAsync(IScreenCaptureSource source) + { + try + { + await source.StopAsync(); + } + catch (Exception ex) + { + AppLog.Write($"ScreenCaptureManager: stopping capture failed: {ex.Message}"); + } + } + + private bool TryGetActiveSession(CaptureSession session) + { + lock (_gate) + return _sessions.TryGetValue(session.Key, out var current) && ReferenceEquals(current, session); + } + + private void OnFrameAvailable(CaptureSession session, VideoFrame frame) + { + if (!TryGetActiveSession(session)) return; + session.LatestFrame = frame; + + if (session.PreviewBitmap == null) + { + if (_uiDispatcher == null) return; + if (_uiDispatcher.CheckAccess()) + EnsurePreviewBitmap(session); + else + _uiDispatcher.BeginInvoke(() => + { + if (TryGetActiveSession(session)) + EnsurePreviewBitmap(session); + }, DispatcherPriority.Render); + return; + } + + if (_uiDispatcher == null || _uiDispatcher.CheckAccess()) + { + CopyFrame(session); + return; + } + + if (session.FramePending) return; + session.FramePending = true; + _uiDispatcher.BeginInvoke(() => + { + session.FramePending = false; + if (TryGetActiveSession(session) && session.PreviewBitmap != null) + CopyFrame(session); + }, DispatcherPriority.Render); + } + + private void EnsurePreviewBitmap(CaptureSession session) + { + if (session.PreviewBitmap != null || session.LatestFrame == null) return; + var frame = session.LatestFrame; + var bitmap = new WriteableBitmap(frame.Width, frame.Height, 96, 96, PixelFormats.Bgra32, null); + bitmap.WritePixels(new Int32Rect(0, 0, frame.Width, frame.Height), frame.BgraPixels, frame.Stride, 0); + session.PreviewBitmap = bitmap; + PreviewBitmapChanged?.Invoke(session.Key, bitmap); + } + + private void CopyFrame(CaptureSession session) + { + var bitmap = session.PreviewBitmap; + var frame = session.LatestFrame; + if (bitmap == null || frame == null) return; + if (frame.Width != bitmap.PixelWidth || frame.Height != bitmap.PixelHeight) return; + bitmap.WritePixels(new Int32Rect(0, 0, frame.Width, frame.Height), frame.BgraPixels, frame.Stride, 0); + } +} diff --git a/Services/ScreenCaptureSourceFactory.cs b/Services/ScreenCaptureSourceFactory.cs new file mode 100644 index 0000000..de255e2 --- /dev/null +++ b/Services/ScreenCaptureSourceFactory.cs @@ -0,0 +1,74 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Threading.Tasks; +using Windows.Graphics.Capture; + +namespace ytLive.Services; + +/// +/// Turns a capture key ("monitor:<n>", "window:<hwnd>", "picker:<...>") +/// into a live , and hosts the OS +/// GraphicsCapturePicker used by "Change Capture…". Picker picks are transient: +/// the picked GraphicsCaptureItem has no HWND/monitor identity, so a +/// "picker:" key resolves only within the current session and re-detection +/// takes over after a reload (documented v1 limit). +/// +public sealed class ScreenCaptureSourceFactory +{ + private readonly Func _ownerHwndProvider; + private readonly Dictionary _picked = new(); + private readonly object _gate = new(); + + public ScreenCaptureSourceFactory(Func ownerHwndProvider) + { + _ownerHwndProvider = ownerHwndProvider; + } + + /// Creates the source for a persisted key, or null when it can't be resolved. + public IScreenCaptureSource? Resolve(string key) + { + if (key.StartsWith("monitor:", StringComparison.OrdinalIgnoreCase)) + { + var indexText = key.AsSpan("monitor:".Length); + if (int.TryParse(indexText, NumberStyles.Integer, CultureInfo.InvariantCulture, out var index)) + return ScreenCaptureFrameSource.CreateForMonitor(index); + return null; + } + + if (key.StartsWith("window:", StringComparison.OrdinalIgnoreCase)) + { + var hexText = key.AsSpan("window:".Length); + if (long.TryParse(hexText, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var hwnd)) + return ScreenCaptureFrameSource.CreateForWindow(new IntPtr(hwnd)); + return null; + } + + if (key.StartsWith("picker:", StringComparison.OrdinalIgnoreCase)) + { + lock (_gate) + return _picked.TryGetValue(key, out var item) + ? ScreenCaptureFrameSource.CreateForPicker(key, item) + : null; + } + + return null; + } + + /// + /// Shows the OS capture picker (must run on the UI thread, owner window set + /// via IInitializeWithWindow). Returns the session key, or null if cancelled. + /// + public async Task PickAsync() + { + var picker = new GraphicsCapturePicker(); + CaptureInterop.SetWindowOwner(picker, _ownerHwndProvider()); + var item = await picker.PickSingleItemAsync(); + if (item == null) return null; + + var key = $"picker:{item.DisplayName}"; + lock (_gate) + _picked[key] = item; + return key; + } +} diff --git a/Services/Win32FullScreenDetector.cs b/Services/Win32FullScreenDetector.cs new file mode 100644 index 0000000..a1d3dc0 --- /dev/null +++ b/Services/Win32FullScreenDetector.cs @@ -0,0 +1,194 @@ +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Windows.Interop; + +namespace ytLive.Services; + +/// A physical display: its capture-key index, geometry, and primary flag. +public sealed record DisplayInfo(int Index, string Name, int Width, int Height, int X, int Y, bool IsPrimary) +{ + public string Label + => IsPrimary ? $"{Name} — {Width}×{Height} (primary)" : $"{Name} — {Width}×{Height}"; +} + +/// +/// Win32 detection: GetForegroundWindow + DwmGetWindowAttribute +/// (DWMWA_EXTENDED_FRAME_BOUNDS) + MonitorFromWindow + GetMonitorInfo. +/// A foreground window whose extended frame bounds cover an entire monitor is +/// treated as a full-screen game/app on that monitor's index (monitor order = +/// EnumDisplayMonitors enumeration order, the same order the capture +/// factory uses to map index → HMONITOR). Windows of our own process are excluded. +/// DRM-protected content captures as black frames — a documented OS limit. +/// +public sealed class Win32FullScreenDetector : IFullScreenDetector +{ + private const int DwmwaExtendedFrameBounds = 9; + private const uint MonitorDefaultToNearest = 2; + private const uint MonitorInfoFPrimary = 0x00000001; + + public int? GetForegroundFullScreenMonitorIndex() + { + var hwnd = GetForegroundWindow(); + if (hwnd == IntPtr.Zero || IsOwnWindow(hwnd)) return null; + if (!TryGetExtendedFrameBounds(hwnd, out var bounds)) return null; + + var monitor = MonitorFromWindow(hwnd, MonitorDefaultToNearest); + if (monitor == IntPtr.Zero || !TryGetMonitorInfo(monitor, out var info)) return null; + + var r = info.RcMonitor; + var coversMonitor = bounds.Left <= r.Left && bounds.Top <= r.Top && + bounds.Right >= r.Right && bounds.Bottom >= r.Bottom; + if (!coversMonitor) return null; + + return MonitorIndex(monitor); + } + + private static bool IsOwnWindow(IntPtr hwnd) + { + _ = GetWindowThreadProcessId(hwnd, out var pid); + return pid == Environment.ProcessId; + } + + private static bool TryGetExtendedFrameBounds(IntPtr hwnd, out Win32Rect bounds) + { + bounds = default; + return DwmGetWindowAttribute(hwnd, DwmwaExtendedFrameBounds, ref bounds, Marshal.SizeOf()) == 0; + } + + private static bool TryGetMonitorInfo(IntPtr monitor, out MonitorInfo info) + { + info = new MonitorInfo { CbSize = Marshal.SizeOf() }; + return GetMonitorInfo(monitor, ref info); + } + + private static int? MonitorIndex(IntPtr monitor) + { + var handles = EnumerateMonitors(); + var index = handles.IndexOf(monitor); + return index >= 0 ? index : null; + } + + public int PrimaryMonitorIndex() + => GetDisplays().FirstOrDefault(d => d.IsPrimary)?.Index ?? 0; + + public IReadOnlyList GetDisplays() + { + var result = new List(); + var handles = EnumerateMonitors(); + for (var i = 0; i < handles.Count; i++) + { + if (!TryGetMonitorInfo(handles[i], out var info)) continue; + var name = GetDisplayName(handles[i]); + result.Add(new DisplayInfo( + i, + name, + info.RcMonitor.Right - info.RcMonitor.Left, + info.RcMonitor.Bottom - info.RcMonitor.Top, + info.RcMonitor.Left, + info.RcMonitor.Top, + (info.DwFlags & MonitorInfoFPrimary) != 0)); + } + return result; + } + + private static string GetDisplayName(IntPtr monitor) + { + var info = new MonitorInfoEx { CbSize = Marshal.SizeOf() }; + if (!GetMonitorInfo(monitor, ref info)) return $"Display {monitor}"; + var deviceName = info.SzDevice.TrimEnd('\0'); + var dev = new DisplayDevice { Cb = Marshal.SizeOf() }; + if (!EnumDisplayDevices(deviceName, 0, ref dev, 0)) return deviceName; + var friendly = dev.DeviceString.TrimEnd('\0'); + return string.IsNullOrWhiteSpace(friendly) ? deviceName : friendly; + } + + public static IntPtr GetMonitorHandle(int index) + { + var handles = EnumerateMonitors(); + return index >= 0 && index < handles.Count ? handles[index] : IntPtr.Zero; + } + + private static List EnumerateMonitors() + { + var handles = new List(); + EnumDisplayMonitors(IntPtr.Zero, IntPtr.Zero, + (IntPtr hMonitor, IntPtr hdcMonitor, ref Win32Rect lprcMonitor, IntPtr dwData) => + { + handles.Add(hMonitor); + return true; + }, IntPtr.Zero); + return handles; + } + + [StructLayout(LayoutKind.Sequential)] + private struct Win32Rect + { + public int Left; + public int Top; + public int Right; + public int Bottom; + } + + [StructLayout(LayoutKind.Sequential)] + private struct MonitorInfo + { + public int CbSize; + public Win32Rect RcMonitor; + public Win32Rect RcWork; + public uint DwFlags; + } + + [StructLayout(LayoutKind.Sequential)] + private struct MonitorInfoEx + { + public int CbSize; + public Win32Rect RcMonitor; + public Win32Rect RcWork; + public uint DwFlags; + [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 32)] + public string SzDevice; + } + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)] + private struct DisplayDevice + { + public int Cb; + [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 32)] + public string DeviceName; + [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 128)] + public string DeviceString; + public uint StateFlags; + [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 128)] + public string DeviceId; + [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 128)] + public string DeviceKey; + } + + [DllImport("user32.dll")] + private static extern IntPtr GetForegroundWindow(); + + [DllImport("user32.dll")] + private static extern IntPtr MonitorFromWindow(IntPtr hwnd, uint dwFlags); + + [DllImport("user32.dll", CharSet = CharSet.Unicode)] + private static extern bool GetMonitorInfo(IntPtr hMonitor, ref MonitorInfo lpmi); + + [DllImport("user32.dll", CharSet = CharSet.Unicode)] + private static extern bool GetMonitorInfo(IntPtr hMonitor, ref MonitorInfoEx lpmi); + + [DllImport("user32.dll", CharSet = CharSet.Unicode)] + private static extern bool EnumDisplayDevices(string lpDevice, uint iDevNum, ref DisplayDevice lpDisplayDevice, uint dwFlags); + + [DllImport("user32.dll")] + private static extern bool EnumDisplayMonitors(IntPtr hdc, IntPtr lprcClip, + MonitorEnumProc lpfnEnum, IntPtr dwData); + + [DllImport("user32.dll")] + private static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint lpdwProcessId); + + [DllImport("dwmapi.dll")] + private static extern int DwmGetWindowAttribute(IntPtr hwnd, int dwAttribute, ref Win32Rect pvAttribute, int cbAttribute); + + private delegate bool MonitorEnumProc(IntPtr hMonitor, IntPtr hdcMonitor, ref Win32Rect lprcMonitor, IntPtr dwData); +} diff --git a/Services/index.md b/Services/index.md index c4fa94d..b989445 100644 --- a/Services/index.md +++ b/Services/index.md @@ -8,14 +8,22 @@ External-facing logic: YouTube API, persistence. See | `YouTubeAuthService.cs` | OAuth2 via Google: loopback callback (`http://localhost:8765/oauth2/callback`), token exchange, refresh, channel fetch. Constructor takes optional `HttpClient` + `sessionChanged` callback (test seam + save hook); session persists via `Helpers/TokenStore` (DPAPI); `ClearSession()` signs out (called by `MainViewModel.StopStream` on End Livestream) | | `YouTubeStreamService.cs` | Broadcast/stream management via the v3 API (`enableAutoStart/Stop`). **Not yet switched to the `variable` reusable stream** | | `YouTubeChatService.cs` | Polls `liveChat/messages`, raises `MessageReceived`; `IDisposable` | -| `LayoutStore.cs` | SQLite persistence (`Microsoft.Data.Sqlite`) at `%APPDATA%\ytLlive\ytLlive.db`; assets stored as BLOBs keyed by SHA-256 content hash; save/open layout files; schema `user_version` 4 (`Source.ClipShape`/`IsMirrored` via `ALTER TABLE` for pre-v2 DBs; v3 = singleton `Webcam` + per-scene `WebcamSceneConfig`, migrated idempotently **without backfill** — the stale `Source.DeviceId` column remains but is no longer read/written; v4 = `WebcamSceneConfig.RectWidth`/`RectHeight`, the pre-Round rect for the round-to-rect restore) | +| `LayoutStore.cs` | SQLite persistence (`Microsoft.Data.Sqlite`) at `%APPDATA%\ytLlive\ytLlive.db`; assets stored as BLOBs keyed by SHA-256 content hash; save/open layout files; schema `user_version` 6 (`Source.ClipShape`/`IsMirrored` via `ALTER TABLE` for pre-v2 DBs; v3 = singleton `Webcam` + per-scene `WebcamSceneConfig`, migrated idempotently **without backfill** — the stale `Source.DeviceId` column remains but is no longer read/written; v4 = `WebcamSceneConfig.RectWidth`/`RectHeight`, the pre-Round rect for the round-to-rect restore; v5 = `Source.IsBackdrop` + `Source.CaptureKey`, the live-capture backdrop; v6 = `Scene.HasBackdrop` — Live-only policy, one-time backfill turns Starting/BRB/Chat/Ending off + drops their backdrop sources; `MainViewModel.EnforceBackdropPolicy` re-normalizes on every load) | | `VideoFrame.cs` | Normalized CPU frame seam (`Width`/`Height`/tightly-packed BGRA `byte[]`) — the only pixel type the rest of the app knows about; future capture sources (screen, background-removed webcam) feed the same seam | | `CameraDeviceInfo.cs` | `(Id, DisplayName)` for a physical capture device | | `ICameraEnumerator.cs` | `GetCamerasAsync()` — seam so the picker/`CameraManager` never touch WinRT (tests inject fakes) | | `ICameraFrameSource.cs` | `StartAsync`/`StopAsync`/`FrameAvailable(VideoFrame)` — seam for a running capture source | | `MediaCaptureCameraEnumerator.cs` | WinRT enumeration via `DeviceInformation.FindAllAsync(DeviceClass.VideoCapture)` | | `MediaCaptureFrameSource.cs` | CPU-first MediaCapture source (`MemoryPreference = Cpu`, BGRA8 via `CreateFrameReaderAsync`); frames arrive on a worker thread | -| `CameraManager.cs` | Webcam capture ownership: refcounted by `DeviceId`, one shared `WriteableBitmap` per camera, dispatcher-coalesced ~render-rate UI updates (latest-frame drop); `PreviewBitmapChanged`/`CameraFailed` events. `ReleaseAsync` decrements a ref (stops at zero); `ReleaseAllAsync` force-drops every ref + stops the source (webcam swap / layout reload) | +| `CameraManager.cs` | Webcam capture ownership: refcounted by `DeviceId`, one shared `WriteableBitmap` per camera, dispatcher-coalesced ~render-rate UI updates (latest-frame drop); `PreviewBitmapChanged`/`CameraFailed` events. `ReleaseAsync` decrements a ref (stops at zero); `ReleaseAllAsync` force-drops every ref + stops the source (webcam swap / layout reload). `GetPreviewBitmap(deviceId)` returns the current shared bitmap so a `WebcamSceneConfig` added mid-session (after the first frame already created the bitmap) still receives the live frames | +| `IFullScreenDetector.cs` | Seam for the win32 full-screen detector: `int? GetForegroundFullScreenMonitorIndex()`, `int PrimaryMonitorIndex()`, `IReadOnlyList GetDisplays()` | +| `Win32FullScreenDetector.cs` | `GetForegroundWindow` + `DwmGetWindowAttribute(DWMWA_EXTENDED_FRAME_BOUNDS)` + `MonitorFromWindow` + `GetMonitorInfo`; monitor covers all four edges → full-screen; own process excluded; monitor order = `EnumDisplayMonitors` order (static `GetMonitorHandle(int)` maps index→HMONITOR for the capture factory). `GetDisplays()` returns `DisplayInfo` (index/name/resolution/bounds/`IsPrimary`, friendly name via `EnumDisplayDevices`) for the in-app "Capture Display" picker; `PrimaryMonitorIndex()` is the auto-key fallback when no full-screen game is detected | +| `IScreenCaptureSource.cs` | `Key` + `StartAsync`/`StopAsync`/`FrameAvailable(VideoFrame)` — seam so `ScreenCaptureManager` never touches WinRT (tests inject fakes) | +| `Direct3D11Helper.cs` | The one shared `IDirect3DDevice` per process: P/Invoke `d3d11.dll!D3D11CreateDevice` (hardware, BGRA_SUPPORT, explicit 11.1-first feature array) → QI `IDXGIDevice` → the WinRT interop export `CreateDirect3D11DeviceFromDXGIDevice` → `MarshalInterface.FromAbi`. CsWinRT does **not** project `Windows.Graphics.Direct3D11.Direct3D11Helper`, so this hand-rolls it. Do **not** revert to QI-for-`IDirect3DDxgiInterfaceAccess` — the device no longer exposes it on newer Windows (E_NOINTERFACE on build 26200) | +| `CaptureInterop.cs` | ComImport bridges the WinRT-projection gap: `IGraphicsCaptureItemInterop` (`CreateForMonitor`/`CreateForWindow`), `IInitializeWithWindow` (`GraphicsCapturePicker` owner window so the OS picker shows) | +| `ScreenCaptureFrameSource.cs` | One `Direct3D11CaptureFramePool` (free-threaded, 2 buffers) + session per target; frames → `SoftwareBitmap.CreateCopyFromSurfaceAsync` (BGRA, alpha ignored) → `VideoFrame`, bytes read via `WindowsRuntimeMarshal.TryGetDataUnsafe` (CsWinRT-safe — the `IMemoryBufferByteAccess` ComImport cast fails on every frame and is gone). Surfaces > 1920×1080 downscaled bilinearly to the master; conversion failures logged ≤ once/5 s. DRM content = black frames (OS limit). `CreateForMonitor`/`CreateForWindow`/`CreateForPicker` | +| `ScreenCaptureManager.cs` | Screen-capture ownership mirroring `CameraManager`: refcounted by target key, one shared `WriteableBitmap`, dispatcher-coalesced latest-frame copies; `PreviewBitmapChanged`/`CaptureFailed` events; `ReleaseAllAsync` used on re-designation | +| `ScreenCaptureSourceFactory.cs` | `Resolve(key)` parses `monitor:` / `window:` / `picker:` into a source; `PickAsync()` shows the OS `GraphicsCapturePicker` and returns the `picker:` key (transient — a reload falls back to auto-detection) | Related: constructed in [`ViewModels/MainViewModel.cs`](../ViewModels/MainViewModel.cs) (no DI container yet). Models in [`Models/index.md`](../Models/index.md). diff --git a/TASKS.md b/TASKS.md index bf2404a..bdf561e 100644 --- a/TASKS.md +++ b/TASKS.md @@ -159,7 +159,7 @@ Preview shows the transition too (WYSIWYG). No wipes/slides/LUTs beyond the four - **Scenes list:** drag rows to reorder scenes - **Sources list:** drag rows to reorder sources (this *is* the z-order) — implemented -### Status: 🔶 In progress — milestone 1 (webcam) shipped; **schema v3 (Ship Branch A) shipped**: multi-scene webcam (singleton `Webcam` + per-scene `WebcamSceneConfig`), right-click border/context menu, static OSB-style borders, 50%-per-dimension webcam size cap, device-swap (`ReleaseAllAsync`); **schema v4**: round→rect restore persisted (`WebcamSceneConfig.RectWidth`/`RectHeight`) + one-time legacy-square 16:9 heal on load — 25 tests passing; scenes/sources UI (add/reorder/rename, image + background overlays with move/resize/opacity/reuse) built; screen capture, compositing, encoding pending +### Status: 🔶 In progress — milestone 1 (webcam) shipped; **schema v3 (Ship Branch A) shipped**: multi-scene webcam (singleton `Webcam` + per-scene `WebcamSceneConfig`), right-click border/context menu, static OSB-style borders, 50%-per-dimension webcam size cap, device-swap (`ReleaseAllAsync`); **schema v4**: round→rect restore persisted (`WebcamSceneConfig.RectWidth`/`RectHeight`) + one-time legacy-square 16:9 heal on load — 25 tests passing; **screen backdrop (ship task #1) shipped**: live desktop/game capture as a permanent, non-deletable bottom layer (`Source.IsBackdrop`, schema v5), auto-detecting the full-screen game at launch/focus (else the **primary display** — never assumed monitor 0) via `Win32FullScreenDetector` (now with `GetDisplays()`/`PrimaryMonitorIndex()` for the in-app display picker), content re-designated via the OS `GraphicsCapturePicker` ("Change Capture…") or the in-app "Capture Display" submenu, refcounted/shared capture sessions in `ScreenCaptureManager` mirroring `CameraManager` — 45 tests passing; **schema v6**: `Scene.HasBackdrop`, now **Live-only by policy** — the backdrop is enforced by scene name on every load (`EnforceBackdropPolicy`: Starting/BRB/Chat/Ending never carry one; the one-time v5→v6 backfill covers all four), the scene context-menu "Backdrop" checkbox is gone (policy owns the flag), preview watermark hides when the backdrop renders, capture changed to `WindowsRuntimeMarshal.TryGetDataUnsafe` (the CsWinRT-safe frame-read) + downscale to the 1920×1080 master + 5s-throttled error logging (was flooding `startup.log` with 5 MB of cast errors and burning CPU), round webcam no longer re-rasterizes an `ImageBrush` every frame (Image + EllipseGeometry clip) — the live-mode stutter fix; **the five-scene catalog (`SceneCatalog`)**: Starting/Live/BRB/Chat/Ending is the product — work with less, never more; the (+) button only shows when a canonical scene is missing and re-adds it (its menu lists only the missing ones); **webcam-after-session-start fix**: a webcam added to a scene after the camera was already running (e.g. Chat) previously rendered a transparent container — `CameraManager.GetPreviewBitmap` + propagation in `AddWebcamToActiveSceneAsync`/`ReacquireWebcam` now hands the running shared frames to any newly added `WebcamSceneConfig` — 62 tests passing; the **Chat scene's webcam size cap** is raised from 50%-per-dimension (960×540) to half the screen AREA (~1358×764 @16:9, `MaxWebcamWidthFor`/`MaxWebcamHeightFor` keyed by canonical name) so the viewer sees the creator better — 65 tests passing; **"Add Webcam" always opens the camera picker** (deleting one scene's webcam then re-adding used to resurrect the old camera when another scene still used it — `SwapWebcamIdentityAsync` now swaps the app-wide identity if a different camera is chosen, same path as "Change Webcam…"); scenes/sources UI (add/reorder/rename, image + background overlays with move/resize/opacity/reuse) built; window capture, compositing, encoding pending --- @@ -178,10 +178,11 @@ Preview shows the transition too (WYSIWYG). No wipes/slides/LUTs beyond the four - **1080p60 @ 8 Mbps** (default — mainstream ceiling, GPU hardware-encoded so the gaming machine never notices; upload headroom stays comfortable) - Vertical 1080×1920 @ 60fps @ 8 Mbps (9:16 phone tier) - - 1440p/4K = the paid unlock tiers (monetization), not the standard offering The composition master is always 1920×1080; a tier is an output rect + target resolution (see `ai.md` "Resolution tiers"). Vertical output = the centered 607×1080 crop of the master scaled to 1080×1920 (semi-crop preview is already implemented; the encoder applies the same rect). + 1080p60 is the ceiling by design — "if you want 1440 or 4K or 8K → OBS is your solution"; the app + targets the most mainstream creator, not power users. Ladder is sculpted by a **cached probe** (IP-only TCP vs public ingest host; no auth required). Quality is greyed out while live because the declared resolution can't change mid-stream — but with `variable`, we can **auto step-down** bitrate/resolution on the fly with zero API calls @@ -237,15 +238,20 @@ Preview shows the transition too (WYSIWYG). No wipes/slides/LUTs beyond the four **Save Layout As… / Open Layout…** switch the active file; auto-save writes to whatever is active. 4. **Auto-save (invisible)** — ~1.5s debounce on scene add/remove/reorder/rename/hide, source add/remove/reorder, and any source transform change; flush on window close. -5. **Schema** — `Scene` (Id, Name, IsHidden, IsChatScene, SortOrder), `Asset` (Id, Hash, Data, +5. **Schema** — `Scene` (Id, Name, IsHidden, IsChatScene, HasBackdrop, SortOrder), `Asset` (Id, Hash, Data, PixelWidth, PixelHeight), `Source` (Id, SceneId FK cascade, AssetId FK, Type, Name, IsEnabled, X/Y/Width/Height/Opacity, MonitorIndex, DeviceId, ClipShape, IsMirrored, SortOrder) — - `user_version` **4** (v1 → v2 = `ALTER TABLE` adds the two webcam columns; v3 = singleton + `user_version` **6** (v1 → v2 = `ALTER TABLE` adds the two webcam columns; v3 = singleton `Webcam` + per-scene `WebcamSceneConfig`; v4 = `WebcamSceneConfig.RectWidth`/`RectHeight` for the - round-to-rect restore). `WindowHandle` stays in-memory (per-session). Save = transactional - rewrite; orphaned assets pruned. -6. **Startup** — load the active file; seed the default five scenes (Starting/Live/BRB/Chat/Ending) - only when the DB is empty. + round-to-rect restore; v5 = `Source.IsBackdrop` + `Source.CaptureKey` for the live-capture + backdrop; v6 = `Scene.HasBackdrop` — the backdrop is **Live-only by policy** + (one-time backfill turns Starting/BRB/Chat/Ending off and drops their backdrop + sources; `EnforceBackdropPolicy` re-normalizes every load). `WindowHandle` stays in-memory + (per-session). Save = transactional rewrite; orphaned assets pruned. +6. **Startup** — load the active file; seed the five canonical scenes + (Starting/Live/BRB/Chat/Ending, `SceneCatalog`) only when the DB is empty. The (+) + button re-adds a missing canonical scene and is hidden once all five are present; + adding beyond the five is rejected — work with less, never more. ### Status: ✅ Implemented diff --git a/ViewModels/MainViewModel.cs b/ViewModels/MainViewModel.cs index 4bee11e..c3d3d0d 100644 --- a/ViewModels/MainViewModel.cs +++ b/ViewModels/MainViewModel.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using System.IO; using System.Windows; using System.Windows.Input; +using System.Windows.Interop; using System.Windows.Media; using System.Windows.Media.Imaging; using System.Windows.Threading; @@ -64,6 +65,17 @@ public class MainViewModel : ViewModelBase private readonly CameraManager _cameraManager; private Webcam? _webcam; + // Screen backdrop: a permanent live capture (desktop/game) that every scene + // shows at the bottom layer. One shared capture session per key — the + // ScreenCaptureManager refcounts by key, mirroring CameraManager. + private readonly IFullScreenDetector _fullScreenDetector; + private readonly ScreenCaptureManager _screenCaptureManager; + private readonly ScreenCaptureSourceFactory _screenCaptureFactory; + private int? _lastForegroundFullScreenMonitor; + private CancellationTokenSource? _deactivateCts; + private ImageSource? _backdropImage; + private HashSet _liveCaptureKeys = new(); + // Branding flash (monetization): a full-frame "made with ytLlive!" shown // for ~1s every 300s on the live output, ~25% opacity. Paid unlock sets // BrandFlashEnabled = false. See ai.md "Monetization". @@ -78,6 +90,7 @@ public class MainViewModel : ViewModelBase public ObservableCollection Scenes { get; } = new(); public ObservableCollection ChatMessages { get; } = new(); + public ObservableCollection Displays { get; } = new(); public string[] Visibilities { get; } = { "Public", "Unlisted", "Private" }; public Scene? ActiveScene @@ -93,9 +106,11 @@ public class MainViewModel : ViewModelBase OnPropertyChanged(nameof(ShowEmptySceneHint)); OnPropertyChanged(nameof(ShowPreviewPlaceholder)); OnPropertyChanged(nameof(ShowSourcesEmptyHint)); + OnPropertyChanged(nameof(CanChangeBackdrop)); OnPropertyChanged(nameof(CanAddWebcamToActiveScene)); OnPropertyChanged(nameof(CanShowWebcamInActiveScene)); UpdateActiveBackground(); + UpdateBackdropImage(); } } } @@ -106,6 +121,18 @@ public class MainViewModel : ViewModelBase private set => SetProperty(ref _activeBackgroundImage, value); } + /// The active scene's live-capture backdrop frame (rendered below the + /// static background). Set from the shared capture bitmap as it arrives. + public ImageSource? BackdropImage + { + get => _backdropImage; + private set + { + if (SetProperty(ref _backdropImage, value)) + OnPropertyChanged(nameof(ShowPreviewPlaceholder)); + } + } + public SceneElement? SelectedElement { get => _selectedElement; @@ -166,9 +193,20 @@ public class MainViewModel : ViewModelBase public bool ShowStartStream => IsOffline; public bool ShowChatInactiveMessage => !IsLive; public bool ShowEmptySceneHint => ActiveScene != null && ActiveScene.Elements.Count == 0; - public bool ShowPreviewPlaceholder => !ShowEmptySceneHint && ActiveBackgroundImage == null; + public bool ShowPreviewPlaceholder => !ShowEmptySceneHint && ActiveBackgroundImage == null && BackdropImage == null; public bool ShowSourcesEmptyHint => ActiveScene == null || ActiveScene.Elements.Count == 0; + /// The canonical scenes (SceneCatalog) not present right now — what the + /// "+" button may re-add. Work with less, never more. + public IEnumerable MissingScenes + => SceneCatalog.All.Where(canonical => Scenes.All(s => !SceneCatalog.Is(s.Name, canonical))); + + /// Only show the "+" button when one of the five canonical scenes is missing. + public bool ShowAddScene => MissingScenes.Any(); + + /// "Change Capture…"/"Refresh Capture" apply to the active scene's backdrop. + public bool CanChangeBackdrop => ActiveScene?.HasBackdrop == true; + // Paid unlock flips this off (see ai.md "Monetization"). When disabled the // cadence timer is stopped and any active flash is hidden immediately. public bool BrandFlashEnabled @@ -205,6 +243,12 @@ public class MainViewModel : ViewModelBase OnPropertyChanged(nameof(ShowPreviewPlaceholder)); } + private void UpdateBackdropImage() + { + var backdrop = ActiveScene?.Elements.OfType().FirstOrDefault(s => s.IsBackdrop); + BackdropImage = backdrop?.DisplaySource; + } + public StreamHealth CurrentHealth { get => _currentHealth; @@ -330,15 +374,28 @@ public class MainViewModel : ViewModelBase // Webcam size safeguard: no more than half the frame in any dimension // (960x540 over the 1920x1080 master), and no less than 10% of it - // (192x108). Enforced at resize and on layout load. + // (192x108). Enforced at resize and on layout load. The Chat scene is + // exempt from the per-dimension half — its webcam may take half the + // screen AREA (~1358x764 @16:9) so the viewer sees the creator better, + // matched by canonical name (SceneCatalog.IsChat). public const double WebcamMaxWidth = 960; public const double WebcamMaxHeight = 540; + public const double WebcamChatMaxWidth = 1358; + public const double WebcamChatMaxHeight = 764; public const double WebcamMinWidth = MasterFrameWidth * 0.1; public const double WebcamMinHeight = MasterFrameHeight * 0.1; - internal static void ClampWebcamToBounds(WebcamSceneConfig config) + public static double MaxWebcamWidthFor(string? sceneName) + => SceneCatalog.IsChat(sceneName) ? WebcamChatMaxWidth : WebcamMaxWidth; + + public static double MaxWebcamHeightFor(string? sceneName) + => SceneCatalog.IsChat(sceneName) ? WebcamChatMaxHeight : WebcamMaxHeight; + + internal static void ClampWebcamToBounds(WebcamSceneConfig config, string? sceneName) { - var scale = Math.Min(WebcamMaxWidth / config.Width, WebcamMaxHeight / config.Height); + var maxWidth = MaxWebcamWidthFor(sceneName); + var maxHeight = MaxWebcamHeightFor(sceneName); + var scale = Math.Min(maxWidth / config.Width, maxHeight / config.Height); if (scale < 1) { config.Width = Math.Round(config.Width * scale); @@ -472,6 +529,9 @@ public class MainViewModel : ViewModelBase public ICommand RemoveSourceCommand { get; } public ICommand ChangeWebcamCommand { get; } public ICommand ShowWebcamCommand { get; } + public ICommand ChangeCaptureCommand { get; } + public ICommand RefreshCaptureCommand { get; } + public ICommand SetBackdropDisplayCommand { get; } public ICommand StartStreamCommand { get; } public ICommand EndStreamCommand { get; } public ICommand OpenSettingsCommand { get; } @@ -513,7 +573,7 @@ public class MainViewModel : ViewModelBase Scenes.CollectionChanged += OnScenesChanged; - AddSceneCommand = new RelayCommand(_ => AddScene()); + AddSceneCommand = new RelayCommand(name => AddScene(name as string ?? string.Empty)); EditSceneCommand = new RelayCommand(scene => BeginEditScene(scene as Scene)); RemoveSceneCommand = new RelayCommand(scene => RemoveScene(scene as Scene)); ToggleSceneVisibilityCommand = new RelayCommand(scene => ToggleSceneVisibility(scene as Scene)); @@ -522,6 +582,9 @@ public class MainViewModel : ViewModelBase RemoveSourceCommand = new RelayCommand(element => RemoveElement(element as SceneElement)); ChangeWebcamCommand = new RelayCommand(_ => _ = ChangeWebcamAsync(), _ => CanChangeWebcam); ShowWebcamCommand = new RelayCommand(_ => ShowWebcamInActiveScene(), _ => CanShowWebcamInActiveScene); + ChangeCaptureCommand = new RelayCommand(_ => _ = ChangeBackdropCaptureAsync()); + RefreshCaptureCommand = new RelayCommand(_ => RefreshBackdropAutoCapture()); + SetBackdropDisplayCommand = new RelayCommand(display => SetBackdropCapture(display as DisplayInfo)); OpenSettingsCommand = new RelayCommand(_ => ShowOverlay(nameof(IsSettingsOpen), "App Settings")); OpenBugCommand = new RelayCommand(_ => ShowOverlay(nameof(IsBugOpen), "Report Bug")); OpenFeatureCommand = new RelayCommand(_ => ShowOverlay(nameof(IsFeatureOpen), "Feature Request")); @@ -546,6 +609,18 @@ public class MainViewModel : ViewModelBase System.Windows.Application.Current?.Dispatcher); _cameraManager.PreviewBitmapChanged += OnCameraPreviewBitmapChanged; + _fullScreenDetector = new Win32FullScreenDetector(); + foreach (var display in _fullScreenDetector.GetDisplays()) + Displays.Add(display); + _screenCaptureFactory = new ScreenCaptureSourceFactory( + () => new WindowInteropHelper(System.Windows.Application.Current?.MainWindow).Handle); + _screenCaptureManager = new ScreenCaptureManager( + _screenCaptureFactory.Resolve, + System.Windows.Application.Current?.Dispatcher); + _screenCaptureManager.PreviewBitmapChanged += OnScreenPreviewBitmapChanged; + _screenCaptureManager.CaptureFailed += (key, message) => + AppLog.Write($"ScreenCaptureManager: capture '{key}' failed: {message}"); + LoadLayout(); _ = LoadSavedSessionAsync(); AppLog.Write("MainViewModel ctor end"); @@ -603,17 +678,18 @@ public class MainViewModel : ViewModelBase Scenes.Add(scene); if (Scenes.Count == 0) - { - AddScene("Starting"); - AddScene("Live"); - AddScene("BRB"); - AddScene("Chat", isChatScene: true); - AddScene("Ending"); - } + foreach (var name in SceneCatalog.All) + AddScene(name); + + // The live backdrop belongs to Live only; a pre-policy DB may have + // backdrops lingering in other scenes — drop them, then + // ReacquireScreenCaptures heals Live's. + EnforceBackdropPolicy(Scenes); + foreach (var scene in Scenes) foreach (var config in scene.Elements.OfType()) { - ClampWebcamToBounds(config); + ClampWebcamToBounds(config, scene.Name); HealLegacySquareRect(config); } AppLog.Write($"LoadLayout: {Scenes.Count} scenes loaded"); @@ -624,7 +700,9 @@ public class MainViewModel : ViewModelBase } ActiveScene = Scenes.FirstOrDefault(); UpdateActiveBackground(); + UpdateBackdropImage(); ReacquireWebcam(); + ReacquireScreenCaptures(); ScheduleSave(); AppLog.Write("LoadLayout end"); } @@ -645,9 +723,16 @@ public class MainViewModel : ViewModelBase OnPropertyChanged(nameof(CanAddWebcamToActiveScene)); if (_webcam == null || string.IsNullOrWhiteSpace(newDevice)) return; - if (previousDevice == newDevice) return; - foreach (var unused in Scenes.SelectMany(s => s.Elements.OfType())) - _ = _cameraManager.AcquireAsync(newDevice); + if (previousDevice != newDevice) + foreach (var unused in Scenes.SelectMany(s => s.Elements.OfType())) + _ = _cameraManager.AcquireAsync(newDevice); + + // A session already running (same device, or one that produced a first + // frame) has its shared bitmap; freshly loaded configs must adopt it here, + // because PreviewBitmapChanged never re-fires for an existing bitmap. + if (_cameraManager.GetPreviewBitmap(newDevice) is { } running) + foreach (var config in Scenes.SelectMany(s => s.Elements.OfType())) + config.VideoImageSource = running; } // CameraManager creates the shared WriteableBitmap on the UI thread at the @@ -660,11 +745,174 @@ public class MainViewModel : ViewModelBase config.VideoImageSource = bitmap; } + // ─── Screen backdrop capture (live desktop/game) ─── + + private const string MonitorKeyPrefix = "monitor:"; + + /// Every backdrop-enabled scene has exactly one backdrop, kept at index 0 (bottom layer). + /// Returns null for a scene with = false. + internal static Source? EnsureBackdrop(Scene scene) + { + if (!scene.HasBackdrop) return null; + var backdrop = scene.Elements.OfType().FirstOrDefault(s => s.IsBackdrop); + if (backdrop != null) return backdrop; + + backdrop = new Source + { + Name = "Backdrop", + Type = SourceType.DisplayCapture, + IsBackdrop = true, + IsEnabled = true, + X = 0, + Y = 0, + Width = MasterFrameWidth, + Height = MasterFrameHeight, + }; + scene.Elements.Insert(0, backdrop); + return backdrop; + } + + // The backdrop belongs to the Live scene alone. Runs after every layout + // load: the flag is normalized by scene name and any backdrop lingering in a + // non-Live scene (from a pre-policy DB) is removed. ReacquireScreenCaptures + // re-heals Live's backdrop right after. + internal static void EnforceBackdropPolicy(IEnumerable scenes) + { + foreach (var scene in scenes) + { + scene.HasBackdrop = SceneCatalog.HasBackdrop(scene.Name); + if (scene.HasBackdrop) continue; + var backdrop = scene.Elements.OfType().FirstOrDefault(s => s.IsBackdrop); + if (backdrop != null) + scene.Elements.Remove(backdrop); + } + } + + private IEnumerable AllBackdrops() + => Scenes.SelectMany(s => s.Elements.OfType().Where(x => x.IsBackdrop)); + + // Full-screen game on its monitor, else the primary display. Called at launch + // (from ReacquireScreenCaptures, before the window steals focus) and on + // focus regain (RefreshBackdropAutoCapture). + private string ResolveAutoCaptureKey() + => $"{MonitorKeyPrefix}{_fullScreenDetector.GetForegroundFullScreenMonitorIndex() ?? _fullScreenDetector.PrimaryMonitorIndex()}"; + + // After a layout load / file open: make sure every backdrop-enabled scene + // has a backdrop, give any backdrop without a persisted key the auto-detected + // one, then acquire one capture per unique backdrop key. ScreenCaptureManager + // refcounts by key, so every scene pointing at the same monitor shares one + // session. Captures no longer referenced by any scene are released. + private void ReacquireScreenCaptures() + { + foreach (var scene in Scenes) + EnsureBackdrop(scene); + + var auto = ResolveAutoCaptureKey(); + foreach (var backdrop in AllBackdrops()) + if (string.IsNullOrWhiteSpace(backdrop.CaptureKey)) + backdrop.CaptureKey = auto; + + var keys = AllBackdrops() + .Select(b => b.CaptureKey!) + .Where(k => !string.IsNullOrWhiteSpace(k)) + .ToHashSet(); + + // A layout reload replaces the scenes; stop captures that are no longer + // referenced (mirrors ReacquireWebcam's device swap). + foreach (var stale in _liveCaptureKeys.Except(keys)) + _ = _screenCaptureManager.ReleaseAllAsync(stale); + _liveCaptureKeys = keys; + + foreach (var key in keys) + _ = _screenCaptureManager.AcquireAsync(key); + } + + // ScreenCaptureManager creates the shared WriteableBitmap on the UI thread; + // every backdrop pointing at that key picks it up. + private void OnScreenPreviewBitmapChanged(string key, WriteableBitmap bitmap) + { + var activeBackdrop = ActiveScene?.Elements.OfType().FirstOrDefault(s => s.IsBackdrop); + if (activeBackdrop?.CaptureKey == key) + BackdropImage = bitmap; + + foreach (var backdrop in AllBackdrops().Where(b => b.CaptureKey == key)) + backdrop.VideoImageSource = bitmap; + } + + // One-shot foreground snapshot ~250ms after we lose focus, so the next + // Activated re-detect can see the full-screen game the user switched to. + // A single sample per deactivation — not a session listener or a poller. + public void NoteBackgroundWindow() + { + _deactivateCts?.Cancel(); + _deactivateCts = new CancellationTokenSource(); + var token = _deactivateCts.Token; + _ = Task.Run(async () => + { + try + { + await Task.Delay(250, token); + var monitor = _fullScreenDetector.GetForegroundFullScreenMonitorIndex(); + if (!token.IsCancellationRequested) + _lastForegroundFullScreenMonitor = monitor; + } + catch (OperationCanceledException) { } + }); + } + + // Re-runs full-screen detection at launch and when the app regains focus. + // A null detection (no full-screen foreground window — our app, the desktop, + // a normal window) leaves the current capture alone. + public void RefreshBackdropAutoCapture() + { + var detected = _lastForegroundFullScreenMonitor ?? _fullScreenDetector.GetForegroundFullScreenMonitorIndex(); + _lastForegroundFullScreenMonitor = null; + if (detected == null) return; + _ = RedesignateBackdropAsync($"{MonitorKeyPrefix}{detected}"); + } + + private async Task RedesignateBackdropAsync(string newKey) + { + var oldKeys = AllBackdrops() + .Select(b => b.CaptureKey!) + .Where(k => !string.IsNullOrWhiteSpace(k)) + .ToHashSet(); + + foreach (var backdrop in AllBackdrops()) + backdrop.CaptureKey = newKey; + + foreach (var oldKey in oldKeys.Where(k => k != newKey)) + await _screenCaptureManager.ReleaseAllAsync(oldKey); + if (!oldKeys.Contains(newKey)) + await _screenCaptureManager.AcquireAsync(newKey); + _liveCaptureKeys = new HashSet { newKey }; + UpdateBackdropImage(); + ScheduleSave(); + } + + // "Change Capture…": the OS GraphicsCapturePicker designates the target. + // Picks are transient (see ScreenCaptureSourceFactory) — a reload falls + // back to auto-detection. + public async Task ChangeBackdropCaptureAsync() + { + var key = await _screenCaptureFactory.PickAsync(); + if (key == null) return; + await RedesignateBackdropAsync(key); + } + + // In-app display picker: point every backdrop at a specific monitor. + private void SetBackdropCapture(DisplayInfo? display) + { + if (display == null) return; + _ = RedesignateBackdropAsync($"{MonitorKeyPrefix}{display.Index}"); + } + public void Shutdown() { _saveDebounce?.Stop(); SaveLayoutNow(); _cameraManager.Dispose(); + _screenCaptureManager.Dispose(); _layoutStore.Dispose(); } @@ -722,6 +970,8 @@ public class MainViewModel : ViewModelBase if (e.OldItems != null) foreach (Scene scene in e.OldItems) UnwireScene(scene); + OnPropertyChanged(nameof(MissingScenes)); + OnPropertyChanged(nameof(ShowAddScene)); ScheduleSave(); } @@ -738,7 +988,12 @@ public class MainViewModel : ViewModelBase } private void OnScenePropertyChanged(object? sender, PropertyChangedEventArgs e) - => ScheduleSave(); + { + // A rename can drop a canonical scene in/out of MissingScenes. + OnPropertyChanged(nameof(MissingScenes)); + OnPropertyChanged(nameof(ShowAddScene)); + ScheduleSave(); + } private void OnElementsChanged(object? sender, NotifyCollectionChangedEventArgs e) { @@ -766,9 +1021,19 @@ public class MainViewModel : ViewModelBase _saveDebounce.Start(); } - private void AddScene(string? name = null, bool isChatScene = false) + // Adds a canonical scene by name — only when it's actually missing. The + // backdrop flag follows the policy (Live yes, everyone else no). + private void AddScene(string name) { - var scene = new Scene { Name = name ?? $"New Scene {Scenes.Count + 1}", IsChatScene = isChatScene }; + if (!SceneCatalog.IsCanonical(name)) return; + if (Scenes.Any(s => SceneCatalog.Is(s.Name, name))) return; + var scene = new Scene + { + Name = name.Trim(), + IsChatScene = SceneCatalog.IsChat(name), + HasBackdrop = SceneCatalog.HasBackdrop(name), + }; + EnsureBackdrop(scene); Scenes.Add(scene); ActiveScene = scene; } @@ -855,25 +1120,33 @@ public class MainViewModel : ViewModelBase UpdateActiveBackground(); } - // Adds the webcam to the active scene. The camera is picked once app-wide - // (first add); afterwards "Add Webcam" just places the existing webcam here - // at the default spot — each scene's config is independent (webcam.{scene}.config). + // Adds the webcam to the active scene. The creator ALWAYS picks from the + // cameras Windows has registered — never silently resurrects the previous + // camera (which is what happened after deleting one scene's webcam while + // another scene still used it). Picking a different camera than the current + // app-wide one swaps it everywhere, so the single-identity model stays honest; + // each scene's placement config is independent (webcam.{scene}.config). private async Task AddWebcamToActiveSceneAsync() { var scene = ActiveScene; if (scene == null || scene.WebcamConfig != null) return; + var dialog = new CameraPickerDialog(new CameraPickerViewModel(_cameraEnumerator)) + { + Owner = Application.Current.MainWindow + }; + if (dialog.ShowDialog() != true || dialog.PickedDevice == null) return; + var device = dialog.PickedDevice; + if (_webcam == null) { - var dialog = new CameraPickerDialog(new CameraPickerViewModel(_cameraEnumerator)) - { - Owner = Application.Current.MainWindow - }; - if (dialog.ShowDialog() != true || dialog.PickedDevice == null) return; - var device = dialog.PickedDevice; _webcam = new Webcam { DeviceId = device.Id, Name = device.DisplayName }; OnPropertyChanged(nameof(CanChangeWebcam)); } + else if (_webcam.DeviceId != device.Id) + { + await SwapWebcamIdentityAsync(device); + } var config = new WebcamSceneConfig { @@ -891,6 +1164,12 @@ public class MainViewModel : ViewModelBase OnPropertyChanged(nameof(ShowSourcesEmptyHint)); UpdateActiveBackground(); + // PreviewBitmapChanged fires only on the camera's first frame, so a config + // added while the session is already running must pick up the shared bitmap + // directly (it's written in place from then on). + if (_cameraManager.GetPreviewBitmap(_webcam.DeviceId) is { } running) + config.VideoImageSource = running; + var started = await _cameraManager.AcquireAsync(_webcam.DeviceId); if (!started) { @@ -902,7 +1181,32 @@ public class MainViewModel : ViewModelBase // Swaps the device on the app-wide webcam identity. The old device is stopped // unconditionally; each scene that uses the webcam re-acquires the new one so - // the per-config refcount stays honest. + // the per-config refcount stays honest. Only called when the device differs. + private async Task SwapWebcamIdentityAsync(CameraDeviceInfo device) + { + var oldDevice = _webcam!.DeviceId; + _webcam.DeviceId = device.Id; + _webcam.Name = device.DisplayName; + ScheduleSave(); + + if (!string.IsNullOrWhiteSpace(oldDevice) && oldDevice != device.Id) + await _cameraManager.ReleaseAllAsync(oldDevice); + + foreach (var unused in Scenes.SelectMany(s => s.Elements.OfType())) + { + var started = await _cameraManager.AcquireAsync(device.Id); + if (!started) + { + MessageBox.Show( + "Couldn't start that camera. It may be in use by another app, or Windows camera access may be turned off.", + "ytLlive", MessageBoxButton.OK, MessageBoxImage.Warning); + break; + } + } + } + + // "Change Webcam…" from the webcam's context menu: picker, then swap the + // app-wide identity if a different device was chosen. private async Task ChangeWebcamAsync() { if (_webcam == null) return; @@ -912,29 +1216,9 @@ public class MainViewModel : ViewModelBase Owner = Application.Current.MainWindow }; if (dialog.ShowDialog() != true || dialog.PickedDevice == null) return; + if (dialog.PickedDevice.Id == _webcam.DeviceId) return; - var oldDevice = _webcam.DeviceId; - var newDevice = dialog.PickedDevice.Id; - if (oldDevice == newDevice) return; - - _webcam.DeviceId = newDevice; - _webcam.Name = dialog.PickedDevice.DisplayName; - ScheduleSave(); - - if (!string.IsNullOrWhiteSpace(oldDevice)) - await _cameraManager.ReleaseAllAsync(oldDevice); - - foreach (var unused in Scenes.SelectMany(s => s.Elements.OfType())) - { - var started = await _cameraManager.AcquireAsync(newDevice); - if (!started) - { - MessageBox.Show( - "Couldn't start that camera. It may be in use by another app, or Windows camera access may be turned off.", - "ytLlive", MessageBoxButton.OK, MessageBoxImage.Warning); - break; - } - } + await SwapWebcamIdentityAsync(dialog.PickedDevice); } // "Show Webcam" from a right-click on the empty preview. Unhides this scene's @@ -1081,6 +1365,7 @@ public class MainViewModel : ViewModelBase { var scene = ActiveScene; if (scene == null || element == null) return; + if (element is Source { IsBackdrop: true }) return; if (element is WebcamSceneConfig && _webcam != null) { if (SelectedElement == element) diff --git a/ViewModels/index.md b/ViewModels/index.md index d39ed5c..3a16bd2 100644 --- a/ViewModels/index.md +++ b/ViewModels/index.md @@ -4,7 +4,7 @@ MVVM layer. See [`schema.md`](../schema.md) for the memory-map conventions. | File | Purpose | |------|---------| -| `MainViewModel.cs` | The app brain: scenes/elements collections + commands, stream state (`IsLive`/`IsOffline`/`IsConnected`), chat feed, overlays, layout save/open, **resolution dropdown** (`QualityOptions`, `SelectedQuality`, `ResolutionHelp`, `ApplyStreamQuality`) that computes the output rect over the 1920×1080 master (`OutputRectX/Y/W/H`, `DimRects`, `IsOutputCropped`, `ResolutionBadgeText`). Auth: loads the saved DPAPI session at startup (`LoadSavedSessionAsync`), `SignInAsync` feeds the GoLive dialog; **End Livestream signs out** (`StopStream` clears session + token — crash-safe). **Webcam:** owns `CameraManager` (MediaCapture), `AddWebcamToActiveSceneAsync` (picker → default 480×270 bottom-right placement → acquire; greys out when the active scene already has a config via `CanAddWebcamToActiveScene`), `ChangeWebcamAsync` (device swap — `ReleaseAllAsync` old + re-acquire), `ShowWebcamInActiveScene` (reveal hidden config / empty-canvas right-click), `ReacquireWebcam` after layout load, `OnCameraPreviewBitmapChanged` forwards the shared bitmap into every `WebcamSceneConfig.VideoImageSource`; `SelectedElement` drives the preview overlay + `internal ClampWebcamToBounds` (50% cap seam) | +| `MainViewModel.cs` | The app brain: scenes/elements collections + commands, stream state (`IsLive`/`IsOffline`/`IsConnected`), chat feed, overlays, layout save/open, **resolution dropdown** (`QualityOptions`, `SelectedQuality`, `ResolutionHelp`, `ApplyStreamQuality`) that computes the output rect over the 1920×1080 master (`OutputRectX/Y/W/H`, `DimRects`, `IsOutputCropped`, `ResolutionBadgeText`). Auth: loads the saved DPAPI session at startup (`LoadSavedSessionAsync`), `SignInAsync` feeds the GoLive dialog; **End Livestream signs out** (`StopStream` clears session + token — crash-safe). **Webcam:** owns `CameraManager` (MediaCapture), `AddWebcamToActiveSceneAsync` (picker → default 480×270 bottom-right placement → acquire; the Windows camera picker ALWAYS opens so the creator chooses — never silently reuses the previous camera; picking a different camera swaps the app-wide identity via `SwapWebcamIdentityAsync`, same path as `ChangeWebcamAsync`; propagates the running shared bitmap via `CameraManager.GetPreviewBitmap` first, so a webcam added mid-session never renders a transparent container; greys out when the active scene already has a config via `CanAddWebcamToActiveScene`), `ChangeWebcamAsync` (device swap — `ReleaseAllAsync` old + re-acquire), `ShowWebcamInActiveScene` (reveal hidden config / empty-canvas right-click), `ReacquireWebcam` after layout load (re-propagates the shared bitmap to every config), `OnCameraPreviewBitmapChanged` forwards the shared bitmap into every `WebcamSceneConfig.VideoImageSource`; `SelectedElement` drives the preview overlay + `internal ClampWebcamToBounds(config, sceneName)` (per-scene size cap seam: 50%-per-dimension everywhere, half-screen-AREA in Chat via `MaxWebcamWidthFor`/`MaxWebcamHeightFor`). **Scenes (five-scene catalog):** empty DB seeds Starting/Live/BRB/Chat/Ending (`SceneCatalog.All`); `AddScene(name)` accepts only canonical, missing names; `MissingScenes`/`ShowAddScene` drive the "+" button (hidden once all five exist; its menu lists only the missing scenes via `AddSceneCommand`). **Screen backdrop (Live-only by policy):** owns `ScreenCaptureManager` + `ScreenCaptureSourceFactory` (WinRT GraphicsCapture), `internal static EnsureBackdrop` heals one per backdrop-enabled scene (gated on `Scene.HasBackdrop`), `ReacquireScreenCaptures`/`RefreshBackdropAutoCapture`/`NoteBackgroundWindow` key by full-screen game monitor or the **primary display** via `Win32FullScreenDetector` (`GetDisplays()`/`PrimaryMonitorIndex()`), `ChangeBackdropCaptureAsync` (OS picker) + `SetBackdropCapture(DisplayInfo)` (in-app "Capture Display"), `RedesignateBackdropAsync` re-targets all backdrops and releases orphaned sessions, `internal static EnforceBackdropPolicy` normalizes `Scene.HasBackdrop` by name on every load + strips lingering non-Live backdrops, `CanChangeBackdrop` gates the capture menu to Live (the only scene with a backdrop), `BackdropImage` hides the preview watermark (`ShowPreviewPlaceholder`) | | `GoLiveViewModel.cs` | Start Stream dialog: **account row** (saved channel shown with Change Account, or Sign in to YouTube; Start gated on `IsSignedIn`/`IsBusy`) + title/description/visibility, start/cancel requests | | `ReuseImageViewModel.cs` | Add Image dialog: candidate list (`ReuseImageCandidate`), reuse/new/cancel | | `CameraPickerViewModel.cs` | Add Webcam dialog: async camera list (loading / has / none states), `UseRequested(CameraDeviceInfo)`/`CancelRequested` | diff --git a/ai.md b/ai.md index 76c8d33..615bb0a 100644 --- a/ai.md +++ b/ai.md @@ -53,8 +53,14 @@ dotnet.exe vstest "C:\...\ytLive.Tests\bin\Debug\net8.0-windows10.0.19041.0\ytLi Good Dog Rule: ONE integration test per branch, ONE test per PR. Current: TokenStore DPAPI roundtrip/corrupt/missing/clear, OAuth exchange/refresh/ClearSession, CameraManager refcount + frame pump + failure handling (fakes for the WinRT seams), real-`MainWindow` round-clip -interaction test, LayoutStore delete roundtrip, LayoutStore pre-round-rect-dims roundtrip — -25 passing. +interaction test, LayoutStore delete roundtrip, LayoutStore pre-round-rect-dims roundtrip, +LayoutStore backdrop roundtrip, LayoutStore HasBackdrop roundtrip + v5→v6 non-Live backfill, +ScreenCaptureManager refcount + shared-bitmap + coalescing +(fake `IScreenCaptureSource` + a real background-STA `Dispatcher`), BackdropTests (EnsureBackdrop +insert/idempotent/heal + HasBackdrop gate, IsLiveCapture, DisplaySource, INPC), SceneCatalogTests +(the five canonical scenes, Live-only backdrop policy, EnforceBackdropPolicy), WebcamSafeguardTests +(the per-scene size clamp incl. the Chat half-screen-area cap) — +65 passing. ### Real-MainWindow tests MUST be hermetic (DB pollution bug) @@ -81,7 +87,7 @@ C# / WPF (.NET 8) following MVVM: |------|------| | `Models/` | Plain data types — Scene, Source (incl. `ClipShape`, `IsMirrored`, `VideoImageSource`), QualityOption, StreamConfig, StreamHealth, YouTubeChannel, ChatMessage | | `ViewModels/` | MainViewModel — exposes collections + commands for the UI; GoLiveViewModel, ReuseImageViewModel, CameraPickerViewModel | -| `Services/` | YouTube OAuth2, stream/broadcast management, live chat polling, LayoutStore (SQLite), **webcam: `VideoFrame` seam + `CameraDeviceInfo`/`ICameraEnumerator`/`ICameraFrameSource` interfaces + `MediaCaptureCameraEnumerator`/`MediaCaptureFrameSource` (WinRT) + `CameraManager`** | +| `Services/` | YouTube OAuth2, stream/broadcast management, live chat polling, LayoutStore (SQLite), **webcam: `VideoFrame` seam + `CameraDeviceInfo`/`ICameraEnumerator`/`ICameraFrameSource` interfaces + `MediaCaptureCameraEnumerator`/`MediaCaptureFrameSource` (WinRT) + `CameraManager`**, **screen capture: `IFullScreenDetector`/`Win32FullScreenDetector` + `IScreenCaptureSource`/`ScreenCaptureFrameSource` (WinRT GraphicsCapture) + `ScreenCaptureManager` + `ScreenCaptureSourceFactory` + `Direct3D11Helper`/`CaptureInterop` (COM bridges)** | | `Helpers/` | ViewModelBase (INotifyPropertyChanged), RelayCommand, ImageCache, AppLog (file logger), FocusPreservingListBox, OAuthCredentials, **TokenStore (DPAPI session persistence)**, visibility converters | | `Themes/` | `Controls.xaml` — the single dark-theme source, merged once in `App.xaml` (see `Themes/index.md`) | | `MainWindow.xaml` | Dark theme; layout: top bar (controls), center (preview), left (scenes/sources), right (chat), bottom (health) | @@ -93,6 +99,7 @@ C# / WPF (.NET 8) following MVVM: - ViewModels are constructed in XAML (`` as DataContext) - Services are currently instantiated in MainViewModel's constructor — no DI container yet - Layout persists to SQLite (`Microsoft.Data.Sqlite`); scenes/sources/asset bytes stored in the DB, asset identity is a SHA-256 content hash (1:M reuse, no file paths — assets are always available) +- **Five-scene catalog (`Models/SceneCatalog.cs`):** the product is exactly Starting/Live/BRB/Chat/Ending — work with less, never more (the escape hatch for "more" is OBS). Scenes are matched **by name** (`SceneCatalog.Is`, case-insensitive trim). Empty DBs seed all five; the scenes-header "+" (`ShowAddScene`/`MissingScenes` on `MainViewModel`) only appears while ≥1 canonical scene is missing and its menu lists only the missing ones, re-adding them by name (`AddSceneCommand`). Renaming a canonical scene makes it missing again; `AddScene` rejects non-canonical names. - Theming: all custom styles live in `Themes/Controls.xaml`, merged in `App.xaml` — never duplicate styles per-window (dialog duplicates were consolidated into this dictionary) - Resolution tiers (bottom bar): 1080p60@8 (default) → 1080p30@8 → 720p60@6 → 720p30@6 → **Vertical 1080p60@8 (9:16, 1080×1920)**. The composition master frame is **always 1920×1080** — a tier is an output rect + target resolution over that master, so source geometry is never rewritten (no rounding drift). 16:9 tiers use the full frame; the vertical tier uses a centered **607×1080** window and the preview dims the cropped side strips at 55% black with an accent outline (semi-crop — the cut area stays visible). A resolution badge in the preview corner shows the active tier; the bottom bar shows bitrate/FPS. A **tooltip** explains finding upload bandwidth — an in-app speed test was deliberately dropped (unreliable). The future encoder crops the master to the rect and scales to the tier's Width×Height - Crash diagnosis: `AppLog` writes startup checkpoints to `%APPDATA%\ytLlive\startup.log`; `App.xaml.cs` logs `DispatcherUnhandledException`/`AppDomain.UnhandledException`. When WPF won't run from WSL, this log is how you find the failure (it caught the `MenuItemRole.Separator` XAML crash and the ComboBox SelectionBoxItem bug) @@ -100,11 +107,80 @@ C# / WPF (.NET 8) following MVVM: ### Current limitations / TODOs - `Helpers/OAuthCredentials.cs` contains the real ClientId/ClientSecret. Auth is complete and the session **persists via Windows DPAPI** (`Helpers/TokenStore.cs` → `%APPDATA%\ytLlive\ytLlive.auth`, CurrentUser scope), reloaded best-effort at startup with a proactive refresh of a near-expiry access token. Sign-in/Change Account lives **inside the Start Stream dialog** (two-state flow — no separate Connect button). A **graceful End Livestream signs out**: `StopStream()` clears the session + token, so the next go-live needs a fresh sign-in; a crash never runs End, so the token survives and the creator stays signed in. `YouTubeAuthService` takes an optional `HttpClient` + `sessionChanged` callback (test seam + save hook; services are still constructed in `MainViewModel`) -- Scene/source/asset layout persists (SQLite, schema v4); the OAuth session persists (DPAPI); the paid-unlock state does not (yet — itch.io key verification pending) +- Scene/source/asset layout persists (SQLite, schema v6); the OAuth session persists (DPAPI); the paid-unlock state does not (yet — itch.io key verification pending) - `YouTubeStreamService` uses hardcoded `1080p`/`60fps` and per-broadcast streams — must switch to the v3 `variable` reusable stream -- Webcam capture is shipped (milestone 1); **screen capture, scene compositing/encoding, RTMP are next** +- Webcam capture is shipped (milestone 1); the live desktop/game backdrop is shipped (ship task #1); **window capture (non-backdrop), scene compositing/encoding, RTMP are next** - `StreamConfig` defaults (`TargetBitrate=6000`, `Resolution="1920x1080"`) are stale — the live dropdown drives `StreamHealth.CurrentBitrate`/`FPS` instead +### Screen backdrop capture (TASK 3 ship task #1) + +The backdrop is the **live desktop/game capture as a permanent, non-deletable bottom layer** +rendered in every scene — the "Screen" source from the minimal set, done as content-swap +instead of a normal draggable source. + +- **Model (schema v6):** `Source.IsBackdrop` (persisted) marks the one backdrop per scene; `Source.CaptureKey` + (persisted) names the target — `monitor:`, `window:`, or `picker:`. The backdrop is + a real `Source` of `Type DisplayCapture`, inserted **first** (`MainViewModel.EnsureBackdrop(scene)`, + internal static — runs on layout load + every `AddScene`, healing any scene missing one), fixed at + X=0/Y=0/1920×1080, and excluded from drag/hit-test/remove/reorder (remove is guarded in `RemoveElement`; + `IsDraggableElement` never matches live types; the element template sets `IsHitTestVisible=false` for + backdrops; the row's remove button and "Remove Source" menu item are hidden). **`Scene.HasBackdrop` + (persisted, default off) is the Live-only policy flag** — the backdrop belongs to the canonical Live + scene alone (see `SceneCatalog`). `EnsureBackdrop` returns null for a flag-less scene, so + Starting/BRB/Chat/Ending compose their own layers. The one-time v5→v6 backfill turns those four scenes + off and drops their backdrop sources, and **`EnforceBackdropPolicy` (internal static, runs after every + load)** re-normalizes the flag by scene name and strips any backdrop that lingers in a non-Live scene — + the flag is owned by policy, never the user. There is no scene-list "Backdrop" checkbox anymore + (the old `ToggleSceneBackdropCommand` is gone); "Change Capture…"/"Refresh Capture"/"Capture Display" + only show in the Live scene's preview menu (`CanChangeBackdrop`). The static Background, if any, + renders **above** the backdrop. +- **Detection (launch + focus only, no live session listener):** `Win32FullScreenDetector` = + `GetForegroundWindow` + `DwmGetWindowAttribute(DWMWA_EXTENDED_FRAME_BOUNDS)` + `MonitorFromWindow` + + `GetMonitorInfo`; a window is full-screen when its frame covers all four monitor edges; own process is + excluded; monitor **index = `EnumDisplayMonitors` enumeration order** (the same order + `ScreenCaptureSourceFactory` uses to map index→HMONITOR via `Win32FullScreenDetector.GetMonitorHandle`). + `IFullScreenDetector` also exposes `GetDisplays()` (`DisplayInfo`: index/name/resolution/bounds/`IsPrimary`, + friendly name via `EnumDisplayDevices`) + `PrimaryMonitorIndex()` for the in-app "Capture Display" + submenu. At launch `ReacquireScreenCaptures` keys the backdrop to the full-screen game's monitor (or the + **primary** display — never assumed monitor 0). + On **Deactivated**, `NoteBackgroundWindow` samples the foreground ~250ms later (so an alt-tab to a game + lands before the app re-activates); on the next **Activated**, `RefreshBackdropAutoCapture` re-runs + detection against that sample and re-designates. Null detection (our app, the desktop, a normal window) + leaves the current capture alone. +- **Capture (WinRT GraphicsCapture):** `ScreenCaptureFrameSource` creates a free-threaded + `Direct3D11CaptureFramePool` (2 buffers, `B8G8R8A8UIntNormalized`) + `GraphicsCaptureSession`; frames → + `SoftwareBitmap.CreateCopyFromSurfaceAsync` (alpha ignored) → `VideoFrame` (BGRA8), bytes read via + `WindowsRuntimeMarshal.TryGetDataUnsafe` (the same CsWinRT-safe read the webcam path uses) — the + `IMemoryBufferByteAccess` ComImport cast threw `Invalid cast` on **every frame** under CsWinRT, which + flooded `startup.log` (~5 MB in a session) and burned CPU, so it is gone. Surfaces larger than the + 1920×1080 master are downscaled bilinearly to the master (`DownscaleBgra`) before the copy, and + per-frame conversion failures are logged at most once per 5 s (`ErrorLogThrottle`). DRM-protected + content delivers black frames (OS limitation, documented). Frame pool + pauses while the app is minimized — capture keeps running, the pool just stops delivering. +- **Ownership:** `ScreenCaptureManager` mirrors `CameraManager` — refcounted by target key, one shared + `WriteableBitmap` per key, dispatcher-coalesced latest-frame copies, `PreviewBitmapChanged`/`CaptureFailed` + events, `ReleaseAllAsync` on re-designation. `ScreenCaptureSourceFactory.Resolve(key)` parses the key into + a source; `PickAsync()` shows the OS `GraphicsCapturePicker` ("Change Capture…", owner window set via the + `IInitializeWithWindow` ComImport) and returns a **transient** `picker:` key — a reload falls back to + auto-detection. +- **CsWinRT projection gaps hand-rolled:** `Windows.Graphics.Direct3D11.Direct3D11Helper` is not projected, + so `Direct3D11Helper` P/Invokes `d3d11.dll!D3D11CreateDevice` (hardware, BGRA_SUPPORT, explicit 11.1-first + feature array) → QI `IDXGIDevice` → the WinRT interop export + `CreateDirect3D11DeviceFromDXGIDevice` → `MarshalInterface.FromAbi` (one shared device per + process). Do **not** switch back to the QI-for-`IDirect3DDxgiInterfaceAccess` trick: the raw D3D11 device + no longer exposes that interface on newer Windows (verified E_NOINTERFACE on build 26200, hardware and + WARP alike) while `CreateDirect3D11DeviceFromDXGIDevice` keeps working. + `IInitializeWithWindow` are ComImports in `CaptureInterop.cs`. All WinRT projections were verified by + reflection against the built `Microsoft.Windows.SDK.NET.dll` before writing the interop. +- **Known v1 limits:** full-desktop captures are CPU-copied at native resolution (GPU downscale = encoder + task); window capture (`window:`) and multi-monitor live re-targeting beyond the auto-detected + game are behind the picker; picker-based captures don't survive reload. +- **GPU posture:** same as webcam — CPU frames, WPF hardware-presents; D3DImage GPU compositing deferred + to the encoder task. +- **Preview watermark:** the "Preview" placeholder hides while a backdrop renders — + `ShowPreviewPlaceholder` now also checks `BackdropImage` (raised on backdrop change), so a scene with + live capture shows the feed instead of the "nothing here" label. + ### Webcam capture (TASK 3 milestone 1) - **Seam-first:** everything above the WinRT layer speaks only `VideoFrame` (normalized tightly-packed @@ -131,11 +207,16 @@ C# / WPF (.NET 8) following MVVM: border/`IsVisible`). `Scene.Elements` holds images (`Source`) and, at most once, the webcam (`WebcamSceneConfig`); `Scene.WebcamConfig` is the accessor. `CameraManager` refcounts capture sessions by `DeviceId` (a session starts at `RefCount = 1`; repeat acquire bumps it; the last - release stops + disposes). The Add Webcam menu greys out when the **active** scene already has a - config (`CanAddWebcamToActiveScene`); showing a hidden webcam reuses the existing config - (`CanShowWebcamInActiveScene` / empty-canvas right-click "Show Webcam"). **Removing the last webcam - config anywhere clears the identity** (`_webcam = null`), so re-adding opens the picker again - instead of resurrecting the old camera. + release stops + disposes). **"Add Webcam" ALWAYS opens the Windows camera picker** — the creator is + never silently handed the previous camera (which used to happen after deleting one scene's webcam + while another scene still used it; that identity survived, so re-adding bypassed the choice). + Picking a different camera than the current app-wide one swaps it everywhere via + `SwapWebcamIdentityAsync` (the same path "Change Webcam…" uses), keeping the singleton honest; + picking the same one just places the config. The Add Webcam menu greys out when the **active** + scene already has a config (`CanAddWebcamToActiveScene`); showing a hidden webcam reuses the + existing config (`CanShowWebcamInActiveScene` / empty-canvas right-click "Show Webcam" — which, on + a config-less canvas, delegates to Add Webcam and so also picks). **Removing the last webcam + config anywhere clears the identity** (`_webcam = null`), which also drops "Change Webcam…". - **Round→rect restores the aspect (persisted, schema v4):** `SceneElement.ToggleClipShape()` snapshots the rectangular Width/Height into public `RectWidth`/`RectHeight` before going Round and restores them when switching back — otherwise the Round resize lock (square) would leave a square @@ -152,6 +233,12 @@ C# / WPF (.NET 8) following MVVM: `PreviewBitmapChanged`. Frames arrive on a worker thread; `CameraManager` coalesces onto the dispatcher (at most one pending copy per session, at `Render` priority, always copying the latest frame) so a 60fps device never drowns the render thread. +- **Webcam added mid-session must get the live frames:** `PreviewBitmapChanged` fires **once** (the first + frame creates the shared bitmap); later frames only mutate that bitmap in place, so a config that didn't + exist at first-frame time would never receive it — the empty/transparent container you'd see adding a + webcam to Chat while Live already had the camera. `CameraManager.GetPreviewBitmap(deviceId)` exposes the + current shared bitmap; `AddWebcamToActiveSceneAsync` assigns it to the new config right before + `AcquireAsync`, and `ReacquireWebcam` re-propagates it to every config after a reload. - **Clip/mirror/border:** per-element `ClipShape` (Traditional rectangle / Round ellipse) + `IsMirrored` (`ScaleX = -1`) + the OSB-standard static border (`BorderColor` `#RRGGBB` or `""`=none, `BorderOpacity` 0–1, `BorderWidth` 0–20, `BorderAnimation` `None|Pulse|Chase|Rainbow|Shimmer|MarchingAnts|Glow| @@ -160,13 +247,19 @@ C# / WPF (.NET 8) following MVVM: rendering stays static until the animation tier ships — Border Color, Opacity/Thickness sliders, Hide in this scene, Remove); persisted in the layout DB. The Add menu shows when no webcam exists; the empty preview canvas has its own Show Webcam entry. - - The Round `Ellipse` is wrapped in a `Viewbox Stretch="Uniform"` holding a `1x1` Grid, so it renders - as a true circle (diameter = the shorter element dimension) instead of an oval stretched to the - element rect — and the traditional `Image` keeps `UniformToFill` over the full rect. The Round - border is a centered `Ellipse` at `Width/Height = RoundBorderSize`. + - The Round webcam is an `Image Stretch="UniformToFill"` with an `EllipseGeometry` clip + (`Center=0.5,0.5` `RadiusX/Y=0.5`), inside a `Viewbox Stretch="Uniform"` holding a `1x1` Grid, so it + renders as a true circle (diameter = the shorter element dimension) instead of an oval stretched to + the element rect — the traditional `Image` keeps `UniformToFill` over the full rect. The clip is + geometry, **not an `ImageBrush`**: a brush re-rasterizes the frequently-updated `WriteableBitmap` per + frame on the render thread, which is what made the live webcam crawl while round. The Round border is + a centered `Ellipse` at `Width/Height = RoundBorderSize`. - Resizing locks to a square (`_resizeAspect = 1`) while `ClipShape == Round`. - - **Webcam size clamp:** `ClampWebcamToBounds` (internal — test seam) enforces 50% of the 1920×1080 - master per dimension (960×540 max) and no less than 10% (192×108) at resize + load; + - **Webcam size clamp:** `ClampWebcamToBounds(config, sceneName)` (internal — test seam) enforces the + max per dimension at resize + load and no less than 10% of the master (192×108). The cap is picked + **by canonical scene name**: 50% per dimension (960×540) everywhere except the **Chat scene**, which + may reach half the screen **area** (~1358×764 @16:9) so the viewer sees the creator better + (`MaxWebcamWidthFor`/`MaxWebcamHeightFor`; a renamed Chat loses the bigger cap). `RoundBorderSize` follows the clamped height. `WebcamSafeguardTests` guards the clamp. - **Hit-testing:** a `Grid` without `Background` only hit-tests where its children draw, so clicks in the empty corners of a round clip fell through to `Window_PreviewMouseLeftButtonDown` and deselected diff --git a/ytLive.Tests/BackdropTests.cs b/ytLive.Tests/BackdropTests.cs new file mode 100644 index 0000000..309da4d --- /dev/null +++ b/ytLive.Tests/BackdropTests.cs @@ -0,0 +1,118 @@ +using Xunit; +using ytLive.Models; +using ytLive.ViewModels; + +namespace ytLive.Tests; + +/// +/// The backdrop is a permanent, non-deletable bottom layer holding live +/// desktop/game capture, and it exists only in the canonical Live scene (see +/// ). Every backdrop-enabled scene +/// () is healed to have exactly one, inserted +/// first; it is never reorderable or removable by the UI. Scenes without the +/// flag never get one. +/// +public class BackdropTests +{ + [Fact] + public void EnsureBackdrop_OnEmptyScene_InsertsFullFrameBackdrop() + { + var scene = new Scene { Name = "S", HasBackdrop = true }; + + var backdrop = MainViewModel.EnsureBackdrop(scene); + + Assert.NotNull(backdrop); + Assert.Single(scene.Elements); + Assert.Same(backdrop, scene.Elements[0]); + Assert.True(backdrop.IsBackdrop); + Assert.True(backdrop.IsEnabled); + Assert.Equal(SourceType.DisplayCapture, backdrop.Type); + Assert.Equal(0, backdrop.X); + Assert.Equal(0, backdrop.Y); + Assert.Equal(1920, backdrop.Width); + Assert.Equal(1080, backdrop.Height); + } + + [Fact] + public void EnsureBackdrop_ExistingBackdrop_IsIdempotent() + { + var scene = new Scene { Name = "S", HasBackdrop = true }; + var first = MainViewModel.EnsureBackdrop(scene); + var second = MainViewModel.EnsureBackdrop(scene); + + Assert.Same(first, second); + Assert.Single(scene.Elements); + } + + [Fact] + public void EnsureBackdrop_HealsMissingBackdropAtIndexZero() + { + var scene = new Scene { Name = "S", HasBackdrop = true }; + var image = new Source { Name = "Logo", Type = SourceType.Image }; + scene.Elements.Add(image); + + MainViewModel.EnsureBackdrop(scene); + + Assert.Equal(2, scene.Elements.Count); + Assert.True(((Source)scene.Elements[0]).IsBackdrop); + Assert.Same(image, scene.Elements[1]); + } + + [Fact] + public void EnsureBackdrop_SceneWithoutHasBackdrop_ReturnsNullAndInsertsNothing() + { + var scene = new Scene { Name = "Ending", HasBackdrop = false }; + + var backdrop = MainViewModel.EnsureBackdrop(scene); + + Assert.Null(backdrop); + Assert.Empty(scene.Elements); + } + + [Theory] + [InlineData(SourceType.DisplayCapture, true)] + [InlineData(SourceType.WindowCapture, true)] + [InlineData(SourceType.Background, false)] + [InlineData(SourceType.Image, false)] + [InlineData(SourceType.TextOverlay, false)] + public void Source_IsLiveCapture_OnlyForDisplayAndWindow(SourceType type, bool expected) + { + Assert.Equal(expected, new Source { Type = type }.IsLiveCapture); + } + + [Fact] + public void Source_DisplaySource_IsVideoImageSource_WhenLiveCapture() + { + var source = new Source { Type = SourceType.DisplayCapture }; + var bitmap = new System.Windows.Media.Imaging.WriteableBitmap( + 2, 2, 96, 96, System.Windows.Media.PixelFormats.Bgra32, null); + source.VideoImageSource = bitmap; + + Assert.Same(bitmap, source.DisplaySource); + } + + [Fact] + public void Source_DisplaySource_IsImageSource_WhenNotLiveCapture() + { + var source = new Source { Type = SourceType.Background }; + var bitmap = new System.Windows.Media.Imaging.WriteableBitmap( + 2, 2, 96, 96, System.Windows.Media.PixelFormats.Bgra32, null); + source.VideoImageSource = bitmap; + + Assert.NotSame(bitmap, source.DisplaySource); + Assert.Null(source.DisplaySource); + } + + [Fact] + public void Source_TypeChange_RaisesDisplaySource() + { + var source = new Source { Type = SourceType.Image }; + var raised = new List(); + source.PropertyChanged += (_, e) => raised.Add(e.PropertyName ?? string.Empty); + + source.Type = SourceType.DisplayCapture; + + Assert.Contains(nameof(Source.IsLiveCapture), raised); + Assert.Contains(nameof(Source.DisplaySource), raised); + } +} diff --git a/ytLive.Tests/LayoutStorePersistenceTests.cs b/ytLive.Tests/LayoutStorePersistenceTests.cs index 69c3454..ca9e552 100644 --- a/ytLive.Tests/LayoutStorePersistenceTests.cs +++ b/ytLive.Tests/LayoutStorePersistenceTests.cs @@ -49,6 +49,151 @@ public class LayoutStorePersistenceTests } } + // The backdrop (schema v5): permanent bottom layer holding live desktop/game + // capture. IsBackdrop + CaptureKey must survive a save + reload so a + // re-designated target isn't lost on restart. + [Fact] + public void Backdrop_IsBackdrop_And_CaptureKey_Survive_Save_And_Reload() + { + var path = Path.Combine(Path.GetTempPath(), $"ytLlive-layout-{Guid.NewGuid():N}.db"); + try + { + using var store = new LayoutStore(path); + var backdrop = new Source + { + Name = "Backdrop", + Type = SourceType.DisplayCapture, + IsBackdrop = true, + IsEnabled = true, + X = 0, + Y = 0, + Width = 1920, + Height = 1080, + MonitorIndex = 1, + CaptureKey = "monitor:1", + }; + var scene = new Scene { Name = "Starting" }; + scene.Elements.Add(backdrop); + store.Save(new[] { scene }, null); + + var reloaded = store.Load(); + var restored = Assert.IsType(Assert.Single(reloaded[0].Elements)); + + Assert.True(restored.IsBackdrop); + Assert.Equal("monitor:1", restored.CaptureKey); + Assert.Equal(1, restored.MonitorIndex); + Assert.Equal(SourceType.DisplayCapture, restored.Type); + } + finally + { + SqliteConnection.ClearAllPools(); + try { File.Delete(path); } catch { /* best-effort cleanup */ } + } + } + + // Per-scene backdrop switch (schema v6): a scene whose backdrop was turned + // off must stay off across a save + reload. + [Fact] + public void Scene_HasBackdrop_Survives_Save_And_Reload() + { + var path = Path.Combine(Path.GetTempPath(), $"ytLlive-layout-{Guid.NewGuid():N}.db"); + try + { + using var store = new LayoutStore(path); + var on = new Scene { Name = "Live", HasBackdrop = true }; + var off = new Scene { Name = "Ending", HasBackdrop = false }; + store.Save(new[] { on, off }, null); + + var reloaded = store.Load(); + + Assert.True(reloaded.First(s => s.Name == "Live").HasBackdrop); + Assert.False(reloaded.First(s => s.Name == "Ending").HasBackdrop); + } + finally + { + SqliteConnection.ClearAllPools(); + try { File.Delete(path); } catch { /* best-effort cleanup */ } + } + } + + // v5 → v6 one-time backfill: pre-v6 DBs already have backdrops sitting in the + // non-Live canonical scenes (Starting/BRB/Chat/Ending). Adding the HasBackdrop + // column turns those scenes off and drops their backdrop sources; Live keeps + // its backdrop. + [Fact] + public void V5_Database_Backfill_Removes_NonLive_Backdrops() + { + var path = Path.Combine(Path.GetTempPath(), $"ytLlive-layout-{Guid.NewGuid():N}.db"); + try + { + using (var conn = new SqliteConnection($"Data Source={path}")) + { + conn.Open(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = """ + CREATE TABLE Scene ( + Id TEXT PRIMARY KEY, + Name TEXT NOT NULL, + IsHidden INTEGER NOT NULL DEFAULT 0, + IsChatScene INTEGER NOT NULL DEFAULT 0, + SortOrder INTEGER NOT NULL DEFAULT 0 + ); + CREATE TABLE Source ( + Id TEXT PRIMARY KEY, + SceneId TEXT NOT NULL REFERENCES Scene(Id) ON DELETE CASCADE, + AssetId TEXT, + Type TEXT NOT NULL, + Name TEXT NOT NULL, + IsEnabled INTEGER NOT NULL DEFAULT 1, + X REAL NOT NULL DEFAULT 0, + Y REAL NOT NULL DEFAULT 0, + Width REAL NOT NULL DEFAULT 0, + Height REAL NOT NULL DEFAULT 0, + Opacity REAL NOT NULL DEFAULT 1, + MonitorIndex INTEGER, + DeviceId TEXT, + ClipShape TEXT NOT NULL DEFAULT 'Traditional', + IsMirrored INTEGER NOT NULL DEFAULT 0, + IsBackdrop INTEGER NOT NULL DEFAULT 0, + CaptureKey TEXT, + SortOrder INTEGER NOT NULL DEFAULT 0 + ); + INSERT INTO Scene (Id, Name, SortOrder) VALUES ('s-starting', 'Starting', 0); + INSERT INTO Scene (Id, Name, SortOrder) VALUES ('s-live', 'Live', 1); + INSERT INTO Scene (Id, Name, SortOrder) VALUES ('s-chat', 'Chat', 2); + INSERT INTO Source (Id, SceneId, Type, Name, IsBackdrop, CaptureKey, SortOrder) + VALUES ('b-starting', 's-starting', 'DisplayCapture', 'Backdrop', 1, 'monitor:0', 0); + INSERT INTO Source (Id, SceneId, Type, Name, IsBackdrop, CaptureKey, SortOrder) + VALUES ('b-live', 's-live', 'DisplayCapture', 'Backdrop', 1, 'monitor:0', 0); + INSERT INTO Source (Id, SceneId, Type, Name, IsBackdrop, CaptureKey, SortOrder) + VALUES ('b-chat', 's-chat', 'DisplayCapture', 'Backdrop', 1, 'monitor:0', 0); + PRAGMA user_version = 5; + """; + cmd.ExecuteNonQuery(); + } + + using var store = new LayoutStore(path); + var scenes = store.Load(); + + var starting = Assert.Single(scenes, s => s.Name == "Starting"); + Assert.False(starting.HasBackdrop); + Assert.DoesNotContain(starting.Elements, e => e is Source { IsBackdrop: true }); + + var chat = Assert.Single(scenes, s => s.Name == "Chat"); + Assert.False(chat.HasBackdrop); + Assert.DoesNotContain(chat.Elements, e => e is Source { IsBackdrop: true }); + + var live = Assert.Single(scenes, s => s.Name == "Live"); + Assert.True(live.HasBackdrop); + Assert.Single(live.Elements, e => e is Source { IsBackdrop: true }); + } + finally + { + SqliteConnection.ClearAllPools(); + try { File.Delete(path); } catch { /* best-effort cleanup */ } + } + } + // Round-to-rect restore is persisted (schema v4): a Round webcam resized to a // square saves its pre-Round rect dims, and a reloaded config restores them on // toggle-back instead of staying square. diff --git a/ytLive.Tests/RoundClipInteractionTests.cs b/ytLive.Tests/RoundClipInteractionTests.cs index 4dd0be8..2dedcd8 100644 --- a/ytLive.Tests/RoundClipInteractionTests.cs +++ b/ytLive.Tests/RoundClipInteractionTests.cs @@ -101,9 +101,14 @@ public sealed class RoundClipInteractionTests // The round clip must render as a circle (square bounding box), not an oval. webcam.ClipShape = ClipShape.Round; window.UpdateLayout(); - var ellipse = FindRoundEllipse(window); - Assert.NotNull(ellipse); - Assert.Equal(ellipse!.RenderSize.Width, ellipse.RenderSize.Height, 1.0); + var clip = FindRoundClip(window); + Assert.NotNull(clip); + var geometry = Assert.IsType(clip!.Clip); + Assert.Equal(0.5, geometry.Center.X, 1.0); + Assert.Equal(0.5, geometry.Center.Y, 1.0); + Assert.Equal(0.5, geometry.RadiusX, 1.0); + Assert.Equal(0.5, geometry.RadiusY, 1.0); + Assert.Equal(clip.ActualWidth, clip.ActualHeight, 1.0); } finally { @@ -114,9 +119,9 @@ public sealed class RoundClipInteractionTests } } - private static Ellipse? FindRoundEllipse(Window window) + private static Image? FindRoundClip(Window window) { - return Walk(window, element => element is Ellipse e && e.IsVisible) as Ellipse; + return Walk(window, element => element is Image i && i.IsVisible && i.Clip is EllipseGeometry) as Image; } private static DependencyObject? Walk(DependencyObject parent, Func predicate) diff --git a/ytLive.Tests/SceneCatalogTests.cs b/ytLive.Tests/SceneCatalogTests.cs new file mode 100644 index 0000000..1dfa551 --- /dev/null +++ b/ytLive.Tests/SceneCatalogTests.cs @@ -0,0 +1,77 @@ +using Xunit; +using ytLive.Models; +using ytLive.ViewModels; + +namespace ytLive.Tests; + +/// +/// The app is a five-scene product: Starting/Live/BRB/Chat/Ending. Scenes are +/// worked on by name; you can delete them but never add beyond the set, and the +/// live backdrop belongs to Live alone. +/// +public class SceneCatalogTests +{ + [Fact] + public void SceneCatalog_Defines_The_Five_Canonical_Scenes() + { + Assert.Equal( + new[] { "Starting", "Live", "BRB", "Chat", "Ending" }, + SceneCatalog.All); + } + + [Theory] + [InlineData("Starting", true)] + [InlineData("Live", true)] + [InlineData("BRB", true)] + [InlineData("Chat", true)] + [InlineData("Ending", true)] + [InlineData("live", true)] + [InlineData("My Scene", false)] + [InlineData("", false)] + [InlineData(null, false)] + public void SceneCatalog_IsCanonical_Matches_By_Name(string? name, bool expected) + { + Assert.Equal(expected, SceneCatalog.IsCanonical(name)); + } + + [Theory] + [InlineData("Starting", false)] + [InlineData("Live", true)] + [InlineData("BRB", false)] + [InlineData("Chat", false)] + [InlineData("Ending", false)] + public void SceneCatalog_HasBackdrop_Only_For_Live(string name, bool expected) + { + Assert.Equal(expected, SceneCatalog.HasBackdrop(name)); + } + + [Fact] + public void EnforceBackdropPolicy_Removes_Backdrop_From_NonLive_Scenes() + { + var chat = new Scene { Name = "Chat", HasBackdrop = true }; + var live = new Scene { Name = "Live", HasBackdrop = true }; + var chatBackdrop = MainViewModel.EnsureBackdrop(chat)!; + var liveBackdrop = MainViewModel.EnsureBackdrop(live)!; + Assert.Contains(chat.Elements, e => ReferenceEquals(e, chatBackdrop)); + + MainViewModel.EnforceBackdropPolicy(new[] { chat, live }); + + Assert.False(chat.HasBackdrop); + Assert.DoesNotContain(chat.Elements, e => e is Source { IsBackdrop: true }); + Assert.True(live.HasBackdrop); + Assert.Contains(live.Elements, e => ReferenceEquals(e, liveBackdrop)); + } + + [Fact] + public void EnforceBackdropPolicy_Marks_Live_And_Clears_A_Pre_Policy_Database() + { + var starting = new Scene { Name = "Starting", HasBackdrop = true }; + MainViewModel.EnsureBackdrop(starting); + var live = new Scene { Name = "Live", HasBackdrop = false }; + + MainViewModel.EnforceBackdropPolicy(new[] { starting, live }); + + Assert.False(starting.HasBackdrop); + Assert.True(live.HasBackdrop); + } +} diff --git a/ytLive.Tests/ScreenCaptureManagerTests.cs b/ytLive.Tests/ScreenCaptureManagerTests.cs new file mode 100644 index 0000000..94ed71a --- /dev/null +++ b/ytLive.Tests/ScreenCaptureManagerTests.cs @@ -0,0 +1,189 @@ +using System.Threading; +using System.Windows.Threading; +using System.Windows.Media.Imaging; +using Xunit; +using ytLive.Services; + +namespace ytLive.Tests; + +/// +/// ScreenCaptureManager is the app-wide owner of screen-capture sessions, +/// refcounted by target key and coalesced onto the UI dispatcher. These tests +/// pin that contract with a fake IScreenCaptureSource (the WinRT pool/session +/// layer is exercised only on Windows at runtime). +/// +public class ScreenCaptureManagerTests +{ + private sealed class FakeScreenSource : IScreenCaptureSource + { + private readonly List? _started; + private readonly List? _stopped; + + public string Key { get; } + public event Action? FrameAvailable; + + public FakeScreenSource(string key, List? started = null, List? stopped = null) + { + Key = key; + _started = started; + _stopped = stopped; + } + + public Task StartAsync() + { + _started?.Add(Key); + return Task.CompletedTask; + } + + public Task StopAsync() + { + _stopped?.Add(Key); + return Task.CompletedTask; + } + + public void Pump(VideoFrame frame) => FrameAvailable?.Invoke(frame); + } + + private sealed class FailingScreenSource : IScreenCaptureSource + { + public string Key { get; } + public event Action? FrameAvailable; + + public FailingScreenSource(string key) => Key = key; + + public Task StartAsync() => throw new InvalidOperationException("access denied"); + + public Task StopAsync() => Task.CompletedTask; + + public void Pump(VideoFrame frame) => FrameAvailable?.Invoke(frame); + } + + // A real Dispatcher pumping on a background STA thread, so the manager's + // BeginInvoke queue can be drained deterministically from the test thread. + private sealed class DispatcherPump : IDisposable + { + private readonly Thread _thread; + private readonly ManualResetEventSlim _ready = new(false); + private Dispatcher? _dispatcher; + + public Dispatcher Dispatcher => _dispatcher!; + + public DispatcherPump() + { + _thread = new Thread(() => + { + _dispatcher = Dispatcher.CurrentDispatcher; + _ready.Set(); + Dispatcher.Run(); + }); + _thread.IsBackground = true; + _thread.SetApartmentState(ApartmentState.STA); + _thread.Start(); + _ready.Wait(); + } + + // Queued behind anything already posted at Render/higher, so pending + // bitmap copies run first. + public void Drain() => Dispatcher.Invoke(() => { }, DispatcherPriority.ContextIdle); + + public void Dispose() + { + Dispatcher.InvokeShutdown(); + _thread.Join(3000); + } + } + + private static byte[] Pixels(params byte[] raw) => raw; + + [Fact] + public async Task Refcount_TwoAcquires_OneCaptureUntilLastRelease() + { + var started = new List(); + var stopped = new List(); + var manager = new ScreenCaptureManager(key => new FakeScreenSource(key, started, stopped)); + + Assert.True(await manager.AcquireAsync("monitor:0")); + Assert.True(await manager.AcquireAsync("monitor:0")); + Assert.Single(started); + + await manager.ReleaseAsync("monitor:0"); + Assert.Empty(stopped); + + await manager.ReleaseAsync("monitor:0"); + Assert.Single(stopped); + } + + [Fact] + public async Task ReleaseAll_StopsEvenWithMultipleRefs() + { + var started = new List(); + var stopped = new List(); + var manager = new ScreenCaptureManager(key => new FakeScreenSource(key, started, stopped)); + + await manager.AcquireAsync("monitor:0"); + await manager.AcquireAsync("monitor:0"); + await manager.ReleaseAllAsync("monitor:0"); + + Assert.Single(stopped); + } + + [Fact] + public async Task Acquire_FailingSource_ReturnsFalseAndRaisesCaptureFailed() + { + var manager = new ScreenCaptureManager(key => new FailingScreenSource(key)); + string? failedKey = null; + manager.CaptureFailed += (key, _) => failedKey = key; + + Assert.False(await manager.AcquireAsync("monitor:0")); + Assert.Equal("monitor:0", failedKey); + } + + [Fact] + public async Task Acquire_EmptyKey_ReturnsFalse() + { + var manager = new ScreenCaptureManager(key => new FakeScreenSource(key)); + Assert.False(await manager.AcquireAsync(" ")); + } + + // The one integration test for this branch: one target creates one shared + // WriteableBitmap, published once, and back-to-back frames coalesce to the + // latest (a single pending UI copy per session). + [Fact] + public async Task Frames_ShareOneBitmap_AndCoalesceToLatest() + { + using var pump = new DispatcherPump(); + FakeScreenSource? captured = null; + var manager = new ScreenCaptureManager(key => captured = new FakeScreenSource(key), pump.Dispatcher); + + WriteableBitmap? published = null; + var publishedCount = 0; + manager.PreviewBitmapChanged += (_, bitmap) => { published = bitmap; publishedCount++; }; + + Assert.True(await manager.AcquireAsync("monitor:0")); + + var first = new VideoFrame(2, 2, Pixels(1, 0, 0, 255, 2, 0, 0, 255, 3, 0, 0, 255, 4, 0, 0, 255)); + captured!.Pump(first); + pump.Drain(); + + Assert.NotNull(published); + Assert.Equal(1, publishedCount); + + var dims = pump.Dispatcher.Invoke(() => new[] { published!.PixelWidth, published.PixelHeight }); + Assert.Equal(new[] { 2, 2 }, dims); + + var second = new VideoFrame(2, 2, Pixels(5, 0, 0, 255, 6, 0, 0, 255, 7, 0, 0, 255, 8, 0, 0, 255)); + var third = new VideoFrame(2, 2, Pixels(9, 0, 0, 255, 10, 0, 0, 255, 11, 0, 0, 255, 12, 0, 0, 255)); + captured.Pump(second); + captured.Pump(third); + pump.Drain(); + + // The bitmap is frozen to the dispatcher thread; copy its pixels there. + var result = pump.Dispatcher.Invoke(() => + { + var bytes = new byte[16]; + published!.CopyPixels(new System.Windows.Int32Rect(0, 0, 2, 2), bytes, 8, 0); + return bytes; + }); + Assert.Equal(third.BgraPixels, result); + } +} diff --git a/ytLive.Tests/WebcamSafeguardTests.cs b/ytLive.Tests/WebcamSafeguardTests.cs index 8430fe1..1b02b3d 100644 --- a/ytLive.Tests/WebcamSafeguardTests.cs +++ b/ytLive.Tests/WebcamSafeguardTests.cs @@ -6,16 +6,21 @@ namespace ytLive.Tests; /// /// The webcam size safeguard: no scene placement may exceed half the 1920×1080 -/// master frame (960×540), nor drop below 10% of it (192×108). Enforced at -/// resize (MainWindow) and defensively again on every layout load. +/// master frame (960×540) — except the Chat scene, where it may reach half the +/// screen AREA (~1358×764 @16:9) — nor drop below 10% of the frame (192×108). +/// The cap is picked by canonical scene name. Enforced at resize (MainWindow) +/// and defensively again on every layout load. /// public class WebcamSafeguardTests { + private const string DefaultScene = "Starting"; + private const string ChatScene = "Chat"; + [Fact] public void Oversize_Webcam_Is_Capped_To_Half_The_Frame() { var webcam = new WebcamSceneConfig { Width = 1920, Height = 1080 }; - MainViewModel.ClampWebcamToBounds(webcam); + MainViewModel.ClampWebcamToBounds(webcam, DefaultScene); Assert.Equal(MainViewModel.WebcamMaxWidth, webcam.Width); Assert.Equal(MainViewModel.WebcamMaxHeight, webcam.Height); } @@ -24,7 +29,7 @@ public class WebcamSafeguardTests public void Wide_Webcam_Scales_To_Fit_Both_Dimensions() { var webcam = new WebcamSceneConfig { Width = 1000, Height = 500 }; - MainViewModel.ClampWebcamToBounds(webcam); + MainViewModel.ClampWebcamToBounds(webcam, DefaultScene); Assert.Equal(960, webcam.Width); Assert.Equal(480, webcam.Height); } @@ -33,7 +38,7 @@ public class WebcamSafeguardTests public void Tiny_Webcam_Is_Brought_Up_To_The_Minimum() { var webcam = new WebcamSceneConfig { Width = 10, Height = 10 }; - MainViewModel.ClampWebcamToBounds(webcam); + MainViewModel.ClampWebcamToBounds(webcam, DefaultScene); Assert.Equal(MainViewModel.WebcamMinWidth, webcam.Width); Assert.Equal(MainViewModel.WebcamMinWidth, webcam.Height); } @@ -42,7 +47,7 @@ public class WebcamSafeguardTests public void Short_Wide_Webcam_Meets_Both_Minimums() { var webcam = new WebcamSceneConfig { Width = 100, Height = 20 }; - MainViewModel.ClampWebcamToBounds(webcam); + MainViewModel.ClampWebcamToBounds(webcam, DefaultScene); Assert.True(webcam.Width >= MainViewModel.WebcamMinWidth); Assert.Equal(MainViewModel.WebcamMinHeight, webcam.Height); } @@ -51,12 +56,39 @@ public class WebcamSafeguardTests public void Round_Border_Size_Tracks_The_Shorter_Clamped_Dimension() { var webcam = new WebcamSceneConfig { Width = 1920, Height = 500 }; - MainViewModel.ClampWebcamToBounds(webcam); + MainViewModel.ClampWebcamToBounds(webcam, DefaultScene); Assert.Equal(960, webcam.Width); Assert.Equal(250, webcam.Height); Assert.Equal(250, webcam.RoundBorderSize); } + [Fact] + public void Chat_Webcam_May_Reach_Half_The_Screen_Area() + { + var webcam = new WebcamSceneConfig { Width = 1920, Height = 1080 }; + MainViewModel.ClampWebcamToBounds(webcam, ChatScene); + Assert.Equal(MainViewModel.WebcamChatMaxWidth, webcam.Width); + Assert.Equal(MainViewModel.WebcamChatMaxHeight, webcam.Height); + } + + [Fact] + public void Chat_Webcam_At_The_Default_Size_Is_Not_Grown_Or_Shrunk() + { + var webcam = new WebcamSceneConfig { Width = 480, Height = 270 }; + MainViewModel.ClampWebcamToBounds(webcam, ChatScene); + Assert.Equal(480, webcam.Width); + Assert.Equal(270, webcam.Height); + } + + [Fact] + public void Renamed_Chat_Scene_Loses_The_Larger_Cap() + { + var webcam = new WebcamSceneConfig { Width = 1920, Height = 1080 }; + MainViewModel.ClampWebcamToBounds(webcam, "chit-chat"); + Assert.Equal(MainViewModel.WebcamMaxWidth, webcam.Width); + Assert.Equal(MainViewModel.WebcamMaxHeight, webcam.Height); + } + [Fact] public void Round_Then_Back_To_Rect_Restores_The_Original_Aspect() {