TASK 51 slice 4: camera-path hookup — CameraManager renders per-scene effect variants (one segmentation per raw frame on a latest-frame-drop worker, raw path untouched), VM declares/adopts variants per config; separable upsampling cuts Remove 46->3 ms, Blur 80->18 ms (Debug, 720p) (Unit: bg-effects hookup)

This commit is contained in:
2026-10-09 10:27:38 -07:00
parent 55210b9a61
commit de22983b36
10 changed files with 524 additions and 78 deletions
+1
View File
@@ -252,6 +252,7 @@ public partial class MainViewModel
if (element is WebcamSceneConfig)
{
SyncWebcamEffects(); // drop a variant nobody shows any more
OnPropertyChanged(nameof(CanAddWebcam));
OnPropertyChanged(nameof(CanShowWebcamInStagedScene));
OnPropertyChanged(nameof(WebcamAddToolTip));
+53 -6
View File
@@ -7,6 +7,7 @@ using ytLive;
using ytLive.Helpers;
using ytLive.Models;
using ytLive.Services;
using ytLive.Services.BackgroundEffects;
namespace ytLive.ViewModels;
@@ -193,9 +194,8 @@ public partial class MainViewModel
// A session already running (same device, or one that produced a first
// frame) has its shared bitmap; freshly loaded configs must adopt it here,
// because PreviewBitmapChanged never re-fires for an existing bitmap.
if (_cameraManager.GetPreviewBitmap(newDevice) is { } running)
foreach (var config in Scenes.SelectMany(s => s.Elements.OfType<WebcamSceneConfig>()))
config.VideoImageSource = running;
// SyncWebcamEffects also declares each config's background-effect variant.
SyncWebcamEffects();
}
// CameraManager creates the shared WriteableBitmap on the UI thread at the
@@ -209,7 +209,52 @@ public partial class MainViewModel
RaiseWebcamGateChanged();
foreach (var scene in Scenes)
foreach (var config in scene.Elements.OfType<WebcamSceneConfig>())
if (EffectKeyFor(config).Effect == BackgroundEffect.None)
config.VideoImageSource = bitmap;
}
// TASK 51: an effect variant's first processed frame created its bitmap — every config
// showing that variant adopts it (later frames write into it in place).
private void OnCameraEffectPreviewBitmapChanged(string deviceId, WebcamEffectKey key, WriteableBitmap bitmap)
{
if (_webcam?.DeviceId != deviceId) return;
foreach (var scene in Scenes)
foreach (var config in scene.Elements.OfType<WebcamSceneConfig>())
if (EffectKeyFor(config) == key)
config.VideoImageSource = bitmap;
}
internal static WebcamEffectKey EffectKeyFor(WebcamSceneConfig config)
=> WebcamEffectKey.For(config.BackgroundEffect, config.BackgroundStrength);
private readonly HashSet<WebcamSceneConfig> _effectWatchedConfigs = new();
/// <summary>TASK 51: declares the union of every scene's webcam effect variant to the camera
/// (unused variants stop costing CPU) and points each config's preview at its variant's
/// bitmap. An effect variant with no processed frame yet shows nothing rather than the raw
/// room. Re-run whenever a config is added/removed/loaded or its effect changes.</summary>
internal void SyncWebcamEffects()
{
if (_webcam == null || string.IsNullOrWhiteSpace(_webcam.DeviceId)) return;
var configs = Scenes.SelectMany(s => s.Elements.OfType<WebcamSceneConfig>()).ToList();
_cameraManager.SetRequestedEffects(_webcam.DeviceId, configs.Select(EffectKeyFor));
foreach (var config in configs)
{
if (_effectWatchedConfigs.Add(config))
config.PropertyChanged += OnWebcamConfigEffectChanged;
var key = EffectKeyFor(config);
var bitmap = _cameraManager.GetPreviewBitmap(_webcam.DeviceId, key);
if (bitmap != null || key.Effect != BackgroundEffect.None)
config.VideoImageSource = bitmap;
}
}
private void OnWebcamConfigEffectChanged(object? sender, System.ComponentModel.PropertyChangedEventArgs e)
{
if (e.PropertyName is not (nameof(WebcamSceneConfig.BackgroundEffect) or nameof(WebcamSceneConfig.BackgroundStrength)))
return;
SyncWebcamEffects();
ScheduleSave();
}
// The camera failed to start or died asynchronously (in use, offline, locked,
@@ -321,9 +366,9 @@ public partial class MainViewModel
// PreviewBitmapChanged fires only on the camera's first frame, so a config
// added while the session is already running must pick up the shared bitmap
// directly (it's written in place from then on).
if (_cameraManager.GetPreviewBitmap(_webcam.DeviceId) is { } running)
config.VideoImageSource = running;
// directly (it's written in place from then on) — SyncWebcamEffects does that,
// per the config's background-effect variant.
SyncWebcamEffects();
var started = await _cameraManager.AcquireAsync(_webcam.DeviceId);
if (!started)
@@ -346,6 +391,8 @@ public partial class MainViewModel
if (!string.IsNullOrWhiteSpace(oldDevice) && oldDevice != device.Id)
await _cameraManager.ReleaseAllAsync(oldDevice);
SyncWebcamEffects(); // the new device inherits the scenes' background effects
foreach (var unused in Scenes.SelectMany(s => s.Elements.OfType<WebcamSceneConfig>()))
{
var started = await _cameraManager.AcquireAsync(device.Id);
+9 -2
View File
@@ -276,8 +276,11 @@ public partial class MainViewModel : ViewModelBase
_cameraManager = new CameraManager(
_cameraEnumerator,
CameraFrameSourceFactoryOverride ?? (id => new MediaCaptureFrameSource(id)),
System.Windows.Application.Current?.Dispatcher);
System.Windows.Application.Current?.Dispatcher,
effects: new Services.BackgroundEffects.BackgroundEffectProcessor(
PersonSegmenterOverride ?? new Services.BackgroundEffects.OnnxPersonSegmenter()));
_cameraManager.PreviewBitmapChanged += OnCameraPreviewBitmapChanged;
_cameraManager.EffectPreviewBitmapChanged += OnCameraEffectPreviewBitmapChanged;
_cameraManager.CameraFailed += OnCameraFailed;
_microphoneEnumerator = new WinRtMicrophoneEnumerator();
@@ -384,6 +387,10 @@ public partial class MainViewModel : ViewModelBase
internal static ICameraEnumerator? CameraEnumeratorOverride { get; set; }
internal static Func<string, ICameraFrameSource>? CameraFrameSourceFactoryOverride { get; set; }
/// <summary>Test seam (TASK 51): a fake person segmenter so hermetic tests never load the
/// real ONNX model. Mirrors <see cref="CameraFrameSourceFactoryOverride"/>.</summary>
internal static Services.BackgroundEffects.IPersonSegmenter? PersonSegmenterOverride { get; set; }
private void LoadLayout()
{
_isLoading = true;
@@ -570,7 +577,7 @@ public partial class MainViewModel : ViewModelBase
// Looking up by the GUID returned null forever — the webcam was structurally
// invisible on the output while the preview (bitmap path) looked fine
// (caught by the resolver logging, 2026-09-01 take two).
WebcamSceneConfig webcam => _cameraManager.GetLatestFrame(DeviceKeyForWebcam(webcam.WebcamId)),
WebcamSceneConfig webcam => _cameraManager.GetLatestFrame(DeviceKeyForWebcam(webcam.WebcamId), EffectKeyFor(webcam)),
Source { IsLiveCapture: true, CaptureKey: not null } live => _screenCaptureManager.GetLatestFrame(live.CaptureKey),
Source { Type: SourceType.ChatBox } chatBox => RenderChatBox(chatBox),
Source { Type: SourceType.AlertBox } alertBox => RenderAlertBox(alertBox),