TASK 51 slice 4: camera-path hookup — CameraManager renders per-scene effect variants (one segmentation per raw frame on a latest-frame-drop worker, raw path untouched), VM declares/adopts variants per config; separable upsampling cuts Remove 46->3 ms, Blur 80->18 ms (Debug, 720p) (Unit: bg-effects hookup)

2026-10-09 10:27:38 -07:00
parent 55210b9a61
commit de22983b36
10 changed files with 524 additions and 78 deletions
@@ -6,6 +6,8 @@ using System.Windows.Media;
 using System.Windows.Media.Imaging;
 using System.Windows.Threading;
 using ytLive.Helpers;
+using ytLive.Models;
+using ytLive.Services.BackgroundEffects;

 namespace ytLive.Services;

@@ -21,9 +23,23 @@ namespace ytLive.Services;
 /// <paramref name="firstFrameTimeout"/>), and an async <see cref="SourceFailed"/>
 /// tears the session down and surfaces <see cref="CameraFailed"/> — no silent
 /// empty box.
+///
+/// Background effects (TASK 51) are the ONE processing point on the camera path: per device,
+/// the set of requested <see cref="WebcamEffectKey"/>s (<see cref="SetRequestedEffects"/>) is
+/// rendered from each raw frame on a background worker — one segmentation per frame, shared by
+/// every variant, latest-frame-drop when the worker is busy — and each variant keeps its own
+/// latest frame (the compositor) and shared WriteableBitmap (the preview). The raw path is
+/// untouched, so a slow or failing model never stalls the plain webcam.
 /// </summary>
 public sealed class CameraManager : IDisposable
 {
+    private sealed class EffectVariant
+    {
+        public WriteableBitmap? PreviewBitmap;
+        public VideoFrame? LatestFrame;
+        public bool FramePending;
+    }
+
    private sealed class CameraSession
    {
        public string DeviceId { get; }
@@ -36,6 +52,9 @@ public sealed class CameraManager : IDisposable
        public VideoFrame? LatestFrame;
        public bool FramePending;
        public TaskCompletionSource<bool>? FirstFrame;
+        public readonly Dictionary<WebcamEffectKey, EffectVariant> Effects = new();
+        public bool EffectWorkerRunning;
+        public VideoFrame? LastEffectSource;

        public CameraSession(string deviceId, ICameraFrameSource source)
        {
@@ -50,11 +69,18 @@ public sealed class CameraManager : IDisposable
    private readonly Dispatcher? _uiDispatcher;
    private readonly TimeSpan _firstFrameTimeout;
    private readonly Dictionary<string, CameraSession> _sessions = new();
+    private readonly Dictionary<string, HashSet<WebcamEffectKey>> _requestedEffects = new();
    private readonly object _gate = new();
+    private readonly BackgroundEffectProcessor? _effects;
+    private readonly EffectTimer _effectTimer = new();

    /// <summary>Raised on the UI thread when a camera's shared preview bitmap is first created.</summary>
    public event Action<string, WriteableBitmap>? PreviewBitmapChanged;

+    /// <summary>Raised on the UI thread when an effect variant's preview bitmap is first created
+    /// (device id, effect key, bitmap) — the effect counterpart of <see cref="PreviewBitmapChanged"/>.</summary>
+    public event Action<string, WebcamEffectKey, WriteableBitmap>? EffectPreviewBitmapChanged;
+
    /// <summary>
    /// Raised when a capture fails: device in use, access denied, no preview source,
    /// or a started reader that never delivers a first frame within the timeout.
@@ -62,12 +88,55 @@ public sealed class CameraManager : IDisposable
    public event Action<string, string>? CameraFailed;

    public CameraManager(ICameraEnumerator enumerator, Func<string, ICameraFrameSource> frameSourceFactory,
-        Dispatcher? uiDispatcher = null, TimeSpan? firstFrameTimeout = null)
+        Dispatcher? uiDispatcher = null, TimeSpan? firstFrameTimeout = null,
+        BackgroundEffectProcessor? effects = null)
    {
        _enumerator = enumerator;
        _frameSourceFactory = frameSourceFactory;
        _uiDispatcher = uiDispatcher;
        _firstFrameTimeout = firstFrameTimeout ?? TimeSpan.FromSeconds(4);
+        _effects = effects;
+    }
+
+    /// <summary>
+    /// Declares which background-effect variants a device must render (the union over every
+    /// scene's webcam config). <see cref="BackgroundEffect.None"/> keys are ignored — that's the
+    /// raw path. Variants no longer requested are dropped (their worker cost stops); new ones
+    /// start rendering from the next camera frame. Survives the session starting later.
+    /// </summary>
+    public void SetRequestedEffects(string deviceId, IEnumerable<WebcamEffectKey> keys)
+    {
+        if (string.IsNullOrWhiteSpace(deviceId)) return;
+        lock (_gate)
+        {
+            var set = new HashSet<WebcamEffectKey>();
+            foreach (var key in keys)
+                if (key.Effect != BackgroundEffect.None) set.Add(key);
+            _requestedEffects[deviceId] = set;
+            if (_sessions.TryGetValue(deviceId, out var session))
+                SyncVariants(session, set);
+        }
+    }
+
+    /// <summary>The latest frame for a variant: the raw frame for <see cref="BackgroundEffect.None"/>,
+    /// otherwise the processed frame — null until the first one is rendered (an un-processed
+    /// frame is never shown in its place, so Remove never flashes the real room).</summary>
+    public VideoFrame? GetLatestFrame(string deviceId, WebcamEffectKey key)
+    {
+        if (key.Effect == BackgroundEffect.None) return GetLatestFrame(deviceId);
+        lock (_gate)
+            return _sessions.TryGetValue(deviceId, out var session)
+                && session.Effects.TryGetValue(key, out var variant) ? variant.LatestFrame : null;
+    }
+
+    /// <summary>The shared preview bitmap for a variant (raw bitmap for None), or null until
+    /// its first frame — then <see cref="EffectPreviewBitmapChanged"/> fires.</summary>
+    public WriteableBitmap? GetPreviewBitmap(string deviceId, WebcamEffectKey key)
+    {
+        if (key.Effect == BackgroundEffect.None) return GetPreviewBitmap(deviceId);
+        lock (_gate)
+            return _sessions.TryGetValue(deviceId, out var session)
+                && session.Effects.TryGetValue(key, out var variant) ? variant.PreviewBitmap : null;
    }

    public ICameraEnumerator Enumerator => _enumerator;
@@ -94,6 +163,8 @@ public sealed class CameraManager : IDisposable
                session.Source.FrameAvailable += session.FrameHandler;
                session.FailureHandler = message => OnSourceFailed(session, message);
                session.Source.SourceFailed += session.FailureHandler;
+                if (_requestedEffects.TryGetValue(deviceId, out var requested))
+                    SyncVariants(session, requested);
                _sessions[deviceId] = session;
                shouldStart = true;
            }
@@ -277,6 +348,7 @@ public sealed class CameraManager : IDisposable
        if (!TryGetActiveSession(session)) return;
        session.LatestFrame = frame;
        session.FirstFrame?.TrySetResult(true);
+        ScheduleEffects(session);

        if (session.PreviewBitmap == null)
        {
@@ -308,6 +380,132 @@ public sealed class CameraManager : IDisposable
        }, DispatcherPriority.Render);
    }

+    // Caller holds _gate.
+    private static void SyncVariants(CameraSession session, HashSet<WebcamEffectKey> keys)
+    {
+        foreach (var stale in session.Effects.Keys.Where(k => !keys.Contains(k)).ToList())
+            session.Effects.Remove(stale);
+        foreach (var key in keys)
+            if (!session.Effects.ContainsKey(key))
+                session.Effects[key] = new EffectVariant();
+    }
+
+    private void ScheduleEffects(CameraSession session)
+    {
+        if (_effects == null) return;
+        lock (_gate)
+        {
+            if (session.Effects.Count == 0 || session.EffectWorkerRunning) return;
+            session.EffectWorkerRunning = true;
+        }
+        _ = Task.Run(() => RunEffectWorker(session));
+    }
+
+    /// <summary>Renders every requested variant from the newest raw frame, then loops while a
+    /// newer frame has arrived; exits (and re-arms on the next frame) when caught up. One
+    /// segmentation per processed frame, whatever the number of variants.</summary>
+    private void RunEffectWorker(CameraSession session)
+    {
+        try
+        {
+            while (true)
+            {
+                VideoFrame? source;
+                List<WebcamEffectKey> keys;
+                lock (_gate)
+                {
+                    source = session.LatestFrame;
+                    if (!TryGetActiveSessionLocked(session) || session.Effects.Count == 0
+                        || source == null || ReferenceEquals(source, session.LastEffectSource))
+                    {
+                        session.EffectWorkerRunning = false;
+                        return;
+                    }
+                    session.LastEffectSource = source;
+                    keys = session.Effects.Keys.ToList();
+                }
+
+                var started = System.Diagnostics.Stopwatch.GetTimestamp();
+                var mask = _effects!.TrySegment(source);
+                var rendered = new List<(WebcamEffectKey Key, VideoFrame Frame)>(keys.Count);
+                if (mask != null)
+                    foreach (var key in keys)
+                        rendered.Add((key, BackgroundEffectProcessor.Render(source, mask, key.Effect, key.Strength)));
+                _effectTimer.Record(System.Diagnostics.Stopwatch.GetElapsedTime(started), source, keys.Count);
+
+                foreach (var (key, frame) in rendered)
+                {
+                    EffectVariant? variant;
+                    lock (_gate)
+                    {
+                        if (session.Effects.TryGetValue(key, out variant))
+                            variant.LatestFrame = frame;
+                    }
+                    if (variant != null) PublishVariantPreview(session, key, variant);
+                }
+            }
+        }
+        catch (Exception ex)
+        {
+            lock (_gate) session.EffectWorkerRunning = false;
+            AppLog.Write($"CameraManager: background effects worker failed: {ex.GetType().Name}: {ex.Message}");
+        }
+    }
+
+    private void PublishVariantPreview(CameraSession session, WebcamEffectKey key, EffectVariant variant)
+    {
+        if (_uiDispatcher == null) return;
+        lock (_gate)
+        {
+            if (variant.FramePending) return;
+            variant.FramePending = true;
+        }
+        _uiDispatcher.BeginInvoke(() =>
+        {
+            VideoFrame? frame;
+            lock (_gate)
+            {
+                variant.FramePending = false;
+                if (!TryGetActiveSessionLocked(session) || !session.Effects.TryGetValue(key, out var current)
+                    || !ReferenceEquals(current, variant)) return;
+                frame = variant.LatestFrame;
+            }
+            if (frame == null) return;
+            var bitmap = variant.PreviewBitmap;
+            if (bitmap == null || bitmap.PixelWidth != frame.Width || bitmap.PixelHeight != frame.Height)
+            {
+                bitmap = new WriteableBitmap(frame.Width, frame.Height, 96, 96, PixelFormats.Bgra32, null);
+                bitmap.WritePixels(new Int32Rect(0, 0, frame.Width, frame.Height), frame.BgraPixels, frame.Stride, 0);
+                variant.PreviewBitmap = bitmap;
+                EffectPreviewBitmapChanged?.Invoke(session.DeviceId, key, bitmap);
+                return;
+            }
+            bitmap.WritePixels(new Int32Rect(0, 0, frame.Width, frame.Height), frame.BgraPixels, frame.Stride, 0);
+        }, DispatcherPriority.Render);
+    }
+
+    private bool TryGetActiveSessionLocked(CameraSession session)
+        => _sessions.TryGetValue(session.DeviceId, out var current) && ReferenceEquals(current, session);
+
+    /// <summary>Perf record for the creator's-box check (TASK 51): logs the average
+    /// segment + render cost once per session after 120 processed frames.</summary>
+    private sealed class EffectTimer
+    {
+        private int _count;
+        private double _totalMs;
+        private bool _logged;
+
+        public void Record(TimeSpan elapsed, VideoFrame frame, int variants)
+        {
+            if (_logged) return;
+            _totalMs += elapsed.TotalMilliseconds;
+            if (++_count < 120) return;
+            _logged = true;
+            AppLog.Write($"CameraManager: background effects avg {_totalMs / _count:0.0} ms/frame " +
+                         $"({frame.Width}x{frame.Height}, {variants} variant(s), {_count} frames)");
+        }
+    }
+
    private void EnsurePreviewBitmap(CameraSession session)
    {
        if (session.PreviewBitmap != null || session.LatestFrame == null) return;