using System; using System.Collections.Generic; using System.Threading.Tasks; using System.Windows; using System.Windows.Media; using System.Windows.Media.Imaging; using System.Windows.Threading; using ytLive.Helpers; using ytLive.Models; using ytLive.Services.BackgroundEffects; namespace ytLive.Services; /// /// Owns webcam capture sessions app-wide, refcounted by DeviceId. One device = /// at most one capture, one shared WriteableBitmap; the last release stops and /// disposes the source. Frames arrive on a worker thread and are coalesced onto /// the UI dispatcher (at most one pending copy per session, using the latest /// frame) so a 60fps device doesn't drown the render thread. /// /// Allocation is proven, never assumed: treats a /// started reader as success only once the first frame actually arrives (within /// ), and an async /// tears the session down and surfaces — no silent /// empty box. /// /// Background effects (TASK 51) are the ONE processing point on the camera path: per device, /// the set of requested s () is /// rendered from each raw frame on a background worker — one segmentation per frame, smoothed /// per camera across frames (see ) and shared by every variant, /// latest-frame-drop when the worker is busy — and each variant keeps its own /// latest frame (the compositor) and shared WriteableBitmap (the preview). The raw path is /// untouched, so a slow or failing model never stalls the plain webcam. /// public sealed class CameraManager : IDisposable { private sealed class EffectVariant { public WriteableBitmap? PreviewBitmap; public VideoFrame? LatestFrame; public bool FramePending; } private sealed class CameraSession { public string DeviceId { get; } public ICameraFrameSource Source { get; } public Action? FrameHandler; public Action? FailureHandler; public int RefCount; public bool Started; public WriteableBitmap? PreviewBitmap; public VideoFrame? LatestFrame; public bool FramePending; public TaskCompletionSource? FirstFrame; public readonly Dictionary Effects = new(); public bool EffectWorkerRunning; public VideoFrame? LastEffectSource; public readonly MaskSmoother Smoother = new(); public CameraSession(string deviceId, ICameraFrameSource source) { DeviceId = deviceId; Source = source; RefCount = 1; } } private readonly ICameraEnumerator _enumerator; private readonly Func _frameSourceFactory; private readonly Dispatcher? _uiDispatcher; private readonly TimeSpan _firstFrameTimeout; private readonly Dictionary _sessions = new(); private readonly Dictionary> _requestedEffects = new(); private readonly object _gate = new(); private readonly BackgroundEffectProcessor? _effects; private readonly EffectTimer _effectTimer = new(); /// Raised on the UI thread when a camera's shared preview bitmap is first created. public event Action? PreviewBitmapChanged; /// Raised on the UI thread when an effect variant's preview bitmap is first created /// (device id, effect key, bitmap) — the effect counterpart of . public event Action? EffectPreviewBitmapChanged; /// /// Raised when a capture fails: device in use, access denied, no preview source, /// or a started reader that never delivers a first frame within the timeout. /// public event Action? CameraFailed; public CameraManager(ICameraEnumerator enumerator, Func frameSourceFactory, Dispatcher? uiDispatcher = null, TimeSpan? firstFrameTimeout = null, BackgroundEffectProcessor? effects = null) { _enumerator = enumerator; _frameSourceFactory = frameSourceFactory; _uiDispatcher = uiDispatcher; _firstFrameTimeout = firstFrameTimeout ?? TimeSpan.FromSeconds(4); _effects = effects; } /// /// Declares which background-effect variants a device must render (the union over every /// scene's webcam config). keys are ignored — that's the /// raw path. Variants no longer requested are dropped (their worker cost stops); new ones /// start rendering from the next camera frame. Survives the session starting later. /// public void SetRequestedEffects(string deviceId, IEnumerable keys) { if (string.IsNullOrWhiteSpace(deviceId)) return; lock (_gate) { var set = new HashSet(); foreach (var key in keys) if (key.Effect != BackgroundEffect.None) set.Add(key); _requestedEffects[deviceId] = set; if (_sessions.TryGetValue(deviceId, out var session)) SyncVariants(session, set); } } /// The latest frame for a variant: the raw frame for , /// otherwise the processed frame — null until the first one is rendered (an un-processed /// frame is never shown in its place, so Remove never flashes the real room). public VideoFrame? GetLatestFrame(string deviceId, WebcamEffectKey key) { if (key.Effect == BackgroundEffect.None) return GetLatestFrame(deviceId); lock (_gate) return _sessions.TryGetValue(deviceId, out var session) && session.Effects.TryGetValue(key, out var variant) ? variant.LatestFrame : null; } /// The shared preview bitmap for a variant (raw bitmap for None), or null until /// its first frame — then fires. public WriteableBitmap? GetPreviewBitmap(string deviceId, WebcamEffectKey key) { if (key.Effect == BackgroundEffect.None) return GetPreviewBitmap(deviceId); lock (_gate) return _sessions.TryGetValue(deviceId, out var session) && session.Effects.TryGetValue(key, out var variant) ? variant.PreviewBitmap : null; } public ICameraEnumerator Enumerator => _enumerator; /// Increments the refcount for a device, starting capture the first time. public async Task AcquireAsync(string deviceId) { if (string.IsNullOrWhiteSpace(deviceId)) return false; CameraSession session; bool shouldStart; lock (_gate) { if (_sessions.TryGetValue(deviceId, out var existing)) { existing.RefCount++; shouldStart = false; session = existing; } else { session = new CameraSession(deviceId, _frameSourceFactory(deviceId)); session.FrameHandler = frame => OnFrameAvailable(session, frame); session.Source.FrameAvailable += session.FrameHandler; session.FailureHandler = message => OnSourceFailed(session, message); session.Source.SourceFailed += session.FailureHandler; if (_requestedEffects.TryGetValue(deviceId, out var requested)) SyncVariants(session, requested); _sessions[deviceId] = session; shouldStart = true; } } if (!shouldStart) return session.Started; session.FirstFrame = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); try { await session.Source.StartAsync(); // First-frame proof: "the reader started" is not "the stream is live". // Success requires an actual frame within the timeout, else the session // is rolled back and reported — never left as a silent empty preview. var proven = _firstFrameTimeout <= TimeSpan.Zero || await WaitForFirstFrameAsync(session, _firstFrameTimeout); if (!proven) { var reason = $"Camera '{deviceId}' started but produced no frames within {_firstFrameTimeout.TotalSeconds:0.#}s."; RollbackSession(session, reason); return false; } // Keep the SourceFailed handler subscribed: an async failure after a // successful start (device lost, stream state Failed) must still surface. session.FirstFrame = null; session.Started = true; return true; } catch (Exception ex) { RollbackSession(session, ex.Message); return false; } } /// Decrements the refcount; stops and disposes the source at zero. public async Task ReleaseAsync(string deviceId) { CameraSession? toStop = null; lock (_gate) { if (!_sessions.TryGetValue(deviceId, out var session)) return; if (--session.RefCount > 0) return; _sessions.Remove(deviceId); toStop = session; } if (toStop == null) return; toStop.Source.FrameAvailable -= toStop.FrameHandler; await SafeStopAsync(toStop.Source); toStop.PreviewBitmap = null; } /// /// Releases a device unconditionally (every ref), regardless of how many scenes /// hold it. Used on identity changes (webcam swap, layout reload) where the old /// device's refcount isn't known after the scenes are replaced. /// public async Task ReleaseAllAsync(string deviceId) { CameraSession? toStop = null; lock (_gate) { if (!_sessions.TryGetValue(deviceId, out var session)) return; session.RefCount = 0; _sessions.Remove(deviceId); toStop = session; } if (toStop == null) return; toStop.Source.FrameAvailable -= toStop.FrameHandler; await SafeStopAsync(toStop.Source); toStop.PreviewBitmap = null; } /// /// True while a session exists for the device — started, or still starting. /// Used by the app's startup resource check to avoid double-acquiring a lock /// that the saved identity's scene configs already hold. A rolled-back (failed) /// session is removed, so this goes false and the startup pass can retry. /// public bool IsRunning(string deviceId) { lock (_gate) return _sessions.ContainsKey(deviceId); } public VideoFrame? GetLatestFrame(string deviceId) { lock (_gate) return _sessions.TryGetValue(deviceId, out var session) ? session.LatestFrame : null; } /// /// The shared WriteableBitmap for a running session, or null when the camera /// hasn't produced its first frame yet. only /// fires once (first frame), so configs created after the session started must /// pick the bitmap up from here. /// public WriteableBitmap? GetPreviewBitmap(string deviceId) { lock (_gate) return _sessions.TryGetValue(deviceId, out var session) ? session.PreviewBitmap : null; } public void Dispose() { List sessions; lock (_gate) { sessions = new List(_sessions.Values); _sessions.Clear(); } foreach (var session in sessions) { session.Source.FrameAvailable -= session.FrameHandler; if (session.FailureHandler != null) session.Source.SourceFailed -= session.FailureHandler; _ = SafeStopAsync(session.Source); } } private static async Task WaitForFirstFrameAsync(CameraSession session, TimeSpan timeout) { var first = session.FirstFrame; if (first == null) return true; var completed = await Task.WhenAny(first.Task, Task.Delay(timeout)); return ReferenceEquals(completed, first.Task) && first.Task.Result; } private void RollbackSession(CameraSession session, string message) { lock (_gate) { if (TryGetActiveSession(session)) _sessions.Remove(session.DeviceId); } session.Source.FrameAvailable -= session.FrameHandler; if (session.FailureHandler != null) session.Source.SourceFailed -= session.FailureHandler; session.FirstFrame?.TrySetResult(false); var enriched = message; var suspects = CameraConflictProbe.GetRunningCameraApps(); if (suspects.Count > 0) enriched += $" Other camera apps running: {string.Join(", ", suspects)}."; AppLog.Write($"CameraManager: camera '{session.DeviceId}' failed: {enriched}", AppLogMessageType.Error, AppLogErrorClass.Class2); CameraFailed?.Invoke(session.DeviceId, enriched); _ = SafeStopAsync(session.Source); } private void OnSourceFailed(CameraSession session, string message) { RollbackSession(session, message); } private static async Task SafeStopAsync(ICameraFrameSource source) { try { await source.StopAsync(); } catch (Exception ex) { AppLog.Write($"CameraManager: stopping camera failed: {ex.Message}", AppLogMessageType.Error, AppLogErrorClass.Class2); } } private bool TryGetActiveSession(CameraSession session) { lock (_gate) return _sessions.TryGetValue(session.DeviceId, out var current) && ReferenceEquals(current, session); } private void OnFrameAvailable(CameraSession session, VideoFrame frame) { if (!TryGetActiveSession(session)) return; session.LatestFrame = frame; session.FirstFrame?.TrySetResult(true); ScheduleEffects(session); if (session.PreviewBitmap == null) { if (_uiDispatcher == null) return; if (_uiDispatcher.CheckAccess()) EnsurePreviewBitmap(session); else _uiDispatcher.BeginInvoke(() => { if (TryGetActiveSession(session)) EnsurePreviewBitmap(session); }, DispatcherPriority.Render); return; } if (_uiDispatcher == null || _uiDispatcher.CheckAccess()) { CopyFrame(session); return; } if (session.FramePending) return; session.FramePending = true; _uiDispatcher.BeginInvoke(() => { session.FramePending = false; if (TryGetActiveSession(session) && session.PreviewBitmap != null) CopyFrame(session); }, DispatcherPriority.Render); } // Caller holds _gate. A new key inherits the last frame + bitmap of a dropped variant with // the SAME effect (a strength change), so neither the stream nor the preview blanks for a // frame while the creator drags the strength slider. private static void SyncVariants(CameraSession session, HashSet keys) { var dropped = session.Effects.Where(kv => !keys.Contains(kv.Key)).ToList(); foreach (var (stale, _) in dropped) session.Effects.Remove(stale); foreach (var key in keys) { if (session.Effects.ContainsKey(key)) continue; var heir = dropped.FirstOrDefault(kv => kv.Key.Effect == key.Effect).Value; session.Effects[key] = new EffectVariant { LatestFrame = heir?.LatestFrame, PreviewBitmap = heir?.PreviewBitmap, }; } } private void ScheduleEffects(CameraSession session) { if (_effects == null) return; lock (_gate) { if (session.Effects.Count == 0 || session.EffectWorkerRunning) return; session.EffectWorkerRunning = true; } _ = Task.Run(() => RunEffectWorker(session)); } /// Renders every requested variant from the newest raw frame, then loops while a /// newer frame has arrived; exits (and re-arms on the next frame) when caught up. One /// segmentation per processed frame, whatever the number of variants. private void RunEffectWorker(CameraSession session) { try { while (true) { VideoFrame? source; List keys; lock (_gate) { source = session.LatestFrame; if (!TryGetActiveSessionLocked(session) || session.Effects.Count == 0 || source == null || ReferenceEquals(source, session.LastEffectSource)) { session.EffectWorkerRunning = false; return; } session.LastEffectSource = source; keys = session.Effects.Keys.ToList(); } var started = System.Diagnostics.Stopwatch.GetTimestamp(); var mask = session.Smoother.Smooth(_effects!.TrySegment(source)); var rendered = new List<(WebcamEffectKey Key, VideoFrame Frame)>(keys.Count); if (mask != null) foreach (var key in keys) rendered.Add((key, BackgroundEffectProcessor.Render(source, mask, key.Effect, key.Strength))); _effectTimer.Record(System.Diagnostics.Stopwatch.GetElapsedTime(started), source, keys.Count); foreach (var (key, frame) in rendered) { EffectVariant? variant; lock (_gate) { if (session.Effects.TryGetValue(key, out variant)) variant.LatestFrame = frame; } if (variant != null) PublishVariantPreview(session, key, variant); } } } catch (Exception ex) { lock (_gate) session.EffectWorkerRunning = false; AppLog.Write($"CameraManager: background effects worker failed: {ex.GetType().Name}: {ex.Message}", AppLogMessageType.Error, AppLogErrorClass.Class2); } } private void PublishVariantPreview(CameraSession session, WebcamEffectKey key, EffectVariant variant) { if (_uiDispatcher == null) return; lock (_gate) { if (variant.FramePending) return; variant.FramePending = true; } _uiDispatcher.BeginInvoke(() => { VideoFrame? frame; lock (_gate) { variant.FramePending = false; if (!TryGetActiveSessionLocked(session) || !session.Effects.TryGetValue(key, out var current) || !ReferenceEquals(current, variant)) return; frame = variant.LatestFrame; } if (frame == null) return; var bitmap = variant.PreviewBitmap; if (bitmap == null || bitmap.PixelWidth != frame.Width || bitmap.PixelHeight != frame.Height) { bitmap = new WriteableBitmap(frame.Width, frame.Height, 96, 96, PixelFormats.Bgra32, null); bitmap.WritePixels(new Int32Rect(0, 0, frame.Width, frame.Height), frame.BgraPixels, frame.Stride, 0); variant.PreviewBitmap = bitmap; EffectPreviewBitmapChanged?.Invoke(session.DeviceId, key, bitmap); return; } bitmap.WritePixels(new Int32Rect(0, 0, frame.Width, frame.Height), frame.BgraPixels, frame.Stride, 0); }, DispatcherPriority.Render); } private bool TryGetActiveSessionLocked(CameraSession session) => _sessions.TryGetValue(session.DeviceId, out var current) && ReferenceEquals(current, session); /// Perf record for the creator's-box check (TASK 51): logs the average /// segment + render cost once per session after 120 processed frames. private sealed class EffectTimer { private int _count; private double _totalMs; private bool _logged; public void Record(TimeSpan elapsed, VideoFrame frame, int variants) { if (_logged) return; _totalMs += elapsed.TotalMilliseconds; if (++_count < 120) return; _logged = true; AppLog.Write($"CameraManager: background effects avg {_totalMs / _count:0.0} ms/frame " + $"({frame.Width}x{frame.Height}, {variants} variant(s), {_count} frames)"); } } private void EnsurePreviewBitmap(CameraSession session) { if (session.PreviewBitmap != null || session.LatestFrame == null) return; var frame = session.LatestFrame; var bitmap = new WriteableBitmap(frame.Width, frame.Height, 96, 96, PixelFormats.Bgra32, null); bitmap.WritePixels(new Int32Rect(0, 0, frame.Width, frame.Height), frame.BgraPixels, frame.Stride, 0); session.PreviewBitmap = bitmap; PreviewBitmapChanged?.Invoke(session.DeviceId, bitmap); } private void CopyFrame(CameraSession session) { var bitmap = session.PreviewBitmap; var frame = session.LatestFrame; if (bitmap == null || frame == null) return; if (frame.Width != bitmap.PixelWidth || frame.Height != bitmap.PixelHeight) return; bitmap.WritePixels(new Int32Rect(0, 0, frame.Width, frame.Height), frame.BgraPixels, frame.Stride, 0); } }