540 lines
22 KiB
C#
540 lines
22 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Threading.Tasks;
|
|
using System.Windows;
|
|
using System.Windows.Media;
|
|
using System.Windows.Media.Imaging;
|
|
using System.Windows.Threading;
|
|
using ytLive.Helpers;
|
|
using ytLive.Models;
|
|
using ytLive.Services.BackgroundEffects;
|
|
|
|
namespace ytLive.Services;
|
|
|
|
/// <summary>
|
|
/// Owns webcam capture sessions app-wide, refcounted by DeviceId. One device =
|
|
/// at most one capture, one shared WriteableBitmap; the last release stops and
|
|
/// disposes the source. Frames arrive on a worker thread and are coalesced onto
|
|
/// the UI dispatcher (at most one pending copy per session, using the latest
|
|
/// frame) so a 60fps device doesn't drown the render thread.
|
|
///
|
|
/// Allocation is proven, never assumed: <see cref="AcquireAsync"/> treats a
|
|
/// started reader as success only once the first frame actually arrives (within
|
|
/// <paramref name="firstFrameTimeout"/>), and an async <see cref="SourceFailed"/>
|
|
/// tears the session down and surfaces <see cref="CameraFailed"/> — no silent
|
|
/// empty box.
|
|
///
|
|
/// Background effects (TASK 51) are the ONE processing point on the camera path: per device,
|
|
/// the set of requested <see cref="WebcamEffectKey"/>s (<see cref="SetRequestedEffects"/>) is
|
|
/// rendered from each raw frame on a background worker — one segmentation per frame, smoothed
|
|
/// per camera across frames (see <see cref="MaskSmoother"/>) and shared by every variant,
|
|
/// latest-frame-drop when the worker is busy — and each variant keeps its own
|
|
/// latest frame (the compositor) and shared WriteableBitmap (the preview). The raw path is
|
|
/// untouched, so a slow or failing model never stalls the plain webcam.
|
|
/// </summary>
|
|
public sealed class CameraManager : IDisposable
|
|
{
|
|
private sealed class EffectVariant
|
|
{
|
|
public WriteableBitmap? PreviewBitmap;
|
|
public VideoFrame? LatestFrame;
|
|
public bool FramePending;
|
|
}
|
|
|
|
private sealed class CameraSession
|
|
{
|
|
public string DeviceId { get; }
|
|
public ICameraFrameSource Source { get; }
|
|
public Action<VideoFrame>? FrameHandler;
|
|
public Action<string>? FailureHandler;
|
|
public int RefCount;
|
|
public bool Started;
|
|
public WriteableBitmap? PreviewBitmap;
|
|
public VideoFrame? LatestFrame;
|
|
public bool FramePending;
|
|
public TaskCompletionSource<bool>? FirstFrame;
|
|
public readonly Dictionary<WebcamEffectKey, EffectVariant> Effects = new();
|
|
public bool EffectWorkerRunning;
|
|
public VideoFrame? LastEffectSource;
|
|
public readonly MaskSmoother Smoother = new();
|
|
|
|
public CameraSession(string deviceId, ICameraFrameSource source)
|
|
{
|
|
DeviceId = deviceId;
|
|
Source = source;
|
|
RefCount = 1;
|
|
}
|
|
}
|
|
|
|
private readonly ICameraEnumerator _enumerator;
|
|
private readonly Func<string, ICameraFrameSource> _frameSourceFactory;
|
|
private readonly Dispatcher? _uiDispatcher;
|
|
private readonly TimeSpan _firstFrameTimeout;
|
|
private readonly Dictionary<string, CameraSession> _sessions = new();
|
|
private readonly Dictionary<string, HashSet<WebcamEffectKey>> _requestedEffects = new();
|
|
private readonly object _gate = new();
|
|
private readonly BackgroundEffectProcessor? _effects;
|
|
private readonly EffectTimer _effectTimer = new();
|
|
|
|
/// <summary>Raised on the UI thread when a camera's shared preview bitmap is first created.</summary>
|
|
public event Action<string, WriteableBitmap>? PreviewBitmapChanged;
|
|
|
|
/// <summary>Raised on the UI thread when an effect variant's preview bitmap is first created
|
|
/// (device id, effect key, bitmap) — the effect counterpart of <see cref="PreviewBitmapChanged"/>.</summary>
|
|
public event Action<string, WebcamEffectKey, WriteableBitmap>? EffectPreviewBitmapChanged;
|
|
|
|
/// <summary>
|
|
/// Raised when a capture fails: device in use, access denied, no preview source,
|
|
/// or a started reader that never delivers a first frame within the timeout.
|
|
/// </summary>
|
|
public event Action<string, string>? CameraFailed;
|
|
|
|
public CameraManager(ICameraEnumerator enumerator, Func<string, ICameraFrameSource> frameSourceFactory,
|
|
Dispatcher? uiDispatcher = null, TimeSpan? firstFrameTimeout = null,
|
|
BackgroundEffectProcessor? effects = null)
|
|
{
|
|
_enumerator = enumerator;
|
|
_frameSourceFactory = frameSourceFactory;
|
|
_uiDispatcher = uiDispatcher;
|
|
_firstFrameTimeout = firstFrameTimeout ?? TimeSpan.FromSeconds(4);
|
|
_effects = effects;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Declares which background-effect variants a device must render (the union over every
|
|
/// scene's webcam config). <see cref="BackgroundEffect.None"/> keys are ignored — that's the
|
|
/// raw path. Variants no longer requested are dropped (their worker cost stops); new ones
|
|
/// start rendering from the next camera frame. Survives the session starting later.
|
|
/// </summary>
|
|
public void SetRequestedEffects(string deviceId, IEnumerable<WebcamEffectKey> keys)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(deviceId)) return;
|
|
lock (_gate)
|
|
{
|
|
var set = new HashSet<WebcamEffectKey>();
|
|
foreach (var key in keys)
|
|
if (key.Effect != BackgroundEffect.None) set.Add(key);
|
|
_requestedEffects[deviceId] = set;
|
|
if (_sessions.TryGetValue(deviceId, out var session))
|
|
SyncVariants(session, set);
|
|
}
|
|
}
|
|
|
|
/// <summary>The latest frame for a variant: the raw frame for <see cref="BackgroundEffect.None"/>,
|
|
/// otherwise the processed frame — null until the first one is rendered (an un-processed
|
|
/// frame is never shown in its place, so Remove never flashes the real room).</summary>
|
|
public VideoFrame? GetLatestFrame(string deviceId, WebcamEffectKey key)
|
|
{
|
|
if (key.Effect == BackgroundEffect.None) return GetLatestFrame(deviceId);
|
|
lock (_gate)
|
|
return _sessions.TryGetValue(deviceId, out var session)
|
|
&& session.Effects.TryGetValue(key, out var variant) ? variant.LatestFrame : null;
|
|
}
|
|
|
|
/// <summary>The shared preview bitmap for a variant (raw bitmap for None), or null until
|
|
/// its first frame — then <see cref="EffectPreviewBitmapChanged"/> fires.</summary>
|
|
public WriteableBitmap? GetPreviewBitmap(string deviceId, WebcamEffectKey key)
|
|
{
|
|
if (key.Effect == BackgroundEffect.None) return GetPreviewBitmap(deviceId);
|
|
lock (_gate)
|
|
return _sessions.TryGetValue(deviceId, out var session)
|
|
&& session.Effects.TryGetValue(key, out var variant) ? variant.PreviewBitmap : null;
|
|
}
|
|
|
|
public ICameraEnumerator Enumerator => _enumerator;
|
|
|
|
/// <summary>Increments the refcount for a device, starting capture the first time.</summary>
|
|
public async Task<bool> AcquireAsync(string deviceId)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(deviceId)) return false;
|
|
|
|
CameraSession session;
|
|
bool shouldStart;
|
|
lock (_gate)
|
|
{
|
|
if (_sessions.TryGetValue(deviceId, out var existing))
|
|
{
|
|
existing.RefCount++;
|
|
shouldStart = false;
|
|
session = existing;
|
|
}
|
|
else
|
|
{
|
|
session = new CameraSession(deviceId, _frameSourceFactory(deviceId));
|
|
session.FrameHandler = frame => OnFrameAvailable(session, frame);
|
|
session.Source.FrameAvailable += session.FrameHandler;
|
|
session.FailureHandler = message => OnSourceFailed(session, message);
|
|
session.Source.SourceFailed += session.FailureHandler;
|
|
if (_requestedEffects.TryGetValue(deviceId, out var requested))
|
|
SyncVariants(session, requested);
|
|
_sessions[deviceId] = session;
|
|
shouldStart = true;
|
|
}
|
|
}
|
|
|
|
if (!shouldStart) return session.Started;
|
|
|
|
session.FirstFrame = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
try
|
|
{
|
|
await session.Source.StartAsync();
|
|
|
|
// First-frame proof: "the reader started" is not "the stream is live".
|
|
// Success requires an actual frame within the timeout, else the session
|
|
// is rolled back and reported — never left as a silent empty preview.
|
|
var proven = _firstFrameTimeout <= TimeSpan.Zero
|
|
|| await WaitForFirstFrameAsync(session, _firstFrameTimeout);
|
|
if (!proven)
|
|
{
|
|
var reason = $"Camera '{deviceId}' started but produced no frames within {_firstFrameTimeout.TotalSeconds:0.#}s.";
|
|
RollbackSession(session, reason);
|
|
return false;
|
|
}
|
|
|
|
// Keep the SourceFailed handler subscribed: an async failure after a
|
|
// successful start (device lost, stream state Failed) must still surface.
|
|
session.FirstFrame = null;
|
|
session.Started = true;
|
|
return true;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
RollbackSession(session, ex.Message);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/// <summary>Decrements the refcount; stops and disposes the source at zero.</summary>
|
|
public async Task ReleaseAsync(string deviceId)
|
|
{
|
|
CameraSession? toStop = null;
|
|
lock (_gate)
|
|
{
|
|
if (!_sessions.TryGetValue(deviceId, out var session)) return;
|
|
if (--session.RefCount > 0) return;
|
|
_sessions.Remove(deviceId);
|
|
toStop = session;
|
|
}
|
|
|
|
if (toStop == null) return;
|
|
toStop.Source.FrameAvailable -= toStop.FrameHandler;
|
|
await SafeStopAsync(toStop.Source);
|
|
toStop.PreviewBitmap = null;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Releases a device unconditionally (every ref), regardless of how many scenes
|
|
/// hold it. Used on identity changes (webcam swap, layout reload) where the old
|
|
/// device's refcount isn't known after the scenes are replaced.
|
|
/// </summary>
|
|
public async Task ReleaseAllAsync(string deviceId)
|
|
{
|
|
CameraSession? toStop = null;
|
|
lock (_gate)
|
|
{
|
|
if (!_sessions.TryGetValue(deviceId, out var session)) return;
|
|
session.RefCount = 0;
|
|
_sessions.Remove(deviceId);
|
|
toStop = session;
|
|
}
|
|
|
|
if (toStop == null) return;
|
|
toStop.Source.FrameAvailable -= toStop.FrameHandler;
|
|
await SafeStopAsync(toStop.Source);
|
|
toStop.PreviewBitmap = null;
|
|
}
|
|
|
|
/// <summary>
|
|
/// True while a session exists for the device — started, or still starting.
|
|
/// Used by the app's startup resource check to avoid double-acquiring a lock
|
|
/// that the saved identity's scene configs already hold. A rolled-back (failed)
|
|
/// session is removed, so this goes false and the startup pass can retry.
|
|
/// </summary>
|
|
public bool IsRunning(string deviceId)
|
|
{
|
|
lock (_gate)
|
|
return _sessions.ContainsKey(deviceId);
|
|
}
|
|
|
|
public VideoFrame? GetLatestFrame(string deviceId)
|
|
{
|
|
lock (_gate)
|
|
return _sessions.TryGetValue(deviceId, out var session) ? session.LatestFrame : null;
|
|
}
|
|
|
|
/// <summary>
|
|
/// The shared WriteableBitmap for a running session, or null when the camera
|
|
/// hasn't produced its first frame yet. <see cref="PreviewBitmapChanged"/> only
|
|
/// fires once (first frame), so configs created after the session started must
|
|
/// pick the bitmap up from here.
|
|
/// </summary>
|
|
public WriteableBitmap? GetPreviewBitmap(string deviceId)
|
|
{
|
|
lock (_gate)
|
|
return _sessions.TryGetValue(deviceId, out var session) ? session.PreviewBitmap : null;
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
List<CameraSession> sessions;
|
|
lock (_gate)
|
|
{
|
|
sessions = new List<CameraSession>(_sessions.Values);
|
|
_sessions.Clear();
|
|
}
|
|
|
|
foreach (var session in sessions)
|
|
{
|
|
session.Source.FrameAvailable -= session.FrameHandler;
|
|
if (session.FailureHandler != null)
|
|
session.Source.SourceFailed -= session.FailureHandler;
|
|
_ = SafeStopAsync(session.Source);
|
|
}
|
|
}
|
|
|
|
private static async Task<bool> WaitForFirstFrameAsync(CameraSession session, TimeSpan timeout)
|
|
{
|
|
var first = session.FirstFrame;
|
|
if (first == null) return true;
|
|
var completed = await Task.WhenAny(first.Task, Task.Delay(timeout));
|
|
return ReferenceEquals(completed, first.Task) && first.Task.Result;
|
|
}
|
|
|
|
private void RollbackSession(CameraSession session, string message)
|
|
{
|
|
lock (_gate)
|
|
{
|
|
if (TryGetActiveSession(session))
|
|
_sessions.Remove(session.DeviceId);
|
|
}
|
|
session.Source.FrameAvailable -= session.FrameHandler;
|
|
if (session.FailureHandler != null)
|
|
session.Source.SourceFailed -= session.FailureHandler;
|
|
session.FirstFrame?.TrySetResult(false);
|
|
|
|
var enriched = message;
|
|
var suspects = CameraConflictProbe.GetRunningCameraApps();
|
|
if (suspects.Count > 0)
|
|
enriched += $" Other camera apps running: {string.Join(", ", suspects)}.";
|
|
|
|
AppLog.Write($"CameraManager: camera '{session.DeviceId}' failed: {enriched}");
|
|
CameraFailed?.Invoke(session.DeviceId, enriched);
|
|
_ = SafeStopAsync(session.Source);
|
|
}
|
|
|
|
private void OnSourceFailed(CameraSession session, string message)
|
|
{
|
|
RollbackSession(session, message);
|
|
}
|
|
|
|
private static async Task SafeStopAsync(ICameraFrameSource source)
|
|
{
|
|
try
|
|
{
|
|
await source.StopAsync();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
AppLog.Write($"CameraManager: stopping camera failed: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
private bool TryGetActiveSession(CameraSession session)
|
|
{
|
|
lock (_gate)
|
|
return _sessions.TryGetValue(session.DeviceId, out var current) && ReferenceEquals(current, session);
|
|
}
|
|
|
|
private void OnFrameAvailable(CameraSession session, VideoFrame frame)
|
|
{
|
|
if (!TryGetActiveSession(session)) return;
|
|
session.LatestFrame = frame;
|
|
session.FirstFrame?.TrySetResult(true);
|
|
ScheduleEffects(session);
|
|
|
|
if (session.PreviewBitmap == null)
|
|
{
|
|
if (_uiDispatcher == null) return;
|
|
if (_uiDispatcher.CheckAccess())
|
|
EnsurePreviewBitmap(session);
|
|
else
|
|
_uiDispatcher.BeginInvoke(() =>
|
|
{
|
|
if (TryGetActiveSession(session))
|
|
EnsurePreviewBitmap(session);
|
|
}, DispatcherPriority.Render);
|
|
return;
|
|
}
|
|
|
|
if (_uiDispatcher == null || _uiDispatcher.CheckAccess())
|
|
{
|
|
CopyFrame(session);
|
|
return;
|
|
}
|
|
|
|
if (session.FramePending) return;
|
|
session.FramePending = true;
|
|
_uiDispatcher.BeginInvoke(() =>
|
|
{
|
|
session.FramePending = false;
|
|
if (TryGetActiveSession(session) && session.PreviewBitmap != null)
|
|
CopyFrame(session);
|
|
}, DispatcherPriority.Render);
|
|
}
|
|
|
|
// Caller holds _gate. A new key inherits the last frame + bitmap of a dropped variant with
|
|
// the SAME effect (a strength change), so neither the stream nor the preview blanks for a
|
|
// frame while the creator drags the strength slider.
|
|
private static void SyncVariants(CameraSession session, HashSet<WebcamEffectKey> keys)
|
|
{
|
|
var dropped = session.Effects.Where(kv => !keys.Contains(kv.Key)).ToList();
|
|
foreach (var (stale, _) in dropped)
|
|
session.Effects.Remove(stale);
|
|
foreach (var key in keys)
|
|
{
|
|
if (session.Effects.ContainsKey(key)) continue;
|
|
var heir = dropped.FirstOrDefault(kv => kv.Key.Effect == key.Effect).Value;
|
|
session.Effects[key] = new EffectVariant
|
|
{
|
|
LatestFrame = heir?.LatestFrame,
|
|
PreviewBitmap = heir?.PreviewBitmap,
|
|
};
|
|
}
|
|
}
|
|
|
|
private void ScheduleEffects(CameraSession session)
|
|
{
|
|
if (_effects == null) return;
|
|
lock (_gate)
|
|
{
|
|
if (session.Effects.Count == 0 || session.EffectWorkerRunning) return;
|
|
session.EffectWorkerRunning = true;
|
|
}
|
|
_ = Task.Run(() => RunEffectWorker(session));
|
|
}
|
|
|
|
/// <summary>Renders every requested variant from the newest raw frame, then loops while a
|
|
/// newer frame has arrived; exits (and re-arms on the next frame) when caught up. One
|
|
/// segmentation per processed frame, whatever the number of variants.</summary>
|
|
private void RunEffectWorker(CameraSession session)
|
|
{
|
|
try
|
|
{
|
|
while (true)
|
|
{
|
|
VideoFrame? source;
|
|
List<WebcamEffectKey> keys;
|
|
lock (_gate)
|
|
{
|
|
source = session.LatestFrame;
|
|
if (!TryGetActiveSessionLocked(session) || session.Effects.Count == 0
|
|
|| source == null || ReferenceEquals(source, session.LastEffectSource))
|
|
{
|
|
session.EffectWorkerRunning = false;
|
|
return;
|
|
}
|
|
session.LastEffectSource = source;
|
|
keys = session.Effects.Keys.ToList();
|
|
}
|
|
|
|
var started = System.Diagnostics.Stopwatch.GetTimestamp();
|
|
var mask = session.Smoother.Smooth(_effects!.TrySegment(source));
|
|
var rendered = new List<(WebcamEffectKey Key, VideoFrame Frame)>(keys.Count);
|
|
if (mask != null)
|
|
foreach (var key in keys)
|
|
rendered.Add((key, BackgroundEffectProcessor.Render(source, mask, key.Effect, key.Strength)));
|
|
_effectTimer.Record(System.Diagnostics.Stopwatch.GetElapsedTime(started), source, keys.Count);
|
|
|
|
foreach (var (key, frame) in rendered)
|
|
{
|
|
EffectVariant? variant;
|
|
lock (_gate)
|
|
{
|
|
if (session.Effects.TryGetValue(key, out variant))
|
|
variant.LatestFrame = frame;
|
|
}
|
|
if (variant != null) PublishVariantPreview(session, key, variant);
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
lock (_gate) session.EffectWorkerRunning = false;
|
|
AppLog.Write($"CameraManager: background effects worker failed: {ex.GetType().Name}: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
private void PublishVariantPreview(CameraSession session, WebcamEffectKey key, EffectVariant variant)
|
|
{
|
|
if (_uiDispatcher == null) return;
|
|
lock (_gate)
|
|
{
|
|
if (variant.FramePending) return;
|
|
variant.FramePending = true;
|
|
}
|
|
_uiDispatcher.BeginInvoke(() =>
|
|
{
|
|
VideoFrame? frame;
|
|
lock (_gate)
|
|
{
|
|
variant.FramePending = false;
|
|
if (!TryGetActiveSessionLocked(session) || !session.Effects.TryGetValue(key, out var current)
|
|
|| !ReferenceEquals(current, variant)) return;
|
|
frame = variant.LatestFrame;
|
|
}
|
|
if (frame == null) return;
|
|
var bitmap = variant.PreviewBitmap;
|
|
if (bitmap == null || bitmap.PixelWidth != frame.Width || bitmap.PixelHeight != frame.Height)
|
|
{
|
|
bitmap = new WriteableBitmap(frame.Width, frame.Height, 96, 96, PixelFormats.Bgra32, null);
|
|
bitmap.WritePixels(new Int32Rect(0, 0, frame.Width, frame.Height), frame.BgraPixels, frame.Stride, 0);
|
|
variant.PreviewBitmap = bitmap;
|
|
EffectPreviewBitmapChanged?.Invoke(session.DeviceId, key, bitmap);
|
|
return;
|
|
}
|
|
bitmap.WritePixels(new Int32Rect(0, 0, frame.Width, frame.Height), frame.BgraPixels, frame.Stride, 0);
|
|
}, DispatcherPriority.Render);
|
|
}
|
|
|
|
private bool TryGetActiveSessionLocked(CameraSession session)
|
|
=> _sessions.TryGetValue(session.DeviceId, out var current) && ReferenceEquals(current, session);
|
|
|
|
/// <summary>Perf record for the creator's-box check (TASK 51): logs the average
|
|
/// segment + render cost once per session after 120 processed frames.</summary>
|
|
private sealed class EffectTimer
|
|
{
|
|
private int _count;
|
|
private double _totalMs;
|
|
private bool _logged;
|
|
|
|
public void Record(TimeSpan elapsed, VideoFrame frame, int variants)
|
|
{
|
|
if (_logged) return;
|
|
_totalMs += elapsed.TotalMilliseconds;
|
|
if (++_count < 120) return;
|
|
_logged = true;
|
|
AppLog.Write($"CameraManager: background effects avg {_totalMs / _count:0.0} ms/frame " +
|
|
$"({frame.Width}x{frame.Height}, {variants} variant(s), {_count} frames)");
|
|
}
|
|
}
|
|
|
|
private void EnsurePreviewBitmap(CameraSession session)
|
|
{
|
|
if (session.PreviewBitmap != null || session.LatestFrame == null) return;
|
|
var frame = session.LatestFrame;
|
|
var bitmap = new WriteableBitmap(frame.Width, frame.Height, 96, 96, PixelFormats.Bgra32, null);
|
|
bitmap.WritePixels(new Int32Rect(0, 0, frame.Width, frame.Height), frame.BgraPixels, frame.Stride, 0);
|
|
session.PreviewBitmap = bitmap;
|
|
PreviewBitmapChanged?.Invoke(session.DeviceId, bitmap);
|
|
}
|
|
|
|
private void CopyFrame(CameraSession session)
|
|
{
|
|
var bitmap = session.PreviewBitmap;
|
|
var frame = session.LatestFrame;
|
|
if (bitmap == null || frame == null) return;
|
|
if (frame.Width != bitmap.PixelWidth || frame.Height != bitmap.PixelHeight) return;
|
|
bitmap.WritePixels(new Int32Rect(0, 0, frame.Width, frame.Height), frame.BgraPixels, frame.Stride, 0);
|
|
}
|
|
}
|