From de22983b365211da25856f51da54339f7bcca1e6 Mon Sep 17 00:00:00 2001 From: gramps Date: Fri, 9 Oct 2026 10:27:38 -0700 Subject: [PATCH] =?UTF-8?q?TASK=2051=20slice=204:=20camera-path=20hookup?= =?UTF-8?q?=20=E2=80=94=20CameraManager=20renders=20per-scene=20effect=20v?= =?UTF-8?q?ariants=20(one=20segmentation=20per=20raw=20frame=20on=20a=20la?= =?UTF-8?q?test-frame-drop=20worker,=20raw=20path=20untouched),=20VM=20dec?= =?UTF-8?q?lares/adopts=20variants=20per=20config;=20separable=20upsamplin?= =?UTF-8?q?g=20cuts=20Remove=2046->3=20ms,=20Blur=2080->18=20ms=20(Debug,?= =?UTF-8?q?=20720p)=20(Unit:=20bg-effects=20hookup)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../BackgroundEffectProcessor.cs | 189 +++++++++++------ Services/BackgroundEffects/WebcamEffectKey.cs | 16 ++ Services/CameraManager.cs | 200 +++++++++++++++++- Services/index.md | 3 +- TASKS/task-51-webcam-bg-effects.md | 10 +- ViewModels/MainViewModel.Sources.cs | 1 + ViewModels/MainViewModel.Webcam.cs | 59 +++++- ViewModels/MainViewModel.cs | 11 +- ai.md | 18 ++ ytLive.Tests/CameraManagerTests.cs | 95 +++++++++ 10 files changed, 524 insertions(+), 78 deletions(-) create mode 100644 Services/BackgroundEffects/WebcamEffectKey.cs diff --git a/Services/BackgroundEffects/BackgroundEffectProcessor.cs b/Services/BackgroundEffects/BackgroundEffectProcessor.cs index e9ee580..09e77d6 100644 --- a/Services/BackgroundEffects/BackgroundEffectProcessor.cs +++ b/Services/BackgroundEffects/BackgroundEffectProcessor.cs @@ -90,26 +90,36 @@ public sealed class BackgroundEffectProcessor var dst = new byte[src.Length]; Buffer.BlockCopy(src, 0, dst, 0, src.Length); - // Edge feather: map the mask's [0.5 - w, 0.5 + w] band to alpha 0..1. - var w = (float)(0.05 + 0.45 * strength); - var lo = 0.5f - w; + // Edge feather: map the mask's [0.5 - w, 0.5 + w] band to alpha 0..1, as a 256-entry LUT + // over the quantized mask value (so the per-pixel work is a lookup + one multiply). + var w = 0.05 + 0.45 * strength; + var lo = 0.5 - w; var span = 2 * w; + var alphaLut = new int[256]; + for (var i = 0; i < 256; i++) + alphaLut[i] = (int)Math.Round(Math.Clamp((i / 255.0 - lo) / span, 0, 1) * 256); var width = frame.Width; var height = frame.Height; var stride = frame.Stride; - for (var y = 0; y < height; y++) + var maskRows = UpsampleMaskRows(mask, width); + var (r0, r1, fy) = AxisLut(mask.Height, height); + + Parallel.For(0, height, RowParallelism, y => { - var my = MaskCoord(y, height, mask.Height); - var row = y * stride; + var a0 = maskRows[r0[y]]; + var a1 = maskRows[r1[y]]; + var t = fy[y]; + var row = y * stride + 3; for (var x = 0; x < width; x++) { - var m = SampleBilinear(mask, MaskCoord(x, width, mask.Width), my); - var a = Math.Clamp((m - lo) / span, 0f, 1f); - var p = row + x * 4 + 3; - dst[p] = (byte)(a * src[p] + 0.5f); + var m = a0[x] + (a1[x] - a0[x]) * t; + var q = (int)(m * 255f + 0.5f); + q = q < 0 ? 0 : q > 255 ? 255 : q; + var p = row + x * 4; + dst[p] = (byte)((src[p] * alphaLut[q] + 128) >> 8); } - } + }); return new VideoFrame(width, height, dst) { Epoch = frame.Epoch, IsOpaque = false }; } @@ -126,7 +136,7 @@ public sealed class BackgroundEffectProcessor var sw = (width + f - 1) / f; var sh = (height + f - 1) / f; var small = new float[sw * sh * 3]; - for (var sy = 0; sy < sh; sy++) + Parallel.For(0, sh, RowParallelism, sy => { var y0 = sy * f; var y1 = Math.Min(height, y0 + f); @@ -148,7 +158,7 @@ public sealed class BackgroundEffectProcessor var o = (sy * sw + sx) * 3; small[o] = b / n; small[o + 1] = g / n; small[o + 2] = r / n; } - } + }); // 2. Three separable box-blur passes ≈ a gaussian. var radius = (int)Math.Round(MinBlurRadius + (MaxBlurRadius - MinBlurRadius) * strength); @@ -159,34 +169,126 @@ public sealed class BackgroundEffectProcessor BoxBlurVertical(scratch, small, sw, sh, radius); } - // 3. Blend: person (mask) keeps the source pixel, background takes the upsampled blur. + // 3. Upsample: horizontal interpolation precomputed once per small row (and per mask row), + // then each output row lerps two precomputed rows — no per-pixel 2-D bilinear. + var blurRows = UpsampleColorRows(small, sw, sh, width); + var (b0, b1, bfy) = AxisLut(sh, height); + var maskRows = UpsampleMaskRows(mask, width); + var (m0, m1, mfy) = AxisLut(mask.Height, height); + + // 4. Blend: person (mask) keeps the source pixel, background takes the blur. var dst = new byte[src.Length]; - for (var y = 0; y < height; y++) + Parallel.For(0, height, RowParallelism, y => { - var my = MaskCoord(y, height, mask.Height); - var by = (y + 0.5f) / f - 0.5f; + var ma = maskRows[m0[y]]; + var mb = maskRows[m1[y]]; + var mt = mfy[y]; + var ba = blurRows[b0[y]]; + var bb = blurRows[b1[y]]; + var bt = bfy[y]; var row = y * stride; for (var x = 0; x < width; x++) { - var m = SampleBilinear(mask, MaskCoord(x, width, mask.Width), my); var p = row + x * 4; - if (m >= 1f) + var m = ma[x] + (mb[x] - ma[x]) * mt; + if (m >= 0.999f) { dst[p] = src[p]; dst[p + 1] = src[p + 1]; dst[p + 2] = src[p + 2]; dst[p + 3] = src[p + 3]; continue; } - SampleSmall(small, sw, sh, (x + 0.5f) / f - 0.5f, by, out var bb, out var bg, out var br); + if (m < 0f) m = 0f; var k = 1f - m; - dst[p] = (byte)Math.Clamp(src[p] * m + bb * k + 0.5f, 0, 255); - dst[p + 1] = (byte)Math.Clamp(src[p + 1] * m + bg * k + 0.5f, 0, 255); - dst[p + 2] = (byte)Math.Clamp(src[p + 2] * m + br * k + 0.5f, 0, 255); + var c = x * 3; + var vb = ba[c] + (bb[c] - ba[c]) * bt; + var vg = ba[c + 1] + (bb[c + 1] - ba[c + 1]) * bt; + var vr = ba[c + 2] + (bb[c + 2] - ba[c + 2]) * bt; + dst[p] = ClampByte(src[p] * m + vb * k); + dst[p + 1] = ClampByte(src[p + 1] * m + vg * k); + dst[p + 2] = ClampByte(src[p + 2] * m + vr * k); dst[p + 3] = src[p + 3]; } - } + }); return new VideoFrame(width, height, dst) { Epoch = frame.Epoch, IsOpaque = frame.IsOpaque }; } + /// Row-parallel pixel loops, capped so the effect worker can't starve the compositor + /// and encoder threads. + private static readonly ParallelOptions RowParallelism = new() + { + MaxDegreeOfParallelism = Math.Clamp(Environment.ProcessorCount / 2, 1, 4), + }; + + private static byte ClampByte(float v) + { + var i = (int)(v + 0.5f); + return (byte)(i < 0 ? 0 : i > 255 ? 255 : i); + } + + /// For each destination index along one axis: the two source indices and the + /// fractional weight — pixel-center aligned, edge-clamped (the same mapping a bilinear + /// sample uses). + private static (int[] I0, int[] I1, float[] F) AxisLut(int srcSize, int dstSize) + { + var i0 = new int[dstSize]; + var i1 = new int[dstSize]; + var fr = new float[dstSize]; + for (var d = 0; d < dstSize; d++) + { + var s = (d + 0.5f) * srcSize / dstSize - 0.5f; + var a = (int)MathF.Floor(s); + var t = s - a; + i0[d] = Math.Clamp(a, 0, srcSize - 1); + i1[d] = Math.Clamp(a + 1, 0, srcSize - 1); + fr[d] = a < 0 ? 0f : a + 1 > srcSize - 1 ? 0f : t; + } + return (i0, i1, fr); + } + + /// Each mask row, horizontally interpolated (and clamped to 0..1) to the frame width. + private static float[][] UpsampleMaskRows(SegmentationMask mask, int width) + { + var (x0, x1, fx) = AxisLut(mask.Width, width); + var rows = new float[mask.Height][]; + var v = mask.Values; + for (var r = 0; r < mask.Height; r++) + { + var row = new float[width]; + var o = r * mask.Width; + for (var x = 0; x < width; x++) + { + var a = v[o + x0[x]]; + var m = a + (v[o + x1[x]] - a) * fx[x]; + row[x] = m < 0f ? 0f : m > 1f ? 1f : m; + } + rows[r] = row; + } + return rows; + } + + /// Each row of a 3-channel float image, horizontally interpolated to . + private static float[][] UpsampleColorRows(float[] img, int w, int h, int width) + { + var (x0, x1, fx) = AxisLut(w, width); + var rows = new float[h][]; + Parallel.For(0, h, RowParallelism, r => + { + var row = new float[width * 3]; + var o = r * w; + for (var x = 0; x < width; x++) + { + var a = (o + x0[x]) * 3; + var b = (o + x1[x]) * 3; + var t = fx[x]; + row[x * 3] = img[a] + (img[b] - img[a]) * t; + row[x * 3 + 1] = img[a + 1] + (img[b + 1] - img[a + 1]) * t; + row[x * 3 + 2] = img[a + 2] + (img[b + 2] - img[a + 2]) * t; + } + rows[r] = row; + }); + return rows; + } + private static void BoxBlurHorizontal(float[] input, float[] output, int w, int h, int r) { var norm = 1f / (2 * r + 1); @@ -230,47 +332,8 @@ public sealed class BackgroundEffectProcessor } } - private static void SampleSmall(float[] img, int w, int h, float x, float y, out float b, out float g, out float r) - { - var x0 = (int)MathF.Floor(x); - var y0 = (int)MathF.Floor(y); - var fx = x - x0; - var fy = y - y0; - var x1 = Math.Clamp(x0 + 1, 0, w - 1); - var y1 = Math.Clamp(y0 + 1, 0, h - 1); - x0 = Math.Clamp(x0, 0, w - 1); - y0 = Math.Clamp(y0, 0, h - 1); - var i00 = (y0 * w + x0) * 3; - var i01 = (y0 * w + x1) * 3; - var i10 = (y1 * w + x0) * 3; - var i11 = (y1 * w + x1) * 3; - var w00 = (1 - fx) * (1 - fy); - var w01 = fx * (1 - fy); - var w10 = (1 - fx) * fy; - var w11 = fx * fy; - b = img[i00] * w00 + img[i01] * w01 + img[i10] * w10 + img[i11] * w11; - g = img[i00 + 1] * w00 + img[i01 + 1] * w01 + img[i10 + 1] * w10 + img[i11 + 1] * w11; - r = img[i00 + 2] * w00 + img[i01 + 2] * w01 + img[i10 + 2] * w10 + img[i11 + 2] * w11; - } - private static float MaskCoord(int pixel, int frameSize, int maskSize) - => (pixel + 0.5f) * maskSize / frameSize - 0.5f; - private static float SampleBilinear(SegmentationMask mask, float x, float y) - { - var x0 = (int)MathF.Floor(x); - var y0 = (int)MathF.Floor(y); - var fx = x - x0; - var fy = y - y0; - var x1 = Math.Clamp(x0 + 1, 0, mask.Width - 1); - var y1 = Math.Clamp(y0 + 1, 0, mask.Height - 1); - x0 = Math.Clamp(x0, 0, mask.Width - 1); - y0 = Math.Clamp(y0, 0, mask.Height - 1); - var v = mask.Values; - var top = v[y0 * mask.Width + x0] * (1 - fx) + v[y0 * mask.Width + x1] * fx; - var bottom = v[y1 * mask.Width + x0] * (1 - fx) + v[y1 * mask.Width + x1] * fx; - return Math.Clamp(top * (1 - fy) + bottom * fy, 0f, 1f); - } private void LogFailureOnce(string reason) { diff --git a/Services/BackgroundEffects/WebcamEffectKey.cs b/Services/BackgroundEffects/WebcamEffectKey.cs new file mode 100644 index 0000000..d09a9f9 --- /dev/null +++ b/Services/BackgroundEffects/WebcamEffectKey.cs @@ -0,0 +1,16 @@ +using ytLive.Models; + +namespace ytLive.Services.BackgroundEffects; + +/// Identifies one rendered variant of a camera's frames (TASK 51): the effect plus its +/// strength, rounded to 2 decimals so a slider drag doesn't spawn a variant per pixel of travel. +/// always normalizes to strength 0 — the raw path. +public readonly record struct WebcamEffectKey(BackgroundEffect Effect, double Strength) +{ + public static WebcamEffectKey None => new(BackgroundEffect.None, 0); + + public static WebcamEffectKey For(BackgroundEffect effect, double strength) + => effect == BackgroundEffect.None + ? None + : new WebcamEffectKey(effect, Math.Round(Math.Clamp(strength, 0, 1), 2)); +} diff --git a/Services/CameraManager.cs b/Services/CameraManager.cs index 2059887..1b4e156 100644 --- a/Services/CameraManager.cs +++ b/Services/CameraManager.cs @@ -6,6 +6,8 @@ using System.Windows.Media; using System.Windows.Media.Imaging; using System.Windows.Threading; using ytLive.Helpers; +using ytLive.Models; +using ytLive.Services.BackgroundEffects; namespace ytLive.Services; @@ -21,9 +23,23 @@ namespace ytLive.Services; /// ), and an async /// tears the session down and surfaces — no silent /// empty box. +/// +/// Background effects (TASK 51) are the ONE processing point on the camera path: per device, +/// the set of requested s () is +/// rendered from each raw frame on a background worker — one segmentation per frame, shared by +/// every variant, latest-frame-drop when the worker is busy — and each variant keeps its own +/// latest frame (the compositor) and shared WriteableBitmap (the preview). The raw path is +/// untouched, so a slow or failing model never stalls the plain webcam. /// public sealed class CameraManager : IDisposable { + private sealed class EffectVariant + { + public WriteableBitmap? PreviewBitmap; + public VideoFrame? LatestFrame; + public bool FramePending; + } + private sealed class CameraSession { public string DeviceId { get; } @@ -36,6 +52,9 @@ public sealed class CameraManager : IDisposable public VideoFrame? LatestFrame; public bool FramePending; public TaskCompletionSource? FirstFrame; + public readonly Dictionary Effects = new(); + public bool EffectWorkerRunning; + public VideoFrame? LastEffectSource; public CameraSession(string deviceId, ICameraFrameSource source) { @@ -50,11 +69,18 @@ public sealed class CameraManager : IDisposable private readonly Dispatcher? _uiDispatcher; private readonly TimeSpan _firstFrameTimeout; private readonly Dictionary _sessions = new(); + private readonly Dictionary> _requestedEffects = new(); private readonly object _gate = new(); + private readonly BackgroundEffectProcessor? _effects; + private readonly EffectTimer _effectTimer = new(); /// Raised on the UI thread when a camera's shared preview bitmap is first created. public event Action? PreviewBitmapChanged; + /// Raised on the UI thread when an effect variant's preview bitmap is first created + /// (device id, effect key, bitmap) — the effect counterpart of . + public event Action? EffectPreviewBitmapChanged; + /// /// Raised when a capture fails: device in use, access denied, no preview source, /// or a started reader that never delivers a first frame within the timeout. @@ -62,12 +88,55 @@ public sealed class CameraManager : IDisposable public event Action? CameraFailed; public CameraManager(ICameraEnumerator enumerator, Func frameSourceFactory, - Dispatcher? uiDispatcher = null, TimeSpan? firstFrameTimeout = null) + Dispatcher? uiDispatcher = null, TimeSpan? firstFrameTimeout = null, + BackgroundEffectProcessor? effects = null) { _enumerator = enumerator; _frameSourceFactory = frameSourceFactory; _uiDispatcher = uiDispatcher; _firstFrameTimeout = firstFrameTimeout ?? TimeSpan.FromSeconds(4); + _effects = effects; + } + + /// + /// Declares which background-effect variants a device must render (the union over every + /// scene's webcam config). keys are ignored — that's the + /// raw path. Variants no longer requested are dropped (their worker cost stops); new ones + /// start rendering from the next camera frame. Survives the session starting later. + /// + public void SetRequestedEffects(string deviceId, IEnumerable keys) + { + if (string.IsNullOrWhiteSpace(deviceId)) return; + lock (_gate) + { + var set = new HashSet(); + foreach (var key in keys) + if (key.Effect != BackgroundEffect.None) set.Add(key); + _requestedEffects[deviceId] = set; + if (_sessions.TryGetValue(deviceId, out var session)) + SyncVariants(session, set); + } + } + + /// The latest frame for a variant: the raw frame for , + /// otherwise the processed frame — null until the first one is rendered (an un-processed + /// frame is never shown in its place, so Remove never flashes the real room). + public VideoFrame? GetLatestFrame(string deviceId, WebcamEffectKey key) + { + if (key.Effect == BackgroundEffect.None) return GetLatestFrame(deviceId); + lock (_gate) + return _sessions.TryGetValue(deviceId, out var session) + && session.Effects.TryGetValue(key, out var variant) ? variant.LatestFrame : null; + } + + /// The shared preview bitmap for a variant (raw bitmap for None), or null until + /// its first frame — then fires. + public WriteableBitmap? GetPreviewBitmap(string deviceId, WebcamEffectKey key) + { + if (key.Effect == BackgroundEffect.None) return GetPreviewBitmap(deviceId); + lock (_gate) + return _sessions.TryGetValue(deviceId, out var session) + && session.Effects.TryGetValue(key, out var variant) ? variant.PreviewBitmap : null; } public ICameraEnumerator Enumerator => _enumerator; @@ -94,6 +163,8 @@ public sealed class CameraManager : IDisposable session.Source.FrameAvailable += session.FrameHandler; session.FailureHandler = message => OnSourceFailed(session, message); session.Source.SourceFailed += session.FailureHandler; + if (_requestedEffects.TryGetValue(deviceId, out var requested)) + SyncVariants(session, requested); _sessions[deviceId] = session; shouldStart = true; } @@ -277,6 +348,7 @@ public sealed class CameraManager : IDisposable if (!TryGetActiveSession(session)) return; session.LatestFrame = frame; session.FirstFrame?.TrySetResult(true); + ScheduleEffects(session); if (session.PreviewBitmap == null) { @@ -308,6 +380,132 @@ public sealed class CameraManager : IDisposable }, DispatcherPriority.Render); } + // Caller holds _gate. + private static void SyncVariants(CameraSession session, HashSet keys) + { + foreach (var stale in session.Effects.Keys.Where(k => !keys.Contains(k)).ToList()) + session.Effects.Remove(stale); + foreach (var key in keys) + if (!session.Effects.ContainsKey(key)) + session.Effects[key] = new EffectVariant(); + } + + private void ScheduleEffects(CameraSession session) + { + if (_effects == null) return; + lock (_gate) + { + if (session.Effects.Count == 0 || session.EffectWorkerRunning) return; + session.EffectWorkerRunning = true; + } + _ = Task.Run(() => RunEffectWorker(session)); + } + + /// Renders every requested variant from the newest raw frame, then loops while a + /// newer frame has arrived; exits (and re-arms on the next frame) when caught up. One + /// segmentation per processed frame, whatever the number of variants. + private void RunEffectWorker(CameraSession session) + { + try + { + while (true) + { + VideoFrame? source; + List keys; + lock (_gate) + { + source = session.LatestFrame; + if (!TryGetActiveSessionLocked(session) || session.Effects.Count == 0 + || source == null || ReferenceEquals(source, session.LastEffectSource)) + { + session.EffectWorkerRunning = false; + return; + } + session.LastEffectSource = source; + keys = session.Effects.Keys.ToList(); + } + + var started = System.Diagnostics.Stopwatch.GetTimestamp(); + var mask = _effects!.TrySegment(source); + var rendered = new List<(WebcamEffectKey Key, VideoFrame Frame)>(keys.Count); + if (mask != null) + foreach (var key in keys) + rendered.Add((key, BackgroundEffectProcessor.Render(source, mask, key.Effect, key.Strength))); + _effectTimer.Record(System.Diagnostics.Stopwatch.GetElapsedTime(started), source, keys.Count); + + foreach (var (key, frame) in rendered) + { + EffectVariant? variant; + lock (_gate) + { + if (session.Effects.TryGetValue(key, out variant)) + variant.LatestFrame = frame; + } + if (variant != null) PublishVariantPreview(session, key, variant); + } + } + } + catch (Exception ex) + { + lock (_gate) session.EffectWorkerRunning = false; + AppLog.Write($"CameraManager: background effects worker failed: {ex.GetType().Name}: {ex.Message}"); + } + } + + private void PublishVariantPreview(CameraSession session, WebcamEffectKey key, EffectVariant variant) + { + if (_uiDispatcher == null) return; + lock (_gate) + { + if (variant.FramePending) return; + variant.FramePending = true; + } + _uiDispatcher.BeginInvoke(() => + { + VideoFrame? frame; + lock (_gate) + { + variant.FramePending = false; + if (!TryGetActiveSessionLocked(session) || !session.Effects.TryGetValue(key, out var current) + || !ReferenceEquals(current, variant)) return; + frame = variant.LatestFrame; + } + if (frame == null) return; + var bitmap = variant.PreviewBitmap; + if (bitmap == null || bitmap.PixelWidth != frame.Width || bitmap.PixelHeight != frame.Height) + { + bitmap = new WriteableBitmap(frame.Width, frame.Height, 96, 96, PixelFormats.Bgra32, null); + bitmap.WritePixels(new Int32Rect(0, 0, frame.Width, frame.Height), frame.BgraPixels, frame.Stride, 0); + variant.PreviewBitmap = bitmap; + EffectPreviewBitmapChanged?.Invoke(session.DeviceId, key, bitmap); + return; + } + bitmap.WritePixels(new Int32Rect(0, 0, frame.Width, frame.Height), frame.BgraPixels, frame.Stride, 0); + }, DispatcherPriority.Render); + } + + private bool TryGetActiveSessionLocked(CameraSession session) + => _sessions.TryGetValue(session.DeviceId, out var current) && ReferenceEquals(current, session); + + /// Perf record for the creator's-box check (TASK 51): logs the average + /// segment + render cost once per session after 120 processed frames. + private sealed class EffectTimer + { + private int _count; + private double _totalMs; + private bool _logged; + + public void Record(TimeSpan elapsed, VideoFrame frame, int variants) + { + if (_logged) return; + _totalMs += elapsed.TotalMilliseconds; + if (++_count < 120) return; + _logged = true; + AppLog.Write($"CameraManager: background effects avg {_totalMs / _count:0.0} ms/frame " + + $"({frame.Width}x{frame.Height}, {variants} variant(s), {_count} frames)"); + } + } + private void EnsurePreviewBitmap(CameraSession session) { if (session.PreviewBitmap != null || session.LatestFrame == null) return; diff --git a/Services/index.md b/Services/index.md index 370f8bd..d2e2627 100644 --- a/Services/index.md +++ b/Services/index.md @@ -18,6 +18,7 @@ External-facing logic: YouTube API, persistence. See | `VideoFrame.cs` | Normalized CPU frame seam (`Width`/`Height`/tightly-packed BGRA `byte[]`) — the only pixel type the rest of the app knows about; future capture sources (screen, background-removed webcam) feed the same seam | | `BackgroundEffects/IPersonSegmenter.cs` | **TASK 51 seam:** `Segment(VideoFrame) → SegmentationMask?` (`SegmentationMask(Width, Height, float[] Values)`, 0 = background, 1 = person); null = no mask → caller passes through. Tests inject fakes | | `BackgroundEffects/OnnxPersonSegmenter.cs` | **TASK 51 default segmenter:** MediaPipe Selfie Segmenter (landscape, Apache-2.0) as an embedded ONNX resource, run by onnxruntime on the CPU (2 intra-op threads); frame stretched to 256×144 RGB/255 NHWC; lazy session; never throws (fail → log once, null forever) | +| `BackgroundEffects/WebcamEffectKey.cs` | **TASK 51:** `(Effect, Strength)` variant key, strength rounded to 0.01; `None` normalizes to strength 0 (the raw path) | | `BackgroundEffects/BackgroundEffectProcessor.cs` | **TASK 51 pure pixel stage:** `Apply(frame, BackgroundEffect)` — None = same instance, Remove = copy with alpha = feathered mask × source alpha (`IsOpaque=false`), Blur = background blurred at ¼ res (3 box passes) and mask-blended, person untouched; strength 0..1; `TrySegment` once + `Render` per effect; no mask/exception → passthrough + one log | | `CameraDeviceInfo.cs` | `(Id, DisplayName)` for a physical capture device | | `ICameraEnumerator.cs` | `GetCamerasAsync()` — seam so the picker/`CameraManager` never touch WinRT (tests inject fakes) | @@ -27,7 +28,7 @@ External-facing logic: YouTube API, persistence. See | `IMicrophoneEnumerator.cs` | `GetMicrophonesAsync()` — seam so the mic picker never touches WinRT (tests inject fakes) | | `WinRtMicrophoneEnumerator.cs` | WinRT mic enumeration via `DeviceInformation.FindAllAsync(DeviceClass.AudioCapture)` (no NAudio needed — capture libs stay deferred to the audio pipeline milestone) | | `MediaCaptureFrameSource.cs` | CPU-first MediaCapture source (`MemoryPreference = Cpu`, BGRA8 via `CreateFrameReaderAsync`). **Validated, not trusted:** post-init checks (`VideoDeviceId` match, `FrameSources` non-empty, `VideoDeviceController.GetAvailableMediaStreamProperties` ≥ 1); `reader.StartAsync()` status read (throws on non-`Success`); `capture.Failed` + `CameraStreamStateChanged` subscribed → `SourceFailed` event. Fallback ladder: VideoPreview → VideoRecord retry. `SharingMode.Exclusive` + `DeviceLost` not in 19041 SDK projection; `CameraStreamState.Failed` compared by `(int)2` (projection omits member names) | -| `CameraManager.cs` | Webcam capture ownership: refcounted by `DeviceId`, one shared `WriteableBitmap` per camera, dispatcher-coalesced ~render-rate UI updates (latest-frame drop); `PreviewBitmapChanged`/`CameraFailed` events. **First-frame proof** (4s timeout, configurable via ctor `TimeSpan?`): `AcquireAsync` returns true only after a real frame arrives — a reader that starts but never delivers (locked, suspended, dead) fails and surfaces `CameraFailed` with suspect-app names (`CameraConflictProbe`) instead of a silent empty box. `SourceFailed` forwarded from the frame source as async death signal | +| `CameraManager.cs` | Webcam capture ownership: refcounted by `DeviceId`, one shared `WriteableBitmap` per camera, dispatcher-coalesced ~render-rate UI updates (latest-frame drop); `PreviewBitmapChanged`/`CameraFailed` events. **First-frame proof** (4s timeout, configurable via ctor `TimeSpan?`): `AcquireAsync` returns true only after a real frame arrives — a reader that starts but never delivers (locked, suspended, dead) fails and surfaces `CameraFailed` with suspect-app names (`CameraConflictProbe`) instead of a silent empty box. `SourceFailed` forwarded from the frame source as async death signal. **TASK 51:** the one background-effects processing point — `SetRequestedEffects(device, keys)` + per-variant `GetLatestFrame`/`GetPreviewBitmap(device, key)` + `EffectPreviewBitmapChanged`; a background worker segments once per raw frame and renders every variant (latest-frame-drop); raw path untouched | | `IFullScreenDetector.cs` | Seam for the win32 full-screen detector: `int? GetForegroundFullScreenMonitorIndex()`, `int PrimaryMonitorIndex()`, `IReadOnlyList GetDisplays()` | | `Win32FullScreenDetector.cs` | `GetForegroundWindow` + `DwmGetWindowAttribute(DWMWA_EXTENDED_FRAME_BOUNDS)` + `MonitorFromWindow` + `GetMonitorInfo`; monitor covers all four edges → full-screen; own process excluded; monitor order = `EnumDisplayMonitors` order (static `GetMonitorHandle(int)` maps index→HMONITOR for the capture factory). `GetDisplays()` returns `DisplayInfo` (index/name/resolution/bounds/`IsPrimary`, friendly name via `EnumDisplayDevices`) for the in-app "Capture Display" picker; `PrimaryMonitorIndex()` is the auto-key fallback when no full-screen game is detected | | `IScreenCaptureSource.cs` | `Key` + `StartAsync`/`StopAsync`/`FrameAvailable(VideoFrame)` — seam so `ScreenCaptureManager` never touches WinRT (tests inject fakes) | diff --git a/TASKS/task-51-webcam-bg-effects.md b/TASKS/task-51-webcam-bg-effects.md index 4bd0e9a..72161c3 100644 --- a/TASKS/task-51-webcam-bg-effects.md +++ b/TASKS/task-51-webcam-bg-effects.md @@ -85,7 +85,7 @@ delivery shape changed: ## 3. ☐ Test seam (Good Dog — one integration test per unit) - ✅ Model-runner interface seam (`IPersonSegmenter`) so tests inject a fake segmenter. -- ☐ Assert the **processed frame reaches both consumers** (preview bitmap + compositor frame). +- ✅ The **processed frame reaches both consumers** (preview bitmap + compositor frame), raw stays raw (`CameraManagerTests`, slice 4). - ✅ `None` = same-instance passthrough of the raw frame (`BackgroundEffectEngineTests`). - ✅ `Remove` = alpha 0 outside the person, RGB untouched, non-opaque (`BackgroundEffectEngineTests`); ✅ `Blur` = background smoothed, person pixel-identical (`BackgroundEffectEngineTests`). @@ -96,14 +96,14 @@ delivery shape changed: - ☐ Engine failure → passthrough + single log, layer never dropped (code path exists; dedicated test lands with the hookup slice). - ☐ Real inference: **manual on the creator's box** (camera + lighting), not in CI. -- ☐ **Perf check in the hookup slice:** time `Segment` + `Apply` per 720p frame on the creator's - box before wiring it into the 60fps path (expect a few ms; the mask can run at a lower rate - than the camera if needed). +- ✅ **Perf, synthetic 720p (Debug):** segment 1.6 ms, Remove 3.3 ms, Blur 17.5 ms (after making the + upsampling separable — naive was 46/80 ms). Runs on its own worker, off the raw path. + ☐ **Real numbers on the creator's box:** `startup.log` → "background effects avg … ms/frame". ## 4. Sequencing - ✅ Separate work unit after the zoom build (zoom shipped 2026-10-08). -- Slice order: 1 engine ✅ → 2 Blur ✅ → 3 config + persistence ✅ → 4 camera-path hookup (one +- Slice order: 1 engine ✅ → 2 Blur ✅ → 3 config + persistence ✅ → 4 camera-path hookup ✅ (one processing point, both consumers) + perf check → 5 properties-panel UI. ## Related (not duplicated here) diff --git a/ViewModels/MainViewModel.Sources.cs b/ViewModels/MainViewModel.Sources.cs index 58b1b0c..761846a 100644 --- a/ViewModels/MainViewModel.Sources.cs +++ b/ViewModels/MainViewModel.Sources.cs @@ -252,6 +252,7 @@ public partial class MainViewModel if (element is WebcamSceneConfig) { + SyncWebcamEffects(); // drop a variant nobody shows any more OnPropertyChanged(nameof(CanAddWebcam)); OnPropertyChanged(nameof(CanShowWebcamInStagedScene)); OnPropertyChanged(nameof(WebcamAddToolTip)); diff --git a/ViewModels/MainViewModel.Webcam.cs b/ViewModels/MainViewModel.Webcam.cs index 858ab4e..f5389aa 100644 --- a/ViewModels/MainViewModel.Webcam.cs +++ b/ViewModels/MainViewModel.Webcam.cs @@ -7,6 +7,7 @@ using ytLive; using ytLive.Helpers; using ytLive.Models; using ytLive.Services; +using ytLive.Services.BackgroundEffects; namespace ytLive.ViewModels; @@ -193,9 +194,8 @@ public partial class MainViewModel // A session already running (same device, or one that produced a first // frame) has its shared bitmap; freshly loaded configs must adopt it here, // because PreviewBitmapChanged never re-fires for an existing bitmap. - if (_cameraManager.GetPreviewBitmap(newDevice) is { } running) - foreach (var config in Scenes.SelectMany(s => s.Elements.OfType())) - config.VideoImageSource = running; + // SyncWebcamEffects also declares each config's background-effect variant. + SyncWebcamEffects(); } // CameraManager creates the shared WriteableBitmap on the UI thread at the @@ -209,7 +209,52 @@ public partial class MainViewModel RaiseWebcamGateChanged(); foreach (var scene in Scenes) foreach (var config in scene.Elements.OfType()) + if (EffectKeyFor(config).Effect == BackgroundEffect.None) + config.VideoImageSource = bitmap; + } + + // TASK 51: an effect variant's first processed frame created its bitmap — every config + // showing that variant adopts it (later frames write into it in place). + private void OnCameraEffectPreviewBitmapChanged(string deviceId, WebcamEffectKey key, WriteableBitmap bitmap) + { + if (_webcam?.DeviceId != deviceId) return; + foreach (var scene in Scenes) + foreach (var config in scene.Elements.OfType()) + if (EffectKeyFor(config) == key) + config.VideoImageSource = bitmap; + } + + internal static WebcamEffectKey EffectKeyFor(WebcamSceneConfig config) + => WebcamEffectKey.For(config.BackgroundEffect, config.BackgroundStrength); + + private readonly HashSet _effectWatchedConfigs = new(); + + /// TASK 51: declares the union of every scene's webcam effect variant to the camera + /// (unused variants stop costing CPU) and points each config's preview at its variant's + /// bitmap. An effect variant with no processed frame yet shows nothing rather than the raw + /// room. Re-run whenever a config is added/removed/loaded or its effect changes. + internal void SyncWebcamEffects() + { + if (_webcam == null || string.IsNullOrWhiteSpace(_webcam.DeviceId)) return; + var configs = Scenes.SelectMany(s => s.Elements.OfType()).ToList(); + _cameraManager.SetRequestedEffects(_webcam.DeviceId, configs.Select(EffectKeyFor)); + foreach (var config in configs) + { + if (_effectWatchedConfigs.Add(config)) + config.PropertyChanged += OnWebcamConfigEffectChanged; + var key = EffectKeyFor(config); + var bitmap = _cameraManager.GetPreviewBitmap(_webcam.DeviceId, key); + if (bitmap != null || key.Effect != BackgroundEffect.None) config.VideoImageSource = bitmap; + } + } + + private void OnWebcamConfigEffectChanged(object? sender, System.ComponentModel.PropertyChangedEventArgs e) + { + if (e.PropertyName is not (nameof(WebcamSceneConfig.BackgroundEffect) or nameof(WebcamSceneConfig.BackgroundStrength))) + return; + SyncWebcamEffects(); + ScheduleSave(); } // The camera failed to start or died asynchronously (in use, offline, locked, @@ -321,9 +366,9 @@ public partial class MainViewModel // PreviewBitmapChanged fires only on the camera's first frame, so a config // added while the session is already running must pick up the shared bitmap - // directly (it's written in place from then on). - if (_cameraManager.GetPreviewBitmap(_webcam.DeviceId) is { } running) - config.VideoImageSource = running; + // directly (it's written in place from then on) — SyncWebcamEffects does that, + // per the config's background-effect variant. + SyncWebcamEffects(); var started = await _cameraManager.AcquireAsync(_webcam.DeviceId); if (!started) @@ -346,6 +391,8 @@ public partial class MainViewModel if (!string.IsNullOrWhiteSpace(oldDevice) && oldDevice != device.Id) await _cameraManager.ReleaseAllAsync(oldDevice); + SyncWebcamEffects(); // the new device inherits the scenes' background effects + foreach (var unused in Scenes.SelectMany(s => s.Elements.OfType())) { var started = await _cameraManager.AcquireAsync(device.Id); diff --git a/ViewModels/MainViewModel.cs b/ViewModels/MainViewModel.cs index 3b8d696..d725538 100644 --- a/ViewModels/MainViewModel.cs +++ b/ViewModels/MainViewModel.cs @@ -276,8 +276,11 @@ public partial class MainViewModel : ViewModelBase _cameraManager = new CameraManager( _cameraEnumerator, CameraFrameSourceFactoryOverride ?? (id => new MediaCaptureFrameSource(id)), - System.Windows.Application.Current?.Dispatcher); + System.Windows.Application.Current?.Dispatcher, + effects: new Services.BackgroundEffects.BackgroundEffectProcessor( + PersonSegmenterOverride ?? new Services.BackgroundEffects.OnnxPersonSegmenter())); _cameraManager.PreviewBitmapChanged += OnCameraPreviewBitmapChanged; + _cameraManager.EffectPreviewBitmapChanged += OnCameraEffectPreviewBitmapChanged; _cameraManager.CameraFailed += OnCameraFailed; _microphoneEnumerator = new WinRtMicrophoneEnumerator(); @@ -384,6 +387,10 @@ public partial class MainViewModel : ViewModelBase internal static ICameraEnumerator? CameraEnumeratorOverride { get; set; } internal static Func? CameraFrameSourceFactoryOverride { get; set; } + /// Test seam (TASK 51): a fake person segmenter so hermetic tests never load the + /// real ONNX model. Mirrors . + internal static Services.BackgroundEffects.IPersonSegmenter? PersonSegmenterOverride { get; set; } + private void LoadLayout() { _isLoading = true; @@ -570,7 +577,7 @@ public partial class MainViewModel : ViewModelBase // Looking up by the GUID returned null forever — the webcam was structurally // invisible on the output while the preview (bitmap path) looked fine // (caught by the resolver logging, 2026-09-01 take two). - WebcamSceneConfig webcam => _cameraManager.GetLatestFrame(DeviceKeyForWebcam(webcam.WebcamId)), + WebcamSceneConfig webcam => _cameraManager.GetLatestFrame(DeviceKeyForWebcam(webcam.WebcamId), EffectKeyFor(webcam)), Source { IsLiveCapture: true, CaptureKey: not null } live => _screenCaptureManager.GetLatestFrame(live.CaptureKey), Source { Type: SourceType.ChatBox } chatBox => RenderChatBox(chatBox), Source { Type: SourceType.AlertBox } alertBox => RenderAlertBox(alertBox), diff --git a/ai.md b/ai.md index 28a304b..a47f0d6 100644 --- a/ai.md +++ b/ai.md @@ -645,6 +645,24 @@ This replaces the old five-seeder cluster (`Seed{Starting,Brb,Ending,Chat}Backgr (0..1, default 0.5), persisted as two additive guarded columns on `WebcamSceneConfig` (`BackgroundEffect TEXT DEFAULT 'None'`, `BackgroundStrength REAL DEFAULT 0.5`, no version bump — the TASK 47 convention). `RemoveBackground`/`BlurBackground` are the radio-pill bools the UI binds. +- **Hookup (slice 4) — the ONE processing point is `CameraManager`.** Effects are per scene but the + camera session is shared, so the manager renders **variants** keyed by `WebcamEffectKey(effect, + strength rounded to 0.01)`. `MainViewModel.SyncWebcamEffects()` declares the union of every + config's key (`SetRequestedEffects`; unused variants are dropped so their CPU stops) and points each + config's `VideoImageSource` at its variant's bitmap; it re-runs on load, add, remove, device swap, + and any `BackgroundEffect`/`BackgroundStrength` change (configs are watched via `PropertyChanged`, + which also `ScheduleSave`s). Per raw frame a background worker (latest-frame-drop, one at a time) + segments ONCE and renders every variant; each variant keeps its own latest frame (the resolver: + `GetLatestFrame(device, key)`) and WriteableBitmap (`EffectPreviewBitmapChanged` on first + creation). **The raw path is untouched** — a slow/failed model never stalls the plain webcam — and + an effect variant shows **nothing** until its first processed frame (Remove never flashes the + real room). The compositor needed no change: its sampler already honours source alpha. +- **Perf (synthetic 1280×720, Debug build):** segment 1.6 ms, Remove 3.3 ms, Blur 17.5 ms per frame + (Release is several × faster). Pixel loops are row-parallel, capped at min(4, cores/2); the + bilinear upsampling is separable (horizontal rows precomputed per mask/small row, one vertical lerp + per pixel) — the first naive per-pixel 2-D bilinear cost 46/80 ms. `CameraManager` logs the real + average once per session after 120 processed frames (`startup.log`: "background effects avg …"). + `MainViewModel.PersonSegmenterOverride` is the hermetic-test seam. - **Tests:** `BackgroundEffectEngineTests` — the Remove integration test (fake segmenter), and a model contract test that runs the BUNDLED model through the C# preprocessing on a fixture and matches the MediaPipe-produced reference mask (≥99.5% agreement; fixture = MediaPipe's public test diff --git a/ytLive.Tests/CameraManagerTests.cs b/ytLive.Tests/CameraManagerTests.cs index bf89b8b..3eb0386 100644 --- a/ytLive.Tests/CameraManagerTests.cs +++ b/ytLive.Tests/CameraManagerTests.cs @@ -1,8 +1,13 @@ using System; using System.Collections.Generic; +using System.Threading; using System.Threading.Tasks; +using System.Windows.Media.Imaging; +using System.Windows.Threading; using Xunit; +using ytLive.Models; using ytLive.Services; +using ytLive.Services.BackgroundEffects; namespace ytLive.Tests; @@ -183,4 +188,94 @@ public class CameraManagerTests Assert.Null(manager.GetLatestFrame("dev1")); Assert.Contains("dev1", stopped); } + + private sealed class HalfMaskSegmenter : IPersonSegmenter + { + public int Calls; + public SegmentationMask? Segment(VideoFrame frame) + { + Interlocked.Increment(ref Calls); + return new SegmentationMask(2, 1, new[] { 1f, 0f }); // left person, right background + } + } + + /// TASK 51 slice 4 integration test — the one processing point: with a Remove + /// variant requested, a raw camera frame reaches BOTH consumers processed — the + /// compositor's latest frame for that variant and the variant's preview bitmap on the UI + /// dispatcher — while the raw path keeps the untouched frame. + [Fact] + public async Task Requested_Remove_Variant_Reaches_Compositor_And_Preview_Raw_Stays_Raw() + { + Dispatcher? ui = null; + using var ready = new ManualResetEventSlim(); + var uiThread = new Thread(() => + { + ui = Dispatcher.CurrentDispatcher; + ready.Set(); + Dispatcher.Run(); + }); + uiThread.SetApartmentState(ApartmentState.STA); + uiThread.IsBackground = true; + uiThread.Start(); + ready.Wait(); + + try + { + FakeFrameSource? source = null; + var segmenter = new HalfMaskSegmenter(); + var manager = new CameraManager( + new FakeEnumerator(), + id => source = new FakeFrameSource(id, new List(), new List()), + ui, TimeSpan.Zero, new BackgroundEffectProcessor(segmenter)); + var key = WebcamEffectKey.For(BackgroundEffect.Remove, 0.5); + manager.SetRequestedEffects("dev1", new[] { key, WebcamEffectKey.None }); + + var previewArrived = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + manager.EffectPreviewBitmapChanged += (device, k, bitmap) => + { + if (device == "dev1" && k == key) previewArrived.TrySetResult(bitmap); + }; + + Assert.True(await manager.AcquireAsync("dev1")); + const int w = 16, h = 4; + var pixels = new byte[w * h * 4]; + for (var i = 0; i < pixels.Length; i += 4) { pixels[i] = 50; pixels[i + 1] = 60; pixels[i + 2] = 70; pixels[i + 3] = 255; } + var raw = new VideoFrame(w, h, pixels) { IsOpaque = true }; + source!.Pump(raw); + + var bitmap = await previewArrived.Task.WaitAsync(TimeSpan.FromSeconds(10)); + + // Raw consumer: the very same frame, untouched. + Assert.Same(raw, manager.GetLatestFrame("dev1")); + Assert.Same(raw, manager.GetLatestFrame("dev1", WebcamEffectKey.None)); + + // Compositor consumer: the processed variant. + var processed = manager.GetLatestFrame("dev1", key); + Assert.NotNull(processed); + Assert.False(processed!.IsOpaque); + Assert.Equal(255, processed.BgraPixels[(1 * w + 1) * 4 + 3]); // person side + Assert.Equal(0, processed.BgraPixels[(1 * w + (w - 2)) * 4 + 3]); // background side + + // Preview consumer: the variant's bitmap carries the same cut-out. + var previewPixels = new byte[w * h * 4]; + var previewWidth = await ui!.InvokeAsync(() => + { + bitmap.CopyPixels(previewPixels, w * 4, 0); + return bitmap.PixelWidth; + }); + Assert.Equal(w, previewWidth); + Assert.Equal(255, previewPixels[(1 * w + 1) * 4 + 3]); + Assert.Equal(0, previewPixels[(1 * w + (w - 2)) * 4 + 3]); + Assert.Same(bitmap, manager.GetPreviewBitmap("dev1", key)); + Assert.True(segmenter.Calls >= 1); + + // Dropping the request stops the variant. + manager.SetRequestedEffects("dev1", new[] { WebcamEffectKey.None }); + Assert.Null(manager.GetLatestFrame("dev1", key)); + } + finally + { + ui?.InvokeShutdown(); + } + } }