TASK 51 jitter fix: temporal mask smoothing. MaskSmoother blends each raw segmentation 0.75xnew + 0.25xprevious at the one CameraManager worker (state per CameraSession, not in the shared segmenter); null resets history, first frame passes through, input array never aliased, ping-pong buffers (no LOH alloc for a 147KB 256x144 mask). Creator's real-camera check: Remove was 'jittery' (per-frame independent segmentation). Weights from the OBS/Jitsi consensus. One integration test asserts the second frame keeps partial alpha (x=6 -> 58) where the raw mask cuts to 0. 416 pass / 2 known env flakes. Refs: https://deepwiki.com/royshil/obs-backgroundremoval/2-core-features https://github.com/jitsi/jitsi-meet/issues/17080 (Unit: bg-removal jitter fix)

2026-10-09 22:15:59 -07:00
parent 5685c5dc3a
commit 305706956e
8 changed files with 252 additions and 63 deletions
@@ -0,0 +1,68 @@
+namespace ytLive.Services.BackgroundEffects;
+
+/// <summary>
+/// Temporal smoothing for the person mask (TASK 51): each raw frame's segmentation is blended
+/// with the previous smoothed mask so the cut-out edge doesn't flutter from frame to frame when
+/// the model's per-frame verdict wavers.
+///
+/// Blend = <see cref="CurrentWeight"/> × new + (1 − <see cref="CurrentWeight"/>) × previous.
+/// Weights come from the established tools rather than a guess: OBS's obs-backgroundremoval
+/// smoke-tests `temporal_smooth_factor` at its 0.85 default, and Jitsi's webcam background
+/// filter (jitsi/jitsi-meet#17080) found 0.7/0.3 transformed best — a midpoint that still
+/// reacts to real movement. 0.5/0.5 is worse: on motion it leaves a smearing edge ghost.
+///
+/// State lives per camera session (the <see cref="CameraManager"/>'s one-per-frame worker), NOT
+/// in the segmenter — the segmenter is shared app-wide and its contract test pins the raw
+/// per-frame mask.
+/// </summary>
+public sealed class MaskSmoother
+{
+    /// <summary>Weight of the newest raw mask; the rest is the previous smoothed mask.</summary>
+    internal const double CurrentWeight = 0.75;
+
+    private float[]? _current;
+    private float[]? _scratch;
+    private int _width;
+    private int _height;
+    private bool _hasHistory;
+
+    /// <summary>
+    /// Returns the temporally-smoothed mask. A null input (segmenter hiccup) resets history so
+    /// the next real mask starts moving again; the first frame and any size change pass through
+    /// un-blended, and the returned mask never aliases the caller's array (the first frame copies
+    /// its history). The two buffers ping-pong so no per-frame allocation is needed — a 256×144
+    /// mask is 147 KB, past the 85 KB LOH threshold.
+    /// </summary>
+    public SegmentationMask? Smooth(SegmentationMask? raw)
+    {
+        if (raw == null)
+        {
+            _hasHistory = false;
+            _width = 0;
+            _height = 0;
+            return null;
+        }
+
+        if (!_hasHistory || _width != raw.Width || _height != raw.Height
+            || _current == null || _scratch == null)
+        {
+            _width = raw.Width;
+            _height = raw.Height;
+            _current = new float[raw.Values.Length];
+            _scratch = new float[raw.Values.Length];
+            Array.Copy(raw.Values, _current, raw.Values.Length);
+            _hasHistory = true;
+            return raw;
+        }
+
+        var v = raw.Values;
+        var next = _scratch;
+        var keep = 1f - (float)CurrentWeight;
+        for (var i = 0; i < v.Length; i++)
+            next[i] = keep * _current[i] + (float)CurrentWeight * v[i];
+
+        _scratch = _current;
+        _current = next;
+        return new SegmentationMask(_width, _height, next);
+    }
+}