TASK 51 jitter fix: temporal mask smoothing. MaskSmoother blends each raw segmentation 0.75xnew + 0.25xprevious at the one CameraManager worker (state per CameraSession, not in the shared segmenter); null resets history, first frame passes through, input array never aliased, ping-pong buffers (no LOH alloc for a 147KB 256x144 mask). Creator's real-camera check: Remove was 'jittery' (per-frame independent segmentation). Weights from the OBS/Jitsi consensus. One integration test asserts the second frame keeps partial alpha (x=6 -> 58) where the raw mask cuts to 0. 416 pass / 2 known env flakes. Refs: https://deepwiki.com/royshil/obs-backgroundremoval/2-core-features https://github.com/jitsi/jitsi-meet/issues/17080 (Unit: bg-removal jitter fix)

This commit is contained in:
2026-10-09 22:15:59 -07:00
parent 5685c5dc3a
commit 305706956e
8 changed files with 252 additions and 63 deletions
+72
View File
@@ -199,6 +199,19 @@ public class CameraManagerTests
}
}
/// <summary>Returns a FRESH mask per call — like the real segmenter, whose contract test
/// pins the raw per-frame output: {person, person, bg, bg} then {person, 0.2, bg, bg}.</summary>
private sealed class SteppingMaskSegmenter : IPersonSegmenter
{
public int Calls;
public SegmentationMask? Segment(VideoFrame _)
{
var n = Interlocked.Increment(ref Calls);
var v = n == 1 ? new[] { 1f, 1f, 0f, 0f } : new[] { 1f, 0.2f, 0f, 0f };
return new SegmentationMask(4, 1, v);
}
}
/// <summary>TASK 51 slice 4 integration test — the one processing point: with a Remove
/// variant requested, a raw camera frame reaches BOTH consumers processed — the
/// compositor's latest frame for that variant and the variant's preview bitmap on the UI
@@ -278,4 +291,63 @@ public class CameraManagerTests
ui?.InvokeShutdown();
}
}
/// <summary>TASK 51 jitter fix integration test — the one processing point: two consecutive
/// frames whose raw masks disagree at an edge. The first frame passes through untouched; the
/// second's blend with history (0.75/0.25) keeps a partial alpha where the raw mask would
/// have snapped to 0 — the per-frame judder the smoother damps.</summary>
[Fact]
public async Task Temporal_Smoothing_Damps_Segmentation_Jitter_Across_Frames()
{
FakeFrameSource? source = null;
var segmenter = new SteppingMaskSegmenter();
var manager = new CameraManager(
new FakeEnumerator(),
id => source = new FakeFrameSource(id, new List<string>(), new List<string>()),
null, TimeSpan.Zero, new BackgroundEffectProcessor(segmenter));
var key = WebcamEffectKey.For(BackgroundEffect.Remove, 0.5);
manager.SetRequestedEffects("dev1", new[] { key });
Assert.True(await manager.AcquireAsync("dev1"));
const int w = 16, h = 4;
var pixels = new byte[w * h * 4];
for (var i = 0; i < pixels.Length; i += 4) { pixels[i] = 50; pixels[i + 1] = 60; pixels[i + 2] = 70; pixels[i + 3] = 255; }
var frame = new VideoFrame(w, h, pixels) { IsOpaque = true };
var edge = (0 * w + 6) * 4 + 3; // row 0, x 6: the contested mask boundary
source!.Pump(frame);
var first = await PollLatestFrameAsync(manager, key, distinctFrom: null);
Assert.NotNull(first);
Assert.Equal(255, first!.BgraPixels[edge]); // maiden frame: raw mask, untouched
// A NEW instance — the worker's latest-frame-drop guard treats a re-queued
// identical reference as "no new frame", which is the real camera behavior.
source.Pump(new VideoFrame(w, h, pixels) { IsOpaque = true });
var second = await PollLatestFrameAsync(manager, key, distinctFrom: first);
Assert.NotNull(second);
Assert.Equal(2, segmenter.Calls);
Assert.False(second!.IsOpaque);
Assert.Equal(58, second.BgraPixels[edge]);
Assert.Equal(255, second.BgraPixels[(0 * w + 1) * 4 + 3]); // person core stays opaque
Assert.Equal(0, second.BgraPixels[(0 * w + 14) * 4 + 3]); // background stays cut
// The raw second mask alone would have cut the same pixel to 0 — exactly the flutter
// the smoother removes.
var rawSecond = BackgroundEffectProcessor.Render(frame,
new SegmentationMask(4, 1, new[] { 1f, 0.2f, 0f, 0f }), BackgroundEffect.Remove, 0.5);
Assert.Equal(0, rawSecond.BgraPixels[edge]);
}
private static async Task<VideoFrame?> PollLatestFrameAsync(CameraManager manager,
WebcamEffectKey key, VideoFrame? distinctFrom)
{
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10);
while (DateTime.UtcNow < deadline)
{
var frame = manager.GetLatestFrame("dev1", key);
if (frame != null && !ReferenceEquals(frame, distinctFrom)) return frame;
await Task.Delay(10);
}
return null;
}
}