TASK 51 jitter fix: temporal mask smoothing. MaskSmoother blends each raw segmentation 0.75xnew + 0.25xprevious at the one CameraManager worker (state per CameraSession, not in the shared segmenter); null resets history, first frame passes through, input array never aliased, ping-pong buffers (no LOH alloc for a 147KB 256x144 mask). Creator's real-camera check: Remove was 'jittery' (per-frame independent segmentation). Weights from the OBS/Jitsi consensus. One integration test asserts the second frame keeps partial alpha (x=6 -> 58) where the raw mask cuts to 0. 416 pass / 2 known env flakes. Refs: https://deepwiki.com/royshil/obs-backgroundremoval/2-core-features https://github.com/jitsi/jitsi-meet/issues/17080 (Unit: bg-removal jitter fix)
This commit is contained in:
@@ -199,6 +199,19 @@ public class CameraManagerTests
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Returns a FRESH mask per call — like the real segmenter, whose contract test
|
||||
/// pins the raw per-frame output: {person, person, bg, bg} then {person, 0.2, bg, bg}.</summary>
|
||||
private sealed class SteppingMaskSegmenter : IPersonSegmenter
|
||||
{
|
||||
public int Calls;
|
||||
public SegmentationMask? Segment(VideoFrame _)
|
||||
{
|
||||
var n = Interlocked.Increment(ref Calls);
|
||||
var v = n == 1 ? new[] { 1f, 1f, 0f, 0f } : new[] { 1f, 0.2f, 0f, 0f };
|
||||
return new SegmentationMask(4, 1, v);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>TASK 51 slice 4 integration test — the one processing point: with a Remove
|
||||
/// variant requested, a raw camera frame reaches BOTH consumers processed — the
|
||||
/// compositor's latest frame for that variant and the variant's preview bitmap on the UI
|
||||
@@ -278,4 +291,63 @@ public class CameraManagerTests
|
||||
ui?.InvokeShutdown();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>TASK 51 jitter fix integration test — the one processing point: two consecutive
|
||||
/// frames whose raw masks disagree at an edge. The first frame passes through untouched; the
|
||||
/// second's blend with history (0.75/0.25) keeps a partial alpha where the raw mask would
|
||||
/// have snapped to 0 — the per-frame judder the smoother damps.</summary>
|
||||
[Fact]
|
||||
public async Task Temporal_Smoothing_Damps_Segmentation_Jitter_Across_Frames()
|
||||
{
|
||||
FakeFrameSource? source = null;
|
||||
var segmenter = new SteppingMaskSegmenter();
|
||||
var manager = new CameraManager(
|
||||
new FakeEnumerator(),
|
||||
id => source = new FakeFrameSource(id, new List<string>(), new List<string>()),
|
||||
null, TimeSpan.Zero, new BackgroundEffectProcessor(segmenter));
|
||||
var key = WebcamEffectKey.For(BackgroundEffect.Remove, 0.5);
|
||||
manager.SetRequestedEffects("dev1", new[] { key });
|
||||
|
||||
Assert.True(await manager.AcquireAsync("dev1"));
|
||||
const int w = 16, h = 4;
|
||||
var pixels = new byte[w * h * 4];
|
||||
for (var i = 0; i < pixels.Length; i += 4) { pixels[i] = 50; pixels[i + 1] = 60; pixels[i + 2] = 70; pixels[i + 3] = 255; }
|
||||
var frame = new VideoFrame(w, h, pixels) { IsOpaque = true };
|
||||
var edge = (0 * w + 6) * 4 + 3; // row 0, x 6: the contested mask boundary
|
||||
|
||||
source!.Pump(frame);
|
||||
var first = await PollLatestFrameAsync(manager, key, distinctFrom: null);
|
||||
Assert.NotNull(first);
|
||||
Assert.Equal(255, first!.BgraPixels[edge]); // maiden frame: raw mask, untouched
|
||||
|
||||
// A NEW instance — the worker's latest-frame-drop guard treats a re-queued
|
||||
// identical reference as "no new frame", which is the real camera behavior.
|
||||
source.Pump(new VideoFrame(w, h, pixels) { IsOpaque = true });
|
||||
var second = await PollLatestFrameAsync(manager, key, distinctFrom: first);
|
||||
Assert.NotNull(second);
|
||||
Assert.Equal(2, segmenter.Calls);
|
||||
Assert.False(second!.IsOpaque);
|
||||
Assert.Equal(58, second.BgraPixels[edge]);
|
||||
Assert.Equal(255, second.BgraPixels[(0 * w + 1) * 4 + 3]); // person core stays opaque
|
||||
Assert.Equal(0, second.BgraPixels[(0 * w + 14) * 4 + 3]); // background stays cut
|
||||
|
||||
// The raw second mask alone would have cut the same pixel to 0 — exactly the flutter
|
||||
// the smoother removes.
|
||||
var rawSecond = BackgroundEffectProcessor.Render(frame,
|
||||
new SegmentationMask(4, 1, new[] { 1f, 0.2f, 0f, 0f }), BackgroundEffect.Remove, 0.5);
|
||||
Assert.Equal(0, rawSecond.BgraPixels[edge]);
|
||||
}
|
||||
|
||||
private static async Task<VideoFrame?> PollLatestFrameAsync(CameraManager manager,
|
||||
WebcamEffectKey key, VideoFrame? distinctFrom)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(10);
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
var frame = manager.GetLatestFrame("dev1", key);
|
||||
if (frame != null && !ReferenceEquals(frame, distinctFrom)) return frame;
|
||||
await Task.Delay(10);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user