TASK 51 slice 1: background-effects engine — bundled MediaPipe Selfie Segmenter (ONNX) + Remove (Unit: bg-effects engine)

- Model: Google's official MediaPipe Selfie Segmenter (landscape, Apache-2.0 per model card), converted to ONNX by tools/selfie-seg/convert_tflite_to_onnx.py (custom op Convolution2DTransposeBias -> ConvTranspose) and verified against MediaPipe's own runtime (verify_against_mediapipe.py: max |diff| 5e-5, 100% binary agreement). Embedded resource, 430 KB. - Services/BackgroundEffects: IPersonSegmenter seam, OnnxPersonSegmenter (onnxruntime 1.31.0 CPU, never throws), BackgroundEffectProcessor (None passthrough, Remove = alpha mask). Not wired yet. - Tests: Remove integration test (fake segmenter) + bundled-model contract test vs MediaPipe reference mask. Full suite 407/408 — the one failure is the documented interactive-desktop flake RealMouseDrag_OnTheLayerList_PersistsTheReorder (passed in the run before; code path untouched). - Docs: ai.md reversal + section + licensing guard, THIRD-PARTY-NOTICES 11-12, TASK 51, indexes, MyMistakes recipe, HANDOFF. References: obs-backgroundremoval (approach only — ONNX Runtime on CPU, 2 threads; GPL-3.0, no code used) https://github.com/royshil/obs-backgroundremoval ; MediaPipe image segmenter https://ai.google.dev/edge/mediapipe/solutions/vision/image_segmenter
2026-10-09 09:24:13 -07:00
parent f8dd1079f3
commit 9f3c154d84
20 changed files with 796 additions and 78 deletions
@@ -0,0 +1,115 @@
+using System.IO;
+using System.Windows.Media;
+using System.Windows.Media.Imaging;
+using Xunit;
+using ytLive.Models;
+using ytLive.Services;
+using ytLive.Services.BackgroundEffects;
+
+namespace ytLive.Tests;
+
+/// <summary>TASK 51 slice 1 — the background-effects engine.</summary>
+public class BackgroundEffectEngineTests
+{
+    private sealed class FakeSegmenter : IPersonSegmenter
+    {
+        private readonly SegmentationMask? _mask;
+        public int Calls;
+        public FakeSegmenter(SegmentationMask? mask) => _mask = mask;
+        public SegmentationMask? Segment(VideoFrame frame)
+        {
+            Calls++;
+            return _mask;
+        }
+    }
+
+    /// <summary>The integration test for this unit: a webcam frame run through the
+    /// processor with Remove comes back with the person kept and the background
+    /// transparent, colors untouched, flagged non-opaque so the compositor blends it.
+    /// None is a true passthrough (same instance, segmenter never called).</summary>
+    [Fact]
+    public void Remove_keeps_the_person_and_makes_the_background_transparent()
+    {
+        const int w = 64, h = 32;
+        var pixels = new byte[w * h * 4];
+        for (var i = 0; i < pixels.Length; i += 4)
+        {
+            pixels[i] = 10; pixels[i + 1] = 20; pixels[i + 2] = 30; pixels[i + 3] = 255;
+        }
+        var frame = new VideoFrame(w, h, pixels) { IsOpaque = true };
+
+        // 4×2 mask: left half person, right half background.
+        var mask = new SegmentationMask(4, 2, new[] { 1f, 1f, 0f, 0f, 1f, 1f, 0f, 0f });
+        var segmenter = new FakeSegmenter(mask);
+        var processor = new BackgroundEffectProcessor(segmenter);
+
+        Assert.Same(frame, processor.Apply(frame, BackgroundEffect.None));
+        Assert.Equal(0, segmenter.Calls);
+
+        var result = processor.Apply(frame, BackgroundEffect.Remove);
+
+        Assert.NotSame(frame, result);
+        Assert.False(result.IsOpaque);
+        Assert.Equal(w, result.Width);
+        Assert.Equal(h, result.Height);
+        for (var y = 0; y < h; y++)
+        {
+            Assert.Equal(255, Alpha(result, 4, y));      // well inside the person half
+            Assert.Equal(0, Alpha(result, w - 5, y));    // well inside the background half
+        }
+        for (var i = 0; i < pixels.Length; i += 4)
+        {
+            Assert.Equal(pixels[i], result.BgraPixels[i]);
+            Assert.Equal(pixels[i + 1], result.BgraPixels[i + 1]);
+            Assert.Equal(pixels[i + 2], result.BgraPixels[i + 2]);
+        }
+        Assert.Equal(255, frame.BgraPixels[(w - 5) * 4 + 3]); // source frame never mutated
+    }
+
+    /// <summary>Model contract: the BUNDLED (embedded) ONNX model, driven through the C#
+    /// preprocessing, reproduces Google's own MediaPipe runtime on a fixed fixture.
+    /// The expected mask was produced by tools/selfie-seg/ running the original .tflite in
+    /// MediaPipe (fixture: MediaPipe's public test image mozart_square.jpg, a public-domain
+    /// painting, cropped to 256×144). Guards the embedding, the input layout (RGB, NHWC,
+    /// /255) and the conversion itself.</summary>
+    [Fact]
+    public void Bundled_model_matches_the_MediaPipe_reference_mask()
+    {
+        var frame = LoadBgra(Fixture("selfie-fixture-256x144.png"));
+        var expected = LoadBgra(Fixture("selfie-fixture-expected-mask.png"));
+
+        using var segmenter = new OnnxPersonSegmenter();
+        var mask = segmenter.Segment(frame);
+
+        Assert.NotNull(mask);
+        Assert.Equal(OnnxPersonSegmenter.ModelWidth, mask!.Width);
+        Assert.Equal(OnnxPersonSegmenter.ModelHeight, mask.Height);
+
+        var agree = 0;
+        var person = 0;
+        for (var i = 0; i < mask.Width * mask.Height; i++)
+        {
+            var expectPerson = expected.BgraPixels[i * 4] > 127;
+            var gotPerson = mask.Values[i] > 0.5f;
+            if (expectPerson == gotPerson) agree++;
+            if (gotPerson) person++;
+        }
+        var total = mask.Width * mask.Height;
+        Assert.True(agree >= total * 0.995, $"binary agreement {100.0 * agree / total:0.00}% < 99.5%");
+        Assert.InRange(100.0 * person / total, 45, 60); // reference: 52.5% person
+    }
+
+    private static int Alpha(VideoFrame f, int x, int y) => f.BgraPixels[(y * f.Width + x) * 4 + 3];
+
+    private static string Fixture(string name) => Path.Combine(AppContext.BaseDirectory, "Fixtures", name);
+
+    private static VideoFrame LoadBgra(string path)
+    {
+        using var stream = File.OpenRead(path);
+        var decoded = BitmapDecoder.Create(stream, BitmapCreateOptions.PreservePixelFormat, BitmapCacheOption.OnLoad).Frames[0];
+        var bgra = new FormatConvertedBitmap(decoded, PixelFormats.Bgra32, null, 0);
+        var pixels = new byte[bgra.PixelWidth * bgra.PixelHeight * 4];
+        bgra.CopyPixels(pixels, bgra.PixelWidth * 4, 0);
+        return new VideoFrame(bgra.PixelWidth, bgra.PixelHeight, pixels);
+    }
+}