TASK 51 slice 1: background-effects engine — bundled MediaPipe Selfie Segmenter (ONNX) + Remove (Unit: bg-effects engine)
- Model: Google's official MediaPipe Selfie Segmenter (landscape, Apache-2.0 per model card), converted to ONNX by tools/selfie-seg/convert_tflite_to_onnx.py (custom op Convolution2DTransposeBias -> ConvTranspose) and verified against MediaPipe's own runtime (verify_against_mediapipe.py: max |diff| 5e-5, 100% binary agreement). Embedded resource, 430 KB. - Services/BackgroundEffects: IPersonSegmenter seam, OnnxPersonSegmenter (onnxruntime 1.31.0 CPU, never throws), BackgroundEffectProcessor (None passthrough, Remove = alpha mask). Not wired yet. - Tests: Remove integration test (fake segmenter) + bundled-model contract test vs MediaPipe reference mask. Full suite 407/408 — the one failure is the documented interactive-desktop flake RealMouseDrag_OnTheLayerList_PersistsTheReorder (passed in the run before; code path untouched). - Docs: ai.md reversal + section + licensing guard, THIRD-PARTY-NOTICES 11-12, TASK 51, indexes, MyMistakes recipe, HANDOFF. References: obs-backgroundremoval (approach only — ONNX Runtime on CPU, 2 threads; GPL-3.0, no code used) https://github.com/royshil/obs-backgroundremoval ; MediaPipe image segmenter https://ai.google.dev/edge/mediapipe/solutions/vision/image_segmenter
This commit is contained in:
@@ -0,0 +1,106 @@
|
||||
using ytLive.Helpers;
|
||||
using ytLive.Models;
|
||||
|
||||
namespace ytLive.Services.BackgroundEffects;
|
||||
|
||||
/// <summary>
|
||||
/// Applies a webcam's <see cref="BackgroundEffect"/> to one captured frame (TASK 51).
|
||||
/// Pure pixel work over the <see cref="VideoFrame"/> seam — WPF-free and thread-agnostic,
|
||||
/// so the camera worker thread can call it before the frame reaches the preview bitmap
|
||||
/// and the compositor.
|
||||
///
|
||||
/// <list type="bullet">
|
||||
/// <item><see cref="BackgroundEffect.None"/> returns the SAME frame instance — a true
|
||||
/// passthrough, no copy.</item>
|
||||
/// <item><see cref="BackgroundEffect.Remove"/> returns a copy whose alpha is the person
|
||||
/// mask (bilinear-upscaled), RGB untouched, <c>IsOpaque = false</c> — the compositor's
|
||||
/// straight-alpha blend then shows the layers below the webcam.</item>
|
||||
/// <item><see cref="BackgroundEffect.Blur"/> is the next slice; until it lands it passes
|
||||
/// through.</item>
|
||||
/// </list>
|
||||
///
|
||||
/// A segmenter that returns no mask (or throws) never drops the layer: the frame passes
|
||||
/// through and the failure is logged once.
|
||||
/// </summary>
|
||||
public sealed class BackgroundEffectProcessor
|
||||
{
|
||||
private readonly IPersonSegmenter _segmenter;
|
||||
private bool _loggedFailure;
|
||||
|
||||
public BackgroundEffectProcessor(IPersonSegmenter segmenter)
|
||||
{
|
||||
_segmenter = segmenter;
|
||||
}
|
||||
|
||||
public VideoFrame Apply(VideoFrame frame, BackgroundEffect effect)
|
||||
{
|
||||
if (effect != BackgroundEffect.Remove) return frame;
|
||||
|
||||
SegmentationMask? mask;
|
||||
try
|
||||
{
|
||||
mask = _segmenter.Segment(frame);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
LogFailureOnce($"{ex.GetType().Name}: {ex.Message}");
|
||||
return frame;
|
||||
}
|
||||
|
||||
if (mask == null || mask.Values.Length < mask.Width * mask.Height || mask.Width < 1 || mask.Height < 1)
|
||||
{
|
||||
LogFailureOnce("segmenter returned no usable mask");
|
||||
return frame;
|
||||
}
|
||||
|
||||
return ApplyRemove(frame, mask);
|
||||
}
|
||||
|
||||
internal static VideoFrame ApplyRemove(VideoFrame frame, SegmentationMask mask)
|
||||
{
|
||||
var src = frame.BgraPixels;
|
||||
var dst = new byte[src.Length];
|
||||
Buffer.BlockCopy(src, 0, dst, 0, src.Length);
|
||||
|
||||
var w = frame.Width;
|
||||
var h = frame.Height;
|
||||
var stride = frame.Stride;
|
||||
for (var y = 0; y < h; y++)
|
||||
{
|
||||
var my = (y + 0.5f) * mask.Height / h - 0.5f;
|
||||
var row = y * stride;
|
||||
for (var x = 0; x < w; x++)
|
||||
{
|
||||
var mx = (x + 0.5f) * mask.Width / w - 0.5f;
|
||||
var a = SampleBilinear(mask, mx, my);
|
||||
var alpha = (int)(a * src[row + x * 4 + 3] + 0.5f);
|
||||
dst[row + x * 4 + 3] = (byte)Math.Clamp(alpha, 0, 255);
|
||||
}
|
||||
}
|
||||
|
||||
return new VideoFrame(w, h, dst) { Epoch = frame.Epoch, IsOpaque = false };
|
||||
}
|
||||
|
||||
private static float SampleBilinear(SegmentationMask mask, float x, float y)
|
||||
{
|
||||
var x0 = (int)MathF.Floor(x);
|
||||
var y0 = (int)MathF.Floor(y);
|
||||
var fx = x - x0;
|
||||
var fy = y - y0;
|
||||
var x1 = Math.Clamp(x0 + 1, 0, mask.Width - 1);
|
||||
var y1 = Math.Clamp(y0 + 1, 0, mask.Height - 1);
|
||||
x0 = Math.Clamp(x0, 0, mask.Width - 1);
|
||||
y0 = Math.Clamp(y0, 0, mask.Height - 1);
|
||||
var v = mask.Values;
|
||||
var top = v[y0 * mask.Width + x0] * (1 - fx) + v[y0 * mask.Width + x1] * fx;
|
||||
var bottom = v[y1 * mask.Width + x0] * (1 - fx) + v[y1 * mask.Width + x1] * fx;
|
||||
return Math.Clamp(top * (1 - fy) + bottom * fy, 0f, 1f);
|
||||
}
|
||||
|
||||
private void LogFailureOnce(string reason)
|
||||
{
|
||||
if (_loggedFailure) return;
|
||||
_loggedFailure = true;
|
||||
AppLog.Write($"BackgroundEffectProcessor: passing webcam through — {reason}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
namespace ytLive.Services.BackgroundEffects;
|
||||
|
||||
/// <summary>A person-probability mask at the segmenter's own resolution: row-major,
|
||||
/// one value per pixel, 0 = background, 1 = person.</summary>
|
||||
public sealed record SegmentationMask(int Width, int Height, float[] Values);
|
||||
|
||||
/// <summary>Seam over the person-segmentation model (TASK 51) so tests inject a fake and
|
||||
/// never run real inference. Returns null when no mask can be produced; callers then pass
|
||||
/// the frame through untouched.</summary>
|
||||
public interface IPersonSegmenter
|
||||
{
|
||||
SegmentationMask? Segment(VideoFrame frame);
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
using System.IO;
|
||||
using Microsoft.ML.OnnxRuntime;
|
||||
using ytLive.Helpers;
|
||||
|
||||
namespace ytLive.Services.BackgroundEffects;
|
||||
|
||||
/// <summary>
|
||||
/// The default <see cref="IPersonSegmenter"/>: Google's MediaPipe Selfie Segmenter
|
||||
/// (landscape, Apache-2.0) converted to ONNX by <c>tools/selfie-seg/</c> and shipped as an
|
||||
/// embedded resource — nothing is downloaded at runtime and no GPU is required.
|
||||
///
|
||||
/// Contract: input [1,144,256,3] RGB in [0,1] (NHWC), output [1,144,256,1] person
|
||||
/// probability. The whole frame is resampled to 256×144 (stretched, not cropped), so the
|
||||
/// mask maps back to the frame by normalized coordinates.
|
||||
///
|
||||
/// Never throws from <see cref="Segment"/>: a model that fails to load or run logs once
|
||||
/// and returns null, and the caller passes the webcam frame through.
|
||||
/// </summary>
|
||||
public sealed class OnnxPersonSegmenter : IPersonSegmenter, IDisposable
|
||||
{
|
||||
public const int ModelWidth = 256;
|
||||
public const int ModelHeight = 144;
|
||||
internal const string ModelResourceName = "ytLive.Assets.Models.selfie_segmenter_landscape.onnx";
|
||||
|
||||
private static readonly long[] InputShape = { 1, ModelHeight, ModelWidth, 3 };
|
||||
|
||||
private readonly object _gate = new();
|
||||
private readonly float[] _input = new float[ModelWidth * ModelHeight * 3];
|
||||
private readonly RunOptions _runOptions = new();
|
||||
private InferenceSession? _session;
|
||||
private bool _failed;
|
||||
private bool _disposed;
|
||||
|
||||
public SegmentationMask? Segment(VideoFrame frame)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (_failed || _disposed) return null;
|
||||
try
|
||||
{
|
||||
_session ??= CreateSession();
|
||||
FillInput(frame, _input);
|
||||
using var input = OrtValue.CreateTensorValueFromMemory(_input, InputShape);
|
||||
using var outputs = _session.Run(_runOptions, new[] { "input" }, new[] { input }, new[] { "mask" });
|
||||
var mask = outputs[0].GetTensorDataAsSpan<float>().ToArray();
|
||||
return new SegmentationMask(ModelWidth, ModelHeight, mask);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_failed = true;
|
||||
AppLog.Write($"OnnxPersonSegmenter: background effects disabled — {ex.GetType().Name}: {ex.Message}");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_disposed = true;
|
||||
_session?.Dispose();
|
||||
_session = null;
|
||||
_runOptions.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private static InferenceSession CreateSession()
|
||||
{
|
||||
using var stream = typeof(OnnxPersonSegmenter).Assembly.GetManifestResourceStream(ModelResourceName)
|
||||
?? throw new InvalidOperationException($"embedded model '{ModelResourceName}' is missing");
|
||||
using var buffer = new MemoryStream();
|
||||
stream.CopyTo(buffer);
|
||||
// Two intra-op threads: the model is tiny, and the webcam must not starve the
|
||||
// compositor/encoder threads (obs-backgroundremoval reports 2 as the sweet spot).
|
||||
using var options = new SessionOptions
|
||||
{
|
||||
IntraOpNumThreads = 2,
|
||||
InterOpNumThreads = 1,
|
||||
GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL,
|
||||
};
|
||||
return new InferenceSession(buffer.ToArray(), options);
|
||||
}
|
||||
|
||||
/// <summary>BGRA frame → model input: nearest-sample stretch to 256×144, RGB order,
|
||||
/// values /255. Nearest is enough — the model input is ~5× smaller than a 720p frame
|
||||
/// and the mask is the only thing that comes back.</summary>
|
||||
internal static void FillInput(VideoFrame frame, float[] input)
|
||||
{
|
||||
var src = frame.BgraPixels;
|
||||
var stride = frame.Stride;
|
||||
var o = 0;
|
||||
for (var y = 0; y < ModelHeight; y++)
|
||||
{
|
||||
var sy = Math.Min(frame.Height - 1, (int)((y + 0.5) * frame.Height / ModelHeight));
|
||||
var row = sy * stride;
|
||||
for (var x = 0; x < ModelWidth; x++)
|
||||
{
|
||||
var sx = Math.Min(frame.Width - 1, (int)((x + 0.5) * frame.Width / ModelWidth));
|
||||
var p = row + sx * 4;
|
||||
input[o++] = src[p + 2] / 255f;
|
||||
input[o++] = src[p + 1] / 255f;
|
||||
input[o++] = src[p] / 255f;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user