TASK 51 slice 1: background-effects engine — bundled MediaPipe Selfie Segmenter (ONNX) + Remove (Unit: bg-effects engine)
- Model: Google's official MediaPipe Selfie Segmenter (landscape, Apache-2.0 per model card), converted to ONNX by tools/selfie-seg/convert_tflite_to_onnx.py (custom op Convolution2DTransposeBias -> ConvTranspose) and verified against MediaPipe's own runtime (verify_against_mediapipe.py: max |diff| 5e-5, 100% binary agreement). Embedded resource, 430 KB. - Services/BackgroundEffects: IPersonSegmenter seam, OnnxPersonSegmenter (onnxruntime 1.31.0 CPU, never throws), BackgroundEffectProcessor (None passthrough, Remove = alpha mask). Not wired yet. - Tests: Remove integration test (fake segmenter) + bundled-model contract test vs MediaPipe reference mask. Full suite 407/408 — the one failure is the documented interactive-desktop flake RealMouseDrag_OnTheLayerList_PersistsTheReorder (passed in the run before; code path untouched). - Docs: ai.md reversal + section + licensing guard, THIRD-PARTY-NOTICES 11-12, TASK 51, indexes, MyMistakes recipe, HANDOFF. References: obs-backgroundremoval (approach only — ONNX Runtime on CPU, 2 threads; GPL-3.0, no code used) https://github.com/royshil/obs-backgroundremoval ; MediaPipe image segmenter https://ai.google.dev/edge/mediapipe/solutions/vision/image_segmenter
This commit is contained in:
@@ -0,0 +1,106 @@
|
||||
using ytLive.Helpers;
|
||||
using ytLive.Models;
|
||||
|
||||
namespace ytLive.Services.BackgroundEffects;
|
||||
|
||||
/// <summary>
|
||||
/// Applies a webcam's <see cref="BackgroundEffect"/> to one captured frame (TASK 51).
|
||||
/// Pure pixel work over the <see cref="VideoFrame"/> seam — WPF-free and thread-agnostic,
|
||||
/// so the camera worker thread can call it before the frame reaches the preview bitmap
|
||||
/// and the compositor.
|
||||
///
|
||||
/// <list type="bullet">
|
||||
/// <item><see cref="BackgroundEffect.None"/> returns the SAME frame instance — a true
|
||||
/// passthrough, no copy.</item>
|
||||
/// <item><see cref="BackgroundEffect.Remove"/> returns a copy whose alpha is the person
|
||||
/// mask (bilinear-upscaled), RGB untouched, <c>IsOpaque = false</c> — the compositor's
|
||||
/// straight-alpha blend then shows the layers below the webcam.</item>
|
||||
/// <item><see cref="BackgroundEffect.Blur"/> is the next slice; until it lands it passes
|
||||
/// through.</item>
|
||||
/// </list>
|
||||
///
|
||||
/// A segmenter that returns no mask (or throws) never drops the layer: the frame passes
|
||||
/// through and the failure is logged once.
|
||||
/// </summary>
|
||||
public sealed class BackgroundEffectProcessor
|
||||
{
|
||||
private readonly IPersonSegmenter _segmenter;
|
||||
private bool _loggedFailure;
|
||||
|
||||
public BackgroundEffectProcessor(IPersonSegmenter segmenter)
|
||||
{
|
||||
_segmenter = segmenter;
|
||||
}
|
||||
|
||||
public VideoFrame Apply(VideoFrame frame, BackgroundEffect effect)
|
||||
{
|
||||
if (effect != BackgroundEffect.Remove) return frame;
|
||||
|
||||
SegmentationMask? mask;
|
||||
try
|
||||
{
|
||||
mask = _segmenter.Segment(frame);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
LogFailureOnce($"{ex.GetType().Name}: {ex.Message}");
|
||||
return frame;
|
||||
}
|
||||
|
||||
if (mask == null || mask.Values.Length < mask.Width * mask.Height || mask.Width < 1 || mask.Height < 1)
|
||||
{
|
||||
LogFailureOnce("segmenter returned no usable mask");
|
||||
return frame;
|
||||
}
|
||||
|
||||
return ApplyRemove(frame, mask);
|
||||
}
|
||||
|
||||
internal static VideoFrame ApplyRemove(VideoFrame frame, SegmentationMask mask)
|
||||
{
|
||||
var src = frame.BgraPixels;
|
||||
var dst = new byte[src.Length];
|
||||
Buffer.BlockCopy(src, 0, dst, 0, src.Length);
|
||||
|
||||
var w = frame.Width;
|
||||
var h = frame.Height;
|
||||
var stride = frame.Stride;
|
||||
for (var y = 0; y < h; y++)
|
||||
{
|
||||
var my = (y + 0.5f) * mask.Height / h - 0.5f;
|
||||
var row = y * stride;
|
||||
for (var x = 0; x < w; x++)
|
||||
{
|
||||
var mx = (x + 0.5f) * mask.Width / w - 0.5f;
|
||||
var a = SampleBilinear(mask, mx, my);
|
||||
var alpha = (int)(a * src[row + x * 4 + 3] + 0.5f);
|
||||
dst[row + x * 4 + 3] = (byte)Math.Clamp(alpha, 0, 255);
|
||||
}
|
||||
}
|
||||
|
||||
return new VideoFrame(w, h, dst) { Epoch = frame.Epoch, IsOpaque = false };
|
||||
}
|
||||
|
||||
private static float SampleBilinear(SegmentationMask mask, float x, float y)
|
||||
{
|
||||
var x0 = (int)MathF.Floor(x);
|
||||
var y0 = (int)MathF.Floor(y);
|
||||
var fx = x - x0;
|
||||
var fy = y - y0;
|
||||
var x1 = Math.Clamp(x0 + 1, 0, mask.Width - 1);
|
||||
var y1 = Math.Clamp(y0 + 1, 0, mask.Height - 1);
|
||||
x0 = Math.Clamp(x0, 0, mask.Width - 1);
|
||||
y0 = Math.Clamp(y0, 0, mask.Height - 1);
|
||||
var v = mask.Values;
|
||||
var top = v[y0 * mask.Width + x0] * (1 - fx) + v[y0 * mask.Width + x1] * fx;
|
||||
var bottom = v[y1 * mask.Width + x0] * (1 - fx) + v[y1 * mask.Width + x1] * fx;
|
||||
return Math.Clamp(top * (1 - fy) + bottom * fy, 0f, 1f);
|
||||
}
|
||||
|
||||
private void LogFailureOnce(string reason)
|
||||
{
|
||||
if (_loggedFailure) return;
|
||||
_loggedFailure = true;
|
||||
AppLog.Write($"BackgroundEffectProcessor: passing webcam through — {reason}");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user