using System.IO; using Microsoft.ML.OnnxRuntime; using ytLive.Helpers; namespace ytLive.Services.BackgroundEffects; /// /// The default : Google's MediaPipe Selfie Segmenter /// (landscape, Apache-2.0) converted to ONNX by tools/selfie-seg/ and shipped as an /// embedded resource — nothing is downloaded at runtime and no GPU is required. /// /// Contract: input [1,144,256,3] RGB in [0,1] (NHWC), output [1,144,256,1] person /// probability. The whole frame is resampled to 256×144 (stretched, not cropped), so the /// mask maps back to the frame by normalized coordinates. /// /// Never throws from : a model that fails to load or run logs once /// and returns null, and the caller passes the webcam frame through. /// public sealed class OnnxPersonSegmenter : IPersonSegmenter, IDisposable { public const int ModelWidth = 256; public const int ModelHeight = 144; internal const string ModelResourceName = "ytLive.Assets.Models.selfie_segmenter_landscape.onnx"; private static readonly long[] InputShape = { 1, ModelHeight, ModelWidth, 3 }; private readonly object _gate = new(); private readonly float[] _input = new float[ModelWidth * ModelHeight * 3]; private readonly RunOptions _runOptions = new(); private InferenceSession? _session; private bool _failed; private bool _disposed; public SegmentationMask? Segment(VideoFrame frame) { lock (_gate) { if (_failed || _disposed) return null; try { _session ??= CreateSession(); FillInput(frame, _input); using var input = OrtValue.CreateTensorValueFromMemory(_input, InputShape); using var outputs = _session.Run(_runOptions, new[] { "input" }, new[] { input }, new[] { "mask" }); var mask = outputs[0].GetTensorDataAsSpan().ToArray(); return new SegmentationMask(ModelWidth, ModelHeight, mask); } catch (Exception ex) { _failed = true; AppLog.Write($"OnnxPersonSegmenter: background effects disabled — {ex.GetType().Name}: {ex.Message}"); return null; } } } public void Dispose() { lock (_gate) { _disposed = true; _session?.Dispose(); _session = null; _runOptions.Dispose(); } } private static InferenceSession CreateSession() { using var stream = typeof(OnnxPersonSegmenter).Assembly.GetManifestResourceStream(ModelResourceName) ?? throw new InvalidOperationException($"embedded model '{ModelResourceName}' is missing"); using var buffer = new MemoryStream(); stream.CopyTo(buffer); // Two intra-op threads: the model is tiny, and the webcam must not starve the // compositor/encoder threads (obs-backgroundremoval reports 2 as the sweet spot). using var options = new SessionOptions { IntraOpNumThreads = 2, InterOpNumThreads = 1, GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL, }; return new InferenceSession(buffer.ToArray(), options); } /// BGRA frame → model input: nearest-sample stretch to 256×144, RGB order, /// values /255. Nearest is enough — the model input is ~5× smaller than a 720p frame /// and the mask is the only thing that comes back. internal static void FillInput(VideoFrame frame, float[] input) { var src = frame.BgraPixels; var stride = frame.Stride; var o = 0; for (var y = 0; y < ModelHeight; y++) { var sy = Math.Min(frame.Height - 1, (int)((y + 0.5) * frame.Height / ModelHeight)); var row = sy * stride; for (var x = 0; x < ModelWidth; x++) { var sx = Math.Min(frame.Width - 1, (int)((x + 0.5) * frame.Width / ModelWidth)); var p = row + sx * 4; input[o++] = src[p + 2] / 255f; input[o++] = src[p + 1] / 255f; input[o++] = src[p] / 255f; } } } }