using ytLive.Helpers; using ytLive.Models; namespace ytLive.Services.BackgroundEffects; /// /// Applies a webcam's to one captured frame (TASK 51). /// Pure pixel work over the seam — WPF-free and thread-agnostic, /// so the camera's processing worker can call it before the frame reaches the preview bitmap /// and the compositor. /// /// /// returns the SAME frame instance — a true /// passthrough, no copy. /// returns a copy whose alpha is the person mask, /// RGB untouched, IsOpaque = false — the compositor's straight-alpha blend then shows /// the layers below the webcam. Strength = edge softness (0 = crisp cut-out, 1 = the model's /// raw soft mask). /// returns a copy where the background is replaced /// by a blurred version of itself and the person stays sharp. Strength = blur amount. /// /// /// The mask is computed ONCE per camera frame () and then rendered for /// every effect variant in use (). A segmenter that returns no mask (or /// throws) never drops the layer: the frame passes through and the failure is logged once. /// public sealed class BackgroundEffectProcessor { public const double DefaultStrength = 0.5; /// The background is blurred at 1/ resolution and /// upsampled while blending — a strong blur at a fraction of the cost. internal const int BlurDownscale = 4; /// Box-blur radius (in downscaled pixels) at strength 0 and 1. internal const int MinBlurRadius = 1; internal const int MaxBlurRadius = 12; private readonly IPersonSegmenter _segmenter; private bool _loggedFailure; public BackgroundEffectProcessor(IPersonSegmenter segmenter) { _segmenter = segmenter; } public VideoFrame Apply(VideoFrame frame, BackgroundEffect effect, double strength = DefaultStrength) { if (effect == BackgroundEffect.None) return frame; var mask = TrySegment(frame); return mask == null ? frame : Render(frame, mask, effect, strength); } /// Runs the segmenter once; null (logged once) when it can't produce a usable mask. public SegmentationMask? TrySegment(VideoFrame frame) { SegmentationMask? mask; try { mask = _segmenter.Segment(frame); } catch (Exception ex) { LogFailureOnce($"{ex.GetType().Name}: {ex.Message}"); return null; } if (mask == null || mask.Width < 1 || mask.Height < 1 || mask.Values.Length < mask.Width * mask.Height) { LogFailureOnce("segmenter returned no usable mask"); return null; } return mask; } public static VideoFrame Render(VideoFrame frame, SegmentationMask mask, BackgroundEffect effect, double strength) { var s = Math.Clamp(strength, 0, 1); return effect switch { BackgroundEffect.Remove => ApplyRemove(frame, mask, s), BackgroundEffect.Blur => ApplyBlur(frame, mask, s), _ => frame, }; } internal static VideoFrame ApplyRemove(VideoFrame frame, SegmentationMask mask, double strength) { var src = frame.BgraPixels; var dst = new byte[src.Length]; Buffer.BlockCopy(src, 0, dst, 0, src.Length); // Edge feather: map the mask's [0.5 - w, 0.5 + w] band to alpha 0..1. var w = (float)(0.05 + 0.45 * strength); var lo = 0.5f - w; var span = 2 * w; var width = frame.Width; var height = frame.Height; var stride = frame.Stride; for (var y = 0; y < height; y++) { var my = MaskCoord(y, height, mask.Height); var row = y * stride; for (var x = 0; x < width; x++) { var m = SampleBilinear(mask, MaskCoord(x, width, mask.Width), my); var a = Math.Clamp((m - lo) / span, 0f, 1f); var p = row + x * 4 + 3; dst[p] = (byte)(a * src[p] + 0.5f); } } return new VideoFrame(width, height, dst) { Epoch = frame.Epoch, IsOpaque = false }; } internal static VideoFrame ApplyBlur(VideoFrame frame, SegmentationMask mask, double strength) { var width = frame.Width; var height = frame.Height; var src = frame.BgraPixels; var stride = frame.Stride; // 1. Downscale by box-averaging BlurDownscale×BlurDownscale blocks (3 float channels). var f = BlurDownscale; var sw = (width + f - 1) / f; var sh = (height + f - 1) / f; var small = new float[sw * sh * 3]; for (var sy = 0; sy < sh; sy++) { var y0 = sy * f; var y1 = Math.Min(height, y0 + f); for (var sx = 0; sx < sw; sx++) { var x0 = sx * f; var x1 = Math.Min(width, x0 + f); float b = 0, g = 0, r = 0; var n = 0; for (var y = y0; y < y1; y++) { var p = y * stride + x0 * 4; for (var x = x0; x < x1; x++, p += 4) { b += src[p]; g += src[p + 1]; r += src[p + 2]; n++; } } var o = (sy * sw + sx) * 3; small[o] = b / n; small[o + 1] = g / n; small[o + 2] = r / n; } } // 2. Three separable box-blur passes ≈ a gaussian. var radius = (int)Math.Round(MinBlurRadius + (MaxBlurRadius - MinBlurRadius) * strength); var scratch = new float[small.Length]; for (var pass = 0; pass < 3; pass++) { BoxBlurHorizontal(small, scratch, sw, sh, radius); BoxBlurVertical(scratch, small, sw, sh, radius); } // 3. Blend: person (mask) keeps the source pixel, background takes the upsampled blur. var dst = new byte[src.Length]; for (var y = 0; y < height; y++) { var my = MaskCoord(y, height, mask.Height); var by = (y + 0.5f) / f - 0.5f; var row = y * stride; for (var x = 0; x < width; x++) { var m = SampleBilinear(mask, MaskCoord(x, width, mask.Width), my); var p = row + x * 4; if (m >= 1f) { dst[p] = src[p]; dst[p + 1] = src[p + 1]; dst[p + 2] = src[p + 2]; dst[p + 3] = src[p + 3]; continue; } SampleSmall(small, sw, sh, (x + 0.5f) / f - 0.5f, by, out var bb, out var bg, out var br); var k = 1f - m; dst[p] = (byte)Math.Clamp(src[p] * m + bb * k + 0.5f, 0, 255); dst[p + 1] = (byte)Math.Clamp(src[p + 1] * m + bg * k + 0.5f, 0, 255); dst[p + 2] = (byte)Math.Clamp(src[p + 2] * m + br * k + 0.5f, 0, 255); dst[p + 3] = src[p + 3]; } } return new VideoFrame(width, height, dst) { Epoch = frame.Epoch, IsOpaque = frame.IsOpaque }; } private static void BoxBlurHorizontal(float[] input, float[] output, int w, int h, int r) { var norm = 1f / (2 * r + 1); for (var y = 0; y < h; y++) { var row = y * w; for (var c = 0; c < 3; c++) { float sum = 0; for (var i = -r; i <= r; i++) sum += input[(row + Math.Clamp(i, 0, w - 1)) * 3 + c]; for (var x = 0; x < w; x++) { output[(row + x) * 3 + c] = sum * norm; var add = Math.Min(w - 1, x + r + 1); var sub = Math.Max(0, x - r); sum += input[(row + add) * 3 + c] - input[(row + sub) * 3 + c]; } } } } private static void BoxBlurVertical(float[] input, float[] output, int w, int h, int r) { var norm = 1f / (2 * r + 1); for (var x = 0; x < w; x++) { for (var c = 0; c < 3; c++) { float sum = 0; for (var i = -r; i <= r; i++) sum += input[(Math.Clamp(i, 0, h - 1) * w + x) * 3 + c]; for (var y = 0; y < h; y++) { output[(y * w + x) * 3 + c] = sum * norm; var add = Math.Min(h - 1, y + r + 1); var sub = Math.Max(0, y - r); sum += input[(add * w + x) * 3 + c] - input[(sub * w + x) * 3 + c]; } } } } private static void SampleSmall(float[] img, int w, int h, float x, float y, out float b, out float g, out float r) { var x0 = (int)MathF.Floor(x); var y0 = (int)MathF.Floor(y); var fx = x - x0; var fy = y - y0; var x1 = Math.Clamp(x0 + 1, 0, w - 1); var y1 = Math.Clamp(y0 + 1, 0, h - 1); x0 = Math.Clamp(x0, 0, w - 1); y0 = Math.Clamp(y0, 0, h - 1); var i00 = (y0 * w + x0) * 3; var i01 = (y0 * w + x1) * 3; var i10 = (y1 * w + x0) * 3; var i11 = (y1 * w + x1) * 3; var w00 = (1 - fx) * (1 - fy); var w01 = fx * (1 - fy); var w10 = (1 - fx) * fy; var w11 = fx * fy; b = img[i00] * w00 + img[i01] * w01 + img[i10] * w10 + img[i11] * w11; g = img[i00 + 1] * w00 + img[i01 + 1] * w01 + img[i10 + 1] * w10 + img[i11 + 1] * w11; r = img[i00 + 2] * w00 + img[i01 + 2] * w01 + img[i10 + 2] * w10 + img[i11 + 2] * w11; } private static float MaskCoord(int pixel, int frameSize, int maskSize) => (pixel + 0.5f) * maskSize / frameSize - 0.5f; private static float SampleBilinear(SegmentationMask mask, float x, float y) { var x0 = (int)MathF.Floor(x); var y0 = (int)MathF.Floor(y); var fx = x - x0; var fy = y - y0; var x1 = Math.Clamp(x0 + 1, 0, mask.Width - 1); var y1 = Math.Clamp(y0 + 1, 0, mask.Height - 1); x0 = Math.Clamp(x0, 0, mask.Width - 1); y0 = Math.Clamp(y0, 0, mask.Height - 1); var v = mask.Values; var top = v[y0 * mask.Width + x0] * (1 - fx) + v[y0 * mask.Width + x1] * fx; var bottom = v[y1 * mask.Width + x0] * (1 - fx) + v[y1 * mask.Width + x1] * fx; return Math.Clamp(top * (1 - fy) + bottom * fy, 0f, 1f); } private void LogFailureOnce(string reason) { if (_loggedFailure) return; _loggedFailure = true; AppLog.Write($"BackgroundEffectProcessor: passing webcam through — {reason}"); } }