gramps/LlamaCasty
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

TASK 51 slice 2: Blur — background at 1/4 res, 3 separable box passes (~gaussian, radius by strength), mask-blended; person untouched; Remove gains strength = edge feather; mask computed once, rendered per effect (Unit: bg blur)

Browse Source
This commit is contained in:
gramps
2026-10-09 10:18:00 -07:00
parent fbeb63ba79
commit 227ee8fca6
5 changed files with 261 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Services/BackgroundEffects/BackgroundEffectProcessor.cs
+201 -26
View File
@@ -6,24 +6,36 @@ namespace ytLive.Services.BackgroundEffects;
/// <summary>
/// Applies a webcam's <see cref="BackgroundEffect"/> to one captured frame (TASK 51).
/// Pure pixel work over the <see cref="VideoFrame"/> seam — WPF-free and thread-agnostic,
/// so the camera worker thread can call it before the frame reaches the preview bitmap
/// so the camera's processing worker can call it before the frame reaches the preview bitmap
/// and the compositor.
///
/// <list type="bullet">
/// <item><see cref="BackgroundEffect.None"/> returns the SAME frame instance — a true
/// passthrough, no copy.</item>
/// <item><see cref="BackgroundEffect.Remove"/> returns a copy whose alpha is the person
/// mask (bilinear-upscaled), RGB untouched, <c>IsOpaque = false</c> — the compositor's
/// straight-alpha blend then shows the layers below the webcam.</item>
/// <item><see cref="BackgroundEffect.Blur"/> is the next slice; until it lands it passes
/// through.</item>
/// <item><see cref="BackgroundEffect.Remove"/> returns a copy whose alpha is the person mask,
/// RGB untouched, <c>IsOpaque = false</c> — the compositor's straight-alpha blend then shows
/// the layers below the webcam. Strength = edge softness (0 = crisp cut-out, 1 = the model's
/// raw soft mask).</item>
/// <item><see cref="BackgroundEffect.Blur"/> returns a copy where the background is replaced
/// by a blurred version of itself and the person stays sharp. Strength = blur amount.</item>
/// </list>
///
/// A segmenter that returns no mask (or throws) never drops the layer: the frame passes
/// through and the failure is logged once.
/// The mask is computed ONCE per camera frame (<see cref="TrySegment"/>) and then rendered for
/// every effect variant in use (<see cref="Render"/>). A segmenter that returns no mask (or
/// throws) never drops the layer: the frame passes through and the failure is logged once.
/// </summary>
public sealed class BackgroundEffectProcessor
{
public const double DefaultStrength = 0.5;
/// <summary>The background is blurred at 1/<see cref="BlurDownscale"/> resolution and
/// upsampled while blending — a strong blur at a fraction of the cost.</summary>
internal const int BlurDownscale = 4;
/// <summary>Box-blur radius (in downscaled pixels) at strength 0 and 1.</summary>
internal const int MinBlurRadius = 1;
internal const int MaxBlurRadius = 12;
private readonly IPersonSegmenter _segmenter;
private bool _loggedFailure;
@@ -32,10 +44,16 @@ public sealed class BackgroundEffectProcessor
_segmenter = segmenter;
}
public VideoFrame Apply(VideoFrame frame, BackgroundEffect effect)
public VideoFrame Apply(VideoFrame frame, BackgroundEffect effect, double strength = DefaultStrength)
{
if (effect != BackgroundEffect.Remove) return frame;
if (effect == BackgroundEffect.None) return frame;
var mask = TrySegment(frame);
return mask == null ? frame : Render(frame, mask, effect, strength);
}
/// <summary>Runs the segmenter once; null (logged once) when it can't produce a usable mask.</summary>
public SegmentationMask? TrySegment(VideoFrame frame)
{
SegmentationMask? mask;
try
{
@@ -44,43 +62,200 @@ public sealed class BackgroundEffectProcessor
catch (Exception ex)
{
LogFailureOnce($"{ex.GetType().Name}: {ex.Message}");
return frame;
return null;
}
if (mask == null || mask.Values.Length < mask.Width * mask.Height || mask.Width < 1 || mask.Height < 1)
if (mask == null || mask.Width < 1 || mask.Height < 1 || mask.Values.Length < mask.Width * mask.Height)
{
LogFailureOnce("segmenter returned no usable mask");
return frame;
return null;
}
return ApplyRemove(frame, mask);
return mask;
}
internal static VideoFrame ApplyRemove(VideoFrame frame, SegmentationMask mask)
public static VideoFrame Render(VideoFrame frame, SegmentationMask mask, BackgroundEffect effect, double strength)
{
var s = Math.Clamp(strength, 0, 1);
return effect switch
{
BackgroundEffect.Remove => ApplyRemove(frame, mask, s),
BackgroundEffect.Blur => ApplyBlur(frame, mask, s),
_ => frame,
};
}
internal static VideoFrame ApplyRemove(VideoFrame frame, SegmentationMask mask, double strength)
{
var src = frame.BgraPixels;
var dst = new byte[src.Length];
Buffer.BlockCopy(src, 0, dst, 0, src.Length);
var w = frame.Width;
var h = frame.Height;
// Edge feather: map the mask's [0.5 - w, 0.5 + w] band to alpha 0..1.
var w = (float)(0.05 + 0.45 * strength);
var lo = 0.5f - w;
var span = 2 * w;
var width = frame.Width;
var height = frame.Height;
var stride = frame.Stride;
for (var y = 0; y < h; y++)
for (var y = 0; y < height; y++)
{
var my = (y + 0.5f) * mask.Height / h - 0.5f;
var my = MaskCoord(y, height, mask.Height);
var row = y * stride;
for (var x = 0; x < w; x++)
for (var x = 0; x < width; x++)
{
var mx = (x + 0.5f) * mask.Width / w - 0.5f;
var a = SampleBilinear(mask, mx, my);
var alpha = (int)(a * src[row + x * 4 + 3] + 0.5f);
dst[row + x * 4 + 3] = (byte)Math.Clamp(alpha, 0, 255);
var m = SampleBilinear(mask, MaskCoord(x, width, mask.Width), my);
var a = Math.Clamp((m - lo) / span, 0f, 1f);
var p = row + x * 4 + 3;
dst[p] = (byte)(a * src[p] + 0.5f);
}
}
return new VideoFrame(w, h, dst) { Epoch = frame.Epoch, IsOpaque = false };
return new VideoFrame(width, height, dst) { Epoch = frame.Epoch, IsOpaque = false };
}
internal static VideoFrame ApplyBlur(VideoFrame frame, SegmentationMask mask, double strength)
{
var width = frame.Width;
var height = frame.Height;
var src = frame.BgraPixels;
var stride = frame.Stride;
// 1. Downscale by box-averaging BlurDownscale×BlurDownscale blocks (3 float channels).
var f = BlurDownscale;
var sw = (width + f - 1) / f;
var sh = (height + f - 1) / f;
var small = new float[sw * sh * 3];
for (var sy = 0; sy < sh; sy++)
{
var y0 = sy * f;
var y1 = Math.Min(height, y0 + f);
for (var sx = 0; sx < sw; sx++)
{
var x0 = sx * f;
var x1 = Math.Min(width, x0 + f);
float b = 0, g = 0, r = 0;
var n = 0;
for (var y = y0; y < y1; y++)
{
var p = y * stride + x0 * 4;
for (var x = x0; x < x1; x++, p += 4)
{
b += src[p]; g += src[p + 1]; r += src[p + 2];
n++;
}
}
var o = (sy * sw + sx) * 3;
small[o] = b / n; small[o + 1] = g / n; small[o + 2] = r / n;
}
}
// 2. Three separable box-blur passes ≈ a gaussian.
var radius = (int)Math.Round(MinBlurRadius + (MaxBlurRadius - MinBlurRadius) * strength);
var scratch = new float[small.Length];
for (var pass = 0; pass < 3; pass++)
{
BoxBlurHorizontal(small, scratch, sw, sh, radius);
BoxBlurVertical(scratch, small, sw, sh, radius);
}
// 3. Blend: person (mask) keeps the source pixel, background takes the upsampled blur.
var dst = new byte[src.Length];
for (var y = 0; y < height; y++)
{
var my = MaskCoord(y, height, mask.Height);
var by = (y + 0.5f) / f - 0.5f;
var row = y * stride;
for (var x = 0; x < width; x++)
{
var m = SampleBilinear(mask, MaskCoord(x, width, mask.Width), my);
var p = row + x * 4;
if (m >= 1f)
{
dst[p] = src[p]; dst[p + 1] = src[p + 1]; dst[p + 2] = src[p + 2]; dst[p + 3] = src[p + 3];
continue;
}
SampleSmall(small, sw, sh, (x + 0.5f) / f - 0.5f, by, out var bb, out var bg, out var br);
var k = 1f - m;
dst[p] = (byte)Math.Clamp(src[p] * m + bb * k + 0.5f, 0, 255);
dst[p + 1] = (byte)Math.Clamp(src[p + 1] * m + bg * k + 0.5f, 0, 255);
dst[p + 2] = (byte)Math.Clamp(src[p + 2] * m + br * k + 0.5f, 0, 255);
dst[p + 3] = src[p + 3];
}
}
return new VideoFrame(width, height, dst) { Epoch = frame.Epoch, IsOpaque = frame.IsOpaque };
}
private static void BoxBlurHorizontal(float[] input, float[] output, int w, int h, int r)
{
var norm = 1f / (2 * r + 1);
for (var y = 0; y < h; y++)
{
var row = y * w;
for (var c = 0; c < 3; c++)
{
float sum = 0;
for (var i = -r; i <= r; i++)
sum += input[(row + Math.Clamp(i, 0, w - 1)) * 3 + c];
for (var x = 0; x < w; x++)
{
output[(row + x) * 3 + c] = sum * norm;
var add = Math.Min(w - 1, x + r + 1);
var sub = Math.Max(0, x - r);
sum += input[(row + add) * 3 + c] - input[(row + sub) * 3 + c];
}
}
}
}
private static void BoxBlurVertical(float[] input, float[] output, int w, int h, int r)
{
var norm = 1f / (2 * r + 1);
for (var x = 0; x < w; x++)
{
for (var c = 0; c < 3; c++)
{
float sum = 0;
for (var i = -r; i <= r; i++)
sum += input[(Math.Clamp(i, 0, h - 1) * w + x) * 3 + c];
for (var y = 0; y < h; y++)
{
output[(y * w + x) * 3 + c] = sum * norm;
var add = Math.Min(h - 1, y + r + 1);
var sub = Math.Max(0, y - r);
sum += input[(add * w + x) * 3 + c] - input[(sub * w + x) * 3 + c];
}
}
}
}
private static void SampleSmall(float[] img, int w, int h, float x, float y, out float b, out float g, out float r)
{
var x0 = (int)MathF.Floor(x);
var y0 = (int)MathF.Floor(y);
var fx = x - x0;
var fy = y - y0;
var x1 = Math.Clamp(x0 + 1, 0, w - 1);
var y1 = Math.Clamp(y0 + 1, 0, h - 1);
x0 = Math.Clamp(x0, 0, w - 1);
y0 = Math.Clamp(y0, 0, h - 1);
var i00 = (y0 * w + x0) * 3;
var i01 = (y0 * w + x1) * 3;
var i10 = (y1 * w + x0) * 3;
var i11 = (y1 * w + x1) * 3;
var w00 = (1 - fx) * (1 - fy);
var w01 = fx * (1 - fy);
var w10 = (1 - fx) * fy;
var w11 = fx * fy;
b = img[i00] * w00 + img[i01] * w01 + img[i10] * w10 + img[i11] * w11;
g = img[i00 + 1] * w00 + img[i01 + 1] * w01 + img[i10 + 1] * w10 + img[i11 + 1] * w11;
r = img[i00 + 2] * w00 + img[i01 + 2] * w01 + img[i10 + 2] * w10 + img[i11 + 2] * w11;
}
private static float MaskCoord(int pixel, int frameSize, int maskSize)
=> (pixel + 0.5f) * maskSize / frameSize - 0.5f;
private static float SampleBilinear(SegmentationMask mask, float x, float y)
{
var x0 = (int)MathF.Floor(x);
Services/index.md
+1 -1
View File
@@ -18,7 +18,7 @@ External-facing logic: YouTube API, persistence. See
| `VideoFrame.cs` | Normalized CPU frame seam (`Width`/`Height`/tightly-packed BGRA `byte[]`) — the only pixel type the rest of the app knows about; future capture sources (screen, background-removed webcam) feed the same seam |
| `BackgroundEffects/IPersonSegmenter.cs` | **TASK 51 seam:** `Segment(VideoFrame) → SegmentationMask?` (`SegmentationMask(Width, Height, float[] Values)`, 0 = background, 1 = person); null = no mask → caller passes through. Tests inject fakes |
| `BackgroundEffects/OnnxPersonSegmenter.cs` | **TASK 51 default segmenter:** MediaPipe Selfie Segmenter (landscape, Apache-2.0) as an embedded ONNX resource, run by onnxruntime on the CPU (2 intra-op threads); frame stretched to 256×144 RGB/255 NHWC; lazy session; never throws (fail → log once, null forever) |
| `BackgroundEffects/BackgroundEffectProcessor.cs` | **TASK 51 pure pixel stage:** `Apply(frame, BackgroundEffect)` — None = same instance, Remove = copy with alpha = bilinear mask × source alpha (`IsOpaque=false`), Blur = passthrough until slice 2; no mask/exception → passthrough + one log. Not wired into the camera path yet |
| `BackgroundEffects/BackgroundEffectProcessor.cs` | **TASK 51 pure pixel stage:** `Apply(frame, BackgroundEffect)` — None = same instance, Remove = copy with alpha = feathered mask × source alpha (`IsOpaque=false`), Blur = background blurred at ¼ res (3 box passes) and mask-blended, person untouched; strength 0..1; `TrySegment` once + `Render` per effect; no mask/exception → passthrough + one log |
| `CameraDeviceInfo.cs` | `(Id, DisplayName)` for a physical capture device |
| `ICameraEnumerator.cs` | `GetCamerasAsync()` — seam so the picker/`CameraManager` never touch WinRT (tests inject fakes) |
| `ICameraFrameSource.cs` | `StartAsync`/`StopAsync`/`FrameAvailable(VideoFrame)` + **`SourceFailed(string)`** — seam for a running capture source (tests inject fakes) |