gramps/LlamaCasty
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

TASK 51 slice 4: camera-path hookup — CameraManager renders per-scene effect variants (one segmentation per raw frame on a latest-frame-drop worker, raw path untouched), VM declares/adopts variants per config; separable upsampling cuts Remove 46->3 ms, Blur 80->18 ms (Debug, 720p) (Unit: bg-effects hookup)

Browse Source
This commit is contained in:
gramps
2026-10-09 10:27:38 -07:00
parent 55210b9a61
commit de22983b36
10 changed files with 524 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Services/BackgroundEffects/BackgroundEffectProcessor.cs
+126 -63
View File
@@ -90,26 +90,36 @@ public sealed class BackgroundEffectProcessor
var dst = new byte[src.Length];
Buffer.BlockCopy(src, 0, dst, 0, src.Length);
// Edge feather: map the mask's [0.5 - w, 0.5 + w] band to alpha 0..1.
var w = (float)(0.05 + 0.45 * strength);
var lo = 0.5f - w;
// Edge feather: map the mask's [0.5 - w, 0.5 + w] band to alpha 0..1, as a 256-entry LUT
// over the quantized mask value (so the per-pixel work is a lookup + one multiply).
var w = 0.05 + 0.45 * strength;
var lo = 0.5 - w;
var span = 2 * w;
var alphaLut = new int[256];
for (var i = 0; i < 256; i++)
alphaLut[i] = (int)Math.Round(Math.Clamp((i / 255.0 - lo) / span, 0, 1) * 256);
var width = frame.Width;
var height = frame.Height;
var stride = frame.Stride;
for (var y = 0; y < height; y++)
var maskRows = UpsampleMaskRows(mask, width);
var (r0, r1, fy) = AxisLut(mask.Height, height);
Parallel.For(0, height, RowParallelism, y =>
{
var my = MaskCoord(y, height, mask.Height);
var row = y * stride;
var a0 = maskRows[r0[y]];
var a1 = maskRows[r1[y]];
var t = fy[y];
var row = y * stride + 3;
for (var x = 0; x < width; x++)
{
var m = SampleBilinear(mask, MaskCoord(x, width, mask.Width), my);
var a = Math.Clamp((m - lo) / span, 0f, 1f);
var p = row + x * 4 + 3;
dst[p] = (byte)(a * src[p] + 0.5f);
var m = a0[x] + (a1[x] - a0[x]) * t;
var q = (int)(m * 255f + 0.5f);
q = q < 0 ? 0 : q > 255 ? 255 : q;
var p = row + x * 4;
dst[p] = (byte)((src[p] * alphaLut[q] + 128) >> 8);
}
}
});
return new VideoFrame(width, height, dst) { Epoch = frame.Epoch, IsOpaque = false };
}
@@ -126,7 +136,7 @@ public sealed class BackgroundEffectProcessor
var sw = (width + f - 1) / f;
var sh = (height + f - 1) / f;
var small = new float[sw * sh * 3];
for (var sy = 0; sy < sh; sy++)
Parallel.For(0, sh, RowParallelism, sy =>
{
var y0 = sy * f;
var y1 = Math.Min(height, y0 + f);
@@ -148,7 +158,7 @@ public sealed class BackgroundEffectProcessor
var o = (sy * sw + sx) * 3;
small[o] = b / n; small[o + 1] = g / n; small[o + 2] = r / n;
}
}
});
// 2. Three separable box-blur passes ≈ a gaussian.
var radius = (int)Math.Round(MinBlurRadius + (MaxBlurRadius - MinBlurRadius) * strength);
@@ -159,34 +169,126 @@ public sealed class BackgroundEffectProcessor
BoxBlurVertical(scratch, small, sw, sh, radius);
}
// 3. Blend: person (mask) keeps the source pixel, background takes the upsampled blur.
// 3. Upsample: horizontal interpolation precomputed once per small row (and per mask row),
// then each output row lerps two precomputed rows — no per-pixel 2-D bilinear.
var blurRows = UpsampleColorRows(small, sw, sh, width);
var (b0, b1, bfy) = AxisLut(sh, height);
var maskRows = UpsampleMaskRows(mask, width);
var (m0, m1, mfy) = AxisLut(mask.Height, height);
// 4. Blend: person (mask) keeps the source pixel, background takes the blur.
var dst = new byte[src.Length];
for (var y = 0; y < height; y++)
Parallel.For(0, height, RowParallelism, y =>
{
var my = MaskCoord(y, height, mask.Height);
var by = (y + 0.5f) / f - 0.5f;
var ma = maskRows[m0[y]];
var mb = maskRows[m1[y]];
var mt = mfy[y];
var ba = blurRows[b0[y]];
var bb = blurRows[b1[y]];
var bt = bfy[y];
var row = y * stride;
for (var x = 0; x < width; x++)
{
var m = SampleBilinear(mask, MaskCoord(x, width, mask.Width), my);
var p = row + x * 4;
if (m >= 1f)
var m = ma[x] + (mb[x] - ma[x]) * mt;
if (m >= 0.999f)
{
dst[p] = src[p]; dst[p + 1] = src[p + 1]; dst[p + 2] = src[p + 2]; dst[p + 3] = src[p + 3];
continue;
}
SampleSmall(small, sw, sh, (x + 0.5f) / f - 0.5f, by, out var bb, out var bg, out var br);
if (m < 0f) m = 0f;
var k = 1f - m;
dst[p] = (byte)Math.Clamp(src[p] * m + bb * k + 0.5f, 0, 255);
dst[p + 1] = (byte)Math.Clamp(src[p + 1] * m + bg * k + 0.5f, 0, 255);
dst[p + 2] = (byte)Math.Clamp(src[p + 2] * m + br * k + 0.5f, 0, 255);
var c = x * 3;
var vb = ba[c] + (bb[c] - ba[c]) * bt;
var vg = ba[c + 1] + (bb[c + 1] - ba[c + 1]) * bt;
var vr = ba[c + 2] + (bb[c + 2] - ba[c + 2]) * bt;
dst[p] = ClampByte(src[p] * m + vb * k);
dst[p + 1] = ClampByte(src[p + 1] * m + vg * k);
dst[p + 2] = ClampByte(src[p + 2] * m + vr * k);
dst[p + 3] = src[p + 3];
}
}
});
return new VideoFrame(width, height, dst) { Epoch = frame.Epoch, IsOpaque = frame.IsOpaque };
}
/// <summary>Row-parallel pixel loops, capped so the effect worker can't starve the compositor
/// and encoder threads.</summary>
private static readonly ParallelOptions RowParallelism = new()
{
MaxDegreeOfParallelism = Math.Clamp(Environment.ProcessorCount / 2, 1, 4),
};
private static byte ClampByte(float v)
{
var i = (int)(v + 0.5f);
return (byte)(i < 0 ? 0 : i > 255 ? 255 : i);
}
/// <summary>For each destination index along one axis: the two source indices and the
/// fractional weight — pixel-center aligned, edge-clamped (the same mapping a bilinear
/// sample uses).</summary>
private static (int[] I0, int[] I1, float[] F) AxisLut(int srcSize, int dstSize)
{
var i0 = new int[dstSize];
var i1 = new int[dstSize];
var fr = new float[dstSize];
for (var d = 0; d < dstSize; d++)
{
var s = (d + 0.5f) * srcSize / dstSize - 0.5f;
var a = (int)MathF.Floor(s);
var t = s - a;
i0[d] = Math.Clamp(a, 0, srcSize - 1);
i1[d] = Math.Clamp(a + 1, 0, srcSize - 1);
fr[d] = a < 0 ? 0f : a + 1 > srcSize - 1 ? 0f : t;
}
return (i0, i1, fr);
}
/// <summary>Each mask row, horizontally interpolated (and clamped to 0..1) to the frame width.</summary>
private static float[][] UpsampleMaskRows(SegmentationMask mask, int width)
{
var (x0, x1, fx) = AxisLut(mask.Width, width);
var rows = new float[mask.Height][];
var v = mask.Values;
for (var r = 0; r < mask.Height; r++)
{
var row = new float[width];
var o = r * mask.Width;
for (var x = 0; x < width; x++)
{
var a = v[o + x0[x]];
var m = a + (v[o + x1[x]] - a) * fx[x];
row[x] = m < 0f ? 0f : m > 1f ? 1f : m;
}
rows[r] = row;
}
return rows;
}
/// <summary>Each row of a 3-channel float image, horizontally interpolated to <paramref name="width"/>.</summary>
private static float[][] UpsampleColorRows(float[] img, int w, int h, int width)
{
var (x0, x1, fx) = AxisLut(w, width);
var rows = new float[h][];
Parallel.For(0, h, RowParallelism, r =>
{
var row = new float[width * 3];
var o = r * w;
for (var x = 0; x < width; x++)
{
var a = (o + x0[x]) * 3;
var b = (o + x1[x]) * 3;
var t = fx[x];
row[x * 3] = img[a] + (img[b] - img[a]) * t;
row[x * 3 + 1] = img[a + 1] + (img[b + 1] - img[a + 1]) * t;
row[x * 3 + 2] = img[a + 2] + (img[b + 2] - img[a + 2]) * t;
}
rows[r] = row;
});
return rows;
}
private static void BoxBlurHorizontal(float[] input, float[] output, int w, int h, int r)
{
var norm = 1f / (2 * r + 1);
@@ -230,47 +332,8 @@ public sealed class BackgroundEffectProcessor
}
}
private static void SampleSmall(float[] img, int w, int h, float x, float y, out float b, out float g, out float r)
{
var x0 = (int)MathF.Floor(x);
var y0 = (int)MathF.Floor(y);
var fx = x - x0;
var fy = y - y0;
var x1 = Math.Clamp(x0 + 1, 0, w - 1);
var y1 = Math.Clamp(y0 + 1, 0, h - 1);
x0 = Math.Clamp(x0, 0, w - 1);
y0 = Math.Clamp(y0, 0, h - 1);
var i00 = (y0 * w + x0) * 3;
var i01 = (y0 * w + x1) * 3;
var i10 = (y1 * w + x0) * 3;
var i11 = (y1 * w + x1) * 3;
var w00 = (1 - fx) * (1 - fy);
var w01 = fx * (1 - fy);
var w10 = (1 - fx) * fy;
var w11 = fx * fy;
b = img[i00] * w00 + img[i01] * w01 + img[i10] * w10 + img[i11] * w11;
g = img[i00 + 1] * w00 + img[i01 + 1] * w01 + img[i10 + 1] * w10 + img[i11 + 1] * w11;
r = img[i00 + 2] * w00 + img[i01 + 2] * w01 + img[i10 + 2] * w10 + img[i11 + 2] * w11;
}
private static float MaskCoord(int pixel, int frameSize, int maskSize)
=> (pixel + 0.5f) * maskSize / frameSize - 0.5f;
private static float SampleBilinear(SegmentationMask mask, float x, float y)
{
var x0 = (int)MathF.Floor(x);
var y0 = (int)MathF.Floor(y);
var fx = x - x0;
var fy = y - y0;
var x1 = Math.Clamp(x0 + 1, 0, mask.Width - 1);
var y1 = Math.Clamp(y0 + 1, 0, mask.Height - 1);
x0 = Math.Clamp(x0, 0, mask.Width - 1);
y0 = Math.Clamp(y0, 0, mask.Height - 1);
var v = mask.Values;
var top = v[y0 * mask.Width + x0] * (1 - fx) + v[y0 * mask.Width + x1] * fx;
var bottom = v[y1 * mask.Width + x0] * (1 - fx) + v[y1 * mask.Width + x1] * fx;
return Math.Clamp(top * (1 - fy) + bottom * fy, 0f, 1f);
}
private void LogFailureOnce(string reason)
{