TASK 51 slice 3: per-scene webcam BackgroundEffect + BackgroundStrength on WebcamSceneConfig, persisted via additive guarded columns; radio-pill bools RemoveBackground/BlurBackground (Unit: bg-effect config)

This commit is contained in:
2026-10-09 10:19:48 -07:00
parent 227ee8fca6
commit 55210b9a61
9 changed files with 126 additions and 13 deletions
+38
View File
@@ -15,4 +15,42 @@ public class WebcamSceneConfig : SceneElement
public override bool IsDraggable => true;
public override ElementKind Kind => ElementKind.Dynamic;
public override ImageSource? DisplaySource => VideoImageSource;
private BackgroundEffect _backgroundEffect = BackgroundEffect.None;
/// <summary>TASK 51: this scene's background effect for the webcam (persisted).</summary>
public BackgroundEffect BackgroundEffect
{
get => _backgroundEffect;
set
{
if (!Set(ref _backgroundEffect, value)) return;
Raise(nameof(RemoveBackground));
Raise(nameof(BlurBackground));
}
}
private double _backgroundStrength = 0.5;
/// <summary>TASK 51: 0..1 — Blur amount, or Remove's edge softness (persisted).</summary>
public double BackgroundStrength
{
get => _backgroundStrength;
set => Set(ref _backgroundStrength, Math.Clamp(value, 0, 1));
}
/// <summary>Pill-switch surface: Remove and Blur behave like radio buttons — turning one
/// on turns the other off; turning the active one off means <see cref="BackgroundEffect.None"/>.</summary>
public bool RemoveBackground
{
get => BackgroundEffect == BackgroundEffect.Remove;
set => BackgroundEffect = value ? BackgroundEffect.Remove
: BackgroundEffect == BackgroundEffect.Remove ? BackgroundEffect.None : BackgroundEffect;
}
/// <inheritdoc cref="RemoveBackground"/>
public bool BlurBackground
{
get => BackgroundEffect == BackgroundEffect.Blur;
set => BackgroundEffect = value ? BackgroundEffect.Blur
: BackgroundEffect == BackgroundEffect.Blur ? BackgroundEffect.None : BackgroundEffect;
}
}
+2 -2
View File
@@ -11,8 +11,8 @@ Plain data types. No logic beyond what a property can carry. See
| `SceneElement.cs` | Base for anything placeable in a scene: shared layout/clip/mirror/border surface, `virtual IsWebcam`/`virtual IsImageSource`/`virtual IsZoomLayer` (TASK 52); `ToggleClipShape` + persisted `RectWidth`/`RectHeight` (pre-Round rect so round→rect restores after reload) |
| `ZoomSceneElement.cs` | **Transient zoom box (TASK 52, creator-corrected 2026-10-08):** carries the box rect AND the anchored `CaptureX`/`CaptureY` (the fixed box/2 capture window set on `ActivateZoom`; moving the box never touches it). `IsZoomActive` flips `Kind` Static→Dynamic (leaves the baked base while active); never persisted (LayoutStore save skips non-Source/non-WebcamSceneConfig rows), never shown in the layers list or the element template (`IsZoomLayer` gates both). The compositor SKIPS the active element in every layer loop and draws its 2x content LAST (layer n+1) via `ZoomMagnifier`. |
| `Webcam.cs` | Singleton webcam identity: `Id`, `DeviceId`, `Name` (one row app-wide) |
| `BackgroundEffect.cs` | **TASK 51:** per-webcam background effect enum `None` / `Remove` / `Blur` (not yet on `WebcamSceneConfig` — config + persistence is slice 3) |
| `WebcamSceneConfig.cs` | Per-scene webcam placement (subclass of `SceneElement`): geometry + `IsVisible` + border (`BorderColor`/`BorderOpacity`/`BorderWidth`/`BorderAnimation`) + `VideoImageSource`; `WebcamId` links to `Webcam` |
| `BackgroundEffect.cs` | **TASK 51:** per-webcam background effect enum `None` / `Remove` / `Blur`, carried by `WebcamSceneConfig` |
| `WebcamSceneConfig.cs` | Per-scene webcam placement (subclass of `SceneElement`): geometry + `IsVisible` + border (`BorderColor`/`BorderOpacity`/`BorderWidth`/`BorderAnimation`) + `VideoImageSource`; `WebcamId` links to `Webcam`; **TASK 51:** `BackgroundEffect` + `BackgroundStrength` (0..1, default 0.5) + the radio-pill surface `RemoveBackground`/`BlurBackground` (one on turns the other off; both off = None) |
| `QualityOption.cs` | Resolution tier record `(Display, Fps, Bitrate, Width, Height)`; `Label` formats as `1080p60 (8Mbps)` — drives the bottom-bar dropdown and the output-rect math |
| `StreamConfig.cs` | `StreamConfig` (note: `TargetBitrate`/`Resolution` defaults are stale — not yet wired to the dropdown), `StreamHealth` (bitrate/FPS/dropped/duration), `StreamStatus` enum |
| `MicStatus.cs` | Footer mic indicator: `NotConnected` / `Connected` / `Problem` — the state behind the MIC button's status dot (red = no device/capture off, green = capturing, yellow = a requested mic failed: in use/unplugged) |
+3 -1
View File
@@ -165,7 +165,7 @@ public partial class LayoutStore : IDisposable
cmd.CommandText = """
SELECT SceneId, WebcamId, IsVisible, X, Y, Width, Height, Opacity,
ClipShape, IsMirrored, BorderColor, BorderOpacity, BorderWidth, BorderAnimation,
RectWidth, RectHeight, SortOrder
RectWidth, RectHeight, SortOrder, BackgroundEffect, BackgroundStrength
FROM WebcamSceneConfig ORDER BY SortOrder
""";
using var reader = cmd.ExecuteReader();
@@ -190,6 +190,8 @@ public partial class LayoutStore : IDisposable
BorderAnimation = Enum.TryParse<BorderAnimation>(reader.GetString(13), out var anim) ? anim : BorderAnimation.None,
RectWidth = reader.IsDBNull(14) ? null : reader.GetDouble(14),
RectHeight = reader.IsDBNull(15) ? null : reader.GetDouble(15),
BackgroundEffect = Enum.TryParse<BackgroundEffect>(reader.GetString(17), out var effect) ? effect : BackgroundEffect.None,
BackgroundStrength = reader.GetDouble(18),
};
if (!configsByScene.TryGetValue(sceneId, out var list))
configsByScene[sceneId] = list = new List<(WebcamSceneConfig, int)>();
+17
View File
@@ -75,6 +75,8 @@ public partial class LayoutStore : IDisposable
BorderOpacity REAL NOT NULL DEFAULT 1,
BorderWidth INTEGER NOT NULL DEFAULT 0,
BorderAnimation TEXT NOT NULL DEFAULT 'None',
BackgroundEffect TEXT NOT NULL DEFAULT 'None',
BackgroundStrength REAL NOT NULL DEFAULT 0.5,
SortOrder INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (SceneId, WebcamId)
);
@@ -360,6 +362,21 @@ public partial class LayoutStore : IDisposable
cmd.CommandText = "ALTER TABLE WebcamSceneConfig ADD COLUMN RectHeight REAL;";
cmd.ExecuteNonQuery();
}
// TASK 51: per-webcam background effect + strength (additive, no version bump).
if (!columns.Contains("BackgroundEffect"))
{
using var cmd = _connection.CreateCommand();
cmd.CommandText = "ALTER TABLE WebcamSceneConfig ADD COLUMN BackgroundEffect TEXT NOT NULL DEFAULT 'None';";
cmd.ExecuteNonQuery();
}
if (!columns.Contains("BackgroundStrength"))
{
using var cmd = _connection.CreateCommand();
cmd.CommandText = "ALTER TABLE WebcamSceneConfig ADD COLUMN BackgroundStrength REAL NOT NULL DEFAULT 0.5;";
cmd.ExecuteNonQuery();
}
}
// v2 → v3: the webcam leaves the Source table. Any webcam Source rows become
+6 -2
View File
@@ -209,10 +209,10 @@ public partial class LayoutStore : IDisposable
INSERT INTO WebcamSceneConfig
(SceneId, WebcamId, IsVisible, X, Y, Width, Height, Opacity,
ClipShape, IsMirrored, BorderColor, BorderOpacity, BorderWidth, BorderAnimation,
RectWidth, RectHeight, SortOrder)
RectWidth, RectHeight, SortOrder, BackgroundEffect, BackgroundStrength)
VALUES ($sceneId, $webcamId, $isVisible, $x, $y, $w, $h, $opacity,
$clip, $mirrored, $borderColor, $borderOpacity, $borderWidth, $borderAnimation,
$rectWidth, $rectHeight, $sort)
$rectWidth, $rectHeight, $sort, $bgEffect, $bgStrength)
""";
cmd.Transaction = tx;
var sceneIdP = cmd.Parameters.Add("$sceneId", SqliteType.Text);
@@ -232,6 +232,8 @@ public partial class LayoutStore : IDisposable
var rectWidthP = cmd.Parameters.Add("$rectWidth", SqliteType.Real);
var rectHeightP = cmd.Parameters.Add("$rectHeight", SqliteType.Real);
var sortP = cmd.Parameters.Add("$sort", SqliteType.Integer);
var bgEffectP = cmd.Parameters.Add("$bgEffect", SqliteType.Text);
var bgStrengthP = cmd.Parameters.Add("$bgStrength", SqliteType.Real);
foreach (var scene in scenes)
{
@@ -239,6 +241,8 @@ public partial class LayoutStore : IDisposable
foreach (var element in scene.Elements)
{
if (element is not WebcamSceneConfig config) { z++; continue; }
bgEffectP.Value = config.BackgroundEffect.ToString();
bgStrengthP.Value = config.BackgroundStrength;
sceneIdP.Value = scene.Id;
webcamIdP.Value = config.WebcamId;
visibleP.Value = config.IsVisible ? 1 : 0;
+1 -1
View File
@@ -14,7 +14,7 @@ External-facing logic: YouTube API, persistence. See
| `AlertRenderer.cs` | **The alert rasterizer (TASK 43, 2026-09-24):** renders one event's celebration animation into a BGRA `VideoFrame` — SIX DISTINCT animations (SuperChat slide-up/shine/count-up, SuperSticker scale-pop, NewMember drop-in/flash-band, MemberGift slide-left/chip-fan, GiftReceived confetti scale-in, MemberMilestone rise/growth-bar), enter/hold/exit phases, brand line on every card. **Still the fallback** when no alert clip is playable (TASK 47) |
| `AlertOverlayLayer.cs` | **The native alert-box component (TASK 43; video branch TASK 47):** event queue (cap 10) + 33ms dispatch ticker + `RenderFrame` cache-first live path + `UpdatePreview`; **`Advance(double)` is the deterministic test clock** (tests never start the timer); `ChatEventKind.None` rows (sub mentions) never enqueue; idle = transparent (null frame). Since TASK 47 a clip branch owns `IAlertClipDecoder` per play — ctor seams `(clipDecoderFactory, clipPathResolver, audioSink)`, fade in/out over `FadeDurationSeconds = 0.30` (straight-alpha copies + volume×fade audio), EOF freeze-frame → drain, and `AlertTickerFrame` composes the scrolling message strip |
| `AlertClipDecoder.cs` | **Per-play alert video decoder (TASK 47):** `IAlertClipDecoder` (Start/Stop/Dispose + `FrameAvailable`/`AudioReady`/`Completed`) + ffmpeg child emitting `rawvideo bgra` (`-vf scale=W:H`, read via `RawVideoFrameReader`) and `f32le -ar 48000 -ac 2` on a **carry-buffer** audio loop (float boundary straddles never drop samples), both real-time paced; bgra is opaque (layer owns alpha). One-shot lifetime — created per alert, disposed at drain; tests inject a fake |
| `LayoutStore.cs` | SQLite persistence (`Microsoft.Data.Sqlite`) at `%APPDATA%\ytLlive\ytLlive.db`; assets stored as BLOBs keyed by SHA-256 content hash; save/open layout files; schema `user_version` 10 (`Source.ClipShape`/`IsMirrored` via `ALTER TABLE` for pre-v2 DBs; v3 = singleton `Webcam` + per-scene `WebcamSceneConfig`, migrated idempotently **without backfill** — the stale `Source.DeviceId` column remains but is no longer read/written; v4 = `WebcamSceneConfig.RectWidth`/`RectHeight`, the pre-Round rect for the round-to-rect restore; v5 = `Source.IsBackdrop` + `Source.CaptureKey`, the live-capture backdrop; v6 = `Scene.HasBackdrop` — Live-only policy, one-time backfill turns Starting/BRB/Chat/Ending off + drops their backdrop sources; v9→v10 (TASK 39) = `YppSnapshot` history table; all later columns stay additive guarded ALTERs — TASK 47 adds the five alert `Source` columns (`AlertVideoAssetId`/`AlertVideoPath`/`AlertUseDefaultVideo`/`AlertShowTicker`/`AlertVideoVolume`) without a version bump; `MainViewModel.NormalizeBackgrounds` re-normalizes on every load — TASK 25: one locked "Background" per canonical screen (static art everywhere, DisplayCapture on Live), duplicates dropped, index 0, orphaned `BackgroundUseDefault_*`/`BackgroundPath_*` Settings keys purged at save). Settings key/value table holds app state: `MicSourceName`, `ReusableStream` (TASK 9 — the cached reusable stream's id/address/name so the pump has its RTMP URL at startup), `AlertDefaultVideoAssetId` (TASK 47 — the stamped built-in alert clip's asset row; save's prune exempts it via UNION) |
| `LayoutStore.cs` | SQLite persistence (`Microsoft.Data.Sqlite`) at `%APPDATA%\ytLlive\ytLlive.db`; assets stored as BLOBs keyed by SHA-256 content hash; save/open layout files; schema `user_version` 10 (`Source.ClipShape`/`IsMirrored` via `ALTER TABLE` for pre-v2 DBs; v3 = singleton `Webcam` + per-scene `WebcamSceneConfig`, migrated idempotently **without backfill** — the stale `Source.DeviceId` column remains but is no longer read/written; v4 = `WebcamSceneConfig.RectWidth`/`RectHeight`, the pre-Round rect for the round-to-rect restore; v5 = `Source.IsBackdrop` + `Source.CaptureKey`, the live-capture backdrop; v6 = `Scene.HasBackdrop` — Live-only policy, one-time backfill turns Starting/BRB/Chat/Ending off + drops their backdrop sources; v9→v10 (TASK 39) = `YppSnapshot` history table; all later columns stay additive guarded ALTERs — TASK 51 adds `WebcamSceneConfig.BackgroundEffect`/`BackgroundStrength`; TASK 47 adds the five alert `Source` columns (`AlertVideoAssetId`/`AlertVideoPath`/`AlertUseDefaultVideo`/`AlertShowTicker`/`AlertVideoVolume`) without a version bump; `MainViewModel.NormalizeBackgrounds` re-normalizes on every load — TASK 25: one locked "Background" per canonical screen (static art everywhere, DisplayCapture on Live), duplicates dropped, index 0, orphaned `BackgroundUseDefault_*`/`BackgroundPath_*` Settings keys purged at save). Settings key/value table holds app state: `MicSourceName`, `ReusableStream` (TASK 9 — the cached reusable stream's id/address/name so the pump has its RTMP URL at startup), `AlertDefaultVideoAssetId` (TASK 47 — the stamped built-in alert clip's asset row; save's prune exempts it via UNION) |
| `VideoFrame.cs` | Normalized CPU frame seam (`Width`/`Height`/tightly-packed BGRA `byte[]`) — the only pixel type the rest of the app knows about; future capture sources (screen, background-removed webcam) feed the same seam |
| `BackgroundEffects/IPersonSegmenter.cs` | **TASK 51 seam:** `Segment(VideoFrame) → SegmentationMask?` (`SegmentationMask(Width, Height, float[] Values)`, 0 = background, 1 = person); null = no mask → caller passes through. Tests inject fakes |
| `BackgroundEffects/OnnxPersonSegmenter.cs` | **TASK 51 default segmenter:** MediaPipe Selfie Segmenter (landscape, Apache-2.0) as an embedded ONNX resource, run by onnxruntime on the CPU (2 intra-op threads); frame stretched to 256×144 RGB/255 NHWC; lazy session; never throws (fail → log once, null forever) |
+4 -4
View File
@@ -69,8 +69,8 @@ delivery shape changed:
## 2. ☐ Config surface + persistence
- ☐ `WebcamSceneConfig` gains `BackgroundEffect` (`None | Remove | Blur`) + `Strength`.
- ☐ Persist via the existing `WebcamSceneConfig` table path in `LayoutStore.Save.cs`/
- ✅ `WebcamSceneConfig` gains `BackgroundEffect` (`None | Remove | Blur`) + `BackgroundStrength` (slice 3).
- ✅ Persist via the existing `WebcamSceneConfig` table path in `LayoutStore.Save.cs`/
`Load.cs` (new columns + migration). Follow `LayoutStore.Migrations.cs` conventions.
- ☐ **Properties UI (creator spec 2026-10-09):** in the webcam's properties in the **Layers
panel**, two **pill switches** — "Remove background" and "Blur background" — using the existing
@@ -92,7 +92,7 @@ delivery shape changed:
- ✅ **Bundled-model contract:** the embedded ONNX model through the C# preprocessing reproduces
MediaPipe's reference mask on a fixture (≥99.5% agreement). This is a fixed, deterministic
contract check of the shipped bytes — not a camera/lighting quality test.
- ☐ Config → persistence roundtrip (per webcam element).
- ✅ Config → persistence roundtrip per scene + legacy-DB migration (`LayoutStorePersistenceTests`).
- ☐ Engine failure → passthrough + single log, layer never dropped (code path exists; dedicated
test lands with the hookup slice).
- ☐ Real inference: **manual on the creator's box** (camera + lighting), not in CI.
@@ -103,7 +103,7 @@ delivery shape changed:
## 4. Sequencing
- ✅ Separate work unit after the zoom build (zoom shipped 2026-10-08).
- Slice order: 1 engine ✅ → 2 Blur ✅ → 3 config + persistence → 4 camera-path hookup (one
- Slice order: 1 engine ✅ → 2 Blur ✅ → 3 config + persistence ✅ → 4 camera-path hookup (one
processing point, both consumers) + perf check → 5 properties-panel UI.
## Related (not duplicated here)
+4 -3
View File
@@ -641,9 +641,10 @@ This replaces the old five-seeder cluster (`Seed{Starting,Brb,Ending,Chat}Backgr
untouched, opacity preserved. The mask is computed once per frame (`TrySegment`) and rendered per
effect (`Render`), so several variants share one inference. No mask / segmenter exception →
passthrough + one log line, never a dropped layer.
- **Not wired yet:** nothing calls the processor — no config, no UI, no `CameraManager` hookup. The
next slices add Blur, the per-webcam `BackgroundEffect` config + persistence, then the single
processing point on the camera path (both preview and compositor consume the processed frame).
- **Config (slice 3):** per scene, on `WebcamSceneConfig` — `BackgroundEffect` + `BackgroundStrength`
(0..1, default 0.5), persisted as two additive guarded columns on `WebcamSceneConfig`
(`BackgroundEffect TEXT DEFAULT 'None'`, `BackgroundStrength REAL DEFAULT 0.5`, no version bump —
the TASK 47 convention). `RemoveBackground`/`BlurBackground` are the radio-pill bools the UI binds.
- **Tests:** `BackgroundEffectEngineTests` — the Remove integration test (fake segmenter), and a
model contract test that runs the BUNDLED model through the C# preprocessing on a fixture and
matches the MediaPipe-produced reference mask (≥99.5% agreement; fixture = MediaPipe's public test
@@ -128,6 +128,57 @@ public class LayoutStorePersistenceTests
}
}
/// <summary>TASK 51 slice 3: each scene's webcam background effect + strength survives a
/// save and a reopen; a pre-TASK-51 DB (no columns) migrates in place and reads None/0.5.</summary>
[Fact]
public void Webcam_Background_Effect_Round_Trips_Per_Scene_And_Legacy_Db_Migrates()
{
var path = Path.Combine(Path.GetTempPath(), $"ytLlive-layout-{Guid.NewGuid():N}.db");
try
{
var webcam = new Webcam { DeviceId = "real-device", Name = "Logitech" };
var live = new Scene { Name = "Live" };
live.Elements.Add(new WebcamSceneConfig { WebcamId = webcam.Id, Width = 480, Height = 270, BackgroundEffect = BackgroundEffect.Remove, BackgroundStrength = 0.2 });
var chat = new Scene { Name = "Chat" };
chat.Elements.Add(new WebcamSceneConfig { WebcamId = webcam.Id, Width = 480, Height = 270, BackgroundEffect = BackgroundEffect.Blur, BackgroundStrength = 0.8 });
using (var store = new LayoutStore(path))
store.Save(new[] { live, chat }, webcam, null);
using (var reopened = new LayoutStore(path))
{
var scenes = reopened.Load();
var liveCfg = scenes.Single(s => s.Name == "Live").Elements.OfType<WebcamSceneConfig>().Single();
var chatCfg = scenes.Single(s => s.Name == "Chat").Elements.OfType<WebcamSceneConfig>().Single();
Assert.Equal(BackgroundEffect.Remove, liveCfg.BackgroundEffect);
Assert.Equal(0.2, liveCfg.BackgroundStrength, 6);
Assert.Equal(BackgroundEffect.Blur, chatCfg.BackgroundEffect);
Assert.Equal(0.8, chatCfg.BackgroundStrength, 6);
}
// Simulate a DB written before TASK 51: drop the two columns, reopen.
SqliteConnection.ClearAllPools();
using (var raw = new SqliteConnection($"Data Source={path}"))
{
raw.Open();
using var cmd = raw.CreateCommand();
cmd.CommandText = "ALTER TABLE WebcamSceneConfig DROP COLUMN BackgroundEffect; ALTER TABLE WebcamSceneConfig DROP COLUMN BackgroundStrength;";
cmd.ExecuteNonQuery();
}
SqliteConnection.ClearAllPools();
using (var legacy = new LayoutStore(path))
{
var cfg = legacy.Load().SelectMany(s => s.Elements).OfType<WebcamSceneConfig>().First();
Assert.Equal(BackgroundEffect.None, cfg.BackgroundEffect);
Assert.Equal(0.5, cfg.BackgroundStrength, 6);
}
}
finally
{
SqliteConnection.ClearAllPools();
try { File.Delete(path); } catch { /* best-effort cleanup */ }
}
}
// The background (schema v5): permanent bottom layer holding live desktop/game
// capture. IsBackground + CaptureKey must survive a save + reload so a
// re-designated target isn't lost on restart.