diff --git a/Models/WebcamSceneConfig.cs b/Models/WebcamSceneConfig.cs index dc82b50..2e3ed75 100644 --- a/Models/WebcamSceneConfig.cs +++ b/Models/WebcamSceneConfig.cs @@ -15,4 +15,42 @@ public class WebcamSceneConfig : SceneElement public override bool IsDraggable => true; public override ElementKind Kind => ElementKind.Dynamic; public override ImageSource? DisplaySource => VideoImageSource; + + private BackgroundEffect _backgroundEffect = BackgroundEffect.None; + /// TASK 51: this scene's background effect for the webcam (persisted). + public BackgroundEffect BackgroundEffect + { + get => _backgroundEffect; + set + { + if (!Set(ref _backgroundEffect, value)) return; + Raise(nameof(RemoveBackground)); + Raise(nameof(BlurBackground)); + } + } + + private double _backgroundStrength = 0.5; + /// TASK 51: 0..1 — Blur amount, or Remove's edge softness (persisted). + public double BackgroundStrength + { + get => _backgroundStrength; + set => Set(ref _backgroundStrength, Math.Clamp(value, 0, 1)); + } + + /// Pill-switch surface: Remove and Blur behave like radio buttons — turning one + /// on turns the other off; turning the active one off means . + public bool RemoveBackground + { + get => BackgroundEffect == BackgroundEffect.Remove; + set => BackgroundEffect = value ? BackgroundEffect.Remove + : BackgroundEffect == BackgroundEffect.Remove ? BackgroundEffect.None : BackgroundEffect; + } + + /// + public bool BlurBackground + { + get => BackgroundEffect == BackgroundEffect.Blur; + set => BackgroundEffect = value ? BackgroundEffect.Blur + : BackgroundEffect == BackgroundEffect.Blur ? BackgroundEffect.None : BackgroundEffect; + } } diff --git a/Models/index.md b/Models/index.md index a22d217..a1915a1 100644 --- a/Models/index.md +++ b/Models/index.md @@ -11,8 +11,8 @@ Plain data types. No logic beyond what a property can carry. See | `SceneElement.cs` | Base for anything placeable in a scene: shared layout/clip/mirror/border surface, `virtual IsWebcam`/`virtual IsImageSource`/`virtual IsZoomLayer` (TASK 52); `ToggleClipShape` + persisted `RectWidth`/`RectHeight` (pre-Round rect so round→rect restores after reload) | | `ZoomSceneElement.cs` | **Transient zoom box (TASK 52, creator-corrected 2026-10-08):** carries the box rect AND the anchored `CaptureX`/`CaptureY` (the fixed box/2 capture window set on `ActivateZoom`; moving the box never touches it). `IsZoomActive` flips `Kind` Static→Dynamic (leaves the baked base while active); never persisted (LayoutStore save skips non-Source/non-WebcamSceneConfig rows), never shown in the layers list or the element template (`IsZoomLayer` gates both). The compositor SKIPS the active element in every layer loop and draws its 2x content LAST (layer n+1) via `ZoomMagnifier`. | | `Webcam.cs` | Singleton webcam identity: `Id`, `DeviceId`, `Name` (one row app-wide) | -| `BackgroundEffect.cs` | **TASK 51:** per-webcam background effect enum `None` / `Remove` / `Blur` (not yet on `WebcamSceneConfig` — config + persistence is slice 3) | -| `WebcamSceneConfig.cs` | Per-scene webcam placement (subclass of `SceneElement`): geometry + `IsVisible` + border (`BorderColor`/`BorderOpacity`/`BorderWidth`/`BorderAnimation`) + `VideoImageSource`; `WebcamId` links to `Webcam` | +| `BackgroundEffect.cs` | **TASK 51:** per-webcam background effect enum `None` / `Remove` / `Blur`, carried by `WebcamSceneConfig` | +| `WebcamSceneConfig.cs` | Per-scene webcam placement (subclass of `SceneElement`): geometry + `IsVisible` + border (`BorderColor`/`BorderOpacity`/`BorderWidth`/`BorderAnimation`) + `VideoImageSource`; `WebcamId` links to `Webcam`; **TASK 51:** `BackgroundEffect` + `BackgroundStrength` (0..1, default 0.5) + the radio-pill surface `RemoveBackground`/`BlurBackground` (one on turns the other off; both off = None) | | `QualityOption.cs` | Resolution tier record `(Display, Fps, Bitrate, Width, Height)`; `Label` formats as `1080p60 (8Mbps)` — drives the bottom-bar dropdown and the output-rect math | | `StreamConfig.cs` | `StreamConfig` (note: `TargetBitrate`/`Resolution` defaults are stale — not yet wired to the dropdown), `StreamHealth` (bitrate/FPS/dropped/duration), `StreamStatus` enum | | `MicStatus.cs` | Footer mic indicator: `NotConnected` / `Connected` / `Problem` — the state behind the MIC button's status dot (red = no device/capture off, green = capturing, yellow = a requested mic failed: in use/unplugged) | diff --git a/Services/LayoutStore.Load.cs b/Services/LayoutStore.Load.cs index 3bbe946..5543862 100644 --- a/Services/LayoutStore.Load.cs +++ b/Services/LayoutStore.Load.cs @@ -165,7 +165,7 @@ public partial class LayoutStore : IDisposable cmd.CommandText = """ SELECT SceneId, WebcamId, IsVisible, X, Y, Width, Height, Opacity, ClipShape, IsMirrored, BorderColor, BorderOpacity, BorderWidth, BorderAnimation, - RectWidth, RectHeight, SortOrder + RectWidth, RectHeight, SortOrder, BackgroundEffect, BackgroundStrength FROM WebcamSceneConfig ORDER BY SortOrder """; using var reader = cmd.ExecuteReader(); @@ -190,6 +190,8 @@ public partial class LayoutStore : IDisposable BorderAnimation = Enum.TryParse(reader.GetString(13), out var anim) ? anim : BorderAnimation.None, RectWidth = reader.IsDBNull(14) ? null : reader.GetDouble(14), RectHeight = reader.IsDBNull(15) ? null : reader.GetDouble(15), + BackgroundEffect = Enum.TryParse(reader.GetString(17), out var effect) ? effect : BackgroundEffect.None, + BackgroundStrength = reader.GetDouble(18), }; if (!configsByScene.TryGetValue(sceneId, out var list)) configsByScene[sceneId] = list = new List<(WebcamSceneConfig, int)>(); diff --git a/Services/LayoutStore.Migrations.cs b/Services/LayoutStore.Migrations.cs index 41f3d7b..2b26f47 100644 --- a/Services/LayoutStore.Migrations.cs +++ b/Services/LayoutStore.Migrations.cs @@ -75,6 +75,8 @@ public partial class LayoutStore : IDisposable BorderOpacity REAL NOT NULL DEFAULT 1, BorderWidth INTEGER NOT NULL DEFAULT 0, BorderAnimation TEXT NOT NULL DEFAULT 'None', + BackgroundEffect TEXT NOT NULL DEFAULT 'None', + BackgroundStrength REAL NOT NULL DEFAULT 0.5, SortOrder INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (SceneId, WebcamId) ); @@ -360,6 +362,21 @@ public partial class LayoutStore : IDisposable cmd.CommandText = "ALTER TABLE WebcamSceneConfig ADD COLUMN RectHeight REAL;"; cmd.ExecuteNonQuery(); } + + // TASK 51: per-webcam background effect + strength (additive, no version bump). + if (!columns.Contains("BackgroundEffect")) + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = "ALTER TABLE WebcamSceneConfig ADD COLUMN BackgroundEffect TEXT NOT NULL DEFAULT 'None';"; + cmd.ExecuteNonQuery(); + } + + if (!columns.Contains("BackgroundStrength")) + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = "ALTER TABLE WebcamSceneConfig ADD COLUMN BackgroundStrength REAL NOT NULL DEFAULT 0.5;"; + cmd.ExecuteNonQuery(); + } } // v2 → v3: the webcam leaves the Source table. Any webcam Source rows become diff --git a/Services/LayoutStore.Save.cs b/Services/LayoutStore.Save.cs index 1d42e1b..7b82e29 100644 --- a/Services/LayoutStore.Save.cs +++ b/Services/LayoutStore.Save.cs @@ -209,10 +209,10 @@ public partial class LayoutStore : IDisposable INSERT INTO WebcamSceneConfig (SceneId, WebcamId, IsVisible, X, Y, Width, Height, Opacity, ClipShape, IsMirrored, BorderColor, BorderOpacity, BorderWidth, BorderAnimation, - RectWidth, RectHeight, SortOrder) + RectWidth, RectHeight, SortOrder, BackgroundEffect, BackgroundStrength) VALUES ($sceneId, $webcamId, $isVisible, $x, $y, $w, $h, $opacity, $clip, $mirrored, $borderColor, $borderOpacity, $borderWidth, $borderAnimation, - $rectWidth, $rectHeight, $sort) + $rectWidth, $rectHeight, $sort, $bgEffect, $bgStrength) """; cmd.Transaction = tx; var sceneIdP = cmd.Parameters.Add("$sceneId", SqliteType.Text); @@ -232,6 +232,8 @@ public partial class LayoutStore : IDisposable var rectWidthP = cmd.Parameters.Add("$rectWidth", SqliteType.Real); var rectHeightP = cmd.Parameters.Add("$rectHeight", SqliteType.Real); var sortP = cmd.Parameters.Add("$sort", SqliteType.Integer); + var bgEffectP = cmd.Parameters.Add("$bgEffect", SqliteType.Text); + var bgStrengthP = cmd.Parameters.Add("$bgStrength", SqliteType.Real); foreach (var scene in scenes) { @@ -239,6 +241,8 @@ public partial class LayoutStore : IDisposable foreach (var element in scene.Elements) { if (element is not WebcamSceneConfig config) { z++; continue; } + bgEffectP.Value = config.BackgroundEffect.ToString(); + bgStrengthP.Value = config.BackgroundStrength; sceneIdP.Value = scene.Id; webcamIdP.Value = config.WebcamId; visibleP.Value = config.IsVisible ? 1 : 0; diff --git a/Services/index.md b/Services/index.md index 6a3add1..370f8bd 100644 --- a/Services/index.md +++ b/Services/index.md @@ -14,7 +14,7 @@ External-facing logic: YouTube API, persistence. See | `AlertRenderer.cs` | **The alert rasterizer (TASK 43, 2026-09-24):** renders one event's celebration animation into a BGRA `VideoFrame` — SIX DISTINCT animations (SuperChat slide-up/shine/count-up, SuperSticker scale-pop, NewMember drop-in/flash-band, MemberGift slide-left/chip-fan, GiftReceived confetti scale-in, MemberMilestone rise/growth-bar), enter/hold/exit phases, brand line on every card. **Still the fallback** when no alert clip is playable (TASK 47) | | `AlertOverlayLayer.cs` | **The native alert-box component (TASK 43; video branch TASK 47):** event queue (cap 10) + 33ms dispatch ticker + `RenderFrame` cache-first live path + `UpdatePreview`; **`Advance(double)` is the deterministic test clock** (tests never start the timer); `ChatEventKind.None` rows (sub mentions) never enqueue; idle = transparent (null frame). Since TASK 47 a clip branch owns `IAlertClipDecoder` per play — ctor seams `(clipDecoderFactory, clipPathResolver, audioSink)`, fade in/out over `FadeDurationSeconds = 0.30` (straight-alpha copies + volume×fade audio), EOF freeze-frame → drain, and `AlertTickerFrame` composes the scrolling message strip | | `AlertClipDecoder.cs` | **Per-play alert video decoder (TASK 47):** `IAlertClipDecoder` (Start/Stop/Dispose + `FrameAvailable`/`AudioReady`/`Completed`) + ffmpeg child emitting `rawvideo bgra` (`-vf scale=W:H`, read via `RawVideoFrameReader`) and `f32le -ar 48000 -ac 2` on a **carry-buffer** audio loop (float boundary straddles never drop samples), both real-time paced; bgra is opaque (layer owns alpha). One-shot lifetime — created per alert, disposed at drain; tests inject a fake | -| `LayoutStore.cs` | SQLite persistence (`Microsoft.Data.Sqlite`) at `%APPDATA%\ytLlive\ytLlive.db`; assets stored as BLOBs keyed by SHA-256 content hash; save/open layout files; schema `user_version` 10 (`Source.ClipShape`/`IsMirrored` via `ALTER TABLE` for pre-v2 DBs; v3 = singleton `Webcam` + per-scene `WebcamSceneConfig`, migrated idempotently **without backfill** — the stale `Source.DeviceId` column remains but is no longer read/written; v4 = `WebcamSceneConfig.RectWidth`/`RectHeight`, the pre-Round rect for the round-to-rect restore; v5 = `Source.IsBackdrop` + `Source.CaptureKey`, the live-capture backdrop; v6 = `Scene.HasBackdrop` — Live-only policy, one-time backfill turns Starting/BRB/Chat/Ending off + drops their backdrop sources; v9→v10 (TASK 39) = `YppSnapshot` history table; all later columns stay additive guarded ALTERs — TASK 47 adds the five alert `Source` columns (`AlertVideoAssetId`/`AlertVideoPath`/`AlertUseDefaultVideo`/`AlertShowTicker`/`AlertVideoVolume`) without a version bump; `MainViewModel.NormalizeBackgrounds` re-normalizes on every load — TASK 25: one locked "Background" per canonical screen (static art everywhere, DisplayCapture on Live), duplicates dropped, index 0, orphaned `BackgroundUseDefault_*`/`BackgroundPath_*` Settings keys purged at save). Settings key/value table holds app state: `MicSourceName`, `ReusableStream` (TASK 9 — the cached reusable stream's id/address/name so the pump has its RTMP URL at startup), `AlertDefaultVideoAssetId` (TASK 47 — the stamped built-in alert clip's asset row; save's prune exempts it via UNION) | +| `LayoutStore.cs` | SQLite persistence (`Microsoft.Data.Sqlite`) at `%APPDATA%\ytLlive\ytLlive.db`; assets stored as BLOBs keyed by SHA-256 content hash; save/open layout files; schema `user_version` 10 (`Source.ClipShape`/`IsMirrored` via `ALTER TABLE` for pre-v2 DBs; v3 = singleton `Webcam` + per-scene `WebcamSceneConfig`, migrated idempotently **without backfill** — the stale `Source.DeviceId` column remains but is no longer read/written; v4 = `WebcamSceneConfig.RectWidth`/`RectHeight`, the pre-Round rect for the round-to-rect restore; v5 = `Source.IsBackdrop` + `Source.CaptureKey`, the live-capture backdrop; v6 = `Scene.HasBackdrop` — Live-only policy, one-time backfill turns Starting/BRB/Chat/Ending off + drops their backdrop sources; v9→v10 (TASK 39) = `YppSnapshot` history table; all later columns stay additive guarded ALTERs — TASK 51 adds `WebcamSceneConfig.BackgroundEffect`/`BackgroundStrength`; TASK 47 adds the five alert `Source` columns (`AlertVideoAssetId`/`AlertVideoPath`/`AlertUseDefaultVideo`/`AlertShowTicker`/`AlertVideoVolume`) without a version bump; `MainViewModel.NormalizeBackgrounds` re-normalizes on every load — TASK 25: one locked "Background" per canonical screen (static art everywhere, DisplayCapture on Live), duplicates dropped, index 0, orphaned `BackgroundUseDefault_*`/`BackgroundPath_*` Settings keys purged at save). Settings key/value table holds app state: `MicSourceName`, `ReusableStream` (TASK 9 — the cached reusable stream's id/address/name so the pump has its RTMP URL at startup), `AlertDefaultVideoAssetId` (TASK 47 — the stamped built-in alert clip's asset row; save's prune exempts it via UNION) | | `VideoFrame.cs` | Normalized CPU frame seam (`Width`/`Height`/tightly-packed BGRA `byte[]`) — the only pixel type the rest of the app knows about; future capture sources (screen, background-removed webcam) feed the same seam | | `BackgroundEffects/IPersonSegmenter.cs` | **TASK 51 seam:** `Segment(VideoFrame) → SegmentationMask?` (`SegmentationMask(Width, Height, float[] Values)`, 0 = background, 1 = person); null = no mask → caller passes through. Tests inject fakes | | `BackgroundEffects/OnnxPersonSegmenter.cs` | **TASK 51 default segmenter:** MediaPipe Selfie Segmenter (landscape, Apache-2.0) as an embedded ONNX resource, run by onnxruntime on the CPU (2 intra-op threads); frame stretched to 256×144 RGB/255 NHWC; lazy session; never throws (fail → log once, null forever) | diff --git a/TASKS/task-51-webcam-bg-effects.md b/TASKS/task-51-webcam-bg-effects.md index 19ce9d6..4bd0e9a 100644 --- a/TASKS/task-51-webcam-bg-effects.md +++ b/TASKS/task-51-webcam-bg-effects.md @@ -69,8 +69,8 @@ delivery shape changed: ## 2. ☐ Config surface + persistence -- ☐ `WebcamSceneConfig` gains `BackgroundEffect` (`None | Remove | Blur`) + `Strength`. -- ☐ Persist via the existing `WebcamSceneConfig` table path in `LayoutStore.Save.cs`/ +- ✅ `WebcamSceneConfig` gains `BackgroundEffect` (`None | Remove | Blur`) + `BackgroundStrength` (slice 3). +- ✅ Persist via the existing `WebcamSceneConfig` table path in `LayoutStore.Save.cs`/ `Load.cs` (new columns + migration). Follow `LayoutStore.Migrations.cs` conventions. - ☐ **Properties UI (creator spec 2026-10-09):** in the webcam's properties in the **Layers panel**, two **pill switches** — "Remove background" and "Blur background" — using the existing @@ -92,7 +92,7 @@ delivery shape changed: - ✅ **Bundled-model contract:** the embedded ONNX model through the C# preprocessing reproduces MediaPipe's reference mask on a fixture (≥99.5% agreement). This is a fixed, deterministic contract check of the shipped bytes — not a camera/lighting quality test. -- ☐ Config → persistence roundtrip (per webcam element). +- ✅ Config → persistence roundtrip per scene + legacy-DB migration (`LayoutStorePersistenceTests`). - ☐ Engine failure → passthrough + single log, layer never dropped (code path exists; dedicated test lands with the hookup slice). - ☐ Real inference: **manual on the creator's box** (camera + lighting), not in CI. @@ -103,7 +103,7 @@ delivery shape changed: ## 4. Sequencing - ✅ Separate work unit after the zoom build (zoom shipped 2026-10-08). -- Slice order: 1 engine ✅ → 2 Blur ✅ → 3 config + persistence → 4 camera-path hookup (one +- Slice order: 1 engine ✅ → 2 Blur ✅ → 3 config + persistence ✅ → 4 camera-path hookup (one processing point, both consumers) + perf check → 5 properties-panel UI. ## Related (not duplicated here) diff --git a/ai.md b/ai.md index cf04c01..28a304b 100644 --- a/ai.md +++ b/ai.md @@ -641,9 +641,10 @@ This replaces the old five-seeder cluster (`Seed{Starting,Brb,Ending,Chat}Backgr untouched, opacity preserved. The mask is computed once per frame (`TrySegment`) and rendered per effect (`Render`), so several variants share one inference. No mask / segmenter exception → passthrough + one log line, never a dropped layer. -- **Not wired yet:** nothing calls the processor — no config, no UI, no `CameraManager` hookup. The - next slices add Blur, the per-webcam `BackgroundEffect` config + persistence, then the single - processing point on the camera path (both preview and compositor consume the processed frame). +- **Config (slice 3):** per scene, on `WebcamSceneConfig` — `BackgroundEffect` + `BackgroundStrength` + (0..1, default 0.5), persisted as two additive guarded columns on `WebcamSceneConfig` + (`BackgroundEffect TEXT DEFAULT 'None'`, `BackgroundStrength REAL DEFAULT 0.5`, no version bump — + the TASK 47 convention). `RemoveBackground`/`BlurBackground` are the radio-pill bools the UI binds. - **Tests:** `BackgroundEffectEngineTests` — the Remove integration test (fake segmenter), and a model contract test that runs the BUNDLED model through the C# preprocessing on a fixture and matches the MediaPipe-produced reference mask (≥99.5% agreement; fixture = MediaPipe's public test diff --git a/ytLive.Tests/LayoutStorePersistenceTests.cs b/ytLive.Tests/LayoutStorePersistenceTests.cs index d779c75..4eb5c9c 100644 --- a/ytLive.Tests/LayoutStorePersistenceTests.cs +++ b/ytLive.Tests/LayoutStorePersistenceTests.cs @@ -128,6 +128,57 @@ public class LayoutStorePersistenceTests } } + /// TASK 51 slice 3: each scene's webcam background effect + strength survives a + /// save and a reopen; a pre-TASK-51 DB (no columns) migrates in place and reads None/0.5. + [Fact] + public void Webcam_Background_Effect_Round_Trips_Per_Scene_And_Legacy_Db_Migrates() + { + var path = Path.Combine(Path.GetTempPath(), $"ytLlive-layout-{Guid.NewGuid():N}.db"); + try + { + var webcam = new Webcam { DeviceId = "real-device", Name = "Logitech" }; + var live = new Scene { Name = "Live" }; + live.Elements.Add(new WebcamSceneConfig { WebcamId = webcam.Id, Width = 480, Height = 270, BackgroundEffect = BackgroundEffect.Remove, BackgroundStrength = 0.2 }); + var chat = new Scene { Name = "Chat" }; + chat.Elements.Add(new WebcamSceneConfig { WebcamId = webcam.Id, Width = 480, Height = 270, BackgroundEffect = BackgroundEffect.Blur, BackgroundStrength = 0.8 }); + using (var store = new LayoutStore(path)) + store.Save(new[] { live, chat }, webcam, null); + + using (var reopened = new LayoutStore(path)) + { + var scenes = reopened.Load(); + var liveCfg = scenes.Single(s => s.Name == "Live").Elements.OfType().Single(); + var chatCfg = scenes.Single(s => s.Name == "Chat").Elements.OfType().Single(); + Assert.Equal(BackgroundEffect.Remove, liveCfg.BackgroundEffect); + Assert.Equal(0.2, liveCfg.BackgroundStrength, 6); + Assert.Equal(BackgroundEffect.Blur, chatCfg.BackgroundEffect); + Assert.Equal(0.8, chatCfg.BackgroundStrength, 6); + } + + // Simulate a DB written before TASK 51: drop the two columns, reopen. + SqliteConnection.ClearAllPools(); + using (var raw = new SqliteConnection($"Data Source={path}")) + { + raw.Open(); + using var cmd = raw.CreateCommand(); + cmd.CommandText = "ALTER TABLE WebcamSceneConfig DROP COLUMN BackgroundEffect; ALTER TABLE WebcamSceneConfig DROP COLUMN BackgroundStrength;"; + cmd.ExecuteNonQuery(); + } + SqliteConnection.ClearAllPools(); + using (var legacy = new LayoutStore(path)) + { + var cfg = legacy.Load().SelectMany(s => s.Elements).OfType().First(); + Assert.Equal(BackgroundEffect.None, cfg.BackgroundEffect); + Assert.Equal(0.5, cfg.BackgroundStrength, 6); + } + } + finally + { + SqliteConnection.ClearAllPools(); + try { File.Delete(path); } catch { /* best-effort cleanup */ } + } + } + // The background (schema v5): permanent bottom layer holding live desktop/game // capture. IsBackground + CaptureKey must survive a save + reload so a // re-designated target isn't lost on restart.