From 7bb1207fb2b7df552a027f613f682a10641eb543 Mon Sep 17 00:00:00 2001 From: gramps Date: Thu, 5 Nov 2026 08:20:33 -0800 Subject: [PATCH] feat(chroma): opt-in auto key, and alert clips key like any other clip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TASK 64 slice F — the two things left after the feature worked by hand. - App Settings → **Video clips → Auto Green/Blue Screen** (off by default). When on, a high-confidence detected screen is keyed as soon as the clip is added; the panel says "Removed automatically" so an automatic change is never silent. Medium confidence still asks, low says nothing — a wrong automatic key eats the picture. - **A clip is a clip:** the alert box's own replacement clip keys through the same settings block (one alert box per layout, so its settings are the ones the next alert uses), and choosing an alert clip runs the same detector. The Green/Blue Screen panel now appears for a video layer *or* an alert box with a custom clip (`Source.ShowsChromaKey`). - Spill ships as the creator-facing **Fringe** switch; nothing further to tune behind it. Test: `ChromaKeyApplyTests` gains the auto-apply case (no click, still explicit in the panel) and an alert-clip case (its pipe asks ffmpeg for colorkey+despill with the blue screen type). Docs: TASK 64 complete in TASKS.md, known limits recorded in the task file, HANDOFF rewritten. verify.sh: 546 passed, 0 failed, 0 warnings. --- Controls/LeftPanel.xaml | 10 ++++- Controls/SettingsDrawer.xaml | 12 ++++++ HANDOFF.md | 12 +++--- Models/Source.cs | 30 ++++++++++++- Services/AlertClipDecoder.cs | 6 ++- Services/LayoutStore.Settings.cs | 6 +++ TASKS.md | 2 +- TASKS/task-64-green-blue-screen.md | 11 ++++- ViewModels/MainViewModel.Chat.cs | 14 ++++++- ViewModels/MainViewModel.Media.cs | 42 +++++++++++++++++-- ViewModels/MainViewModel.cs | 1 + ytLive.Tests/ChromaKeyApplyTests.cs | 65 +++++++++++++++++++++++++++++ 12 files changed, 192 insertions(+), 19 deletions(-) diff --git a/Controls/LeftPanel.xaml b/Controls/LeftPanel.xaml index fd054b8..43da574 100644 --- a/Controls/LeftPanel.xaml +++ b/Controls/LeftPanel.xaml @@ -762,8 +762,11 @@ - + + + + @@ -787,6 +790,9 @@ + + + + + + + + + + + diff --git a/HANDOFF.md b/HANDOFF.md index 521d6e8..b26fe7e 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -9,15 +9,15 @@ Sessions are keyed by **path + look**, so identical looks share one decoder. - Detection stays silent on lawns/pitches (confidence = coverage × hue tightness) and ignores letterbox bars. It never acts on its own. -- Tests: `ChromaKeyDetectorTests` (pure) + `ChromaKeyApplyTests` (pipe asks ffmpeg to key; offer → - accept → keyed session feeds the resolver; survives save). App build 0 warnings; full suite green - apart from the two documented pre-existing failures. +- Slice F shipped too: **App Settings → Video clips → "Auto Green/Blue Screen"** (off by default, + high confidence only, and the panel states when a key was automatic) and **alert clips** key through + the same settings block (one alert box per layout). +- Tests: `ChromaKeyDetectorTests` + `ChromaKeyApplyTests` (pipe asks ffmpeg to key; offer → accept → + keyed session feeds the resolver; survives save; auto-apply; alert clip keyed). App build 0 warnings. ## Next action -- TASK 64 slice F: the **"Auto Green/Blue Screen"** opt-in setting (App Settings, high confidence - only) and wiring alert clips to the same settings block. -- Then TASK 49 (chat profanity filter). Day 3's TASK 32/33 resilience slices remain outstanding — +- TASK 64 is complete. Next queued: TASK 49 (chat profanity filter). Day 3's TASK 32/33 resilience slices remain outstanding — 32·4 (fragmented MP4) is the smallest. - Do not push absent a checkpoint request. diff --git a/Models/Source.cs b/Models/Source.cs index 4636c2b..d6bf940 100644 --- a/Models/Source.cs +++ b/Models/Source.cs @@ -111,6 +111,7 @@ public class Source : SceneElement Raise(nameof(IsTextOverlay)); Raise(nameof(IsScoreboard)); Raise(nameof(IsTicker)); + Raise(nameof(ShowsChromaKey)); } } } @@ -379,6 +380,25 @@ public class Source : SceneElement /// True while there is an offer the creator has not accepted. public bool HasChromaSuggestion => !ChromaKeyEnabled && ChromaSuggestion.Screen != ChromaScreen.None; + private bool _chromaKeyAutoApplied; + /// True when the app applied the key on its own (the "Auto Green/Blue Screen" + /// setting). The panel says so, so an automatic change is never silent. + public bool ChromaKeyAutoApplied + { + get => _chromaKeyAutoApplied; + set + { + if (Set(ref _chromaKeyAutoApplied, value)) Raise(nameof(HasAutoChromaKey)); + } + } + + public bool HasAutoChromaKey => ChromaKeyAutoApplied && ChromaKeyEnabled; + + /// Which layers get the Green/Blue Screen block: a video layer, or the alert box's + /// own clip (TASK 64 — a clip is a clip). + public bool ShowsChromaKey => IsMediaSource + || (Type == SourceType.AlertBox && !AlertUseDefaultVideo && !string.IsNullOrWhiteSpace(AlertVideoPath)); + public string ChromaSuggestionText => ChromaSuggestion.Screen switch { ChromaScreen.Green => "Looks like a green screen.", @@ -406,14 +426,20 @@ public class Source : SceneElement public string? AlertVideoPath { get => _alertVideoPath; - set => Set(ref _alertVideoPath, value); + set + { + if (Set(ref _alertVideoPath, value)) Raise(nameof(ShowsChromaKey)); + } } private bool _alertUseDefaultVideo = true; public bool AlertUseDefaultVideo { get => _alertUseDefaultVideo; - set => Set(ref _alertUseDefaultVideo, value); + set + { + if (Set(ref _alertUseDefaultVideo, value)) Raise(nameof(ShowsChromaKey)); + } } public bool AlertShowTicker diff --git a/Services/AlertClipDecoder.cs b/Services/AlertClipDecoder.cs index 62af420..5ee7937 100644 --- a/Services/AlertClipDecoder.cs +++ b/Services/AlertClipDecoder.cs @@ -50,10 +50,12 @@ public sealed class AlertClipDecoder : IAlertClipDecoder IFfmpegLocator locator, Func processFactory, IFrameRateProbe? frameRateProbe = null, - Func? delay = null) + Func? delay = null, + ChromaKeyOptions? chromaKey = null) { _clip = new VideoClipPipe( - path, width, height, locator, processFactory, frameRateProbe, delay, audio: true); + path, width, height, locator, processFactory, frameRateProbe, delay, + audio: true, chromaKey: chromaKey); _clip.FrameAvailable += frame => FrameAvailable?.Invoke(frame); _clip.AudioAvailable += sample => AudioReady?.Invoke(sample); _clip.Completed += () => Completed?.Invoke(); diff --git a/Services/LayoutStore.Settings.cs b/Services/LayoutStore.Settings.cs index a1d302d..65652e0 100644 --- a/Services/LayoutStore.Settings.cs +++ b/Services/LayoutStore.Settings.cs @@ -48,6 +48,12 @@ public partial class LayoutStore : IDisposable /// Zoom box follow mode (creator ruling 2026-11-01): false = ANCHOR (the capture /// stays on the point that was clicked, the default), true = FOLLOW (the capture re-anchors /// to the box centre while it is dragged). + /// TASK 64: key a detected green/blue screen without asking. Off by default — the + /// app offers, the creator accepts; this switch is for the creator who is tired of clicking. + public bool LoadAutoChromaScreen() => GetSetting("Video.AutoChromaScreen") == "1"; + + public void SaveAutoChromaScreen(bool enabled) => UpsertSetting("Video.AutoChromaScreen", enabled ? "1" : "0"); + public bool LoadZoomFollowBox() => GetSetting("Zoom.FollowBox") == "1"; public void SaveZoomFollowBox(bool follow) => UpsertSetting("Zoom.FollowBox", follow ? "1" : "0"); diff --git a/TASKS.md b/TASKS.md index 723e978..8ccae71 100644 --- a/TASKS.md +++ b/TASKS.md @@ -43,7 +43,7 @@ blocker; the smallest slice is 32·4 (fragmented MP4 on the record block). | — | ✅ | Output entitlement code: free at 30 fps / paid at 60 fps regardless of resolution; Steam App ID is still pending | [`commercial`](docs/ai/commercial.md) | | 3·17 | ✅ | **Text layer** — content, font family/size/weight/style, colour, alignment, drag/resize, persisted (done 2026-11-04) | [`task-03`](TASKS/task-03-capture-pipeline.md) | | 21 | ✅ | Media source: video file added from the (+) menu decodes, composites, plays its soundtrack, loops, and stops with its last reference (done 2026-11-05) | [`task-21`](TASKS/task-21-media-source.md) | -| 64 | 🔶 | **Green/Blue Screen** — detect + key shipped 2026-11-05 (offer on add, Amount/Softness/Fringe, keyed in the decode pipe, persisted); **remaining: opt-in auto-apply setting + alert clips** | [`task-64`](TASKS/task-64-green-blue-screen.md) | +| 64 | ✅ | **Green/Blue Screen** — detect + key from the decode pipe, offer on add, Amount/Softness/Fringe, opt-in auto-apply, alert clips included, persisted (done 2026-11-05) | [`task-64`](TASKS/task-64-green-blue-screen.md) | | — | ☐ | Transitions **Move**: per-layer carry-across-scenes flag, per-scene placement, fixed z band | [`ui-surface`](docs/ai/ui-surface.md) | | 55·2–3 | ☐ | Disk hygiene: delete `%TEMP%` alert clips on close + sweep at startup; low-disk guard before REC (slice 1 shipped) | [`task-55`](TASKS/task-55-disk-hygiene.md) | | 40·A/C/D | ☐ | App Settings: active-camera picker; persist output resolution + surface transition defaults + confirm-before-End; accent consolidation | [`task-40`](TASKS/task-40-app-settings-round.md) | diff --git a/TASKS/task-64-green-blue-screen.md b/TASKS/task-64-green-blue-screen.md index fba5576..fdd6b92 100644 --- a/TASKS/task-64-green-blue-screen.md +++ b/TASKS/task-64-green-blue-screen.md @@ -48,7 +48,7 @@ lighting to blame. The reversal is recorded there. | C | ✅ Pure `Services/VideoFilterChain.cs`: options → `-vf` string (`scale → colorkey → despill → format=bgra`) | exact string assertions (encoder `FfmpegArgs` style) | | D | ✅ Detect on add: one `-frames:v 1` probe (mirror `IFrameRateProbe`) → suggestion in the panel; no rendering change | add a green-screen file → suggestion appears with its colour | | E | ✅ Apply: session key `path|key-settings`, filter chain in the pipe, level/mute untouched | keyed layer over art shows the art through keyed pixels **and** the preview bitmap carries alpha | -| F | ☐ **Remaining:** "Auto Green/Blue Screen" opt-in in App Settings (high confidence only), alert clips wired to the same settings block, spill quality pass | slider reaches the chain and the restart preserves loop state | +| F | ✅ "Auto Green/Blue Screen" opt-in (App Settings; high confidence only, and the panel says it was automatic), alert clips keyed through the same settings block, spill shipped as the **Fringe** switch | slider reaches the chain and the restart preserves loop state | ## 3. Landmines for whoever builds this @@ -61,3 +61,12 @@ lighting to blame. The reversal is recorded there. - **Detection is a second reader of the file** — one frame, async, never on the UI thread; a failed probe means "no suggestion", never a broken layer. - **Letterboxing** (black bars) breaks naive border sampling — detect and exclude them first. + +## 4. Known limits (honest) + +- The offer is made **when a clip is added or changed** — not at every launch. The settings persist, + so a keyed layer comes back keyed; it just does not re-offer the probe. +- Auto-apply fires only at high confidence. Medium confidence always asks, low says nothing. +- The key is the layer's, but the decode is shared per **path + look**: two layers on one file with + different Amount/Softness values are two decoders (and therefore two audio streams, muted upstream + unless enabled) — rare, and the alternative is silently sharing the wrong picture. diff --git a/ViewModels/MainViewModel.Chat.cs b/ViewModels/MainViewModel.Chat.cs index f327118..d3582df 100644 --- a/ViewModels/MainViewModel.Chat.cs +++ b/ViewModels/MainViewModel.Chat.cs @@ -140,7 +140,10 @@ public partial class MainViewModel : ViewModelBase path, width, height, new FfmpegLocator(), () => new FfmpegDecodeProcess(), - frameRateProbe: new FfmpegFrameRateProbe(new FfmpegLocator(), () => new FfmpegDecodeProcess())); + frameRateProbe: new FfmpegFrameRateProbe(new FfmpegLocator(), () => new FfmpegDecodeProcess()), + // TASK 64: an alert clip is a clip — it keys through the same settings block. The + // layout has one alert box, so its settings are the ones the next alert uses. + chromaKey: AlertBoxChromaKey()); /// Stamp the shipped llama placeholder into the Asset table at startup /// (idempotent — hash-keyed UpsertAsset dedupes). Best-effort: if it fails the @@ -179,6 +182,13 @@ public partial class MainViewModel : ViewModelBase } } + /// The alert zone's own green/blue screen settings (one alert box per layout). + private ChromaKeyOptions? AlertBoxChromaKey() + => Scenes.SelectMany(scene => scene.Elements).OfType() + .FirstOrDefault(source => source.IsAlertBox)?.ChromaKey is { Enabled: true } key + ? key + : null; + private void BrowseAlertVideo(Source? box) { if (box == null || box.Type != SourceType.AlertBox) return; @@ -190,6 +200,8 @@ public partial class MainViewModel : ViewModelBase if (dialog.ShowDialog() != true) return; box.AlertVideoPath = dialog.FileName; box.AlertUseDefaultVideo = false; + // Offer the key for the chosen clip (same detector as a media layer). + _ = SuggestChromaKeyAsync(box); } private void BrowseAlertAudio(Source? box) diff --git a/ViewModels/MainViewModel.Media.cs b/ViewModels/MainViewModel.Media.cs index 59b3489..b3e853f 100644 --- a/ViewModels/MainViewModel.Media.cs +++ b/ViewModels/MainViewModel.Media.cs @@ -37,6 +37,21 @@ public partial class MainViewModel // One probe per file per launch — the answer belongs to the file, not to the layer. private readonly HashSet _chromaProbedPaths = new(StringComparer.OrdinalIgnoreCase); + private bool _autoChromaScreen; + + /// "Auto Green/Blue Screen": apply a detected screen without asking. Off by default — + /// the app offers, and only a creator who turns this on gets it done for them. Fires only on + /// high-confidence detections, because a wrong automatic key eats the picture. + public bool AutoChromaScreen + { + get => _autoChromaScreen; + set + { + if (!SetProperty(ref _autoChromaScreen, value)) return; + _layoutStore.SaveAutoChromaScreen(value); + } + } + // One subscription per media Source, so removing a layer stops the listener as // well as the decoder (a scene element outlives its layer list row). private readonly Dictionary _mediaHandlers = new(); @@ -102,19 +117,37 @@ public partial class MainViewModel // ---------- green / blue screen detection (TASK 64) ---------- /// Looks at one frame of the clip and stores the offer on the layer. Runs off the - /// UI thread through the probe seam; a file that cannot be read simply gets no offer. + /// UI thread through the probe seam; a file that cannot be read simply gets no offer. + /// Works for a video layer and for the alert box's own clip — a clip is a clip. private async System.Threading.Tasks.Task SuggestChromaKeyAsync(Source source) { - if (!source.IsMediaSource || source.MediaPath is not { } path) return; + if (source.ChromaKeyEnabled) return; + if (ClipPathFor(source) is not { } path) return; if (_chromaProbe == null) return; var frame = await _chromaProbe.ProbeAsync(path).ConfigureAwait(true); if (frame == null) return; - if (!string.Equals(source.MediaPath, path, StringComparison.OrdinalIgnoreCase)) return; + if (!string.Equals(ClipPathFor(source), path, StringComparison.OrdinalIgnoreCase)) return; - source.ChromaSuggestion = ChromaKeyDetector.Detect(frame); + var suggestion = ChromaKeyDetector.Detect(frame); + source.ChromaSuggestion = suggestion; + + if (AutoChromaScreen && suggestion.IsHighConfidence) + { + AcceptChromaSuggestion(source); + source.ChromaKeyAutoApplied = true; + } } + /// The file a layer plays: a video layer's clip, or the alert box's replacement clip. + /// The shipped alert default is not probed — it is ours, and it is not a screen shoot. + private static string? ClipPathFor(Source source) + => source.IsMediaSource + ? source.MediaPath + : source.Type == SourceType.AlertBox && !source.AlertUseDefaultVideo + ? source.AlertVideoPath + : null; + /// Accepts the detector's offer: the layer keys the colour the camera actually saw. private void AcceptChromaSuggestion(Source? source) { @@ -126,6 +159,7 @@ public partial class MainViewModel source.ChromaKeySoftness = key.Softness; source.ChromaKeySpill = key.Spill; source.ChromaScreen = key.Screen; + source.ChromaKeyAutoApplied = false; source.ChromaKeyEnabled = true; } diff --git a/ViewModels/MainViewModel.cs b/ViewModels/MainViewModel.cs index 8d9f618..59570a3 100644 --- a/ViewModels/MainViewModel.cs +++ b/ViewModels/MainViewModel.cs @@ -344,6 +344,7 @@ public partial class MainViewModel : ViewModelBase MicSourceName = _layoutStore.LoadMicSourceName(); AudioSyncOffsetMs = _layoutStore.LoadAudioSyncOffsetMs(); _zoomFollowsBox = _layoutStore.LoadZoomFollowBox(); + _autoChromaScreen = _layoutStore.LoadAutoChromaScreen(); MonoOutput = _layoutStore.LoadMonoOutput(); // applied to the mixer by the setter _musicPlayer.UserGain = (float)Helpers.GainDb.FromDb(TraxDefaultDb); // background by default InitScoreboard(); diff --git a/ytLive.Tests/ChromaKeyApplyTests.cs b/ytLive.Tests/ChromaKeyApplyTests.cs index 7e7f140..2fd6550 100644 --- a/ytLive.Tests/ChromaKeyApplyTests.cs +++ b/ytLive.Tests/ChromaKeyApplyTests.cs @@ -134,6 +134,71 @@ public sealed class ChromaKeyApplyTests }); } + // ─── the opt-in setting ─── + + [Fact] + public void AutoGreenBlueScreen_KeysAHighConfidenceClipWithoutAsking() + { + _app.Run(() => + { + var tempDb = Path.Combine(Path.GetTempPath(), $"ytLlive-test-{Guid.NewGuid():N}.db"); + var videoPath = Path.Combine(Path.GetTempPath(), "auto-keyed.mp4"); + var decoders = new List(); + MainViewModel.LayoutPathOverride = tempDb; + MainViewModel.MediaFrameSourceFactoryOverride = key => + { + var decoder = new FakeMediaSource(key); + decoders.Add(decoder); + return decoder; + }; + MainViewModel.ChromaProbeOverride = new FakeProbe(GreenScreenFrame()); + + var window = new MainWindow(); + var vm = (MainViewModel)window.DataContext; + try + { + window.Show(); + vm.AutoChromaScreen = true; + + vm.AddMediaSource(videoPath); + + var layer = vm.StagedScene!.Elements.OfType().Single(s => s.IsMediaSource); + Assert.True(layer.ChromaKeyEnabled, "the auto setting did not key a high-confidence screen"); + Assert.True(layer.ChromaKeyAutoApplied, "an automatic key must say that it was automatic"); + Assert.True(layer.HasAutoChromaKey); + Assert.Equal(2, decoders.Count); // plain session released, keyed one started + } + finally + { + window.Close(); + MainViewModel.ChromaProbeOverride = null; + MainViewModel.MediaFrameSourceFactoryOverride = null; + MainViewModel.LayoutPathOverride = null; + SqliteConnection.ClearAllPools(); + try { File.Delete(tempDb); } catch { /* best-effort cleanup */ } + } + }); + } + + // ─── an alert clip is a clip ─── + + [Fact] + public async Task AlertClip_KeyedThroughTheSameSettingsBlock() + { + var seen = new List(); + using var decoder = new AlertClipDecoder( + "alert-clip.mp4", 320, 180, new FakeLocator(), + () => new CapturingProcess(seen), + delay: (_, _) => Task.CompletedTask, + chromaKey: new ChromaKeyOptions(true, "#0047BB", 0.1, 0.05, true, ChromaScreen.Blue)); + + decoder.Start(); + await Task.Delay(150); + decoder.Stop(); + + Assert.Contains(seen, args => args.Contains("colorkey=0x0047BB") && args.Contains("despill=type=blue")); + } + private static bool ReadChromaEnabled(string dbPath) { using var connection = new SqliteConnection($"Data Source={dbPath}");