diff --git a/Services/ChromaKeyDetector.cs b/Services/ChromaKeyDetector.cs new file mode 100644 index 0000000..21358f0 --- /dev/null +++ b/Services/ChromaKeyDetector.cs @@ -0,0 +1,155 @@ +namespace ytLive.Services; + +/// What a clip's background looks like, as far as keying goes. +public enum ChromaScreen +{ + /// No usable backdrop — say nothing to the creator. + None, + Green, + Blue, +} + +/// +/// TASK 64: "this clip was shot against a green screen". Coverage × Tightness: +/// how much of the frame is the colour, and how consistent that colour is. A painted screen +/// scores near 1; a lawn or a football pitch scores low because the greens disagree. +/// +public readonly record struct ChromaKeySuggestion( + ChromaScreen Screen, + string? ColorHex, + double Confidence) +{ + /// The screen is uniform enough to key without the creator touching a slider. + public bool IsHighConfidence => Screen != ChromaScreen.None && Confidence >= HighConfidenceFloor; + + public const double HighConfidenceFloor = 0.85; + public const double SuggestFloor = 0.65; +} + +/// +/// TASK 64: finds a green or blue screen in a decoded frame — the "clueless creator" half of +/// the feature. Pure and allocation-light (it only samples an outer ring and a coarse grid), so +/// it runs on the probe frame without touching ffmpeg twice. +/// It never decides anything. A suggestion is offered to the creator; auto-apply is +/// an explicit setting and only fires at . +/// Wrongly keying a lawn is destructive, so "None" is a normal, expected answer. +/// +public static class ChromaKeyDetector +{ + // Hue windows in degrees. Green sits 75–165 (a real screen reads ~120–145), blue 185–265. + private const double GreenHueFrom = 75, GreenHueTo = 165; + private const double BlueHueFrom = 185, BlueHueTo = 265; + + // A screen is saturated and not dark. These also exclude letterbox bars (near-black, + // no saturation) and blown-out highlights from the denominator, so black bars do not + // count against an otherwise perfect screen. + private const double MinSaturation = 0.35; + private const double MinValue = 0.15; + private const double MaxValue = 0.98; + + // The screen's own pixels must agree with each other: a painted surface is flat, a lawn + // is not. This is what keeps "green field" from being keyed. + private const double HueTightness = 12.0; + + /// Samples an outer ring (the subject sits in the middle, the screen fills the + /// edges) and reports what it saw. + public static ChromaKeySuggestion Detect(VideoFrame frame) + { + if (frame.Width < 8 || frame.Height < 8) return Empty; + + var ring = Math.Max(2, (int)Math.Round(Math.Min(frame.Width, frame.Height) * 0.15)); + var step = Math.Max(1, Math.Min(frame.Width, frame.Height) / 96); + + var hues = new List(); + var sameHue = 0; + double hueSum = 0; + long rSum = 0, gSum = 0, bSum = 0; + var considered = 0; + var green = 0; + var blue = 0; + + for (var y = 0; y < frame.Height; y += step) + { + var inVerticalRing = y < ring || y >= frame.Height - ring; + for (var x = 0; x < frame.Width; x += step) + { + if (!inVerticalRing && x >= ring && x < frame.Width - ring) continue; + + var i = y * frame.Stride + x * 4; + var b = frame.BgraPixels[i]; + var g = frame.BgraPixels[i + 1]; + var r = frame.BgraPixels[i + 2]; + if (frame.BgraPixels[i + 3] < 8) continue; // transparent: not the backdrop + + var (hue, saturation, value) = ToHsv(r, g, b); + if (saturation < MinSaturation || value < MinValue || value > MaxValue) continue; + + considered++; + if (hue is >= GreenHueFrom and <= GreenHueTo) green++; + else if (hue is >= BlueHueFrom and <= BlueHueTo) blue++; + else continue; + + hues.Add(hue); + hueSum += hue; + rSum += r; + gSum += g; + bSum += b; + } + } + + if (considered < 64) return Empty; + + var screen = green >= blue + ? (green == 0 ? ChromaScreen.None : ChromaScreen.Green) + : ChromaScreen.Blue; + if (screen == ChromaScreen.None) return Empty; + + var accepted = hues.Count; + var coverage = (double)accepted / considered; + + // Tint stability: how many of the accepted pixels sit within ±HueTightness of the + // median hue. A painted screen ≈ 1.0; grass, foliage and sunlight ≈ far less. + hues.Sort(); + var median = hues[hues.Count / 2]; + foreach (var hue in hues) + { + var delta = Math.Abs(hue - median); + if (delta > 180) delta = 360 - delta; + if (delta <= HueTightness) sameHue++; + } + var tightness = (double)sameHue / accepted; + + var confidence = coverage * tightness; + if (confidence < ChromaKeySuggestion.SuggestFloor) return Empty; + + // The keyed colour is the screen as it was actually shot, not a canonical green: + // similarity is a distance, so the median of the sampled pixels keys far better. + var keyed = Math.Max(1, accepted); + var color = $"#{rSum / keyed:X2}{gSum / keyed:X2}{bSum / keyed:X2}"; + return new ChromaKeySuggestion(screen, color, confidence); + } + + private static readonly ChromaKeySuggestion Empty = new(ChromaScreen.None, null, 0); + + private static (double Hue, double Saturation, double Value) ToHsv(byte r, byte g, byte b) + { + var rf = r / 255.0; + var gf = g / 255.0; + var bf = b / 255.0; + var max = Math.Max(rf, Math.Max(gf, bf)); + var min = Math.Min(rf, Math.Min(gf, bf)); + var delta = max - min; + + double hue = 0; + if (delta > 1e-6) + { + if (max == rf) hue = 60 * (((gf - bf) / delta) % 6); + else if (max == gf) hue = 60 * (((bf - rf) / delta) + 2); + else hue = 60 * (((rf - gf) / delta) + 4); + } + if (hue < 0) hue += 360; + + var saturation = max <= 0 ? 0 : delta / max; + return (hue, saturation, max); + } +} diff --git a/Services/VideoFilterChain.cs b/Services/VideoFilterChain.cs new file mode 100644 index 0000000..3041280 --- /dev/null +++ b/Services/VideoFilterChain.cs @@ -0,0 +1,75 @@ +using System.Globalization; + +namespace ytLive.Services; + +/// +/// TASK 64: the key's settings for one layer, in the app's own terms. is the +/// screen as the creator's camera saw it (whatever the detector measured), not a canonical green. +/// +public sealed record ChromaKeyOptions( + bool Enabled, + string Color, + double Similarity, + double Softness, + bool Spill, + ChromaScreen Screen = ChromaScreen.Green) +{ + public static readonly ChromaKeyOptions Off = new(false, "#00B140", 0.12, 0.08, true); + + /// Creator-facing defaults when a detector suggestion is accepted: a little + /// tolerance around the measured colour, a soft edge, and spill cleanup on. + public static ChromaKeyOptions For(ChromaKeySuggestion suggestion) + => new(true, suggestion.ColorHex ?? "#00B140", 0.12, 0.08, true, suggestion.Screen); + + /// Identifies this look, so decode sessions can be shared per look rather than + /// duplicated per layer (two layers keyed identically reuse one decoder). + public string Signature => Enabled + ? FormattableString.Invariant( + $"{Color.ToUpperInvariant()}|{Similarity:0.###}|{Softness:0.###}|{Spill}|{Screen}") + : "off"; +} + +/// +/// TASK 64: builds the ffmpeg filter chain for a clip. Keying happens IN THE DECODE PIPE, not in +/// this process: the bundled ffmpeg keys with slices of its own CPU cost, emits straight-alpha +/// BGRA (verified against the shipped binary: keyed pixels come out with alpha 0), and — the +/// point — the pipe is upstream of BOTH consumers (the shared preview bitmap and the compositor's +/// frame), so the creator's preview and the stream cannot disagree. +/// Order matters: scale first (cheaper, and the edge is computed at output size), then key, +/// then despill, then force BGRA so the alpha survives the pipe. +/// +public static class VideoFilterChain +{ + /// The video half of a clip's `-vf`. Unkeyed clips keep the old single-scale chain. + public static string BuildVideo(int width, int height, ChromaKeyOptions? key = null) + { + var scale = $"scale={width}:{height}"; + if (key is not { Enabled: true }) return scale; + + var parts = new List(4) + { + scale, + $"colorkey={NormalizeColor(key.Color)}:{Number(key.Similarity)}:{Number(key.Softness)}", + }; + if (key.Spill) + parts.Add($"despill=type={(key.Screen == ChromaScreen.Blue ? "blue" : "green")}"); + parts.Add("format=bgra"); + return string.Join(",", parts); + } + + /// ffmpeg takes `0xRRGGBB`. Anything unparseable falls back to the app default + /// rather than producing a filter string ffmpeg would reject at spawn time. + private static string NormalizeColor(string? hex) + { + var value = (hex ?? string.Empty).Trim(); + if (value.StartsWith("0x", StringComparison.OrdinalIgnoreCase)) value = value[2..]; + if (value.StartsWith('#')) value = value[1..]; + if (value.Length != 6) return "0x00B140"; + foreach (var c in value) + if (!Uri.IsHexDigit(c)) return "0x00B140"; + return "0x" + value.ToUpperInvariant(); + } + + private static string Number(double value) + => Math.Clamp(value, 0, 1).ToString("0.###", CultureInfo.InvariantCulture); +} diff --git a/TASKS.md b/TASKS.md index f6de53e..573d143 100644 --- a/TASKS.md +++ b/TASKS.md @@ -42,7 +42,8 @@ blocker; the smallest slice is 32·4 (fragmented MP4 on the record block). | 53 | ✅ | Output-format contract — six size/FPS modes, vertical YouTube playback, and recordings visually accepted | [`task-53`](TASKS/task-53-output-formats.md) | | — | ✅ | Output entitlement code: free at 30 fps / paid at 60 fps regardless of resolution; Steam App ID is still pending | [`commercial`](docs/ai/commercial.md) | | 3·17 | ✅ | **Text layer** — content, font family/size/weight/style, colour, alignment, drag/resize, persisted (done 2026-11-04) | [`task-03`](TASKS/task-03-capture-pipeline.md) | -| 21 | ✅ | Media source: video file added from the (+) menu decodes, composites, loops, and stops with its last reference (done 2026-11-05) | [`task-21`](TASKS/task-21-media-source.md) | +| 21 | ✅ | Media source: video file added from the (+) menu decodes, composites, plays its soundtrack, loops, and stops with its last reference (done 2026-11-05) | [`task-21`](TASKS/task-21-media-source.md) | +| 64 | 🔶 **NEXT** | **Green/Blue Screen** — detect a green/blue backdrop in a clip and key it out through the decode pipe; suggest-only, opt-in auto-apply, alert clips included | [`task-64`](TASKS/task-64-green-blue-screen.md) | | — | ☐ | Transitions **Move**: per-layer carry-across-scenes flag, per-scene placement, fixed z band | [`ui-surface`](docs/ai/ui-surface.md) | | 55·2–3 | ☐ | Disk hygiene: delete `%TEMP%` alert clips on close + sweep at startup; low-disk guard before REC (slice 1 shipped) | [`task-55`](TASKS/task-55-disk-hygiene.md) | | 40·A/C/D | ☐ | App Settings: active-camera picker; persist output resolution + surface transition defaults + confirm-before-End; accent consolidation | [`task-40`](TASKS/task-40-app-settings-round.md) | diff --git a/TASKS/task-64-green-blue-screen.md b/TASKS/task-64-green-blue-screen.md new file mode 100644 index 0000000..d91b444 --- /dev/null +++ b/TASKS/task-64-green-blue-screen.md @@ -0,0 +1,63 @@ +# TASK 64 — Green/blue screen (chroma) for video clips + +> Catalog: [`TASKS.md`](../TASKS.md) — status and requirements live here. +> Queued 2026-11-05 at the creator's direction, straight after TASK 21's audio slice. + +**Goal:** a creator who drops in a clip shot against a green or blue screen gets the background +removed — without knowing what "chroma key", "similarity" or "spill" mean. The app *notices* the +screen and offers to remove it; the creator never has to pick a colour. + +**Why this is not the retired "colour-key" item:** `docs/ai/decisions.md` buried chroma key in the +webcam background-removal context (ML model, lighting, NVIDIA gives it free — support hell). This is a +different job: the creator supplies the clip, ffmpeg does the arithmetic, there is no model and no +lighting to blame. The reversal is recorded there. + +--- + +## 0. What already exists (verified 2026-11-05, bundled ffmpeg N-126342) + +- `colorkey` / `chromakey` (colour, `similarity`, `blend`) and `despill` (`green`/`blue`, `mix`) are all + present in the shipped binary and slice-threaded. +- A probe of the bundled binary showed keyed pixels come out of the pipe with **alpha 0** and straight + (un-premultiplied) colour — `colorkey=…,format=bgra` → `63 175 0 0` vs `63 175 0 255` unkeyed. The + compositor already blends straight alpha, and media frames are not flagged `IsOpaque`, so no + compositor change is needed. +- Consequence: keying costs **zero app-side CPU** — it is one more term in the decode pipe's `-vf`, + which is also the only place that feeds **both** consumers (the shared preview bitmap and the + compositor's `LatestFrame`). A C# post-process would have to be wired twice. + +## 1. Decisions (creator, 2026-11-05) + +- **Suggest, never override.** Detection offers; the creator clicks. Opt-in setting for auto-apply. +- **Green and blue.** The keyed colour is the *detected* screen colour (real screens read 0x00B140, + not 0x00FF00), not a canonical green. +- **"Auto Green/Blue Screen"** is the setting name. No engineering words in the UI (never "chroma", + never "colour key"): the panel section is **Green/Blue Screen**, the sliders are **Amount**, + **Softness**, **Fringe**. +- **A clip is a clip** — alert clips get the same treatment from the same settings block. +- **High confidence only** may auto-apply; medium shows the offer; low says nothing. +- Settings are **per layer** (they belong to the layer in the UI). Sessions are keyed by + **path + settings signature**, so two layers with identical settings still share one decoder. + +## 2. Slices (one Good Dog each) + +| # | Slice | Test | +|---|---|---| +| A | ✅ media audio + shared clip pipe (TASK 21) | shipped `2670ac0` | +| B | ✅ Pure `Services/ChromaKeyDetector.cs`: sample a `VideoFrame` → green / blue / none + suggested colour + confidence | synthetic frames: green, blue, gradient, letterboxed, lawn (must say none) | +| C | ✅ Pure `Services/VideoFilterChain.cs`: options → `-vf` string (`scale → colorkey → despill → format=bgra`) | exact string assertions (encoder `FfmpegArgs` style) | +| D | Detect on add: one `-frames:v 1` probe (mirror `IFrameRateProbe`) → suggestion in the panel; no rendering change | add a green-screen file → suggestion appears with its colour | +| E | Apply: session key `path|key-settings`, filter chain in the pipe, level/mute untouched | keyed layer over art shows the art through keyed pixels **and** the preview bitmap carries alpha | +| F | Alert clips (falls out of E), tuning sliders, "Auto Green/Blue Screen" setting (App Settings), spill | slider reaches the chain and the restart preserves loop state | + +## 3. Landmines for whoever builds this + +- **Two consumers.** The preview bitmap and `LatestFrame` fork inside `MediaVideoSourceManager`; keying + must happen upstream (in the pipe), or one path shows green. +- **Parameter change = decoder restart** (`-vf` is fixed at spawn). Debounce, and carry loop + refcount + across the restart instead of tearing the session down. +- **Session identity.** `ResolveOutputFrame` and the preview-bitmap adoption must derive the session key + from the same helper as the panel, or a keyed and an unkeyed layer on one file fight over one decoder. +- **Detection is a second reader of the file** — one frame, async, never on the UI thread; a failed + probe means "no suggestion", never a broken layer. +- **Letterboxing** (black bars) breaks naive border sampling — detect and exclude them first. diff --git a/docs/ai/decisions.md b/docs/ai/decisions.md index abfab21..c7e8a0a 100644 --- a/docs/ai/decisions.md +++ b/docs/ai/decisions.md @@ -204,6 +204,11 @@ Do not build these; each was explicitly ruled out. - Per-source audio sync offset — global offset shipped (TASK 22); per-source is the OBS mixer rabbit hole and breaks the one-knob audio rule. - Chroma key / NVIDIA-only background removal (the bundled ONNX path is the one shipped). + **Partly reversed 2026-11-05 (TASK 64):** a *video clip* may be keyed — green or blue, detected + from the clip, performed by the bundled ffmpeg `colorkey`/`despill` in the decode pipe, offered + never forced. The original objection was webcam background removal (ML model, lighting, support + weight); none of it applies to a clip the creator supplied. The webcam still gets the shipped ONNX + path, and NVIDIA-only effects stay out. - Virtual camera output — a different product and a device-driver swamp; our output is one YouTube stream. - Replay buffer, stream clipping, multi-destination restreaming, an in-app bug reporter, and an diff --git a/ytLive.Tests/ChromaKeyDetectorTests.cs b/ytLive.Tests/ChromaKeyDetectorTests.cs new file mode 100644 index 0000000..0173816 --- /dev/null +++ b/ytLive.Tests/ChromaKeyDetectorTests.cs @@ -0,0 +1,157 @@ +using System; +using System.Collections.Generic; +using Xunit; +using ytLive.Services; + +namespace ytLive.Tests; + +/// +/// TASK 64 unit tests (pure): does the detector recognise a real green/blue screen, and does it +/// keep its mouth shut about things that merely contain green? Plus the decoder filter chain the +/// key turns into — asserted as a string, the same way the encoder's ffmpeg args are pinned. +/// +public class ChromaKeyDetectorTests +{ + private const byte GreenR = 0x00, GreenG = 0xB1, GreenB = 0x40; // #00B140 — a real screen + private const byte BlueR = 0x00, BlueG = 0x47, BlueB = 0xBB; // #0047BB + + [Fact] + public void GreenScreen_IsDetected_WithTheColourActuallyShot() + { + var frame = Build(120, 68, (x, y) => InCentre(x, y, 120, 68) ? Skin() : (GreenR, GreenG, GreenB)); + + var suggestion = ChromaKeyDetector.Detect(frame); + + Assert.Equal(ChromaScreen.Green, suggestion.Screen); + Assert.True(suggestion.IsHighConfidence, $"confidence {suggestion.Confidence:0.00} should be high"); + Assert.Equal("#00B140", suggestion.ColorHex); + } + + [Fact] + public void BlueScreen_IsDetected() + { + var frame = Build(120, 68, (x, y) => InCentre(x, y, 120, 68) ? Skin() : (BlueR, BlueG, BlueB)); + + var suggestion = ChromaKeyDetector.Detect(frame); + + Assert.Equal(ChromaScreen.Blue, suggestion.Screen); + Assert.True(suggestion.IsHighConfidence); + } + + [Fact] + public void LetterboxedScreen_IsStillDetected() + { + // The clip is wider than the screen: black bars top and bottom. Bars must not count + // against the screen, or every letterboxed clip reads as "no backdrop". + var frame = Build(120, 68, (x, y) => + y < 12 || y >= 56 ? ((byte)0, (byte)0, (byte)0) : (GreenR, GreenG, GreenB)); + + var suggestion = ChromaKeyDetector.Detect(frame); + + Assert.Equal(ChromaScreen.Green, suggestion.Screen); + Assert.True(suggestion.Confidence >= ChromaKeySuggestion.SuggestFloor, + $"letterboxing dropped confidence to {suggestion.Confidence:0.00}"); + } + + [Fact] + public void LawnOrPitch_SaysNothing() + { + // Green, but the greens disagree with each other (hue 100 vs 145 alternating). Keying + // this would eat the picture, so "None" is the correct answer. + var frame = Build(120, 68, (x, y) => (x / 4 + y / 4) % 2 == 0 + ? ((byte)20, (byte)150, (byte)30) // hue ≈ 125, dark-ish grass + : ((byte)80, (byte)180, (byte)10)); // hue ≈ 95, sunlit grass + + var suggestion = ChromaKeyDetector.Detect(frame); + + Assert.Equal(ChromaScreen.None, suggestion.Screen); + } + + [Fact] + public void MixedScene_SaysNothing() + { + var frame = Build(120, 68, (x, y) => ((byte)(x % 255), (byte)(y % 255), (byte)((x + y) % 255))); + + var suggestion = ChromaKeyDetector.Detect(frame); + + Assert.Equal(ChromaScreen.None, suggestion.Screen); + } + + [Fact] + public void DarkFrames_AreNotAScreen() + { + var frame = Build(120, 68, (x, y) => ((byte)0, (byte)(y % 20), (byte)0)); + + Assert.Equal(ChromaScreen.None, ChromaKeyDetector.Detect(frame).Screen); + } + + // ─── the filter chain the key turns into ─── + + [Fact] + public void UnkeyedClip_KeepsThePlainScaleChain() + => Assert.Equal("scale=1920:1080", VideoFilterChain.BuildVideo(1920, 1080, ChromaKeyOptions.Off)); + + [Fact] + public void KeyedClip_ScalesThenKeysThenDespillsThenForcesBgra() + { + var options = new ChromaKeyOptions(true, "#00B140", 0.12, 0.08, Spill: true); + + Assert.Equal( + "scale=1920:1080,colorkey=0x00B140:0.12:0.08,despill=type=green,format=bgra", + VideoFilterChain.BuildVideo(1920, 1080, options)); + } + + [Fact] + public void BlueScreen_DespillsBlue() + { + var options = new ChromaKeyOptions(true, "#0047BB", 0.1, 0.05, Spill: true, ChromaScreen.Blue); + + Assert.Contains("despill=type=blue", VideoFilterChain.BuildVideo(1920, 1080, options)); + } + + [Fact] + public void SpillOff_AndBadColour_StillProduceAValidChain() + { + var noSpill = new ChromaKeyOptions(true, "not-a-colour", 2.0, -1, Spill: false); + var chain = VideoFilterChain.BuildVideo(1280, 720, noSpill); + + // Out-of-range numbers clamp, an unparseable colour falls back to the app default, and + // no despill term is appended — ffmpeg must never be handed a filter it will reject. + Assert.Equal("scale=1280:720,colorkey=0x00B140:1:0,format=bgra", chain); + } + + [Fact] + public void Signature_IsTheSameForIdenticalLooks_AndDiffersOtherwise() + { + var a = new ChromaKeyOptions(true, "#00b140", 0.12, 0.08, true); + var b = new ChromaKeyOptions(true, "#00B140", 0.12, 0.08, true); + var c = new ChromaKeyOptions(true, "#00B140", 0.30, 0.08, true); + + Assert.Equal(a.Signature, b.Signature); // shared decoder for identical looks + Assert.NotEqual(a.Signature, c.Signature); + Assert.Equal("off", ChromaKeyOptions.Off.Signature); + } + + // ─── helpers ─── + + private static bool InCentre(int x, int y, int w, int h) + => x > w * 0.35 && x < w * 0.65 && y > h * 0.25 && y < h * 0.75; + + private static (byte R, byte G, byte B) Skin() => (0xE0, 0xB0, 0x90); + + private static VideoFrame Build(int width, int height, Func pixel) + { + var bytes = new byte[width * height * 4]; + for (var y = 0; y < height; y++) + for (var x = 0; x < width; x++) + { + var (r, g, b) = pixel(x, y); + var i = (y * width + x) * 4; + bytes[i] = b; + bytes[i + 1] = g; + bytes[i + 2] = r; + bytes[i + 3] = 255; + } + return new VideoFrame(width, height, bytes); + } +}