namespace ytLive.Services; /// What a clip's background looks like, as far as keying goes. public enum ChromaScreen { /// No usable backdrop — say nothing to the creator. None, Green, Blue, } /// /// TASK 64: "this clip was shot against a green screen". Coverage × Tightness: /// how much of the frame is the colour, and how consistent that colour is. A painted screen /// scores near 1; a lawn or a football pitch scores low because the greens disagree. /// public readonly record struct ChromaKeySuggestion( ChromaScreen Screen, string? ColorHex, double Confidence) { /// The screen is uniform enough to key without the creator touching a slider. public bool IsHighConfidence => Screen != ChromaScreen.None && Confidence >= HighConfidenceFloor; public const double HighConfidenceFloor = 0.85; public const double SuggestFloor = 0.65; } /// /// TASK 64: finds a green or blue screen in a decoded frame — the "clueless creator" half of /// the feature. Pure and allocation-light (it only samples an outer ring and a coarse grid), so /// it runs on the probe frame without touching ffmpeg twice. /// It never decides anything. A suggestion is offered to the creator; auto-apply is /// an explicit setting and only fires at . /// Wrongly keying a lawn is destructive, so "None" is a normal, expected answer. /// public static class ChromaKeyDetector { // Hue windows in degrees. Green sits 75–165 (a real screen reads ~120–145), blue 185–265. private const double GreenHueFrom = 75, GreenHueTo = 165; private const double BlueHueFrom = 185, BlueHueTo = 265; // A screen is saturated and not dark. These also exclude letterbox bars (near-black, // no saturation) and blown-out highlights from the denominator, so black bars do not // count against an otherwise perfect screen. private const double MinSaturation = 0.35; private const double MinValue = 0.15; private const double MaxValue = 0.98; // The screen's own pixels must agree with each other: a painted surface is flat, a lawn // is not. This is what keeps "green field" from being keyed. private const double HueTightness = 12.0; /// Samples an outer ring (the subject sits in the middle, the screen fills the /// edges) and reports what it saw. public static ChromaKeySuggestion Detect(VideoFrame frame) { if (frame.Width < 8 || frame.Height < 8) return Empty; var ring = Math.Max(2, (int)Math.Round(Math.Min(frame.Width, frame.Height) * 0.15)); var step = Math.Max(1, Math.Min(frame.Width, frame.Height) / 96); var hues = new List(); var sameHue = 0; double hueSum = 0; long rSum = 0, gSum = 0, bSum = 0; var considered = 0; var green = 0; var blue = 0; for (var y = 0; y < frame.Height; y += step) { var inVerticalRing = y < ring || y >= frame.Height - ring; for (var x = 0; x < frame.Width; x += step) { if (!inVerticalRing && x >= ring && x < frame.Width - ring) continue; var i = y * frame.Stride + x * 4; var b = frame.BgraPixels[i]; var g = frame.BgraPixels[i + 1]; var r = frame.BgraPixels[i + 2]; if (frame.BgraPixels[i + 3] < 8) continue; // transparent: not the backdrop var (hue, saturation, value) = ToHsv(r, g, b); if (saturation < MinSaturation || value < MinValue || value > MaxValue) continue; considered++; if (hue is >= GreenHueFrom and <= GreenHueTo) green++; else if (hue is >= BlueHueFrom and <= BlueHueTo) blue++; else continue; hues.Add(hue); hueSum += hue; rSum += r; gSum += g; bSum += b; } } if (considered < 64) return Empty; var screen = green >= blue ? (green == 0 ? ChromaScreen.None : ChromaScreen.Green) : ChromaScreen.Blue; if (screen == ChromaScreen.None) return Empty; var accepted = hues.Count; var coverage = (double)accepted / considered; // Tint stability: how many of the accepted pixels sit within ±HueTightness of the // median hue. A painted screen ≈ 1.0; grass, foliage and sunlight ≈ far less. hues.Sort(); var median = hues[hues.Count / 2]; foreach (var hue in hues) { var delta = Math.Abs(hue - median); if (delta > 180) delta = 360 - delta; if (delta <= HueTightness) sameHue++; } var tightness = (double)sameHue / accepted; var confidence = coverage * tightness; if (confidence < ChromaKeySuggestion.SuggestFloor) return Empty; // The keyed colour is the screen as it was actually shot, not a canonical green: // similarity is a distance, so the median of the sampled pixels keys far better. var keyed = Math.Max(1, accepted); var color = $"#{rSum / keyed:X2}{gSum / keyed:X2}{bSum / keyed:X2}"; return new ChromaKeySuggestion(screen, color, confidence); } private static readonly ChromaKeySuggestion Empty = new(ChromaScreen.None, null, 0); private static (double Hue, double Saturation, double Value) ToHsv(byte r, byte g, byte b) { var rf = r / 255.0; var gf = g / 255.0; var bf = b / 255.0; var max = Math.Max(rf, Math.Max(gf, bf)); var min = Math.Min(rf, Math.Min(gf, bf)); var delta = max - min; double hue = 0; if (delta > 1e-6) { if (max == rf) hue = 60 * (((gf - bf) / delta) % 6); else if (max == gf) hue = 60 * (((bf - rf) / delta) + 2); else hue = 60 * (((rf - gf) / delta) + 4); } if (hue < 0) hue += 360; var saturation = max <= 0 ? 0 : delta / max; return (hue, saturation, max); } }