feat(chroma): offer to remove a green or blue screen, and key it in the decode
TASK 64 slices D and E: the feature is usable end to end. - Add a video file → one frame is probed (new `IChromaProbe`/`FfmpegChromaProbe`, one ffmpeg pass at 160x90, off the UI thread) and the layer gets an offer: "Looks like a green screen — Remove it", with the measured colour as a swatch. - Accepting it turns the offer into settings (Amount / Softness / Fringe) that reach ffmpeg as `colorkey` + `despill` + `format=bgra`, so the key is computed in the pipe and both consumers (the shared preview bitmap and the compositor's frame) carry the same alpha. - Media sessions are now keyed by **path + look** (`MainViewModel.MediaSessionKey`), so two layers with the same settings still share one decoder, and changing the look releases the old session and starts the new one. Removal, load reconciliation, loop and audio settings all move to the same key. - Settings persist on the `Source` row (six ALTER TABLE guards + SELECT/INSERT), so a keyed layer comes back keyed. - Panel section is plain creator words: Green/Blue Screen, Amount, Softness, Fringe. Test: `ChromaKeyApplyTests` — the pipe asks ffmpeg to key (and does not when off); a green-screen clip is offered, accepting it starts a keyed session, the layer's resolver then reads that session rather than the old one; the setting survives a save. Docs: TASK 64 D/E ticked with slice F (auto-apply setting + alert clips) left open; frame-pipeline records the decode-option design; HANDOFF rewritten. verify.sh: 544 passed, 0 failed, 0 warnings.
This commit is contained in:
@@ -15,6 +15,10 @@ namespace ytLive.ViewModels;
|
||||
// partials: the readonly manager field is assigned in the core constructor, the
|
||||
// shared WriteableBitmap published per file is adopted by every media Source whose
|
||||
// MediaPath matches, and ResolveOutputFrame feeds the live compositor.
|
||||
//
|
||||
// TASK 64 changed what a "session" is: it is now keyed by file path + the layer's LOOK
|
||||
// (green/blue screen settings), because the key is applied inside the decode pipe. Two
|
||||
// layers with the same look share one decoder; two different looks get two.
|
||||
public partial class MainViewModel
|
||||
{
|
||||
// Decode resolution: the 1920x1080 master. Media frames are scaled to the
|
||||
@@ -27,23 +31,37 @@ public partial class MainViewModel
|
||||
|
||||
private readonly MediaVideoSourceManager _mediaManager;
|
||||
|
||||
// Reads one frame of a clip so the layer can offer "looks like a green screen" (TASK 64).
|
||||
private readonly IChromaProbe? _chromaProbe;
|
||||
|
||||
// One probe per file per launch — the answer belongs to the file, not to the layer.
|
||||
private readonly HashSet<string> _chromaProbedPaths = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// One subscription per media Source, so removing a layer stops the listener as
|
||||
// well as the decoder (a scene element outlives its layer list row).
|
||||
private readonly Dictionary<string, PropertyChangedEventHandler> _mediaHandlers = new();
|
||||
|
||||
// MediaVideoSourceManager publishes one shared WriteableBitmap per file; every
|
||||
// media Source pointing at that path picks it up for the preview/canvas.
|
||||
private void OnMediaPreviewBitmapChanged(string path, WriteableBitmap bitmap)
|
||||
/// <summary>What a decode session is: the file plus the look applied in the pipe (TASK 64).
|
||||
/// A path cannot contain '|', so the key is unambiguous and human-readable in logs.</summary>
|
||||
internal static string MediaSessionKey(string path, ChromaKeyOptions key)
|
||||
=> $"{path}|{key.Signature}";
|
||||
|
||||
internal static string MediaSessionKey(Source source)
|
||||
=> source.MediaPath is { } path ? MediaSessionKey(path, source.ChromaKey) : string.Empty;
|
||||
|
||||
// MediaVideoSourceManager publishes one shared WriteableBitmap per session; every
|
||||
// media Source pointing at that path+look picks it up for the preview/canvas.
|
||||
private void OnMediaPreviewBitmapChanged(string sessionKey, WriteableBitmap bitmap)
|
||||
{
|
||||
foreach (var source in Scenes.SelectMany(s => s.Elements.OfType<Source>())
|
||||
.Where(s => s.IsMediaSource && s.MediaPath == path))
|
||||
source.VideoImageSource = bitmap;
|
||||
foreach (var source in Scenes.SelectMany(s => s.Elements).OfType<Source>())
|
||||
if (source.IsMediaSource && MediaSessionKey(source) == sessionKey)
|
||||
source.VideoImageSource = bitmap;
|
||||
}
|
||||
|
||||
private void OnMediaFailed(string path, string message)
|
||||
{
|
||||
Helpers.AppLog.Write($"MediaSource: '{path}' failed: {message}");
|
||||
_notifications.Warning("Couldn't play that video", System.IO.Path.GetFileName(path));
|
||||
_notifications.Warning("Couldn't play that video", System.IO.Path.GetFileName(SessionPath(path)));
|
||||
}
|
||||
|
||||
// ---------- add / change the file ----------
|
||||
@@ -77,45 +95,93 @@ public partial class MainViewModel
|
||||
if (source is not { IsMediaSource: true }) return;
|
||||
if (PickVideoFile("Replace the video") is not { } path) return;
|
||||
PushPreviewUndo();
|
||||
source.ChromaSuggestion = default;
|
||||
source.MediaPath = path;
|
||||
}
|
||||
|
||||
// ---------- session lifecycle (one session per distinct path) ----------
|
||||
// ---------- green / blue screen detection (TASK 64) ----------
|
||||
|
||||
/// <summary>Looks at one frame of the clip and stores the offer on the layer. Runs off the
|
||||
/// UI thread through the probe seam; a file that cannot be read simply gets no offer.</summary>
|
||||
private async System.Threading.Tasks.Task SuggestChromaKeyAsync(Source source)
|
||||
{
|
||||
if (!source.IsMediaSource || source.MediaPath is not { } path) return;
|
||||
if (_chromaProbe == null) return;
|
||||
|
||||
var frame = await _chromaProbe.ProbeAsync(path).ConfigureAwait(true);
|
||||
if (frame == null) return;
|
||||
if (!string.Equals(source.MediaPath, path, StringComparison.OrdinalIgnoreCase)) return;
|
||||
|
||||
source.ChromaSuggestion = ChromaKeyDetector.Detect(frame);
|
||||
}
|
||||
|
||||
/// <summary>Accepts the detector's offer: the layer keys the colour the camera actually saw.</summary>
|
||||
private void AcceptChromaSuggestion(Source? source)
|
||||
{
|
||||
if (source is not { IsMediaSource: true } || source.ChromaSuggestion.Screen == ChromaScreen.None) return;
|
||||
PushPreviewUndo();
|
||||
var key = ChromaKeyOptions.For(source.ChromaSuggestion);
|
||||
source.ChromaKeyColor = key.Color;
|
||||
source.ChromaKeySimilarity = key.Similarity;
|
||||
source.ChromaKeySoftness = key.Softness;
|
||||
source.ChromaKeySpill = key.Spill;
|
||||
source.ChromaScreen = key.Screen;
|
||||
source.ChromaKeyEnabled = true;
|
||||
}
|
||||
|
||||
// ---------- session lifecycle (one session per path + look) ----------
|
||||
|
||||
/// <summary>Starts decoding the file if nothing else already is, and subscribes to the
|
||||
/// layer so a file swap or a loop flip reaches the session.</summary>
|
||||
/// layer so a file swap, a loop flip or a look change reaches the right session.</summary>
|
||||
private void RegisterMediaSource(Source source)
|
||||
{
|
||||
if (_mediaHandlers.ContainsKey(source.Id)) return;
|
||||
|
||||
string? lastPath = source.MediaPath;
|
||||
var lastKey = MediaSessionKey(source);
|
||||
|
||||
void OnSourceChanged(object? _, PropertyChangedEventArgs e)
|
||||
{
|
||||
if (e.PropertyName == nameof(Source.MediaPath))
|
||||
if (e.PropertyName is nameof(Source.MediaPath) or nameof(Source.ChromaKeyEnabled)
|
||||
or nameof(Source.ChromaKeyColor) or nameof(Source.ChromaKeySimilarity)
|
||||
or nameof(Source.ChromaKeySoftness) or nameof(Source.ChromaKeySpill)
|
||||
or nameof(Source.ChromaScreen))
|
||||
{
|
||||
var previous = lastPath;
|
||||
lastPath = source.MediaPath;
|
||||
if (string.Equals(previous, source.MediaPath, StringComparison.OrdinalIgnoreCase)) return;
|
||||
// The layer now points elsewhere, so the old file may be unreferenced —
|
||||
// release it before acquiring the replacement.
|
||||
var next = MediaSessionKey(source);
|
||||
if (string.Equals(next, lastKey, StringComparison.OrdinalIgnoreCase)) return;
|
||||
var previous = lastKey;
|
||||
lastKey = next;
|
||||
// A look change is a different decode: let the old session go (if nothing
|
||||
// else uses it) and start the new one.
|
||||
ReleaseMediaIfUnused(previous);
|
||||
EnsureMediaSession(source.MediaPath);
|
||||
EnsureMediaSession(next);
|
||||
return;
|
||||
}
|
||||
else if (e.PropertyName == nameof(Source.MediaIsLooping) && source.MediaPath is { } path)
|
||||
|
||||
if (e.PropertyName == nameof(Source.MediaIsLooping) && MediaSessionKey(source) is { Length: > 0 } key)
|
||||
{
|
||||
_mediaManager.SetLooping(path, AnyMediaReferenceLoops(path));
|
||||
_mediaManager.SetLooping(key, AnyMediaReferenceLoops(key));
|
||||
}
|
||||
else if (e.PropertyName is nameof(Source.MediaVolume) or nameof(Source.MediaMuted)
|
||||
&& source.MediaPath is { } audioPath)
|
||||
&& MediaSessionKey(source) is { Length: > 0 } audioKey)
|
||||
{
|
||||
ApplyMediaAudioSettings(audioPath);
|
||||
ApplyMediaAudioSettings(audioKey);
|
||||
}
|
||||
}
|
||||
|
||||
_mediaHandlers[source.Id] = OnSourceChanged;
|
||||
source.PropertyChanged += OnSourceChanged;
|
||||
EnsureMediaSession(source.MediaPath);
|
||||
|
||||
if (source.MediaPath is { Length: > 0 } path)
|
||||
{
|
||||
EnsureMediaSession(MediaSessionKey(path, source.ChromaKey));
|
||||
|
||||
// One probe per file per launch: the answer depends on the file, not the layer.
|
||||
if (source.ChromaSuggestion.Screen == ChromaScreen.None && !_chromaProbedPaths.Contains(path))
|
||||
{
|
||||
_chromaProbedPaths.Add(path);
|
||||
_ = SuggestChromaKeyAsync(source);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void UnregisterMediaSource(Source source)
|
||||
@@ -124,64 +190,68 @@ public partial class MainViewModel
|
||||
source.PropertyChanged -= handler;
|
||||
}
|
||||
|
||||
private void EnsureMediaSession(string? path)
|
||||
private void EnsureMediaSession(string? sessionKey)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path)) return;
|
||||
if (_mediaManager.HasSession(path))
|
||||
if (string.IsNullOrWhiteSpace(sessionKey)) return;
|
||||
if (_mediaManager.HasSession(sessionKey))
|
||||
{
|
||||
ApplyMediaAudioSettings(path);
|
||||
ApplyMediaAudioSettings(sessionKey);
|
||||
return;
|
||||
}
|
||||
_ = AcquireMediaAsync(path);
|
||||
_ = AcquireMediaSessionAsync(sessionKey);
|
||||
}
|
||||
|
||||
private async System.Threading.Tasks.Task AcquireMediaAsync(string path)
|
||||
{
|
||||
if (!await _mediaManager.AcquireAsync(path)) return;
|
||||
|
||||
// A session can be running before this layer existed (shared path), in which
|
||||
// case the bitmap is already published and will not be raised again for us.
|
||||
if (_mediaManager.GetPreviewBitmap(path) is { } bitmap)
|
||||
OnMediaPreviewBitmapChanged(path, bitmap);
|
||||
|
||||
ApplyMediaAudioSettings(path);
|
||||
}
|
||||
|
||||
/// <summary>The file's sound controls: one decoder is shared per path, so the level and
|
||||
/// mute are per session — the layer that was last touched is the one that wins.</summary>
|
||||
private void ApplyMediaAudioSettings(string path)
|
||||
private async System.Threading.Tasks.Task AcquireMediaSessionAsync(string sessionKey)
|
||||
{
|
||||
var source = Scenes.SelectMany(s => s.Elements).OfType<Source>()
|
||||
.FirstOrDefault(s => s.IsMediaSource && s.MediaPath == path);
|
||||
.FirstOrDefault(s => s.IsMediaSource && MediaSessionKey(s) == sessionKey);
|
||||
if (source?.MediaPath is not { } path) return;
|
||||
|
||||
if (!await _mediaManager.AcquireAsync(sessionKey, path, source.ChromaKey)) return;
|
||||
|
||||
// A session can be running before this layer existed (shared look), in which
|
||||
// case the bitmap is already published and will not be raised again for us.
|
||||
if (_mediaManager.GetPreviewBitmap(sessionKey) is { } bitmap)
|
||||
OnMediaPreviewBitmapChanged(sessionKey, bitmap);
|
||||
|
||||
ApplyMediaAudioSettings(sessionKey);
|
||||
}
|
||||
|
||||
/// <summary>The session's sound controls: one decoder is shared per path+look, so the level
|
||||
/// and mute are per session — the layer that was last touched is the one that wins.</summary>
|
||||
private void ApplyMediaAudioSettings(string sessionKey)
|
||||
{
|
||||
var source = Scenes.SelectMany(s => s.Elements).OfType<Source>()
|
||||
.FirstOrDefault(s => s.IsMediaSource && MediaSessionKey(s) == sessionKey);
|
||||
if (source == null) return;
|
||||
_mediaManager.SetAudioSettings(path, source.MediaVolume, source.MediaMuted);
|
||||
_mediaManager.SetAudioSettings(sessionKey, source.MediaVolume, source.MediaMuted);
|
||||
}
|
||||
|
||||
/// <summary>A clip's own soundtrack joins the desktop/game group, so the desktop fader and
|
||||
/// the voice ducker treat it exactly like game audio (TASK 21 audio slice).</summary>
|
||||
private void OnMediaAudioAvailable(string path, AudioSample sample)
|
||||
private void OnMediaAudioAvailable(string sessionKey, AudioSample sample)
|
||||
=> _audioMixer.EnqueueDesktopMediaAudio(sample);
|
||||
|
||||
/// <summary>Shared sessions or not, the loop state is the path's: any live media
|
||||
/// reference asking for a loop makes the one decoder loop (recorded rule, TASK 21).</summary>
|
||||
private bool AnyMediaReferenceLoops(string path)
|
||||
/// <summary>Shared sessions or not, the loop state is the session's: any live reference
|
||||
/// asking for a loop makes that one decoder loop (recorded rule, TASK 21).</summary>
|
||||
private bool AnyMediaReferenceLoops(string sessionKey)
|
||||
=> Scenes.SelectMany(s => s.Elements).OfType<Source>()
|
||||
.Any(s => s.IsMediaSource && s.MediaPath == path && s.MediaIsLooping);
|
||||
.Any(s => s.IsMediaSource && MediaSessionKey(s) == sessionKey && s.MediaIsLooping);
|
||||
|
||||
private bool HasMediaReference(string path)
|
||||
private bool HasMediaReference(string sessionKey)
|
||||
=> Scenes.SelectMany(s => s.Elements).OfType<Source>()
|
||||
.Any(s => s.IsMediaSource && s.MediaPath == path);
|
||||
.Any(s => s.IsMediaSource && string.Equals(MediaSessionKey(s), sessionKey, StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
private void ReleaseMediaIfUnused(string? path)
|
||||
private void ReleaseMediaIfUnused(string? sessionKey)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path)) return;
|
||||
if (!_mediaManager.HasSession(path)) return;
|
||||
if (HasMediaReference(path)) return;
|
||||
_ = _mediaManager.ReleaseAsync(path);
|
||||
if (string.IsNullOrWhiteSpace(sessionKey)) return;
|
||||
if (!_mediaManager.HasSession(sessionKey)) return;
|
||||
if (HasMediaReference(sessionKey)) return;
|
||||
_ = _mediaManager.ReleaseAsync(sessionKey);
|
||||
}
|
||||
|
||||
/// <summary>After a layout load: subscribe every persisted media layer and start one
|
||||
/// session per distinct file; release sessions for files the layout no longer uses.</summary>
|
||||
/// session per distinct path+look; release sessions the layout no longer uses.</summary>
|
||||
private void ReacquireMediaSources()
|
||||
{
|
||||
var sources = Scenes.SelectMany(s => s.Elements).OfType<Source>()
|
||||
@@ -191,10 +261,17 @@ public partial class MainViewModel
|
||||
foreach (var source in sources)
|
||||
RegisterMediaSource(source);
|
||||
|
||||
var wanted = sources.Select(s => s.MediaPath!).Distinct(StringComparer.OrdinalIgnoreCase).ToList();
|
||||
var wanted = sources.Select(MediaSessionKey)
|
||||
.Where(key => key.Length > 0)
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
foreach (var stale in _mediaManager.GetActiveKeys()
|
||||
.Where(key => !wanted.Contains(key, StringComparer.OrdinalIgnoreCase))
|
||||
.ToList())
|
||||
_ = _mediaManager.ReleaseAllAsync(stale);
|
||||
}
|
||||
|
||||
/// <summary>A session key carries a file path and a look; the toast wants the file name.</summary>
|
||||
private static string SessionPath(string sessionKey)
|
||||
=> sessionKey.Split('|')[0];
|
||||
}
|
||||
|
||||
@@ -300,16 +300,16 @@ public partial class MainViewModel
|
||||
_webView2Manager?.Unregister(webSource.Id);
|
||||
|
||||
// Media: the layer stops listening, and its decode session stops once no scene
|
||||
// references the file any more (sessions are shared per path).
|
||||
string? mediaPath = null;
|
||||
// references the same file + look any more (TASK 64: sessions are per path + look).
|
||||
string? mediaSessionKey = null;
|
||||
if (element is Source mediaSource && mediaSource.IsMediaSource)
|
||||
{
|
||||
mediaSessionKey = MediaSessionKey(mediaSource);
|
||||
UnregisterMediaSource(mediaSource);
|
||||
mediaPath = mediaSource.MediaPath;
|
||||
}
|
||||
|
||||
_sceneGraph.RemoveElement(scene, element);
|
||||
ReleaseMediaIfUnused(mediaPath);
|
||||
ReleaseMediaIfUnused(mediaSessionKey);
|
||||
|
||||
if (element is WebcamSceneConfig)
|
||||
{
|
||||
|
||||
@@ -190,6 +190,8 @@ public partial class MainViewModel : ViewModelBase
|
||||
public ICommand AddImageCommand { get; }
|
||||
public ICommand AddMediaCommand { get; }
|
||||
public ICommand ChangeMediaFileCommand { get; }
|
||||
/// <summary>Accepts the detector's green/blue-screen offer on a clip (TASK 64).</summary>
|
||||
public ICommand AcceptChromaSuggestionCommand { get; }
|
||||
public ICommand RemoveSourceCommand { get; }
|
||||
public ICommand EditElementCommand { get; }
|
||||
public ICommand ToggleElementVisibilityCommand { get; }
|
||||
@@ -266,6 +268,9 @@ public partial class MainViewModel : ViewModelBase
|
||||
ChangeMediaFileCommand = new RelayCommand(
|
||||
element => ChangeMediaFile(element as Source),
|
||||
element => element is Source { IsMediaSource: true });
|
||||
AcceptChromaSuggestionCommand = new RelayCommand(
|
||||
element => AcceptChromaSuggestion(element as Source),
|
||||
element => element is Source { HasChromaSuggestion: true });
|
||||
RemoveSourceCommand = new RelayCommand(element => { PushPreviewUndo(); RemoveElement(element as SceneElement); });
|
||||
EditElementCommand = new RelayCommand(element => BeginEditElement(element as SceneElement));
|
||||
ToggleElementVisibilityCommand = new RelayCommand(element => { PushPreviewUndo(); ToggleElementVisibility(element as SceneElement); });
|
||||
@@ -401,12 +406,18 @@ public partial class MainViewModel : ViewModelBase
|
||||
_screenCaptureManager.PreviewBitmapChanged += OnScreenPreviewBitmapChanged;
|
||||
_screenCaptureManager.CaptureFailed += OnScreenCaptureFailed;
|
||||
|
||||
_chromaProbe = ChromaProbeOverride
|
||||
?? new FfmpegChromaProbe(new FfmpegLocator(), () => new FfmpegDecodeProcess());
|
||||
|
||||
_mediaManager = new MediaVideoSourceManager(
|
||||
MediaFrameSourceFactoryOverride ?? (path => new MediaVideoSource(
|
||||
path, MediaDecodeWidth, MediaDecodeHeight,
|
||||
new FfmpegLocator(), () => new FfmpegDecodeProcess(),
|
||||
frameRateProbe: new FfmpegFrameRateProbe(
|
||||
new FfmpegLocator(), () => new FfmpegDecodeProcess()))),
|
||||
MediaFrameSourceFactoryOverride is { } overrideFactory
|
||||
? (path, _) => overrideFactory(path)
|
||||
: (path, chromaKey) => new MediaVideoSource(
|
||||
path, MediaDecodeWidth, MediaDecodeHeight,
|
||||
new FfmpegLocator(), () => new FfmpegDecodeProcess(),
|
||||
frameRateProbe: new FfmpegFrameRateProbe(
|
||||
new FfmpegLocator(), () => new FfmpegDecodeProcess()),
|
||||
chromaKey: chromaKey),
|
||||
System.Windows.Application.Current?.Dispatcher,
|
||||
// Loop intent is per Source; the manager needs the per-*path* answer.
|
||||
AnyMediaReferenceLoops);
|
||||
@@ -479,6 +490,10 @@ public partial class MainViewModel : ViewModelBase
|
||||
/// lifecycle without spawning ffmpeg. Mirrors <see cref="CameraFrameSourceFactoryOverride"/>.</summary>
|
||||
internal static Func<string, IMediaFrameSource?>? MediaFrameSourceFactoryOverride { get; set; }
|
||||
|
||||
/// <summary>Test seam (TASK 64): a fake chroma probe so tests never spawn ffmpeg to look at
|
||||
/// a clip's backdrop.</summary>
|
||||
internal static IChromaProbe? ChromaProbeOverride { get; set; }
|
||||
|
||||
private void LoadLayout()
|
||||
{
|
||||
_isLoading = true;
|
||||
@@ -678,7 +693,8 @@ public partial class MainViewModel : ViewModelBase
|
||||
Source { Type: SourceType.Scoreboard } => _scoreboardPresenter?.Frame,
|
||||
Source { Type: SourceType.AlertBox } alertBox => RenderAlertBox(alertBox),
|
||||
Source { Type: SourceType.WebSource } web => _webView2Manager?.GetLatestFrame(web.Id),
|
||||
Source { IsMediaSource: true, MediaPath: not null } media => _mediaManager.GetLatestFrame(media.MediaPath),
|
||||
Source { IsMediaSource: true, MediaPath: not null } media
|
||||
=> _mediaManager.GetLatestFrame(MediaSessionKey(media)),
|
||||
Source { AssetId: not null } image => StaticPixelCache.Get(image.AssetId),
|
||||
_ => null,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user