fix(preview): draw video layers, unlink Play from Loop, geometry last

Three reports from the creator, plus two bugs the real clip exposed.

1. A video layer was INVISIBLE in the preview. The pane's per-layer Image is Collapsed by
   default and switched on by a whitelist of DataTriggers; MediaSource matched none, so the clip
   decoded into a bitmap that nothing displayed. Same defect the alert box hit on 2026-09-26 and
   the scoreboard after it — now recorded at the top of docs/ai/ui-surface.md with a test that
   asserts the bound Image is Visible (`MediaPreviewDisplayTests`).

2. Play is no longer tied to Loop: the button is always enabled (creator ruling — Loop must not
   toggle Play). Loop still restarts a finished clip, which is the behaviour that was broken.

3. Geometry is now the LAST section of the properties list for every draggable element, with
   opacity at the top; the web source's duplicate geometry block is gone (it now uses the shared
   one, which had been sitting above the media panel).

4. The physical mouse-drag test is deleted. It injected real cursor movement, hijacked whatever
   machine ran it, and failed whenever the desktop was busy. The arithmetic it covered now lives
   in `Helpers/DragReorderMath.cs` — pure, and covered by `DragReorderMathTests` (threshold,
   up/down moves, no-op and out-of-range). The two persistence tests were already synthetic.

5. Real-clip check (creator's `test_greenscreen_moving_subject.mp4`, 1920×1080): the detector
   REJECTED it — a fully saturated green sits at value 1.0 and the detector's 0.98 brightness
   ceiling threw it away. White highlights are excluded by the saturation floor, which is the
   property that actually separates them, so the ceiling is gone. With it fixed, the clip is
   detected as green (high confidence), and keyed through our own pipe 97.8% of pixels go
   transparent. Measured: keying with the canonical #00B140 keys 0.0% of this clip — which is why
   the key colour comes from the measured screen.

Tests: 567 passed, 0 failed, clean build 0 warnings. The gate's scope check flags
`Assets/Videos/test_greenscreen_moving_subject.mp4` — the creator's working copy, deliberately not
committed; the fixture lives under ytLive.Tests/Fixtures.
This commit is contained in:
2026-11-05 09:38:18 -08:00
parent 242a13ca30
commit 0f4a035376
14 changed files with 401 additions and 183 deletions
+87
View File
@@ -0,0 +1,87 @@
using System;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Xunit;
using ytLive.Services;
using ytLive.Services.Encoder;
namespace ytLive.Tests;
/// <summary>
/// TASK 64 against a REAL clip (creator's `test_greenscreen_moving_subject.mp4`, 1920×1080, 5 s, a
/// subject moving over a green field). The detector had only ever seen synthetic frames; this runs
/// the bundled ffmpeg, our probe, our detector, and our keyed decode on actual video.
/// <para>It also pins the decision that the key colour comes from the measured screen rather than a
/// canonical green — on this clip the field is #00FE00 and a hardcoded #00B140 keys <b>nothing</b>
/// (measured: 0.0% of pixels transparent), while the detected colour keys 97.8%.</para>
/// </summary>
public class ChromaKeyRealClipTests
{
private const string ClipName = "test_greenscreen_moving_subject.mp4";
private static string ClipPath => Path.Combine(AppContext.BaseDirectory, "Fixtures", ClipName);
private static string FfmpegPath => Path.Combine(AppContext.BaseDirectory, "ffmpeg", "ffmpeg.exe");
[Fact]
public async Task RealGreenScreenClip_IsDetected_AndKeysWithTheMeasuredColour()
{
Assert.True(File.Exists(ClipPath), $"missing fixture: {ClipPath}");
Assert.True(File.Exists(FfmpegPath), $"the bundled ffmpeg should sit beside the test binary: {FfmpegPath}");
// 1. The probe reads one real frame of the clip.
var probe = new FfmpegChromaProbe(new FixedLocator(FfmpegPath), () => new FfmpegDecodeProcess());
var frame = await probe.ProbeAsync(ClipPath);
Assert.NotNull(frame);
// 2. The detector says "green screen", with high confidence.
var suggestion = ChromaKeyDetector.Detect(frame!);
Assert.Equal(ChromaScreen.Green, suggestion.Screen);
Assert.True(suggestion.IsHighConfidence,
$"a clean green field should be high confidence, got {suggestion.Confidence:0.00}");
Assert.NotNull(suggestion.ColorHex);
// …and the colour it names is the field as shot: green-dominant, almost no red or blue.
var (r, g, b) = ParseHex(suggestion.ColorHex!);
Assert.True(g > 0xE0 && r < 0x20 && b < 0x20,
$"the detected colour should be the measured field, got {suggestion.ColorHex}");
// 3. That colour actually keys the clip through our own pipe: most pixels go transparent,
// the moving subject survives.
var keyed = ChromaKeyOptions.For(suggestion);
var firstFrame = new TaskCompletionSource<VideoFrame>(TaskCreationOptions.RunContinuationsAsynchronously);
using var pipe = new VideoClipPipe(
ClipPath, 320, 180, new FixedLocator(FfmpegPath), () => new FfmpegDecodeProcess(),
audio: false, chromaKey: keyed);
pipe.FrameAvailable += f => firstFrame.TrySetResult(f);
await pipe.StartAsync();
var keyedFrame = await firstFrame.Task.WaitAsync(TimeSpan.FromSeconds(30));
await pipe.StopAsync();
var transparent = 0;
for (var i = 3; i < keyedFrame.BgraPixels.Length; i += 4)
if (keyedFrame.BgraPixels[i] == 0) transparent++;
var ratio = (double)transparent / (keyedFrame.Width * keyedFrame.Height);
Assert.True(ratio > 0.5,
$"the keyed decode left only {ratio:P1} transparent — the filter chain is not keying this clip");
Assert.True(ratio < 1.0, "everything vanished; the subject should survive the key");
}
private static (int R, int G, int B) ParseHex(string hex)
{
var value = hex.TrimStart('#');
return (Convert.ToInt32(value[..2], 16), Convert.ToInt32(value.Substring(2, 2), 16),
Convert.ToInt32(value.Substring(4, 2), 16));
}
private sealed class FixedLocator : IFfmpegLocator
{
private readonly string _path;
public FixedLocator(string path) => _path = path;
public Task<string> LocateAsync(CancellationToken ct = default) => Task.FromResult(_path);
}
}
+70
View File
@@ -0,0 +1,70 @@
using System.Collections;
using Xunit;
using ytLive.Helpers;
namespace ytLive.Tests;
/// <summary>
/// The layer-list drag arithmetic, tested without a mouse. It replaced a test that injected REAL
/// cursor movement into the desktop — which hijacked the machine and failed whenever it was busy
/// (deleted 2026-11-05). Same behaviour covered here, in microseconds, with no input device.
/// </summary>
public class DragReorderMathTests
{
[Theory]
[InlineData(0, 0, false)] // a click, not a drag
[InlineData(3, 2, false)] // inside both thresholds (4 px is WPF's default)
[InlineData(4, 0, true)] // exactly the threshold counts — WPF's own rule
[InlineData(0, 4, true)]
[InlineData(-9, 1, true)] // direction is irrelevant
public void Threshold_DecidesDragVersusClick(double dx, double dy, bool dragging)
=> Assert.Equal(dragging, DragReorderMath.ExceededThreshold(dx, dy, 4, 4));
[Fact]
public void MovingDown_LandsOnTheTargetsIndex()
{
var items = new ArrayList { "A", "B", "C" };
var landed = DragReorderMath.TryMove(items, 0, 2);
Assert.Equal(2, landed);
Assert.Equal(new[] { "B", "C", "A" }, items.Cast<string>());
}
[Fact]
public void MovingUp_LandsOnTheTargetsIndex()
{
var items = new ArrayList { "A", "B", "C" };
var landed = DragReorderMath.TryMove(items, 2, 0);
Assert.Equal(0, landed);
Assert.Equal(new[] { "C", "A", "B" }, items.Cast<string>());
}
[Fact]
public void DraggingOntoItself_IsANoOp()
{
var items = new ArrayList { "A", "B", "C" };
Assert.Equal(-1, DragReorderMath.TryMove(items, 1, 1));
Assert.Equal(new[] { "A", "B", "C" }, items.Cast<string>());
}
[Theory]
[InlineData(-1, 0)]
[InlineData(0, -1)]
[InlineData(3, 0)]
[InlineData(0, 3)]
public void OutOfRangeIndices_ChangeNothing(int from, int to)
{
var items = new ArrayList { "A", "B", "C" };
Assert.Equal(-1, DragReorderMath.TryMove(items, from, to));
Assert.Equal(new[] { "A", "B", "C" }, items.Cast<string>());
}
[Fact]
public void EmptyList_IsSafe()
=> Assert.Equal(-1, DragReorderMath.TryMove(new ArrayList(), 0, 0));
}
@@ -54,122 +54,6 @@ public sealed class LayerReorderPersistenceTests
_app.Run(RunCrossTypeReload);
}
/// <summary>
/// Same use case as above but driven with REAL injected mouse input
/// (SetCursorPos + mouse_event against the desktop) so the actual
/// List_PreviewMouseLeftButtonDown/Move/Up handlers — the seam no synthetic
/// event can exercise, since e.GetPosition reads the physical cursor — must
/// run. Requires an interactive desktop session: if the window is covered or
/// the session is locked, the pointer no-ops and the drag never lands.
/// </summary>
[Fact]
public void RealMouseDrag_OnTheLayerList_PersistsTheReorder()
{
_app.Run(RunRealDrag);
}
private void RunRealDrag()
{
var tempDb = Path.Combine(Path.GetTempPath(), $"ytLlive-realdrag-{Guid.NewGuid():N}.db");
MainViewModel.LayoutPathOverride = tempDb;
var window = default(MainWindow);
try
{
using (var schema = new LayoutStore(tempDb)) { }
SqliteConnection.ClearAllPools();
SeedThreeLayers(tempDb);
window = new MainWindow();
var vm = (MainViewModel)window.DataContext;
vm.StagedScene = vm.Scenes.Single(s => SceneCatalog.Is(s.Name, SceneCatalog.Live));
Assert.True(vm.StagedScene!.Elements.OfType<Source>().Count() >= 4,
"3 seeded layers + the healed Background + Brand logo must load");
window.Show();
window.Activate();
window.Topmost = true;
window.UpdateLayout();
var panel = (UserControl)window.FindName("LeftPanel")!;
var list = (FocusPreservingListBox)panel.FindName("SourceList")!;
list.UpdateLayout();
// Brand logo is a permanent layer between Background and user layers.
// Physically drag ImgC above ImgA; the background and logo keep their
// relative positions while EndListDrag persists the user-layer reorder.
var sources = vm.StagedScene.Elements.OfType<Source>().ToArray();
var from = RowScreenPoint(list, sources.First(s => s.Name == "ImgC"));
var to = RowScreenPoint(list, sources.First(s => s.Name == "ImgA"));
SetCursorPos((int)from.X, (int)from.Y);
LetInputSettle(80);
mouse_event(MOUSEEVENTF_LEFTDOWN, 0, 0, 0, UIntPtr.Zero);
LetInputSettle(120);
SetCursorPos((int)to.X, (int)to.Y);
LetInputSettle(250);
mouse_event(MOUSEEVENTF_LEFTUP, 0, 0, 0, UIntPtr.Zero);
LetInputSettle(400);
var expected = vm.StagedScene.Elements.OfType<Source>().Select(s => s.Name).ToArray();
Assert.Equal("Background", expected[0]);
Assert.Equal("Brand logo", expected[1]);
Assert.Equal("ImgC", expected[2]);
Assert.True(Array.IndexOf(expected, "ImgC") < Array.IndexOf(expected, "ImgA"));
Assert.True(PumpUntil(() => SourceNamesInDb(tempDb).SequenceEqual(expected),
TimeSpan.FromSeconds(10)),
"a real mouse drag-and-drop must be persisted by the background save");
Assert.Equal(expected, SourceNamesInDb(tempDb));
// The same drop must persist the PREVIEW LAYOUT too — each layer's
// geometry (X/Y/W/H) rides in the same Source rows as the SortOrder.
var dragged = vm.StagedScene.Elements.OfType<Source>().Single(s => s.Name == "ImgC");
Assert.Equal((dragged.X, dragged.Y, dragged.Width, dragged.Height),
GeometryInDb(tempDb, "ImgC"));
}
finally
{
if (window != null) window.Close();
MainViewModel.LayoutPathOverride = null;
SqliteConnection.ClearAllPools();
try { File.Delete(tempDb); } catch { /* best-effort cleanup */ }
}
}
/// <summary>Screen (device-pixel) point one-third into an item's list row — over
/// the name TextBlock, away from the per-row buttons/rename TextBox.</summary>
private static Point RowScreenPoint(FocusPreservingListBox list, Models.SceneElement item)
{
var container = (ListBoxItem)list.ItemContainerGenerator.ContainerFromItem(item)!;
var inList = container.TransformToVisual(list)
.Transform(new Point(Math.Min(30, container.ActualWidth - 12), container.ActualHeight / 2));
return list.PointToScreen(inList);
}
/// <summary>Pumps the app dispatcher while sleeping in small steps, so real
/// input messages posted to the thread queue are processed mid-gesture.</summary>
private static void LetInputSettle(int milliseconds)
{
var frame = new DispatcherFrame();
var timer = new DispatcherTimer { Interval = TimeSpan.FromMilliseconds(milliseconds) };
timer.Tick += (_, _) =>
{
timer.Stop();
frame.Continue = false;
};
timer.Start();
Dispatcher.PushFrame(frame);
}
private const uint MOUSEEVENTF_LEFTDOWN = 0x0002;
private const uint MOUSEEVENTF_LEFTUP = 0x0004;
[DllImport("user32.dll")]
private static extern bool SetCursorPos(int x, int y);
[DllImport("user32.dll")]
private static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint dwData, UIntPtr dwExtraInfo);
private void Run()
{
var tempDb = Path.Combine(Path.GetTempPath(), $"ytLlive-reorder-{Guid.NewGuid():N}.db");
+127
View File
@@ -0,0 +1,127 @@
using System;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Controls;
using System.Windows.Media;
using Microsoft.Data.Sqlite;
using Xunit;
using ytLive.Models;
using ytLive.Services;
using ytLive.ViewModels;
namespace ytLive.Tests;
/// <summary>
/// 2026-11-05 report: a video layer added and playing showed NOTHING in the preview pane. The decode
/// was healthy — the preview's <c>Image</c> is switched on by a whitelist of DataTriggers (image,
/// chat, alert, ticker, scoreboard, text, web, webcam) and a video layer matched none of them, so its
/// Image stayed Collapsed. Same class of defect as the invisible alert box (2026-09-26).
/// <para>These assert the two ends: the layer receives frames, and the pane actually shows it.</para>
/// </summary>
[Collection("RealApp")]
public sealed class MediaPreviewDisplayTests
{
private readonly RealAppHost _app;
public MediaPreviewDisplayTests(RealAppHost app) => _app = app;
[Fact]
public void VideoLayer_IsDrawnByThePreviewPane()
{
_app.Run(() =>
{
var tempDb = Path.Combine(Path.GetTempPath(), $"ytLlive-test-{Guid.NewGuid():N}.db");
var videoPath = Path.Combine(Path.GetTempPath(), "preview-clip.mp4");
var decoders = new System.Collections.Generic.List<FakeMediaSource>();
MainViewModel.LayoutPathOverride = tempDb;
MainViewModel.MediaFrameSourceFactoryOverride = key =>
{
var decoder = new FakeMediaSource(key);
decoders.Add(decoder);
return decoder;
};
MainViewModel.ChromaProbeOverride = new NoProbe();
var window = new MainWindow();
var vm = (MainViewModel)window.DataContext;
try
{
window.Show();
window.UpdateLayout();
vm.AddMediaSource(videoPath);
var layer = vm.StagedScene!.Elements.OfType<Source>().Single(s => s.IsMediaSource);
// The layer receives a decoded frame…
var frame = SolidFrame(64, 36);
decoders.Single().Emit(frame);
Assert.NotNull(layer.DisplaySource);
window.UpdateLayout();
// …and the preview pane actually renders it. Without the IsMediaSource trigger the
// Image is Collapsed and the clip is invisible while everything else looks healthy.
var pane = (UserControl)window.FindName("PreviewPane")!;
var image = FindImageBoundTo(pane, layer.DisplaySource!);
Assert.True(image != null,
"the preview pane has no Image bound to the video layer's frame — the layer is invisible");
Assert.Equal(Visibility.Visible, image!.Visibility);
Assert.True(image.ActualWidth > 0 && image.ActualHeight > 0,
$"the video layer's preview Image has no size ({image.ActualWidth}x{image.ActualHeight})");
}
finally
{
window.Close();
MainViewModel.ChromaProbeOverride = null;
MainViewModel.MediaFrameSourceFactoryOverride = null;
MainViewModel.LayoutPathOverride = null;
SqliteConnection.ClearAllPools();
try { File.Delete(tempDb); } catch { /* best-effort cleanup */ }
}
});
}
private static Image? FindImageBoundTo(DependencyObject parent, ImageSource source)
{
for (var i = 0; i < VisualTreeHelper.GetChildrenCount(parent); i++)
{
var child = VisualTreeHelper.GetChild(parent, i);
if (child is Image image && ReferenceEquals(image.Source, source)) return image;
if (FindImageBoundTo(child, source) is { } found) return found;
}
return null;
}
private static VideoFrame SolidFrame(int w, int h)
{
var bytes = new byte[w * h * 4];
for (var i = 0; i < bytes.Length; i += 4)
{
bytes[i] = 40; bytes[i + 1] = 200; bytes[i + 2] = 40; bytes[i + 3] = 255;
}
return new VideoFrame(w, h, bytes);
}
private sealed class FakeMediaSource : IMediaFrameSource
{
public FakeMediaSource(string key) => Key = key;
public string Key { get; }
public bool Looping { get; set; }
public event Action<VideoFrame>? FrameAvailable;
public event Action<ytLive.Services.Audio.AudioSample>? AudioAvailable { add { } remove { } }
public event Action? Completed { add { } remove { } }
public void Emit(VideoFrame frame) => FrameAvailable?.Invoke(frame);
public Task StartAsync(CancellationToken cancellationToken = default) => Task.CompletedTask;
public Task StopAsync() => Task.CompletedTask;
public void Dispose() { }
}
private sealed class NoProbe : IChromaProbe
{
public Task<VideoFrame?> ProbeAsync(string path, CancellationToken cancellationToken = default)
=> Task.FromResult<VideoFrame?>(null);
}
}