feat: web sources frame-captured via composition (CoreWebView2CompositionController → Windows.Graphics.Capture); PNG poll + CaptureScheduler deleted
The ~30Hz CapturePreviewAsync PNG poll capped real cadence at ~20Hz (35–165ms full-HD encode+decode), so a 60fps widget still juddered at ~1/6 speed. Replaced polling with frame-driven capture of the composition controller's root visual — the mechanism WebView2CompositionControl and Flutter's webview_windows use (graphics_context.cc captures the root surface_ visual via CreateGraphicsCaptureItemFromVisual; reference: github.com/microsoft/Windows.UI.Composition.WinUI / flutter-internal webview_windows). Frames now arrive at the renderer's own pace; capture memory is epoch'd ring reuse + one crop-sized shared WriteableBitmap. New Services/WebCaptureFrameSource.cs owns GraphicsCaptureItem + free- threaded Direct3D11CaptureFramePool + session (Straight alpha readback, per-frame FindContentBounds → CropBounds). WebView2Manager reworked around per-session composition controllers + one UI-thread Compositor created via the CoreMessaging CreateDispatcherQueueController P/Invoke (the 19041 projection lacks CreateOnCurrentThread); internal seam ctor (Dispatcher, Func<string,IScreenCaptureSource>?) for hermetic tests. CaptureScheduler.cs deleted; the three SetCaptureInterval cadence hooks removed; InitWebView2() moved from MainWindow ctor to Loaded (a parent HWND must exist for the composition controller); the hidden WebViewHostPanel overlay deleted. TransparentBackgroundScript unchanged. Tests: WebView2ManagerTests reworked — 4 control-size + scheduler tests dropped, FindContentBounds tests moved to WebCaptureFrameSource, ONE integration test (Frames_PublishCroppedPreview_And_CoalesceToLatest_CarryingCropBounds) drives the seam with a FakeWebSource + background-STA DispatcherPump. Suite 293/293, 0 warnings. NOTE: composition path NOT yet verified on a device — the take is the next step. Web work committed locally only (no push per standing rule). Docs same-commit: ai.md Slice 14 + supersede marker on Slice 11, HANDOFF, MyMistakes (CoreMessaging DQ + namespace-landmine recipe), TASK 17, Controls/ViewModels/Services indexes.
This commit is contained in:
@@ -84,10 +84,11 @@ public sealed class RoundClipInteractionTests
|
||||
// of PreviewGrid); otherwise Window_PreviewMouseLeftButtonDown
|
||||
// deselects the source and the handle can never be grabbed.
|
||||
// UIElement.InputHitTest IS the input pipeline — what Mouse uses.
|
||||
// Do NOT use VisualTreeHelper.HitTest here: it returned the window's
|
||||
// non-hit-testable WebViewHostPanel overlay for EVERY point (it does
|
||||
// not model IsHitTestVisible input semantics) — the bug this test
|
||||
// "found" for months was this wrong API (root-caused 2026-09-01).
|
||||
// Do NOT use VisualTreeHelper.HitTest here: it ignores IsHitTestVisible
|
||||
// input semantics (the old invisible WebView2 host overlay for web
|
||||
// sources used to win every visual hit — the wrong-API false bug this
|
||||
// test "found" for months, root-caused 2026-09-01; the overlay is gone
|
||||
// with the composition-based web render but the API lesson stays).
|
||||
var hit = window.InputHitTest(cornerInWindow) as DependencyObject;
|
||||
var grabbable = hit != null && IsDescendantOf(hit, previewGrid);
|
||||
|
||||
|
||||
@@ -1,114 +1,72 @@
|
||||
using System.Drawing;
|
||||
using System.Linq;
|
||||
using System.Windows.Controls;
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Windows.Threading;
|
||||
using Microsoft.Web.WebView2.Wpf;
|
||||
using System.Windows.Media.Imaging;
|
||||
using Xunit;
|
||||
using ytLive.Models;
|
||||
using ytLive.Services;
|
||||
|
||||
namespace ytLive.Tests;
|
||||
|
||||
[Collection("RealApp")]
|
||||
public sealed class WebView2ManagerTests
|
||||
/// <summary>
|
||||
/// WebView2Manager owns the frame-driven web sources (composition-captured, see
|
||||
/// Services/WebCaptureFrameSource.cs). The controller/visual/GraphicCaptureItem
|
||||
/// layer is WinRT and only exercisable on Windows at runtime, so these tests pin
|
||||
/// the pure pieces (transparency script, the alpha-bbox math) and the ONE
|
||||
/// integration test drives the session plumbing with a fake source: registration,
|
||||
/// crop-stamped frames, the shared cropped preview bitmap, and coalescing to the
|
||||
/// latest frame.
|
||||
/// </summary>
|
||||
public class WebView2ManagerTests
|
||||
{
|
||||
private readonly RealAppHost _app;
|
||||
|
||||
public WebView2ManagerTests(RealAppHost app)
|
||||
private sealed class FakeWebSource : IScreenCaptureSource
|
||||
{
|
||||
_app = app;
|
||||
public string Key { get; }
|
||||
public event Action<VideoFrame>? FrameAvailable;
|
||||
|
||||
public FakeWebSource(string key) => Key = key;
|
||||
|
||||
public Task StartAsync() => Task.CompletedTask;
|
||||
public Task StopAsync() => Task.CompletedTask;
|
||||
|
||||
public void Pump(VideoFrame frame) => FrameAvailable?.Invoke(frame);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WebView2_Control_Size_Is_Canvas_Resolution_Regardless_Of_Source()
|
||||
// A real Dispatcher pumping on a background STA thread, so the manager's
|
||||
// BeginInvoke queue can be drained deterministically from the test thread
|
||||
// (same harness as ScreenCaptureManagerTests).
|
||||
private sealed class DispatcherPump : IDisposable
|
||||
{
|
||||
_app.Run(() =>
|
||||
private readonly Thread _thread;
|
||||
private readonly ManualResetEventSlim _ready = new(false);
|
||||
private Dispatcher? _dispatcher;
|
||||
|
||||
public Dispatcher Dispatcher => _dispatcher!;
|
||||
|
||||
public DispatcherPump()
|
||||
{
|
||||
var panel = new Canvas();
|
||||
var source = new Source
|
||||
_thread = new Thread(() =>
|
||||
{
|
||||
Type = SourceType.WebSource,
|
||||
Width = 640,
|
||||
Height = 480,
|
||||
};
|
||||
_dispatcher = Dispatcher.CurrentDispatcher;
|
||||
_ready.Set();
|
||||
Dispatcher.Run();
|
||||
});
|
||||
_thread.IsBackground = true;
|
||||
_thread.SetApartmentState(ApartmentState.STA);
|
||||
_thread.Start();
|
||||
_ready.Wait();
|
||||
}
|
||||
|
||||
using var manager = new WebView2Manager(panel, Dispatcher.CurrentDispatcher);
|
||||
manager.Register(source);
|
||||
// Queued behind anything already posted at Render/higher, so pending
|
||||
// bitmap copies run first.
|
||||
public void Drain() => Dispatcher.Invoke(() => { }, DispatcherPriority.ContextIdle);
|
||||
|
||||
var webView = panel.Children.OfType<WebView2>().First();
|
||||
Assert.Equal(1920.0, webView.Width);
|
||||
Assert.Equal(1080.0, webView.Height);
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WebView2_Control_Size_Does_Not_Track_Box_After_Register()
|
||||
{
|
||||
_app.Run(() =>
|
||||
public void Dispose()
|
||||
{
|
||||
var panel = new Canvas();
|
||||
var source = new Source
|
||||
{
|
||||
Type = SourceType.WebSource,
|
||||
Width = 640,
|
||||
Height = 480,
|
||||
};
|
||||
|
||||
using var manager = new WebView2Manager(panel, Dispatcher.CurrentDispatcher);
|
||||
manager.Register(source);
|
||||
|
||||
var webView = panel.Children.OfType<WebView2>().First();
|
||||
|
||||
source.Width = 1024;
|
||||
source.Height = 768;
|
||||
|
||||
Assert.Equal(1920.0, webView.Width);
|
||||
Assert.Equal(1080.0, webView.Height);
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WebView2_Control_Size_Ignores_Zero_Sized_Source()
|
||||
{
|
||||
_app.Run(() =>
|
||||
{
|
||||
var panel = new Canvas();
|
||||
var source = new Source
|
||||
{
|
||||
Type = SourceType.WebSource,
|
||||
Width = 0,
|
||||
Height = 0,
|
||||
};
|
||||
|
||||
using var manager = new WebView2Manager(panel, Dispatcher.CurrentDispatcher);
|
||||
manager.Register(source);
|
||||
|
||||
var webView = panel.Children.OfType<WebView2>().First();
|
||||
Assert.Equal(1920.0, webView.Width);
|
||||
Assert.Equal(1080.0, webView.Height);
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WebView2_DefaultBackgroundColor_Is_Transparent()
|
||||
{
|
||||
_app.Run(() =>
|
||||
{
|
||||
var panel = new Canvas();
|
||||
var source = new Source
|
||||
{
|
||||
Type = SourceType.WebSource,
|
||||
Width = 640,
|
||||
Height = 480,
|
||||
};
|
||||
|
||||
using var manager = new WebView2Manager(panel, Dispatcher.CurrentDispatcher);
|
||||
manager.Register(source);
|
||||
|
||||
var webView = panel.Children.OfType<WebView2>().First();
|
||||
|
||||
Assert.Equal(System.Drawing.Color.Transparent, webView.DefaultBackgroundColor);
|
||||
});
|
||||
Dispatcher.InvokeShutdown();
|
||||
_thread.Join(3000);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -142,7 +100,7 @@ public sealed class WebView2ManagerTests
|
||||
pixels[i + 3] = 255; // A
|
||||
}
|
||||
|
||||
var (x, y, bw, bh) = WebView2Manager.FindContentBounds(pixels, w, h, w * 4);
|
||||
var (x, y, bw, bh) = WebCaptureFrameSource.FindContentBounds(pixels, w, h, w * 4);
|
||||
|
||||
Assert.Equal((3, 2, 4, 3), (x, y, bw, bh));
|
||||
}
|
||||
@@ -153,7 +111,7 @@ public sealed class WebView2ManagerTests
|
||||
const int w = 10, h = 6;
|
||||
var pixels = new byte[w * h * 4];
|
||||
|
||||
var (x, y, bw, bh) = WebView2Manager.FindContentBounds(pixels, w, h, w * 4);
|
||||
var (x, y, bw, bh) = WebCaptureFrameSource.FindContentBounds(pixels, w, h, w * 4);
|
||||
|
||||
Assert.Equal((0, 0, w, h), (x, y, bw, bh));
|
||||
}
|
||||
@@ -166,36 +124,108 @@ public sealed class WebView2ManagerTests
|
||||
for (var i = 3; i < pixels.Length; i += 4)
|
||||
pixels[i] = 255;
|
||||
|
||||
var (x, y, bw, bh) = WebView2Manager.FindContentBounds(pixels, w, h, w * 4);
|
||||
var (x, y, bw, bh) = WebCaptureFrameSource.FindContentBounds(pixels, w, h, w * 4);
|
||||
|
||||
Assert.Equal((0, 0, w, h), (x, y, bw, bh));
|
||||
}
|
||||
|
||||
// The ONE integration test for this change: a registered session creates one
|
||||
// shared WriteableBitmap sized to the widget's CropBounds, publishes it once,
|
||||
// and back-to-back frames coalesce to the latest (a single pending UI copy,
|
||||
// mirror of ScreenCaptureManager) — the frame-driven contract the PNG-poll
|
||||
// redesign replaced the CaptureScheduler cadence with.
|
||||
[Fact]
|
||||
public async Task CaptureScheduler_Drops_Ticks_While_Capture_InFlight_And_Resumes()
|
||||
public void Frames_PublishCroppedPreview_And_CoalesceToLatest_CarryingCropBounds()
|
||||
{
|
||||
// The ~30Hz recording cadence would stack concurrent CapturePreviewAsync calls
|
||||
// (full-HD PNG encode is ~10-30ms) without the latest-wins drop: ticks that
|
||||
// arrive while a capture is in flight must be skipped, never queued, and a
|
||||
// capture can always start once the previous one has finished.
|
||||
var calls = 0;
|
||||
var signal = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
var scheduler = new CaptureScheduler(Dispatcher.CurrentDispatcher, 200, async () =>
|
||||
using var pump = new DispatcherPump();
|
||||
FakeWebSource? fake = null;
|
||||
using var manager = new WebView2Manager(pump.Dispatcher, key => fake = new FakeWebSource(key));
|
||||
|
||||
WriteableBitmap? published = null;
|
||||
var publishedCount = 0;
|
||||
manager.PreviewBitmapChanged += (_, bitmap) => { published = bitmap; publishedCount++; };
|
||||
|
||||
var source = new Source { Type = SourceType.WebSource, Width = 640, Height = 480 };
|
||||
manager.Register(source);
|
||||
Assert.NotNull(fake); // the seam instantiated + started the source inline
|
||||
Assert.Null(manager.GetLatestFrame(source.Id)); // nothing pumped yet
|
||||
|
||||
// 4x4 canvas; alpha 255 only in bbox (1,1)-(2,2) → the widget crops to (1,1,2,2).
|
||||
var first = WidgetFrame(4, 4, frameSeed: 42);
|
||||
fake!.Pump(first);
|
||||
pump.Drain();
|
||||
|
||||
Assert.NotNull(published);
|
||||
Assert.Equal(1, publishedCount); // one shared bitmap, published once
|
||||
var dims = pump.Dispatcher.Invoke(() => new[] { published!.PixelWidth, published.PixelHeight });
|
||||
Assert.Equal(new[] { 2, 2 }, dims);
|
||||
|
||||
var latest = manager.GetLatestFrame(source.Id);
|
||||
Assert.NotNull(latest);
|
||||
Assert.Equal((1, 1, 2, 2), latest!.CropBounds); // the source stamps the alpha bbox
|
||||
Assert.False(latest.IsOpaque); // straight-alpha web contract
|
||||
|
||||
var previewBytes = pump.Dispatcher.Invoke(() =>
|
||||
{
|
||||
calls++;
|
||||
await signal.Task;
|
||||
var bytes = new byte[16];
|
||||
published!.CopyPixels(new System.Windows.Int32Rect(0, 0, 2, 2), bytes, 8, 0);
|
||||
return bytes;
|
||||
});
|
||||
// The preview is a DENSE 2x2 rect (bbox 1,1..2,2 sliced out of the 4x4
|
||||
// canvas) — rows y=1 and y=2, columns x=1..2.
|
||||
var expected = CropBytes(first, w: 4, x0: 1, y0: 1, w2: 2, h2: 2);
|
||||
Assert.Equal(expected, previewBytes);
|
||||
|
||||
scheduler.Tick();
|
||||
scheduler.Tick();
|
||||
scheduler.Tick();
|
||||
Assert.Equal(1, calls); // overlapping ticks dropped, never queued
|
||||
// Second + third back-to-back: only the latest lands in the preview bitmap.
|
||||
var second = WidgetFrame(4, 4, frameSeed: 77);
|
||||
var third = WidgetFrame(4, 4, frameSeed: 99);
|
||||
fake.Pump(second);
|
||||
fake.Pump(third);
|
||||
pump.Drain();
|
||||
|
||||
signal.SetResult();
|
||||
await scheduler.Pending; // durable: Pending completes only after the drop resets
|
||||
|
||||
scheduler.Tick();
|
||||
Assert.Equal(2, calls); // once idle, the next tick captures again
|
||||
await scheduler.Pending;
|
||||
Assert.Equal(1, publishedCount); // same crop box → no new bitmap
|
||||
Assert.Same(third, manager.GetLatestFrame(source.Id));
|
||||
var coalesced = pump.Dispatcher.Invoke(() =>
|
||||
{
|
||||
var bytes = new byte[16];
|
||||
published!.CopyPixels(new System.Windows.Int32Rect(0, 0, 2, 2), bytes, 8, 0);
|
||||
return bytes;
|
||||
});
|
||||
Assert.Equal(CropBytes(third, 4, 1, 1, 2, 2), coalesced);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Dense little-endian slice of a w-wide frame for the rect
|
||||
/// (x0, y0)-(x0+w2-1, y0+h2-1); matches the manager's crop readback so the
|
||||
/// byte comparison is against the same layout, not the full frame stride.</summary>
|
||||
private static byte[] CropBytes(VideoFrame frame, int w, int x0, int y0, int w2, int h2)
|
||||
{
|
||||
var src = frame.BgraPixels;
|
||||
var dst = new byte[w2 * h2 * 4];
|
||||
for (var y = 0; y < h2; y++)
|
||||
System.Buffer.BlockCopy(src, ((y0 + y) * w + x0) * 4, dst, y * w2 * 4, w2 * 4);
|
||||
return dst;
|
||||
}
|
||||
|
||||
/// <summary>A 4x4 full canvas where the widget content is bbox (1,1,2,2):
|
||||
/// alpha 255 there, 0 elsewhere; the content pixels carry a seed so a
|
||||
/// coalesce mix-up is visible in the byte assertion.</summary>
|
||||
private static VideoFrame WidgetFrame(int w, int h, byte frameSeed)
|
||||
{
|
||||
var pixels = new byte[w * h * 4];
|
||||
for (var y = 0; y < h; y++)
|
||||
for (var x = 0; x < w; x++)
|
||||
{
|
||||
if (x < 1 || x > 2 || y < 1 || y > 2) continue;
|
||||
var i = (y * w + x) * 4;
|
||||
pixels[i] = frameSeed; // B
|
||||
pixels[i + 1] = (byte)(frameSeed / 3); // G
|
||||
pixels[i + 2] = 0; // R
|
||||
pixels[i + 3] = 255; // A
|
||||
}
|
||||
return new VideoFrame(w, h, pixels)
|
||||
{
|
||||
IsOpaque = false,
|
||||
CropBounds = (1, 1, 2, 2),
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user