gramps/LlamaCasty
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: web sources frame-captured via composition (CoreWebView2CompositionController → Windows.Graphics.Capture); PNG poll + CaptureScheduler deleted

Browse Source 
The ~30Hz CapturePreviewAsync PNG poll capped real cadence at ~20Hz
(35–165ms full-HD encode+decode), so a 60fps widget still juddered at ~1/6
speed. Replaced polling with frame-driven capture of the composition
controller's root visual — the mechanism WebView2CompositionControl and
Flutter's webview_windows use (graphics_context.cc captures the root
surface_ visual via CreateGraphicsCaptureItemFromVisual; reference:
github.com/microsoft/Windows.UI.Composition.WinUI / flutter-internal
webview_windows). Frames now arrive at the renderer's own pace; capture
memory is epoch'd ring reuse + one crop-sized shared WriteableBitmap.

New Services/WebCaptureFrameSource.cs owns GraphicsCaptureItem + free-
threaded Direct3D11CaptureFramePool + session (Straight alpha readback,
per-frame FindContentBounds → CropBounds). WebView2Manager reworked around
per-session composition controllers + one UI-thread Compositor created via
the CoreMessaging CreateDispatcherQueueController P/Invoke (the 19041
projection lacks CreateOnCurrentThread); internal seam ctor
(Dispatcher, Func<string,IScreenCaptureSource>?) for hermetic tests.
CaptureScheduler.cs deleted; the three SetCaptureInterval cadence hooks
removed; InitWebView2() moved from MainWindow ctor to Loaded (a parent
HWND must exist for the composition controller); the hidden WebViewHostPanel
overlay deleted. TransparentBackgroundScript unchanged.

Tests: WebView2ManagerTests reworked — 4 control-size + scheduler tests
dropped, FindContentBounds tests moved to WebCaptureFrameSource, ONE
integration test (Frames_PublishCroppedPreview_And_CoalesceToLatest_CarryingCropBounds)
drives the seam with a FakeWebSource + background-STA DispatcherPump.
Suite 293/293, 0 warnings.

NOTE: composition path NOT yet verified on a device — the take is the
next step. Web work committed locally only (no push per standing rule).

Docs same-commit: ai.md Slice 14 + supersede marker on Slice 11, HANDOFF,
MyMistakes (CoreMessaging DQ + namespace-landmine recipe), TASK 17,
Controls/ViewModels/Services indexes.
This commit is contained in:
Llama Chile Shop
2026-09-14 16:14:00 -07:00
parent 40c453805d
commit c01206fb8a
16 changed files with 890 additions and 581 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ytLive.Tests/WebView2ManagerTests.cs
+148 -118
View File
@@ -1,114 +1,72 @@
using System.Drawing;
using System.Linq;
using System.Windows.Controls;
using System;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Threading;
using Microsoft.Web.WebView2.Wpf;
using System.Windows.Media.Imaging;
using Xunit;
using ytLive.Models;
using ytLive.Services;
namespace ytLive.Tests;
[Collection("RealApp")]
public sealed class WebView2ManagerTests
/// <summary>
/// WebView2Manager owns the frame-driven web sources (composition-captured, see
/// Services/WebCaptureFrameSource.cs). The controller/visual/GraphicCaptureItem
/// layer is WinRT and only exercisable on Windows at runtime, so these tests pin
/// the pure pieces (transparency script, the alpha-bbox math) and the ONE
/// integration test drives the session plumbing with a fake source: registration,
/// crop-stamped frames, the shared cropped preview bitmap, and coalescing to the
/// latest frame.
/// </summary>
public class WebView2ManagerTests
{
private readonly RealAppHost _app;
public WebView2ManagerTests(RealAppHost app)
private sealed class FakeWebSource : IScreenCaptureSource
{
_app = app;
public string Key { get; }
public event Action<VideoFrame>? FrameAvailable;
public FakeWebSource(string key) => Key = key;
public Task StartAsync() => Task.CompletedTask;
public Task StopAsync() => Task.CompletedTask;
public void Pump(VideoFrame frame) => FrameAvailable?.Invoke(frame);
}
[Fact]
public void WebView2_Control_Size_Is_Canvas_Resolution_Regardless_Of_Source()
// A real Dispatcher pumping on a background STA thread, so the manager's
// BeginInvoke queue can be drained deterministically from the test thread
// (same harness as ScreenCaptureManagerTests).
private sealed class DispatcherPump : IDisposable
{
_app.Run(() =>
private readonly Thread _thread;
private readonly ManualResetEventSlim _ready = new(false);
private Dispatcher? _dispatcher;
public Dispatcher Dispatcher => _dispatcher!;
public DispatcherPump()
{
var panel = new Canvas();
var source = new Source
_thread = new Thread(() =>
{
Type = SourceType.WebSource,
Width = 640,
Height = 480,
};
_dispatcher = Dispatcher.CurrentDispatcher;
_ready.Set();
Dispatcher.Run();
});
_thread.IsBackground = true;
_thread.SetApartmentState(ApartmentState.STA);
_thread.Start();
_ready.Wait();
}
using var manager = new WebView2Manager(panel, Dispatcher.CurrentDispatcher);
manager.Register(source);
// Queued behind anything already posted at Render/higher, so pending
// bitmap copies run first.
public void Drain() => Dispatcher.Invoke(() => { }, DispatcherPriority.ContextIdle);
var webView = panel.Children.OfType<WebView2>().First();
Assert.Equal(1920.0, webView.Width);
Assert.Equal(1080.0, webView.Height);
});
}
[Fact]
public void WebView2_Control_Size_Does_Not_Track_Box_After_Register()
{
_app.Run(() =>
public void Dispose()
{
var panel = new Canvas();
var source = new Source
{
Type = SourceType.WebSource,
Width = 640,
Height = 480,
};
using var manager = new WebView2Manager(panel, Dispatcher.CurrentDispatcher);
manager.Register(source);
var webView = panel.Children.OfType<WebView2>().First();
source.Width = 1024;
source.Height = 768;
Assert.Equal(1920.0, webView.Width);
Assert.Equal(1080.0, webView.Height);
});
}
[Fact]
public void WebView2_Control_Size_Ignores_Zero_Sized_Source()
{
_app.Run(() =>
{
var panel = new Canvas();
var source = new Source
{
Type = SourceType.WebSource,
Width = 0,
Height = 0,
};
using var manager = new WebView2Manager(panel, Dispatcher.CurrentDispatcher);
manager.Register(source);
var webView = panel.Children.OfType<WebView2>().First();
Assert.Equal(1920.0, webView.Width);
Assert.Equal(1080.0, webView.Height);
});
}
[Fact]
public void WebView2_DefaultBackgroundColor_Is_Transparent()
{
_app.Run(() =>
{
var panel = new Canvas();
var source = new Source
{
Type = SourceType.WebSource,
Width = 640,
Height = 480,
};
using var manager = new WebView2Manager(panel, Dispatcher.CurrentDispatcher);
manager.Register(source);
var webView = panel.Children.OfType<WebView2>().First();
Assert.Equal(System.Drawing.Color.Transparent, webView.DefaultBackgroundColor);
});
Dispatcher.InvokeShutdown();
_thread.Join(3000);
}
}
[Fact]
@@ -142,7 +100,7 @@ public sealed class WebView2ManagerTests
pixels[i + 3] = 255; // A
}
var (x, y, bw, bh) = WebView2Manager.FindContentBounds(pixels, w, h, w * 4);
var (x, y, bw, bh) = WebCaptureFrameSource.FindContentBounds(pixels, w, h, w * 4);
Assert.Equal((3, 2, 4, 3), (x, y, bw, bh));
}
@@ -153,7 +111,7 @@ public sealed class WebView2ManagerTests
const int w = 10, h = 6;
var pixels = new byte[w * h * 4];
var (x, y, bw, bh) = WebView2Manager.FindContentBounds(pixels, w, h, w * 4);
var (x, y, bw, bh) = WebCaptureFrameSource.FindContentBounds(pixels, w, h, w * 4);
Assert.Equal((0, 0, w, h), (x, y, bw, bh));
}
@@ -166,36 +124,108 @@ public sealed class WebView2ManagerTests
for (var i = 3; i < pixels.Length; i += 4)
pixels[i] = 255;
var (x, y, bw, bh) = WebView2Manager.FindContentBounds(pixels, w, h, w * 4);
var (x, y, bw, bh) = WebCaptureFrameSource.FindContentBounds(pixels, w, h, w * 4);
Assert.Equal((0, 0, w, h), (x, y, bw, bh));
}
// The ONE integration test for this change: a registered session creates one
// shared WriteableBitmap sized to the widget's CropBounds, publishes it once,
// and back-to-back frames coalesce to the latest (a single pending UI copy,
// mirror of ScreenCaptureManager) — the frame-driven contract the PNG-poll
// redesign replaced the CaptureScheduler cadence with.
[Fact]
public async Task CaptureScheduler_Drops_Ticks_While_Capture_InFlight_And_Resumes()
public void Frames_PublishCroppedPreview_And_CoalesceToLatest_CarryingCropBounds()
{
// The ~30Hz recording cadence would stack concurrent CapturePreviewAsync calls
// (full-HD PNG encode is ~10-30ms) without the latest-wins drop: ticks that
// arrive while a capture is in flight must be skipped, never queued, and a
// capture can always start once the previous one has finished.
var calls = 0;
var signal = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var scheduler = new CaptureScheduler(Dispatcher.CurrentDispatcher, 200, async () =>
using var pump = new DispatcherPump();
FakeWebSource? fake = null;
using var manager = new WebView2Manager(pump.Dispatcher, key => fake = new FakeWebSource(key));
WriteableBitmap? published = null;
var publishedCount = 0;
manager.PreviewBitmapChanged += (_, bitmap) => { published = bitmap; publishedCount++; };
var source = new Source { Type = SourceType.WebSource, Width = 640, Height = 480 };
manager.Register(source);
Assert.NotNull(fake); // the seam instantiated + started the source inline
Assert.Null(manager.GetLatestFrame(source.Id)); // nothing pumped yet
// 4x4 canvas; alpha 255 only in bbox (1,1)-(2,2) → the widget crops to (1,1,2,2).
var first = WidgetFrame(4, 4, frameSeed: 42);
fake!.Pump(first);
pump.Drain();
Assert.NotNull(published);
Assert.Equal(1, publishedCount); // one shared bitmap, published once
var dims = pump.Dispatcher.Invoke(() => new[] { published!.PixelWidth, published.PixelHeight });
Assert.Equal(new[] { 2, 2 }, dims);
var latest = manager.GetLatestFrame(source.Id);
Assert.NotNull(latest);
Assert.Equal((1, 1, 2, 2), latest!.CropBounds); // the source stamps the alpha bbox
Assert.False(latest.IsOpaque); // straight-alpha web contract
var previewBytes = pump.Dispatcher.Invoke(() =>
{
calls++;
await signal.Task;
var bytes = new byte[16];
published!.CopyPixels(new System.Windows.Int32Rect(0, 0, 2, 2), bytes, 8, 0);
return bytes;
});
// The preview is a DENSE 2x2 rect (bbox 1,1..2,2 sliced out of the 4x4
// canvas) — rows y=1 and y=2, columns x=1..2.
var expected = CropBytes(first, w: 4, x0: 1, y0: 1, w2: 2, h2: 2);
Assert.Equal(expected, previewBytes);
scheduler.Tick();
scheduler.Tick();
scheduler.Tick();
Assert.Equal(1, calls); // overlapping ticks dropped, never queued
// Second + third back-to-back: only the latest lands in the preview bitmap.
var second = WidgetFrame(4, 4, frameSeed: 77);
var third = WidgetFrame(4, 4, frameSeed: 99);
fake.Pump(second);
fake.Pump(third);
pump.Drain();
signal.SetResult();
await scheduler.Pending; // durable: Pending completes only after the drop resets
scheduler.Tick();
Assert.Equal(2, calls); // once idle, the next tick captures again
await scheduler.Pending;
Assert.Equal(1, publishedCount); // same crop box → no new bitmap
Assert.Same(third, manager.GetLatestFrame(source.Id));
var coalesced = pump.Dispatcher.Invoke(() =>
{
var bytes = new byte[16];
published!.CopyPixels(new System.Windows.Int32Rect(0, 0, 2, 2), bytes, 8, 0);
return bytes;
});
Assert.Equal(CropBytes(third, 4, 1, 1, 2, 2), coalesced);
}
}
/// <summary>Dense little-endian slice of a w-wide frame for the rect
/// (x0, y0)-(x0+w2-1, y0+h2-1); matches the manager's crop readback so the
/// byte comparison is against the same layout, not the full frame stride.</summary>
private static byte[] CropBytes(VideoFrame frame, int w, int x0, int y0, int w2, int h2)
{
var src = frame.BgraPixels;
var dst = new byte[w2 * h2 * 4];
for (var y = 0; y < h2; y++)
System.Buffer.BlockCopy(src, ((y0 + y) * w + x0) * 4, dst, y * w2 * 4, w2 * 4);
return dst;
}
/// <summary>A 4x4 full canvas where the widget content is bbox (1,1,2,2):
/// alpha 255 there, 0 elsewhere; the content pixels carry a seed so a
/// coalesce mix-up is visible in the byte assertion.</summary>
private static VideoFrame WidgetFrame(int w, int h, byte frameSeed)
{
var pixels = new byte[w * h * 4];
for (var y = 0; y < h; y++)
for (var x = 0; x < w; x++)
{
if (x < 1 || x > 2 || y < 1 || y > 2) continue;
var i = (y * w + x) * 4;
pixels[i] = frameSeed; // B
pixels[i + 1] = (byte)(frameSeed / 3); // G
pixels[i + 2] = 0; // R
pixels[i + 3] = 255; // A
}
return new VideoFrame(w, h, pixels)
{
IsOpaque = false,
CropBounds = (1, 1, 2, 2),
};
}
}