feat: web sources frame-captured via composition (CoreWebView2CompositionController → Windows.Graphics.Capture); PNG poll + CaptureScheduler deleted
The ~30Hz CapturePreviewAsync PNG poll capped real cadence at ~20Hz (35–165ms full-HD encode+decode), so a 60fps widget still juddered at ~1/6 speed. Replaced polling with frame-driven capture of the composition controller's root visual — the mechanism WebView2CompositionControl and Flutter's webview_windows use (graphics_context.cc captures the root surface_ visual via CreateGraphicsCaptureItemFromVisual; reference: github.com/microsoft/Windows.UI.Composition.WinUI / flutter-internal webview_windows). Frames now arrive at the renderer's own pace; capture memory is epoch'd ring reuse + one crop-sized shared WriteableBitmap. New Services/WebCaptureFrameSource.cs owns GraphicsCaptureItem + free- threaded Direct3D11CaptureFramePool + session (Straight alpha readback, per-frame FindContentBounds → CropBounds). WebView2Manager reworked around per-session composition controllers + one UI-thread Compositor created via the CoreMessaging CreateDispatcherQueueController P/Invoke (the 19041 projection lacks CreateOnCurrentThread); internal seam ctor (Dispatcher, Func<string,IScreenCaptureSource>?) for hermetic tests. CaptureScheduler.cs deleted; the three SetCaptureInterval cadence hooks removed; InitWebView2() moved from MainWindow ctor to Loaded (a parent HWND must exist for the composition controller); the hidden WebViewHostPanel overlay deleted. TransparentBackgroundScript unchanged. Tests: WebView2ManagerTests reworked — 4 control-size + scheduler tests dropped, FindContentBounds tests moved to WebCaptureFrameSource, ONE integration test (Frames_PublishCroppedPreview_And_CoalesceToLatest_CarryingCropBounds) drives the seam with a FakeWebSource + background-STA DispatcherPump. Suite 293/293, 0 warnings. NOTE: composition path NOT yet verified on a device — the take is the next step. Web work committed locally only (no push per standing rule). Docs same-commit: ai.md Slice 14 + supersede marker on Slice 11, HANDOFF, MyMistakes (CoreMessaging DQ + namespace-landmine recipe), TASK 17, Controls/ViewModels/Services indexes.
This commit is contained in:
@@ -0,0 +1,232 @@
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Runtime.InteropServices.WindowsRuntime;
|
||||
using System.Threading.Tasks;
|
||||
using Windows.Graphics;
|
||||
using Windows.Graphics.Capture;
|
||||
using Windows.Graphics.DirectX;
|
||||
using Windows.Graphics.DirectX.Direct3D11;
|
||||
using Windows.Graphics.Imaging;
|
||||
using ytLive.Helpers;
|
||||
|
||||
namespace ytLive.Services;
|
||||
|
||||
/// <summary>
|
||||
/// The frame-driven web capture for one widget: a GraphicsCaptureItem backed by
|
||||
/// the widget's Windows.UI.Composition visual (see WebView2Manager). Owns the
|
||||
/// item, a free-threaded Direct3D11CaptureFramePool and the capture session;
|
||||
/// frames arrive in step with Chromium's own presentation (~60Hz on a real
|
||||
/// animation), replacing the old CapturePreviewAsync PNG poll whose ~35-165ms
|
||||
/// encode cost capped the recording at ~10Hz and the preview at ~5fps.
|
||||
///
|
||||
/// The capture surface holds a REAL alpha channel (the controller renders onto
|
||||
/// a transparent composition visual), so the GPU→CPU copy uses
|
||||
/// <see cref="BitmapAlphaMode.Straight"/> and the frame is
|
||||
/// <see cref="VideoFrame.IsOpaque"/> = false — the compositor's straight-alpha
|
||||
/// blend math ("web layer transparency", 2026-09-13) requires the readback to
|
||||
/// keep premultiplication OFF. Unlike the screen source there is no master-size
|
||||
/// downscale: the controller renders at the RAW 1920×1080 master canvas, so the
|
||||
/// CPU never holds a larger frame.
|
||||
///
|
||||
/// Per-frame the alpha bounding box of the widget (<see cref="FindContentBounds"/>)
|
||||
/// is measured on this capture worker thread and stamped onto the frame as
|
||||
/// <see cref="VideoFrame.CropBounds"/> — same alpha-bbox contract as the old
|
||||
/// PNG path, but off the UI thread (a 2M-pixel scan per frame there is exactly
|
||||
/// the cost this redesign exists to remove).
|
||||
/// </summary>
|
||||
public sealed class WebCaptureFrameSource : IScreenCaptureSource
|
||||
{
|
||||
private readonly object _gate = new();
|
||||
private GraphicsCaptureItem _item;
|
||||
private Direct3D11CaptureFramePool? _framePool;
|
||||
private GraphicsCaptureSession? _session;
|
||||
private SizeInt32 _poolSize;
|
||||
private bool _started;
|
||||
private bool _framePending;
|
||||
private DateTime _lastErrorLog = DateTime.MinValue;
|
||||
|
||||
// Buffer recycling: identical to the screen source's ring+Epoch contract
|
||||
// (see ScreenCaptureFrameSource) — the compositor's paste cache is keyed on
|
||||
// array identity, so recycled buffers must carry a monotonic Epoch or a
|
||||
// reused array false-hits with stale content.
|
||||
private readonly byte[]?[] _frameRing = new byte[8][];
|
||||
private int _ringNext;
|
||||
private long _epoch;
|
||||
|
||||
private static readonly TimeSpan ErrorLogThrottle = TimeSpan.FromSeconds(5);
|
||||
|
||||
public string Key { get; }
|
||||
public event Action<VideoFrame>? FrameAvailable;
|
||||
|
||||
internal WebCaptureFrameSource(string key, GraphicsCaptureItem item)
|
||||
{
|
||||
Key = key;
|
||||
_item = item;
|
||||
}
|
||||
|
||||
/// <summary>Wraps an item created from a composition visual by the web manager.</summary>
|
||||
public static WebCaptureFrameSource CreateForVisual(string key, GraphicsCaptureItem item)
|
||||
=> new(key, item);
|
||||
|
||||
public Task StartAsync()
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (_started) return Task.CompletedTask;
|
||||
var item = _item;
|
||||
var device = Direct3D11Helper.CreateDevice();
|
||||
var framePool = Direct3D11CaptureFramePool.CreateFreeThreaded(
|
||||
device, DirectXPixelFormat.B8G8R8A8UIntNormalized, 2, item.Size);
|
||||
_poolSize = item.Size;
|
||||
var session = framePool.CreateCaptureSession(item);
|
||||
framePool.FrameArrived += OnFrameArrived;
|
||||
session.StartCapture();
|
||||
_framePool = framePool;
|
||||
_session = session;
|
||||
_started = true;
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task StopAsync()
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
_started = false;
|
||||
_framePending = false;
|
||||
if (_framePool != null)
|
||||
_framePool.FrameArrived -= OnFrameArrived;
|
||||
_session?.Dispose();
|
||||
_session = null;
|
||||
_framePool?.Dispose();
|
||||
_framePool = null;
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private void OnFrameArrived(Direct3D11CaptureFramePool sender, object args)
|
||||
{
|
||||
var frame = sender.TryGetNextFrame();
|
||||
if (frame == null) return;
|
||||
lock (_gate)
|
||||
{
|
||||
if (!_started)
|
||||
{
|
||||
frame.Dispose();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (frame.ContentSize.Width != _poolSize.Width || frame.ContentSize.Height != _poolSize.Height)
|
||||
{
|
||||
sender.Recreate(Direct3D11Helper.CreateDevice(),
|
||||
DirectXPixelFormat.B8G8R8A8UIntNormalized, 2, frame.ContentSize);
|
||||
_poolSize = frame.ContentSize;
|
||||
}
|
||||
|
||||
if (_framePending)
|
||||
{
|
||||
frame.Dispose();
|
||||
return;
|
||||
}
|
||||
_framePending = true;
|
||||
_ = ProcessFrameAsync(frame);
|
||||
}
|
||||
|
||||
private async Task ProcessFrameAsync(Direct3D11CaptureFrame frame)
|
||||
{
|
||||
try
|
||||
{
|
||||
using (frame)
|
||||
using (var softwareBitmap = await SoftwareBitmap.CreateCopyFromSurfaceAsync(
|
||||
frame.Surface, BitmapAlphaMode.Straight))
|
||||
{
|
||||
FrameAvailable?.Invoke(CopyToVideoFrame(softwareBitmap));
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
if (now - _lastErrorLog >= ErrorLogThrottle)
|
||||
{
|
||||
_lastErrorLog = now;
|
||||
AppLog.Write($"WebCaptureFrameSource: frame conversion failed: {ex.Message}");
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_framePending = false;
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] RentRingBuffer(int size)
|
||||
{
|
||||
for (var tries = 0; tries < _frameRing.Length; tries++)
|
||||
{
|
||||
var idx = (_ringNext + tries) % _frameRing.Length;
|
||||
var buf = _frameRing[idx];
|
||||
if (buf is { Length: var len } && len == size)
|
||||
{
|
||||
_ringNext = (idx + 1) % _frameRing.Length;
|
||||
return buf;
|
||||
}
|
||||
}
|
||||
var slot = _ringNext;
|
||||
_ringNext = (slot + 1) % _frameRing.Length;
|
||||
var fresh = new byte[size];
|
||||
_frameRing[slot] = fresh;
|
||||
return fresh;
|
||||
}
|
||||
|
||||
private VideoFrame CopyToVideoFrame(SoftwareBitmap bitmap)
|
||||
{
|
||||
var sw = bitmap.PixelWidth;
|
||||
var sh = bitmap.PixelHeight;
|
||||
using var buffer = bitmap.LockBuffer(BitmapBufferAccessMode.Read);
|
||||
using var reference = buffer.CreateReference();
|
||||
if (!WindowsRuntimeMarshal.TryGetDataUnsafe(reference, out var data, out var capacity))
|
||||
throw new InvalidOperationException("Could not access the frame buffer.");
|
||||
|
||||
var srcStride = sw * 4;
|
||||
var count = (int)Math.Min(capacity, (uint)(sh * srcStride));
|
||||
var pixels = RentRingBuffer(count);
|
||||
Marshal.Copy(data, pixels, 0, pixels.Length);
|
||||
|
||||
var (cropX, cropY, cropW, cropH) = FindContentBounds(pixels, sw, sh, srcStride);
|
||||
return new VideoFrame(sw, sh, pixels)
|
||||
{
|
||||
IsOpaque = false,
|
||||
Epoch = ++_epoch,
|
||||
CropBounds = (cropX, cropY, cropW, cropH),
|
||||
};
|
||||
}
|
||||
|
||||
// The widget's extent is measured from the captured frame itself: the
|
||||
// bounding box of non-transparent pixels (the background is injected
|
||||
// transparent, so the alpha channel IS the widget). Immune to DOM timing
|
||||
// (fonts/images/animations), needs no JS round-trip, and CANNOT break the
|
||||
// render: a widget that fills the canvas gets the full frame; a widget that
|
||||
// floats in transparent margin gets exactly its content box — used for the
|
||||
// preview bitmap / selection. The frame handed to the compositor is always
|
||||
// the FULL canvas, so the page's transparent margins keep revealing the
|
||||
// layers beneath (the "black square over webcam" was the crop fed to the
|
||||
// compositor: UniformToFill zoomed the opaque content box over the element).
|
||||
internal static (int X, int Y, int W, int H) FindContentBounds(byte[] pixels, int pixW, int pixH, int stride)
|
||||
{
|
||||
int minX = pixW, minY = pixH, maxX = -1, maxY = -1;
|
||||
for (var y = 0; y < pixH; y++)
|
||||
{
|
||||
var row = y * stride;
|
||||
for (var x = 0; x < pixW; x++)
|
||||
{
|
||||
if (pixels[row + x * 4 + 3] == 0) continue;
|
||||
if (x < minX) minX = x;
|
||||
if (x > maxX) maxX = x;
|
||||
if (y < minY) minY = y;
|
||||
if (y > maxY) maxY = y;
|
||||
}
|
||||
}
|
||||
if (maxX < minX || maxY < minY) return (0, 0, pixW, pixH);
|
||||
return (minX, minY, maxX - minX + 1, maxY - minY + 1);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user