feat: web sources frame-captured via composition (CoreWebView2CompositionController → Windows.Graphics.Capture); PNG poll + CaptureScheduler deleted
The ~30Hz CapturePreviewAsync PNG poll capped real cadence at ~20Hz (35–165ms full-HD encode+decode), so a 60fps widget still juddered at ~1/6 speed. Replaced polling with frame-driven capture of the composition controller's root visual — the mechanism WebView2CompositionControl and Flutter's webview_windows use (graphics_context.cc captures the root surface_ visual via CreateGraphicsCaptureItemFromVisual; reference: github.com/microsoft/Windows.UI.Composition.WinUI / flutter-internal webview_windows). Frames now arrive at the renderer's own pace; capture memory is epoch'd ring reuse + one crop-sized shared WriteableBitmap. New Services/WebCaptureFrameSource.cs owns GraphicsCaptureItem + free- threaded Direct3D11CaptureFramePool + session (Straight alpha readback, per-frame FindContentBounds → CropBounds). WebView2Manager reworked around per-session composition controllers + one UI-thread Compositor created via the CoreMessaging CreateDispatcherQueueController P/Invoke (the 19041 projection lacks CreateOnCurrentThread); internal seam ctor (Dispatcher, Func<string,IScreenCaptureSource>?) for hermetic tests. CaptureScheduler.cs deleted; the three SetCaptureInterval cadence hooks removed; InitWebView2() moved from MainWindow ctor to Loaded (a parent HWND must exist for the composition controller); the hidden WebViewHostPanel overlay deleted. TransparentBackgroundScript unchanged. Tests: WebView2ManagerTests reworked — 4 control-size + scheduler tests dropped, FindContentBounds tests moved to WebCaptureFrameSource, ONE integration test (Frames_PublishCroppedPreview_And_CoalesceToLatest_CarryingCropBounds) drives the seam with a FakeWebSource + background-STA DispatcherPump. Suite 293/293, 0 warnings. NOTE: composition path NOT yet verified on a device — the take is the next step. Web work committed locally only (no push per standing rule). Docs same-commit: ai.md Slice 14 + supersede marker on Slice 11, HANDOFF, MyMistakes (CoreMessaging DQ + namespace-landmine recipe), TASK 17, Controls/ViewModels/Services indexes.
This commit is contained in:
+308
-310
@@ -1,129 +1,92 @@
|
||||
using System;
|
||||
using System.Diagnostics;
|
||||
using System.Collections.Generic;
|
||||
using System.Drawing;
|
||||
using System.IO;
|
||||
using System.Text.Json;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Threading.Tasks;
|
||||
using System.Windows;
|
||||
using System.Windows.Controls;
|
||||
using System.Windows.Media;
|
||||
using System.Windows.Media.Imaging;
|
||||
using System.Windows.Threading;
|
||||
using Microsoft.Web.WebView2.Core;
|
||||
using Microsoft.Web.WebView2.Wpf;
|
||||
using Windows.Graphics.Capture;
|
||||
using Windows.System;
|
||||
using Windows.UI.Composition;
|
||||
using ytLive.Helpers;
|
||||
using ytLive.Models;
|
||||
|
||||
namespace ytLive.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Owns the composition-based web sources (one per widget). A session renders
|
||||
/// its widget through a <see cref="CoreWebView2CompositionController"/> whose
|
||||
/// visual tree lives off-screen on the WPF UI thread (OBS web-source model:
|
||||
/// the page renders at the MASTER CANVAS size 1920×1080, stable, never tracked);
|
||||
/// the controller's root visual is captured by a <see cref="WebCaptureFrameSource"/>
|
||||
/// at Chromium's own presentation cadence. The preview is crop-copied to a shared
|
||||
/// WriteableBitmap on the UI thread (dispatcher-coalesced latest-wins, mirror of
|
||||
/// ScreenCaptureManager); the FramePump reads <see cref="GetLatestFrame"/>.
|
||||
///
|
||||
/// The composition stack needs a DispatcherQueue on this thread (CsWinRT does not
|
||||
/// project <c>CreateOnCurrentThread</c> on the 19041 SDK — only the dedicated
|
||||
/// variant — so it is installed via a CoreMessaging P/Invoke, the documented
|
||||
/// fallback the WebView2CompositionControl / webview_windows embedders use).
|
||||
/// Everything composition-side therefore runs on the WPF UI thread; only the
|
||||
/// frame conversion and alpha-bbox scan run on the capture worker.
|
||||
///
|
||||
/// Seam: an internal <c>Func<string, IScreenCaptureSource></c> source
|
||||
/// factory substitutes the whole WebView2+item creation in tests — the manager
|
||||
/// then owns the session/coalesce/crop/preview plumbing exactly as in production.
|
||||
/// </summary>
|
||||
public sealed class WebView2Manager : IDisposable
|
||||
{
|
||||
private readonly Panel _hostPanel;
|
||||
private readonly Dispatcher _dispatcher;
|
||||
private readonly Func<string, IScreenCaptureSource>? _sessionSourceFactory;
|
||||
private readonly Dictionary<string, WebSourceSession> _sessions = new();
|
||||
private Task<CoreWebView2Environment>? _environmentTask;
|
||||
private int _captureIntervalMs = 200; // idle pace; recording drops it to ~30Hz via SetCaptureInterval
|
||||
private DispatcherQueueController? _dispatcherQueueController;
|
||||
private Windows.UI.Composition.Compositor? _compositor;
|
||||
|
||||
public event Action<string, WriteableBitmap>? PreviewBitmapChanged;
|
||||
|
||||
private sealed class WebSourceSession
|
||||
{
|
||||
public required WebView2 Control { get; init; }
|
||||
public required CaptureScheduler Scheduler { get; init; }
|
||||
public required string Key { get; init; }
|
||||
public CoreWebView2? Web;
|
||||
public CoreWebView2CompositionController? Controller;
|
||||
public IScreenCaptureSource? Source;
|
||||
public Action<VideoFrame>? FrameHandler;
|
||||
public VideoFrame? LatestFrame;
|
||||
public WriteableBitmap? PreviewBitmap;
|
||||
|
||||
// Buffer reuse (2026-09-04 lesson): the preview crop copy reuses one
|
||||
// scratch array while the widget's extent holds — no per-frame LOH mint.
|
||||
public byte[]? CropScratch;
|
||||
public bool Disposed;
|
||||
public bool Initialized;
|
||||
|
||||
// Buffer reuse (2026-09-04): every capture tick used to mint a fresh
|
||||
// 8.3MB canvas array + a fresh crop array (~83MB/s of LOH at 10Hz →
|
||||
// gen2 pauses surfaced as the pump's 35-40ms "worst render" spikes).
|
||||
// CanvasScratch is reused while the size holds. The frame handed to the
|
||||
// compositor is always the FULL canvas (transparent margins reveal layers
|
||||
// beneath); the crop is only for the preview WriteableBitmap. Epoch is
|
||||
// monotonic per canvas generation (identity-keyed consumers see each frame).
|
||||
public byte[]? CanvasScratch;
|
||||
public readonly byte[]?[] OutRing = new byte[8][];
|
||||
public int OutRingNext;
|
||||
public long Epoch;
|
||||
public WriteableBitmap? PreviewBitmap;
|
||||
public bool DebugPngWritten;
|
||||
public int WidgetDumpRemaining;
|
||||
public int TelemetryCaptureCount;
|
||||
|
||||
public byte[] RentOutBuffer(int size)
|
||||
{
|
||||
for (var tries = 0; tries < OutRing.Length; tries++)
|
||||
{
|
||||
var idx = (OutRingNext + tries) % OutRing.Length;
|
||||
var buf = OutRing[idx];
|
||||
if (buf != null && buf.Length == size)
|
||||
{
|
||||
OutRingNext = (idx + 1) % OutRing.Length;
|
||||
return buf;
|
||||
}
|
||||
}
|
||||
var slot = OutRingNext;
|
||||
OutRingNext = (slot + 1) % OutRing.Length;
|
||||
var fresh = new byte[size];
|
||||
OutRing[slot] = fresh;
|
||||
return fresh;
|
||||
}
|
||||
public bool FramePending;
|
||||
public bool FirstCaptureLogged;
|
||||
public int WidgetLogsRemaining;
|
||||
}
|
||||
|
||||
public WebView2Manager(Panel hostPanel, Dispatcher dispatcher)
|
||||
public WebView2Manager(Dispatcher dispatcher)
|
||||
: this(dispatcher, null)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>Test seam: the source factory replaces WebView2+composition
|
||||
/// creation, so the session/coalesce/crop/preview plumbing is testable
|
||||
/// without the runtime.</summary>
|
||||
internal WebView2Manager(Dispatcher dispatcher, Func<string, IScreenCaptureSource>? sessionSourceFactory)
|
||||
{
|
||||
_hostPanel = hostPanel;
|
||||
_dispatcher = dispatcher;
|
||||
_sessionSourceFactory = sessionSourceFactory;
|
||||
}
|
||||
|
||||
public void Register(Source source)
|
||||
{
|
||||
if (_sessions.ContainsKey(source.Id)) return;
|
||||
|
||||
// OBS model: the page renders at the MASTER CANVAS size (1920×1080),
|
||||
// stable, never tracked. Full-bleed widgets (designed for the canvas)
|
||||
// render fully instead of being clipped to a small box-sized viewport;
|
||||
// dragging the box never reflows the page. The capture is tightly
|
||||
// cropped to the widget's ALPHA bounding box (FindContentBounds) and
|
||||
// the display (Stretch=Fill) maps it flush under the box, anchored at
|
||||
// (0,0) — no DOM query, so no timing fragility; worst case is the
|
||||
// full frame, which is the confirmed-good rendering.
|
||||
var webView = new WebView2
|
||||
{
|
||||
Width = 1920,
|
||||
Height = 1080,
|
||||
DefaultBackgroundColor = System.Drawing.Color.Transparent,
|
||||
Visibility = System.Windows.Visibility.Visible,
|
||||
};
|
||||
Canvas.SetLeft(webView, -5000);
|
||||
Canvas.SetTop(webView, -5000);
|
||||
|
||||
_hostPanel.Children.Add(webView);
|
||||
|
||||
var scheduler = new CaptureScheduler(
|
||||
_dispatcher,
|
||||
_captureIntervalMs,
|
||||
() => CaptureFrame(source.Id));
|
||||
|
||||
var session = new WebSourceSession
|
||||
{
|
||||
Control = webView,
|
||||
Scheduler = scheduler,
|
||||
};
|
||||
|
||||
_sessions[source.Id] = session;
|
||||
|
||||
_ = InitializeAsync(source, session);
|
||||
}
|
||||
|
||||
/// <summary>Changes the capture cadence for every session. Recording runs at
|
||||
/// ~30Hz (33ms) — the practical PNG-capture ceiling — idle drops back to 200ms so
|
||||
/// the preview alone doesn't burn a core decoding full-HD captures.</summary>
|
||||
public void SetCaptureInterval(int milliseconds)
|
||||
{
|
||||
_captureIntervalMs = milliseconds;
|
||||
foreach (var session in _sessions.Values)
|
||||
if (!session.Disposed)
|
||||
session.Scheduler.SetInterval(milliseconds);
|
||||
_sessions[source.Id] = new WebSourceSession { Key = source.Id };
|
||||
_ = InitializeAsync(source, _sessions[source.Id]);
|
||||
}
|
||||
|
||||
public void Unregister(string sourceId) => RemoveSession(sourceId);
|
||||
@@ -136,7 +99,7 @@ public sealed class WebView2Manager : IDisposable
|
||||
if (string.IsNullOrEmpty(source.WebUri))
|
||||
{
|
||||
session.LatestFrame = null;
|
||||
session.Control.CoreWebView2?.NavigateToString(
|
||||
session.Web?.NavigateToString(
|
||||
"<html><body style='background:transparent;margin:0'></body></html>");
|
||||
return;
|
||||
}
|
||||
@@ -152,9 +115,8 @@ public sealed class WebView2Manager : IDisposable
|
||||
// Injected BEFORE the document is parsed (AddScriptToExecuteOnDocumentCreatedAsync) so
|
||||
// the widget page's own CSS/scripts cannot repaint html/body opaque — the reason the
|
||||
// late NavigationStarting/NavigationCompleted ExecuteScriptAsync injection lost the
|
||||
// fight (page styles ran first). CapturePreviewAsync "will always honor a webpage's
|
||||
// background content" (MicrosoftEdge/WebView2Feedback specs/BackgroundColor.md), so a
|
||||
// transparent capture REQUIRES the transparency to be in place before the page paints.
|
||||
// fight (page styles ran first). A transparent capture REQUIRES the transparency in
|
||||
// place before the page paints, on the composition surface too.
|
||||
//
|
||||
// The OLD inline `element.style.background='transparent'` form lost to any page CSS:
|
||||
// a widget that paints a background-COLOR on a container (or html/body with stronger
|
||||
@@ -191,55 +153,75 @@ public sealed class WebView2Manager : IDisposable
|
||||
{
|
||||
try
|
||||
{
|
||||
await session.Control.EnsureCoreWebView2Async(await GetEnvironmentAsync());
|
||||
|
||||
var cws = session.Control.CoreWebView2!;
|
||||
|
||||
// Pre-page hook: runs after the global object exists but BEFORE the document
|
||||
// parses / any page script runs — must complete before navigating away from
|
||||
// about:blank for it to apply to the widget document.
|
||||
await cws.AddScriptToExecuteOnDocumentCreatedAsync(TransparentBackgroundScript);
|
||||
|
||||
cws.NavigationStarting += (_, _) =>
|
||||
IScreenCaptureSource sourceImpl;
|
||||
if (_sessionSourceFactory != null)
|
||||
{
|
||||
_ = cws.ExecuteScriptAsync(TransparentBackgroundScript);
|
||||
};
|
||||
|
||||
cws.NavigationCompleted += (_, e) =>
|
||||
// Test path: the factory hands over a fully-configured source; the
|
||||
// branch has no awaits, so Register() completes the session inline.
|
||||
sourceImpl = _sessionSourceFactory(source.Id);
|
||||
}
|
||||
else
|
||||
{
|
||||
_ = cws.ExecuteScriptAsync(TransparentBackgroundScript);
|
||||
if (e.IsSuccess && session.Control.CoreWebView2?.Source is { } src && src != "about:blank")
|
||||
var controller = await CreateControllerAsync();
|
||||
if (controller == null)
|
||||
{
|
||||
// Arm a one-shot diagnostic dump of the REAL widget document
|
||||
// (a few post-paint captures so styles/backgrounds settle) —
|
||||
// the old first-capture dump only ever saw the about:blank
|
||||
// document, so whether this widget's capture actually HAS the
|
||||
// transparent margins the injection promises has never been
|
||||
// verified (MyMistakes: "? VERIFY").
|
||||
session.WidgetDumpRemaining = 5;
|
||||
// The first five dumps fire within ~1s of completion (usually BEFORE the
|
||||
// recording starts) — re-arm ~30s in so the same real document gets dumped
|
||||
// INSIDE a recording's window too. Next take therefore nukes the gap that
|
||||
// hid what the page actually painted while recording.
|
||||
var rearm = session;
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(30));
|
||||
rearm.WidgetDumpRemaining = 5;
|
||||
});
|
||||
RemoveSession(source.Id);
|
||||
return;
|
||||
}
|
||||
_ = CaptureFrame(source.Id);
|
||||
};
|
||||
|
||||
session.Controller = controller;
|
||||
session.Web = controller.CoreWebView2;
|
||||
if (session.Web == null)
|
||||
{
|
||||
RemoveSession(source.Id);
|
||||
return;
|
||||
}
|
||||
|
||||
// Pre-page hook: runs after the global object exists but BEFORE the document
|
||||
// parses / any page script runs — must complete before navigating away from
|
||||
// about:blank for it to apply to the widget document.
|
||||
await session.Web.AddScriptToExecuteOnDocumentCreatedAsync(TransparentBackgroundScript);
|
||||
|
||||
session.Web.NavigationStarting += (_, _) =>
|
||||
{
|
||||
_ = session.Web?.ExecuteScriptAsync(TransparentBackgroundScript);
|
||||
};
|
||||
|
||||
session.Web.NavigationCompleted += (_, e) =>
|
||||
{
|
||||
_ = session.Web?.ExecuteScriptAsync(TransparentBackgroundScript);
|
||||
if (e.IsSuccess && session.Web?.Source is { } src && src != "about:blank")
|
||||
{
|
||||
// Log-only diagnostic on the REAL widget document (a few post-paint
|
||||
// frames so styles/backgrounds settle). The first-capture dump only
|
||||
// ever saw about:blank; the old PNG widget dumps are gone with the
|
||||
// PNG path — alpha stats now ride the frame-driven channel.
|
||||
session.WidgetLogsRemaining = 3;
|
||||
var rearm = session;
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(30));
|
||||
if (!rearm.Disposed) rearm.WidgetLogsRemaining = 3;
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
session.Web.NavigateToString(
|
||||
"<html><body style='background:transparent;margin:0'></body></html>");
|
||||
|
||||
if (!string.IsNullOrEmpty(source.WebUri))
|
||||
Navigate(source.WebUri, session);
|
||||
|
||||
session.Initialized = true;
|
||||
|
||||
sourceImpl = new WebCaptureFrameSource(source.Id, CreateCaptureItem(controller));
|
||||
}
|
||||
|
||||
session.Source = sourceImpl;
|
||||
session.FrameHandler = frame => OnFrameAvailable(session, frame);
|
||||
sourceImpl.FrameAvailable += session.FrameHandler;
|
||||
await sourceImpl.StartAsync();
|
||||
session.Initialized = true;
|
||||
|
||||
session.Control.CoreWebView2.NavigateToString(
|
||||
"<html><body style='background:transparent;margin:0'></body></html>");
|
||||
|
||||
if (!string.IsNullOrEmpty(source.WebUri))
|
||||
Navigate(source.WebUri, session);
|
||||
|
||||
session.Scheduler.Start();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -248,14 +230,13 @@ public sealed class WebView2Manager : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
// The widget page is rendered off-screen; Chromium throttles hidden pages
|
||||
// (rAF parked, timers clamped to ~1s) the moment the app window is unfocused or
|
||||
// covered — WebView2Feedback#1172/#3070, Chrome 88 timer throttling. These flags
|
||||
// (the Electron-class embedder answer) keep the renderer at full rate:
|
||||
// --disable-backgrounding-occluded-windows, --disable-renderer-backgrounding,
|
||||
// and disable the native-window occlusion detector that marks the page hidden.
|
||||
// The environment MUST exist before EnsureCoreWebView2Async — it's created once
|
||||
// and shared by every session (one browser process).
|
||||
// The environment MUST exist before the controller is created — it's created once
|
||||
// and shared by every session (one browser process). These flags (the
|
||||
// Electron-class embedder answer to Chromium throttling hidden pages — the moment
|
||||
// the app window is unfocused or covered, rAF parks and timers clamp to ~1s) keep
|
||||
// the renderer at full rate: --disable-backgrounding-occluded-windows,
|
||||
// --disable-renderer-backgrounding, and disable the native-window occlusion
|
||||
// detector that marks the page hidden.
|
||||
private Task<CoreWebView2Environment> GetEnvironmentAsync()
|
||||
{
|
||||
if (_environmentTask != null) return _environmentTask;
|
||||
@@ -267,45 +248,177 @@ public sealed class WebView2Manager : IDisposable
|
||||
return _environmentTask = CoreWebView2Environment.CreateAsync(null, null, options);
|
||||
}
|
||||
|
||||
/// <summary>Creates the composition controller off-screen under the main
|
||||
/// window (the parent HWND is required, the visual never mounts a window
|
||||
/// target). Set up is copied from the WebView2CompositionControl /
|
||||
/// webview_windows embedders: raw-pixel bounds at the master canvas,
|
||||
/// fixed rasterization scale, transparent default background.</summary>
|
||||
private async Task<CoreWebView2CompositionController?> CreateControllerAsync()
|
||||
{
|
||||
var mainWindow = Application.Current?.MainWindow;
|
||||
var hwnd = mainWindow == null
|
||||
? IntPtr.Zero
|
||||
: new System.Windows.Interop.WindowInteropHelper(mainWindow).Handle;
|
||||
if (hwnd == IntPtr.Zero)
|
||||
{
|
||||
AppLog.Write("WebView2Manager: main window handle unavailable — web source not created");
|
||||
return null;
|
||||
}
|
||||
|
||||
var env = await GetEnvironmentAsync();
|
||||
if (_compositor == null) _compositor = CreateCompositorForCurrentThread();
|
||||
|
||||
var controller = await env.CreateCoreWebView2CompositionControllerAsync(hwnd);
|
||||
controller.Bounds = new Rectangle(0, 0, 1920, 1080);
|
||||
controller.BoundsMode = CoreWebView2BoundsMode.UseRawPixels;
|
||||
controller.ShouldDetectMonitorScaleChanges = false;
|
||||
controller.RasterizationScale = 1.0;
|
||||
controller.IsVisible = true;
|
||||
controller.DefaultBackgroundColor = System.Drawing.Color.Transparent;
|
||||
return controller;
|
||||
}
|
||||
|
||||
private void Navigate(string url, WebSourceSession session)
|
||||
{
|
||||
try { session.Control.CoreWebView2?.Navigate(url); }
|
||||
try { session.Web?.Navigate(url); }
|
||||
catch (Exception ex) { AppLog.Write($"WebView2Manager: navigate failed: {ex.Message}"); }
|
||||
}
|
||||
|
||||
// The widget's extent is measured from the captured frame itself: the
|
||||
// bounding box of non-transparent pixels (the background is injected
|
||||
// transparent, so the alpha channel IS the widget). This is immune to
|
||||
// DOM timing (fonts/images/animations), needs no JS round-trip, and
|
||||
// CANNOT break the render: if the widget fills the canvas the box equals
|
||||
// the full frame; if it floats in transparent margin the box is the
|
||||
// measured content extent — used ONLY for the preview bitmap / selection.
|
||||
// The frame handed to the compositor is always the FULL canvas, so the
|
||||
// page's transparent margins keep revealing the layers beneath (the
|
||||
// "black square over webcam" was the crop fed to the compositor:
|
||||
// UniformToFill zoomed the opaque content box to cover the whole element).
|
||||
internal static (int X, int Y, int W, int H) FindContentBounds(byte[] pixels, int pixW, int pixH, int stride)
|
||||
// The visual tree webview_windows wires (offscreen mode): a root container
|
||||
// visual sized to the master canvas (the captured subject — the Flutter
|
||||
// plugin calls GraphicsCaptureItem.CreateFromVisual on this root), with the
|
||||
// controller's root visual target set to a child that fills the root via
|
||||
// RelativeSizeAdjustment. BoundsMode raw pixels + RasterizationScale 1.0 keep
|
||||
// the capture at exactly 1920×1080 regardless of monitor scale.
|
||||
private GraphicsCaptureItem CreateCaptureItem(CoreWebView2CompositionController controller)
|
||||
{
|
||||
int minX = pixW, minY = pixH, maxX = -1, maxY = -1;
|
||||
for (var y = 0; y < pixH; y++)
|
||||
{
|
||||
var row = y * stride;
|
||||
for (var x = 0; x < pixW; x++)
|
||||
{
|
||||
if (pixels[row + x * 4 + 3] == 0) continue;
|
||||
if (x < minX) minX = x;
|
||||
if (x > maxX) maxX = x;
|
||||
if (y < minY) minY = y;
|
||||
if (y > maxY) maxY = y;
|
||||
}
|
||||
}
|
||||
if (maxX < minX || maxY < minY) return (0, 0, pixW, pixH);
|
||||
return (minX, minY, maxX - minX + 1, maxY - minY + 1);
|
||||
var compositor = _compositor!;
|
||||
var root = compositor.CreateContainerVisual();
|
||||
root.Size = new System.Numerics.Vector2(1920f, 1080f);
|
||||
root.IsVisible = true;
|
||||
|
||||
var webViewVisual = compositor.CreateContainerVisual();
|
||||
webViewVisual.RelativeSizeAdjustment = new System.Numerics.Vector2(1f, 1f);
|
||||
root.Children.InsertAtTop(webViewVisual);
|
||||
|
||||
controller.RootVisualTarget = webViewVisual;
|
||||
return GraphicsCaptureItem.CreateFromVisual(root);
|
||||
}
|
||||
|
||||
// CsWinRT does not project DispatcherQueueController.CreateOnCurrentThread
|
||||
// on the 19041 SDK (only CreateOnDedicatedThread [CS0117, verified by probe]);
|
||||
// the documented fallback — "how to create a DispatcherQueue on the current
|
||||
// thread" (learn.microsoft.com CoreMessaging docs) — is CreateDispatcherQueueController
|
||||
// from CoreMessaging.dll, wrapped via FromAbi (the CaptureInterop pattern). One
|
||||
// per UI thread, then new Compositor() (which requires the DQ) works here.
|
||||
private Windows.UI.Composition.Compositor CreateCompositorForCurrentThread()
|
||||
{
|
||||
var options = new DispatcherQueueOptions
|
||||
{
|
||||
DwSize = Marshal.SizeOf<DispatcherQueueOptions>(),
|
||||
ThreadType = 2, // DQTYPE_THREAD_CURRENT
|
||||
ApartmentType = 2, // DQTAT_COM_STA — the WPF UI thread is STA
|
||||
};
|
||||
var hr = CreateDispatcherQueueController(options, out var controllerPtr);
|
||||
if (hr != 0)
|
||||
throw Marshal.GetExceptionForHR(hr)!;
|
||||
_dispatcherQueueController = DispatcherQueueController.FromAbi(controllerPtr);
|
||||
return new Windows.UI.Composition.Compositor();
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
private struct DispatcherQueueOptions
|
||||
{
|
||||
public int DwSize;
|
||||
public int ThreadType;
|
||||
public int ApartmentType;
|
||||
}
|
||||
|
||||
[DllImport("CoreMessaging.dll")]
|
||||
private static extern int CreateDispatcherQueueController(
|
||||
DispatcherQueueOptions options, out IntPtr dispatcherQueueController);
|
||||
|
||||
private void OnFrameAvailable(WebSourceSession session, VideoFrame frame)
|
||||
{
|
||||
if (session.Disposed) return;
|
||||
session.LatestFrame = frame;
|
||||
|
||||
if (_dispatcher.CheckAccess())
|
||||
{
|
||||
UpdatePreview(session);
|
||||
return;
|
||||
}
|
||||
if (session.FramePending) return;
|
||||
session.FramePending = true;
|
||||
_dispatcher.BeginInvoke(() =>
|
||||
{
|
||||
session.FramePending = false;
|
||||
if (!session.Disposed)
|
||||
UpdatePreview(session);
|
||||
}, DispatcherPriority.Render);
|
||||
}
|
||||
|
||||
private void UpdatePreview(WebSourceSession session)
|
||||
{
|
||||
var frame = session.LatestFrame;
|
||||
if (frame == null) return;
|
||||
|
||||
var (cropX, cropY, cropW, cropH) = frame.CropBounds ?? (0, 0, frame.Width, frame.Height);
|
||||
if (cropW < 1 || cropH < 1)
|
||||
{
|
||||
cropX = 0; cropY = 0;
|
||||
cropW = frame.Width; cropH = frame.Height;
|
||||
}
|
||||
|
||||
// The preview is sized to the widget's alpha bounding box (the compositor
|
||||
// gets the FULL canvas + CropBounds; the crop is only for the preview).
|
||||
// A fresh box (widget resizes / first content paints) mints a new bitmap
|
||||
// once — WPF re-renders a reused mutable bitmap after WritePixels, so
|
||||
// later frames only re-copy into the same one.
|
||||
if (session.PreviewBitmap == null
|
||||
|| session.PreviewBitmap.PixelWidth != cropW
|
||||
|| session.PreviewBitmap.PixelHeight != cropH)
|
||||
{
|
||||
session.PreviewBitmap = new WriteableBitmap(cropW, cropH, 96, 96, PixelFormats.Bgra32, null);
|
||||
PreviewBitmapChanged?.Invoke(session.Key, session.PreviewBitmap);
|
||||
}
|
||||
|
||||
var need = cropW * cropH * 4;
|
||||
var scratch = session.CropScratch is { } s && s.Length >= need
|
||||
? s
|
||||
: (session.CropScratch = new byte[need]);
|
||||
var stride = frame.Stride;
|
||||
if (cropW == frame.Width && cropH == frame.Height && cropX == 0 && cropY == 0)
|
||||
Buffer.BlockCopy(frame.BgraPixels, 0, scratch, 0, need);
|
||||
else
|
||||
for (var r = 0; r < cropH; r++)
|
||||
Buffer.BlockCopy(frame.BgraPixels, (cropY + r) * stride + cropX * 4, scratch, r * cropW * 4, cropW * 4);
|
||||
|
||||
session.PreviewBitmap.WritePixels(new Int32Rect(0, 0, cropW, cropH), scratch, cropW * 4, 0);
|
||||
|
||||
// alpha[min/max/mean/zero%] — the one number that decides whether a widget's
|
||||
// capture really has transparent margins (0% = opaque, ~100% if the page
|
||||
// painted transparent-only). Sampled every 16th pixel like the original log.
|
||||
if (!session.FirstCaptureLogged)
|
||||
{
|
||||
session.FirstCaptureLogged = true;
|
||||
AppLog.Write(
|
||||
$"WebView2Manager: first capture for '{session.Key}' {frame.Width}x{frame.Height} " +
|
||||
$"{AlphaStats(frame.BgraPixels, frame.Width, frame.Height, stride)} " +
|
||||
$"contentBounds=({cropX},{cropY},{cropW},{cropH})");
|
||||
}
|
||||
|
||||
if (session.WidgetLogsRemaining > 0)
|
||||
{
|
||||
var dumpNo = 3 - session.WidgetLogsRemaining + 1;
|
||||
session.WidgetLogsRemaining--;
|
||||
AppLog.Write(
|
||||
$"WebView2Manager: widget capture [{dumpNo}/3] for '{session.Key}' {frame.Width}x{frame.Height} " +
|
||||
$"{AlphaStats(frame.BgraPixels, frame.Width, frame.Height, stride)} " +
|
||||
$"contentBounds=({cropX},{cropY},{cropW},{cropH})");
|
||||
}
|
||||
}
|
||||
|
||||
// alpha[min/max/mean/zero%] — the one number that decides whether a widget's
|
||||
// capture really has transparent margins (0% = opaque, ~100% if the page
|
||||
// painted transparent-only). Sampled every 16th pixel like the original log.
|
||||
private static string AlphaStats(byte[] pixels, int pixW, int pixH, int stride)
|
||||
{
|
||||
int minA = 255, maxA = 0;
|
||||
@@ -326,147 +439,32 @@ public sealed class WebView2Manager : IDisposable
|
||||
return $"alpha[min={minA},max={maxA},mean={sumA / (double)nA:F1},zero={100.0 * zeroA / nA:F1}%]";
|
||||
}
|
||||
|
||||
private async Task CaptureFrame(string sourceId)
|
||||
{
|
||||
if (!_sessions.TryGetValue(sourceId, out var session) || session.Disposed) return;
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
var webView = session.Control;
|
||||
if (webView.CoreWebView2 == null) return;
|
||||
|
||||
using var ms = new MemoryStream();
|
||||
await webView.CoreWebView2.CapturePreviewAsync(
|
||||
CoreWebView2CapturePreviewImageFormat.Png, ms);
|
||||
|
||||
if (!session.DebugPngWritten)
|
||||
{
|
||||
session.DebugPngWritten = true;
|
||||
try
|
||||
{
|
||||
ms.Position = 0;
|
||||
var dbg = Path.Combine(Path.GetTempPath(), $"ytLive-web-{sourceId}.png");
|
||||
File.WriteAllBytes(dbg, ms.ToArray());
|
||||
AppLog.Write($"WebView2Manager: debug capture dump -> {dbg}");
|
||||
}
|
||||
catch (Exception dex) { AppLog.Write($"WebView2Manager: debug dump failed: {dex.Message}"); }
|
||||
}
|
||||
|
||||
ms.Position = 0;
|
||||
|
||||
var bitmap = new BitmapImage();
|
||||
bitmap.BeginInit();
|
||||
bitmap.StreamSource = ms;
|
||||
bitmap.CacheOption = BitmapCacheOption.OnLoad;
|
||||
bitmap.EndInit();
|
||||
bitmap.Freeze();
|
||||
|
||||
var formatted = new FormatConvertedBitmap(bitmap,
|
||||
PixelFormats.Bgra32, null, 0);
|
||||
|
||||
var pixW = formatted.PixelWidth;
|
||||
var pixH = formatted.PixelHeight;
|
||||
|
||||
if (pixW < 1 || pixH < 1) return;
|
||||
|
||||
var need = pixW * pixH * 4;
|
||||
var pixels = session.CanvasScratch is { } scratch && scratch.Length >= need
|
||||
? scratch
|
||||
: (session.CanvasScratch = new byte[need]);
|
||||
formatted.CopyPixels(pixels, pixW * 4, 0);
|
||||
|
||||
var stride = pixW * 4;
|
||||
|
||||
var (cropX, cropY, cropW, cropH) = FindContentBounds(pixels, pixW, pixH, stride);
|
||||
|
||||
if (session.Epoch == 0)
|
||||
{
|
||||
AppLog.Write(
|
||||
$"WebView2Manager: first capture for '{sourceId}' {pixW}x{pixH} " +
|
||||
$"{AlphaStats(pixels, pixW, pixH, stride)} " +
|
||||
$"contentBounds=({cropX},{cropY},{cropW},{cropH})");
|
||||
}
|
||||
|
||||
if (session.WidgetDumpRemaining > 0)
|
||||
{
|
||||
var dumpNo = 5 - session.WidgetDumpRemaining + 1;
|
||||
session.WidgetDumpRemaining--;
|
||||
try
|
||||
{
|
||||
var dbg = Path.Combine(Path.GetTempPath(), $"ytLive-web-{sourceId}-w{dumpNo}.png");
|
||||
File.WriteAllBytes(dbg, ms.ToArray());
|
||||
AppLog.Write(
|
||||
$"WebView2Manager: widget capture [{dumpNo}/5] for '{sourceId}' {pixW}x{pixH} " +
|
||||
$"{AlphaStats(pixels, pixW, pixH, stride)} " +
|
||||
$"contentBounds=({cropX},{cropY},{cropW},{cropH}) -> {dbg}");
|
||||
}
|
||||
catch (Exception dex)
|
||||
{
|
||||
AppLog.Write($"WebView2Manager: widget dump failed: {dex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
var outPixels = session.RentOutBuffer(cropW * cropH * 4);
|
||||
if (cropW == pixW && cropH == pixH && cropX == 0 && cropY == 0)
|
||||
Buffer.BlockCopy(pixels, 0, outPixels, 0, outPixels.Length);
|
||||
else
|
||||
for (var r = 0; r < cropH; r++)
|
||||
Buffer.BlockCopy(pixels, (cropY + r) * stride + cropX * 4, outPixels, r * cropW * 4, cropW * 4);
|
||||
|
||||
if (session.PreviewBitmap == null
|
||||
|| session.PreviewBitmap.PixelWidth != cropW || session.PreviewBitmap.PixelHeight != cropH)
|
||||
session.PreviewBitmap = new WriteableBitmap(cropW, cropH, 96, 96, PixelFormats.Bgra32, null);
|
||||
session.PreviewBitmap.WritePixels(new Int32Rect(0, 0, cropW, cropH), outPixels, cropW * 4, 0);
|
||||
|
||||
session.LatestFrame = new VideoFrame(pixW, pixH, pixels)
|
||||
{
|
||||
Epoch = ++session.Epoch,
|
||||
CropBounds = (cropX, cropY, cropW, cropH),
|
||||
};
|
||||
|
||||
// The preview bitmap is now MUTABLE and reused (CameraManager precedent):
|
||||
// WPF re-renders it after WritePixels; handlers must not Freeze it.
|
||||
var preview = session.PreviewBitmap;
|
||||
_ = _dispatcher.BeginInvoke(() =>
|
||||
{
|
||||
PreviewBitmapChanged?.Invoke(sourceId, preview);
|
||||
});
|
||||
|
||||
// Capture-cost telemetry for the first ~30 captures of a session: PNG encode
|
||||
// (CapturePreviewAsync) + decode cost decide whether ~30Hz recording cadence
|
||||
// is affordable or must drop to ~20Hz. The FramePump drops frames (never
|
||||
// time-lapses) if the UI thread's GC churn starves it, so startup.log shows
|
||||
// the real budget before anything is tuned further.
|
||||
if (session.TelemetryCaptureCount < 30)
|
||||
{
|
||||
session.TelemetryCaptureCount++;
|
||||
AppLog.Write(
|
||||
$"WebView2Manager: capture #{session.TelemetryCaptureCount} for '{sourceId}' " +
|
||||
$"{pixW}x{pixH} took {sw.ElapsedMilliseconds}ms");
|
||||
}
|
||||
}
|
||||
catch (ObjectDisposedException) { }
|
||||
catch (InvalidOperationException) { }
|
||||
catch (Exception ex)
|
||||
{
|
||||
AppLog.Write($"WebView2Manager: capture failed for '{sourceId}': {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private void RemoveSession(string sourceId)
|
||||
{
|
||||
if (!_sessions.TryGetValue(sourceId, out var session)) return;
|
||||
session.Disposed = true;
|
||||
session.Scheduler.Stop();
|
||||
_hostPanel.Children.Remove(session.Control);
|
||||
try { session.Control.Dispose(); } catch { }
|
||||
|
||||
var src = session.Source;
|
||||
if (src != null)
|
||||
{
|
||||
if (session.FrameHandler != null)
|
||||
src.FrameAvailable -= session.FrameHandler;
|
||||
_ = SafeStopAsync(src);
|
||||
}
|
||||
try { session.Controller?.Close(); } catch { }
|
||||
session.Web = null;
|
||||
_sessions.Remove(sourceId);
|
||||
}
|
||||
|
||||
private static async Task SafeStopAsync(IScreenCaptureSource source)
|
||||
{
|
||||
try { await source.StopAsync(); }
|
||||
catch (Exception ex) { AppLog.Write($"WebView2Manager: stopping web capture failed: {ex.Message}"); }
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
foreach (var id in _sessions.Keys.ToList())
|
||||
RemoveSession(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user