LlamaCasty/Services/WebCaptureFrameSource.cs

using System;
using System.Runtime.InteropServices;
using System.Runtime.InteropServices.WindowsRuntime;
using System.Threading.Tasks;
using Windows.Graphics;
using Windows.Graphics.Capture;
using Windows.Graphics.DirectX;
using Windows.Graphics.DirectX.Direct3D11;
using Windows.Graphics.Imaging;
using ytLive.Helpers;

namespace ytLive.Services;

/// <summary>
/// The frame-driven web capture for one widget: a GraphicsCaptureItem backed by
/// the widget's Windows.UI.Composition visual (see WebView2Manager). Owns the
/// item, a free-threaded Direct3D11CaptureFramePool and the capture session;
/// frames arrive in step with Chromium's own presentation (~60Hz on a real
/// animation), replacing the old CapturePreviewAsync PNG poll whose ~35-165ms
/// encode cost capped the recording at ~10Hz and the preview at ~5fps.
///
/// The capture surface holds a REAL alpha channel (the controller renders onto
/// a transparent composition visual), so the GPU→CPU copy uses
/// <see cref="BitmapAlphaMode.Straight"/> and the frame is
/// <see cref="VideoFrame.IsOpaque"/> = false — the compositor's straight-alpha
/// blend math ("web layer transparency", 2026-09-13) requires the readback to
/// keep premultiplication OFF. Unlike the screen source there is no master-size
/// downscale: the controller renders at the RAW 1920×1080 master canvas, so the
/// CPU never holds a larger frame.
///
/// Per-frame the alpha bounding box of the widget (<see cref="FindContentBounds"/>)
/// is measured on this capture worker thread and stamped onto the frame as
/// <see cref="VideoFrame.CropBounds"/> — same alpha-bbox contract as the old
/// PNG path, but off the UI thread (a 2M-pixel scan per frame there is exactly
/// the cost this redesign exists to remove).
/// </summary>
public sealed class WebCaptureFrameSource : IScreenCaptureSource
{
    private readonly object _gate = new();
    private GraphicsCaptureItem _item;
    private Direct3D11CaptureFramePool? _framePool;
    private GraphicsCaptureSession? _session;
    private SizeInt32 _poolSize;
    private bool _started;
    private bool _framePending;
    private DateTime _lastErrorLog = DateTime.MinValue;

    // Buffer recycling: identical to the screen source's ring+Epoch contract
    // (see ScreenCaptureFrameSource) — the compositor's paste cache is keyed on
    // array identity, so recycled buffers must carry a monotonic Epoch or a
    // reused array false-hits with stale content.
    private readonly byte[]?[] _frameRing = new byte[8][];
    private int _ringNext;
    private long _epoch;

    private static readonly TimeSpan ErrorLogThrottle = TimeSpan.FromSeconds(5);

    public string Key { get; }
    public event Action<VideoFrame>? FrameAvailable;

    internal WebCaptureFrameSource(string key, GraphicsCaptureItem item)
    {
        Key = key;
        _item = item;
    }

    /// <summary>Wraps an item created from a composition visual by the web manager.</summary>
    public static WebCaptureFrameSource CreateForVisual(string key, GraphicsCaptureItem item)
        => new(key, item);

    public Task StartAsync()
    {
        lock (_gate)
        {
            if (_started) return Task.CompletedTask;
            var item = _item;
            var device = Direct3D11Helper.CreateDevice();
            var framePool = Direct3D11CaptureFramePool.CreateFreeThreaded(
                device, DirectXPixelFormat.B8G8R8A8UIntNormalized, 2, item.Size);
            _poolSize = item.Size;
            var session = framePool.CreateCaptureSession(item);
            framePool.FrameArrived += OnFrameArrived;
            session.StartCapture();
            _framePool = framePool;
            _session = session;
            _started = true;
        }
        return Task.CompletedTask;
    }

    public Task StopAsync()
    {
        lock (_gate)
        {
            _started = false;
            _framePending = false;
            if (_framePool != null)
                _framePool.FrameArrived -= OnFrameArrived;
            _session?.Dispose();
            _session = null;
            _framePool?.Dispose();
            _framePool = null;
        }
        return Task.CompletedTask;
    }

    private void OnFrameArrived(Direct3D11CaptureFramePool sender, object args)
    {
        var frame = sender.TryGetNextFrame();
        if (frame == null) return;
        lock (_gate)
        {
            if (!_started)
            {
                frame.Dispose();
                return;
            }
        }

        if (frame.ContentSize.Width != _poolSize.Width || frame.ContentSize.Height != _poolSize.Height)
        {
            sender.Recreate(Direct3D11Helper.CreateDevice(),
                DirectXPixelFormat.B8G8R8A8UIntNormalized, 2, frame.ContentSize);
            _poolSize = frame.ContentSize;
        }

        if (_framePending)
        {
            frame.Dispose();
            return;
        }
        _framePending = true;
        _ = ProcessFrameAsync(frame);
    }

    private async Task ProcessFrameAsync(Direct3D11CaptureFrame frame)
    {
        try
        {
            using (frame)
            using (var softwareBitmap = await SoftwareBitmap.CreateCopyFromSurfaceAsync(
                frame.Surface, BitmapAlphaMode.Straight))
            {
                FrameAvailable?.Invoke(CopyToVideoFrame(softwareBitmap));
            }
        }
        catch (Exception ex)
        {
            var now = DateTime.UtcNow;
            if (now - _lastErrorLog >= ErrorLogThrottle)
            {
                _lastErrorLog = now;
                AppLog.Write($"WebCaptureFrameSource: frame conversion failed: {ex.Message}");
            }
        }
        finally
        {
            _framePending = false;
        }
    }

    private byte[] RentRingBuffer(int size)
    {
        for (var tries = 0; tries < _frameRing.Length; tries++)
        {
            var idx = (_ringNext + tries) % _frameRing.Length;
            var buf = _frameRing[idx];
            if (buf is { Length: var len } && len == size)
            {
                _ringNext = (idx + 1) % _frameRing.Length;
                return buf;
            }
        }
        var slot = _ringNext;
        _ringNext = (slot + 1) % _frameRing.Length;
        var fresh = new byte[size];
        _frameRing[slot] = fresh;
        return fresh;
    }

    private VideoFrame CopyToVideoFrame(SoftwareBitmap bitmap)
    {
        var sw = bitmap.PixelWidth;
        var sh = bitmap.PixelHeight;
        using var buffer = bitmap.LockBuffer(BitmapBufferAccessMode.Read);
        using var reference = buffer.CreateReference();
        if (!WindowsRuntimeMarshal.TryGetDataUnsafe(reference, out var data, out var capacity))
            throw new InvalidOperationException("Could not access the frame buffer.");

        var srcStride = sw * 4;
        var count = (int)Math.Min(capacity, (uint)(sh * srcStride));
        var pixels = RentRingBuffer(count);
        Marshal.Copy(data, pixels, 0, pixels.Length);

        var (cropX, cropY, cropW, cropH) = FindContentBounds(pixels, sw, sh, srcStride);
        return new VideoFrame(sw, sh, pixels)
        {
            IsOpaque = false,
            Epoch = ++_epoch,
            CropBounds = (cropX, cropY, cropW, cropH),
        };
    }

    // The widget's extent is measured from the captured frame itself: the
    // bounding box of non-transparent pixels (the background is injected
    // transparent, so the alpha channel IS the widget). Immune to DOM timing
    // (fonts/images/animations), needs no JS round-trip, and CANNOT break the
    // render: a widget that fills the canvas gets the full frame; a widget that
    // floats in transparent margin gets exactly its content box — used for the
    // preview bitmap / selection. The frame handed to the compositor is always
    // the FULL canvas, so the page's transparent margins keep revealing the
    // layers beneath (the "black square over webcam" was the crop fed to the
    // compositor: UniformToFill zoomed the opaque content box over the element).
    internal static (int X, int Y, int W, int H) FindContentBounds(byte[] pixels, int pixW, int pixH, int stride)
    {
        int minX = pixW, minY = pixH, maxX = -1, maxY = -1;
        for (var y = 0; y < pixH; y++)
        {
            var row = y * stride;
            for (var x = 0; x < pixW; x++)
            {
                if (pixels[row + x * 4 + 3] == 0) continue;
                if (x < minX) minX = x;
                if (x > maxX) maxX = x;
                if (y < minY) minY = y;
                if (y > maxY) maxY = y;
            }
        }
        if (maxX < minX || maxY < minY) return (0, 0, pixW, pixH);
        return (minX, minY, maxX - minX + 1, maxY - minY + 1);
    }
}