LlamaCasty/Services/ScreenCaptureFrameSource.cs

using System;
using System.Diagnostics;
using System.Runtime.InteropServices;
using System.Runtime.InteropServices.WindowsRuntime;
using System.Threading;
using System.Threading.Tasks;
using Windows.Graphics;
using Windows.Graphics.Capture;
using Windows.Graphics.DirectX;
using Windows.Graphics.DirectX.Direct3D11;
using Windows.Graphics.Imaging;
using ytLive.Helpers;

namespace ytLive.Services;

/// <summary>
/// A live screen capture for one target key ("monitor:&lt;n&gt;" or
/// "window:&lt;hwnd&gt;"). Owns the GraphicsCaptureItem, a free-threaded
/// Direct3D11CaptureFramePool and the capture session; frames are converted on
/// the capture worker thread from the GPU surface to a CPU BGRA VideoFrame.
/// Known OS limits: DRM content captures as black frames; capture pauses while
/// the app is minimized (the frame pool simply stops delivering frames).
/// </summary>
public sealed class ScreenCaptureFrameSource : IScreenCaptureSource
{
    private readonly object _gate = new();
    private GraphicsCaptureItem _item;
    private Direct3D11CaptureFramePool? _framePool;
    private GraphicsCaptureSession? _session;
    private SizeInt32 _poolSize;
    private bool _started;
    private int _convertInFlight;
    private DateTime _lastErrorLog = DateTime.MinValue;

    // Buffer recycling (take-11 spikes, 2026-09-04; slice 16, 2026-09-14): a fresh
    // ~8.3MB byte[] per DWM frame ≈ 500MB/s of LOH churn — the gen2 pauses it forced
    // surfaced as the "worst render 35-65ms" spikes that capped fps at ~42. The pool
    // is a reuse-distance ring: a buffer is only rewritten after ≥ redLine rents have
    // cycled since its last hand-out (FrameRingBuffer), so a frame a consumer still
    // holds — session.LatestFrame survives across conversions and the dispatcher
    // preview copy lags — can never be overwritten in place (the 1742 tear, a ring
    // slot rewritten under the consumer's read, handed the compositor one
    // new-top/old-bottom frame). Each hand-out still carries an Epoch so identity-keyed
    // consumers (the compositor's paste cache) cannot false-hit a recycled array.
    private readonly FrameRingBuffer _ring = new(8, redLine: 4);
    private long _epoch;
    private readonly MonotonicGate _publishGate = new();

    // The composition master frame (see ai.md "Resolution tiers"): the background
    // is an input layer, so we never hold a CPU frame bigger than the master.
    private const int MaxBackgroundWidth = 1920;
    private const int MaxBackgroundHeight = 1080;

    // A failing conversion must not re-flood the log at frame rate.
    private static readonly TimeSpan ErrorLogThrottle = TimeSpan.FromSeconds(5);

    // Conversion cadence (slice 16, 2026-09-14; slice 17, 2026-09-14): the monitor
    // delivers at the 240Hz DWM cadence (~4.2ms) while slots are 16.6ms. A 10ms floor
    // between conversion starts keeps the open edge above the ~60 conversions/s the pump
    // can use. What actually bounded the desktop feed on takes 1742/1824 was the SERIAL
    // GPU→CPU readback (`CreateCopyFromSurfaceAsync` ≈ 40-50ms of the ~47ms conversion
    // on a 240Hz-HDR box shared with the encoder), capping captures at ~17-20/s.
    // Slice 17 overlaps up to MaxConcurrentConversions readbacks (pool sized to
    // accommodate in-flight frames) and publishes only monotonically newer frames
    // (MonotonicGate — a slow older completion must never overwrite a newer LatestFrame).
    private static readonly TimeSpan MinConvertInterval = TimeSpan.FromMilliseconds(10);
    private static readonly TimeSpan TelemetryInterval = TimeSpan.FromSeconds(2);
    private const int MaxConcurrentConversions = 3;
    private const int PoolBufferCount = 5;
    private DateTime _lastConvertAt = DateTime.MinValue;
    private DateTime _telemetryFrom = DateTime.UtcNow;
    private DateTime _lastTelemetry = DateTime.UtcNow;
    private long _skippedCadence;
    private long _skippedBusy;
    private long _conversions;
    private long _convertMsTotal;
    private long _convertMsMax;

    public string Key { get; }
    public event Action<VideoFrame>? FrameAvailable;

    internal ScreenCaptureFrameSource(string key, GraphicsCaptureItem item)
    {
        Key = key;
        _item = item;
    }

    /// <summary>Wraps an item chosen through the OS GraphicsCapturePicker.</summary>
    public static ScreenCaptureFrameSource CreateForPicker(string key, GraphicsCaptureItem item)
        => new(key, item);

    public static ScreenCaptureFrameSource CreateForMonitor(int monitorIndex)
    {
        var hmonitor = Win32FullScreenDetector.GetMonitorHandle(monitorIndex);
        if (hmonitor == IntPtr.Zero)
            throw new InvalidOperationException($"Monitor {monitorIndex} is not connected");
        var item = CaptureInterop.CreateForMonitor(hmonitor);
        return new ScreenCaptureFrameSource($"monitor:{monitorIndex}", item);
    }

    public static ScreenCaptureFrameSource CreateForWindow(IntPtr hwnd)
    {
        var item = CaptureInterop.CreateForWindow(hwnd);
        return new ScreenCaptureFrameSource($"window:0x{hwnd.ToInt64():X}", item);
    }

    public Task StartAsync()
    {
        lock (_gate)
        {
            if (_started) return Task.CompletedTask;
            var item = _item;
            var device = Direct3D11Helper.CreateDevice();
            var framePool = Direct3D11CaptureFramePool.CreateFreeThreaded(
                device, DirectXPixelFormat.B8G8R8A8UIntNormalized, PoolBufferCount, item.Size);
            _poolSize = item.Size;
            var session = framePool.CreateCaptureSession(item);
            framePool.FrameArrived += OnFrameArrived;
            session.StartCapture();
            _framePool = framePool;
            _session = session;
            _started = true;
        }
        return Task.CompletedTask;
    }

    public Task StopAsync()
    {
        lock (_gate)
        {
            _started = false;
            _convertInFlight = 0;
            if (_framePool != null)
                _framePool.FrameArrived -= OnFrameArrived;
            _session?.Dispose();
            _session = null;
            _framePool?.Dispose();
            _framePool = null;
        }
        return Task.CompletedTask;
    }

    private void OnFrameArrived(Direct3D11CaptureFramePool sender, object args)
    {
        var frame = sender.TryGetNextFrame();
        if (frame == null) return;

        lock (_gate)
        {
            if (!_started)
            {
                frame.Dispose();
                return;
            }

            if (frame.ContentSize.Width != _poolSize.Width || frame.ContentSize.Height != _poolSize.Height)
            {
                sender.Recreate(Direct3D11Helper.CreateDevice(),
                    DirectXPixelFormat.B8G8R8A8UIntNormalized, PoolBufferCount, frame.ContentSize);
                _poolSize = frame.ContentSize;
            }

            // Up to MaxConcurrentConversions readbacks in flight (slice 17), spaced by
            // MinConvertInterval (slice 16): the 240Hz delivery otherwise queued one
            // conversion every ~4.2ms, and the serial ~47ms readback pinned the desktop
            // layer to ~17 updates/s on the 1824 take.
            if (_convertInFlight >= MaxConcurrentConversions)
            {
                _skippedBusy++;
                frame.Dispose();
                return;
            }
            var now = DateTime.UtcNow;
            if (now - _lastConvertAt < MinConvertInterval)
            {
                _skippedCadence++;
                frame.Dispose();
                return;
            }
            _lastConvertAt = now;
            _convertInFlight++;
        }
        _ = ProcessFrameAsync(frame);
    }

    private async Task ProcessFrameAsync(Direct3D11CaptureFrame frame)
    {
        var sw = Stopwatch.StartNew();
        try
        {
            using (frame)
            using (var softwareBitmap = await SoftwareBitmap.CreateCopyFromSurfaceAsync(
                frame.Surface, BitmapAlphaMode.Ignore))
            {
                // Monotonic sequencing across the overlapping conversions (slice 17): a
                // completed readback is published only if strictly newer than the last
                // one published — a slow older completion must never overwrite a newer
                // LatestFrame (that would be a time hole in the other direction).
                var epoch = Interlocked.Increment(ref _epoch);
                var videoFrame = CopyToVideoFrame(softwareBitmap, epoch);
                if (_publishGate.TryPublish(epoch))
                    FrameAvailable?.Invoke(videoFrame);
            }
        }
        catch (Exception ex)
        {
            var now = DateTime.UtcNow;
            if (now - _lastErrorLog >= ErrorLogThrottle)
            {
                _lastErrorLog = now;
                AppLog.Write($"ScreenCaptureFrameSource: frame conversion failed: {ex.Message}");
            }
        }
        finally
        {
            lock (_gate)
            {
                EmitTelemetry(sw.ElapsedMilliseconds);
                _convertInFlight--;
            }
        }
    }

    private void EmitTelemetry(long conversionMs)
    {
        _conversions++;
        _convertMsTotal += conversionMs;
        if (conversionMs > _convertMsMax) _convertMsMax = conversionMs;

        var now = DateTime.UtcNow;
        if (now - _lastTelemetry < TelemetryInterval) return;

        var span = now - _telemetryFrom;
        var perSecond = span.TotalSeconds > 0 ? _conversions / span.TotalSeconds : 0;
        AppLog.Write(
            $"ScreenCapture telemetry [{Key}]: {_conversions} frames in {span.TotalSeconds:F1}s " +
            $"({perSecond:F0}/s), conv avg {( _conversions == 0 ? 0 : _convertMsTotal / _conversions )}ms " +
            $"max {_convertMsMax}ms, skip busy {_skippedBusy} cadence {_skippedCadence}, " +
            $"ring allocs {_ring.ConsumeAllocations()}");

        _lastTelemetry = now;
        _telemetryFrom = now;
        _conversions = 0;
        _convertMsTotal = 0;
        _convertMsMax = 0;
        _skippedBusy = 0;
        _skippedCadence = 0;
    }

    private VideoFrame CopyToVideoFrame(SoftwareBitmap bitmap, long epoch)
    {
        var sw = bitmap.PixelWidth;
        var sh = bitmap.PixelHeight;
        using var buffer = bitmap.LockBuffer(BitmapBufferAccessMode.Read);
        using var reference = buffer.CreateReference();
        if (!WindowsRuntimeMarshal.TryGetDataUnsafe(reference, out var data, out var capacity))
            throw new InvalidOperationException("Could not access the frame buffer.");

        var srcStride = sw * 4;
        var count = (int)Math.Min(capacity, (uint)(sh * srcStride));

        // The master frame is 1920×1080; a larger monitor is scaled down here so
        // the CPU never holds a frame above the master (the rendered layer fills
        // the frame with UniformToFill regardless).
        if (sw > MaxBackgroundWidth || sh > MaxBackgroundHeight)
        {
            var scale = Math.Min(MaxBackgroundWidth / (double)sw, MaxBackgroundHeight / (double)sh);
            var dw = Math.Max(1, (int)(sw * scale));
            var dh = Math.Max(1, (int)(sh * scale));
            // DWM delivers an opaque surface (alpha 255); bilinear keeps it 255.
            var scaled = _ring.Rent(dw * dh * 4);
            return new VideoFrame(dw, dh, DownscaleBgra(data, sw, sh, srcStride, dw, dh, scaled))
                { IsOpaque = true, Epoch = epoch };
        }

        var pixels = _ring.Rent(count);
        Marshal.Copy(data, pixels, 0, pixels.Length);
        return new VideoFrame(sw, sh, pixels) { IsOpaque = true, Epoch = epoch };
    }

    // Integer 8.8 fixed-point bilinear downscale to the master frame (slice 16,
    // 2026-09-14: the two-stage math from SceneCompositor.Bilinear — the same "shift
    // only at the end" rule from MyMistakes item on fixed-point blending). The previous
    // double-per-pixel version ran ~30-45ms per 2.5K→1080p frame on a quiet desktop
    // and the 1742 take's stall measured ~150ms/frame under load (90%-frozen desktop,
    // ~6 fresh frames/s); integer math keeps the same bilinear result within ±1 while
    // dropping the cost to the row-pair Marshal.Copy. Rows are read once per source
    // row pair through Marshal.Copy (no unsafe), writing tightly packed BGRA output.
    private byte[] DownscaleBgra(IntPtr src, int sw, int sh, int srcStride, int dw, int dh, byte[] dst)
    {
        // Row scratch is per-conversion (overlapping conversions since slice 17 each
        // bring their own — two 10KB arrays, no shared state). Size: a 2560-wide row
        // pair, the largest the monitor path delivers before the downscale.
        var row0 = new byte[srcStride];
        var row1 = new byte[srcStride];

        for (var y = 0; y < dh; y++)
        {
            var sy8 = (int)((long)y * sh * 256 / dh);
            var sy = sy8 >> 8;
            var sy1 = Math.Min(sh - 1, sy + 1);
            var fy8 = sy8 & 255;
            var fyInv = 256 - fy8;
            Marshal.Copy(IntPtr.Add(src, sy * srcStride), row0, 0, srcStride);
            Marshal.Copy(IntPtr.Add(src, sy1 * srcStride), row1, 0, srcStride);

            var dRow = y * dw * 4;
            for (var x = 0; x < dw; x++)
            {
                var sx8 = (int)((long)x * sw * 256 / dw);
                var sx = Math.Min(sw - 1, sx8 >> 8);
                var sx1 = Math.Min(sw - 1, sx + 1);
                var fx8 = sx8 & 255;
                var fxInv = 256 - fx8;

                var i0 = sx * 4;
                var i1 = sx1 * 4;
                var j = dRow + x * 4;
                for (var c = 0; c < 4; c++, i0++, i1++, j++)
                {
                    // Two-stage in one 16.8 scale: top/bot ≤ 65280, ×256 + round ≤ 33.5M — int-safe.
                    var top = row0[i0] * fxInv + row0[i1] * fx8;
                    var bot = row1[i0] * fxInv + row1[i1] * fx8;
                    dst[j] = (byte)((top * fyInv + bot * fy8 + 32768) >> 16);
                }
            }
        }
        return dst;
    }
}