using System; using System.Diagnostics; using System.Runtime.InteropServices; using System.Runtime.InteropServices.WindowsRuntime; using System.Threading.Tasks; using Windows.Graphics; using Windows.Graphics.Capture; using Windows.Graphics.DirectX; using Windows.Graphics.DirectX.Direct3D11; using Windows.Graphics.Imaging; using ytLive.Helpers; namespace ytLive.Services; /// /// A live screen capture for one target key ("monitor:<n>" or /// "window:<hwnd>"). Owns the GraphicsCaptureItem, a free-threaded /// Direct3D11CaptureFramePool and the capture session; frames are converted on /// the capture worker thread from the GPU surface to a CPU BGRA VideoFrame. /// Known OS limits: DRM content captures as black frames; capture pauses while /// the app is minimized (the frame pool simply stops delivering frames). /// public sealed class ScreenCaptureFrameSource : IScreenCaptureSource { private readonly object _gate = new(); private GraphicsCaptureItem _item; private Direct3D11CaptureFramePool? _framePool; private GraphicsCaptureSession? _session; private SizeInt32 _poolSize; private bool _started; private bool _framePending; private DateTime _lastErrorLog = DateTime.MinValue; // Buffer recycling (take-11 spikes, 2026-09-04; slice 16, 2026-09-14): a fresh // ~8.3MB byte[] per DWM frame ≈ 500MB/s of LOH churn — the gen2 pauses it forced // surfaced as the "worst render 35-65ms" spikes that capped fps at ~42. The pool // is a reuse-distance ring: a buffer is only rewritten after ≥ redLine rents have // cycled since its last hand-out (FrameRingBuffer), so a frame a consumer still // holds — session.LatestFrame survives across conversions and the dispatcher // preview copy lags — can never be overwritten in place (the 1742 tear, a ring // slot rewritten under the consumer's read, handed the compositor one // new-top/old-bottom frame). Each hand-out still carries an Epoch so identity-keyed // consumers (the compositor's paste cache) cannot false-hit a recycled array. private readonly FrameRingBuffer _ring = new(8, redLine: 4); private long _epoch; private byte[]? _row0; private byte[]? _row1; // The composition master frame (see ai.md "Resolution tiers"): the background // is an input layer, so we never hold a CPU frame bigger than the master. private const int MaxBackgroundWidth = 1920; private const int MaxBackgroundHeight = 1080; // A failing conversion must not re-flood the log at frame rate. private static readonly TimeSpan ErrorLogThrottle = TimeSpan.FromSeconds(5); // Conversion cadence (slice 16, 2026-09-14): the monitor delivers at the 240Hz DWM // cadence (~4.2ms) while slots are 16.6ms. A 10ms floor between conversion starts // keeps the open edge above the ~60 conversions/s the pump can actually use, so the // 240Hz tail stops chewing a conversion thread that the 1742 take measured at // ~150ms/frame (a 90%-frozen desktop, ~6 fresh frames/s). Drop counters and the // rolling conversion stats feed the 2-second startup.log telemetry line. private static readonly TimeSpan MinConvertInterval = TimeSpan.FromMilliseconds(10); private static readonly TimeSpan TelemetryInterval = TimeSpan.FromSeconds(2); private DateTime _lastConvertAt = DateTime.MinValue; private DateTime _telemetryFrom = DateTime.UtcNow; private DateTime _lastTelemetry = DateTime.UtcNow; private long _skippedCadence; private long _skippedBusy; private long _conversions; private long _convertMsTotal; private long _convertMsMax; public string Key { get; } public event Action? FrameAvailable; internal ScreenCaptureFrameSource(string key, GraphicsCaptureItem item) { Key = key; _item = item; } /// Wraps an item chosen through the OS GraphicsCapturePicker. public static ScreenCaptureFrameSource CreateForPicker(string key, GraphicsCaptureItem item) => new(key, item); public static ScreenCaptureFrameSource CreateForMonitor(int monitorIndex) { var hmonitor = Win32FullScreenDetector.GetMonitorHandle(monitorIndex); if (hmonitor == IntPtr.Zero) throw new InvalidOperationException($"Monitor {monitorIndex} is not connected"); var item = CaptureInterop.CreateForMonitor(hmonitor); return new ScreenCaptureFrameSource($"monitor:{monitorIndex}", item); } public static ScreenCaptureFrameSource CreateForWindow(IntPtr hwnd) { var item = CaptureInterop.CreateForWindow(hwnd); return new ScreenCaptureFrameSource($"window:0x{hwnd.ToInt64():X}", item); } public Task StartAsync() { lock (_gate) { if (_started) return Task.CompletedTask; var item = _item; var device = Direct3D11Helper.CreateDevice(); var framePool = Direct3D11CaptureFramePool.CreateFreeThreaded( device, DirectXPixelFormat.B8G8R8A8UIntNormalized, 2, item.Size); _poolSize = item.Size; var session = framePool.CreateCaptureSession(item); framePool.FrameArrived += OnFrameArrived; session.StartCapture(); _framePool = framePool; _session = session; _started = true; } return Task.CompletedTask; } public Task StopAsync() { lock (_gate) { _started = false; _framePending = false; if (_framePool != null) _framePool.FrameArrived -= OnFrameArrived; _session?.Dispose(); _session = null; _framePool?.Dispose(); _framePool = null; } return Task.CompletedTask; } private void OnFrameArrived(Direct3D11CaptureFramePool sender, object args) { var frame = sender.TryGetNextFrame(); if (frame == null) return; lock (_gate) { if (!_started) { frame.Dispose(); return; } if (frame.ContentSize.Width != _poolSize.Width || frame.ContentSize.Height != _poolSize.Height) { sender.Recreate(Direct3D11Helper.CreateDevice(), DirectXPixelFormat.B8G8R8A8UIntNormalized, 2, frame.ContentSize); _poolSize = frame.ContentSize; } // One conversion at a time, spaced by MinConvertInterval (slice 16): the // 240Hz delivery otherwise queued a conversion every ~4.2ms and the // 1742 take's ~150ms conversion pinned the desktop layer ~90% frozen. if (_framePending) { _skippedBusy++; frame.Dispose(); return; } var now = DateTime.UtcNow; if (now - _lastConvertAt < MinConvertInterval) { _skippedCadence++; frame.Dispose(); return; } _lastConvertAt = now; _framePending = true; } _ = ProcessFrameAsync(frame); } private async Task ProcessFrameAsync(Direct3D11CaptureFrame frame) { var sw = Stopwatch.StartNew(); try { using (frame) using (var softwareBitmap = await SoftwareBitmap.CreateCopyFromSurfaceAsync( frame.Surface, BitmapAlphaMode.Ignore)) { FrameAvailable?.Invoke(CopyToVideoFrame(softwareBitmap)); } } catch (Exception ex) { var now = DateTime.UtcNow; if (now - _lastErrorLog >= ErrorLogThrottle) { _lastErrorLog = now; AppLog.Write($"ScreenCaptureFrameSource: frame conversion failed: {ex.Message}"); } } finally { lock (_gate) { EmitTelemetry(sw.ElapsedMilliseconds); } _framePending = false; } } private void EmitTelemetry(long conversionMs) { _conversions++; _convertMsTotal += conversionMs; if (conversionMs > _convertMsMax) _convertMsMax = conversionMs; var now = DateTime.UtcNow; if (now - _lastTelemetry < TelemetryInterval) return; var span = now - _telemetryFrom; var perSecond = span.TotalSeconds > 0 ? _conversions / span.TotalSeconds : 0; AppLog.Write( $"ScreenCapture telemetry [{Key}]: {_conversions} frames in {span.TotalSeconds:F1}s " + $"({perSecond:F0}/s), conv avg {( _conversions == 0 ? 0 : _convertMsTotal / _conversions )}ms " + $"max {_convertMsMax}ms, skip busy {_skippedBusy} cadence {_skippedCadence}, " + $"ring allocs {_ring.ConsumeAllocations()}"); _lastTelemetry = now; _telemetryFrom = now; _conversions = 0; _convertMsTotal = 0; _convertMsMax = 0; _skippedBusy = 0; _skippedCadence = 0; } private VideoFrame CopyToVideoFrame(SoftwareBitmap bitmap) { var sw = bitmap.PixelWidth; var sh = bitmap.PixelHeight; using var buffer = bitmap.LockBuffer(BitmapBufferAccessMode.Read); using var reference = buffer.CreateReference(); if (!WindowsRuntimeMarshal.TryGetDataUnsafe(reference, out var data, out var capacity)) throw new InvalidOperationException("Could not access the frame buffer."); var srcStride = sw * 4; var count = (int)Math.Min(capacity, (uint)(sh * srcStride)); // The master frame is 1920×1080; a larger monitor is scaled down here so // the CPU never holds a frame above the master (the rendered layer fills // the frame with UniformToFill regardless). if (sw > MaxBackgroundWidth || sh > MaxBackgroundHeight) { var scale = Math.Min(MaxBackgroundWidth / (double)sw, MaxBackgroundHeight / (double)sh); var dw = Math.Max(1, (int)(sw * scale)); var dh = Math.Max(1, (int)(sh * scale)); // DWM delivers an opaque surface (alpha 255); bilinear keeps it 255. var scaled = _ring.Rent(dw * dh * 4); return new VideoFrame(dw, dh, DownscaleBgra(data, sw, sh, srcStride, dw, dh, scaled)) { IsOpaque = true, Epoch = ++_epoch }; } var pixels = _ring.Rent(count); Marshal.Copy(data, pixels, 0, pixels.Length); return new VideoFrame(sw, sh, pixels) { IsOpaque = true, Epoch = ++_epoch }; } // Integer 8.8 fixed-point bilinear downscale to the master frame (slice 16, // 2026-09-14: the two-stage math from SceneCompositor.Bilinear — the same "shift // only at the end" rule from MyMistakes item on fixed-point blending). The previous // double-per-pixel version ran ~30-45ms per 2.5K→1080p frame on a quiet desktop // and the 1742 take's stall measured ~150ms/frame under load (90%-frozen desktop, // ~6 fresh frames/s); integer math keeps the same bilinear result within ±1 while // dropping the cost to the row-pair Marshal.Copy. Rows are read once per source // row pair through Marshal.Copy (no unsafe), writing tightly packed BGRA output. private byte[] DownscaleBgra(IntPtr src, int sw, int sh, int srcStride, int dw, int dh, byte[] dst) { // Row scratch is per-capture-thread and reused across frames (same churn lesson). var row0 = _row0 != null && _row0.Length >= srcStride ? _row0 : (_row0 = new byte[srcStride]); var row1 = _row1 != null && _row1.Length >= srcStride ? _row1 : (_row1 = new byte[srcStride]); for (var y = 0; y < dh; y++) { var sy8 = (int)((long)y * sh * 256 / dh); var sy = sy8 >> 8; var sy1 = Math.Min(sh - 1, sy + 1); var fy8 = sy8 & 255; var fyInv = 256 - fy8; Marshal.Copy(IntPtr.Add(src, sy * srcStride), row0, 0, srcStride); Marshal.Copy(IntPtr.Add(src, sy1 * srcStride), row1, 0, srcStride); var dRow = y * dw * 4; for (var x = 0; x < dw; x++) { var sx8 = (int)((long)x * sw * 256 / dw); var sx = Math.Min(sw - 1, sx8 >> 8); var sx1 = Math.Min(sw - 1, sx + 1); var fx8 = sx8 & 255; var fxInv = 256 - fx8; var i0 = sx * 4; var i1 = sx1 * 4; var j = dRow + x * 4; for (var c = 0; c < 4; c++, i0++, i1++, j++) { // Two-stage in one 16.8 scale: top/bot ≤ 65280, ×256 + round ≤ 33.5M — int-safe. var top = row0[i0] * fxInv + row0[i1] * fx8; var bot = row1[i0] * fxInv + row1[i1] * fx8; dst[j] = (byte)((top * fyInv + bot * fy8 + 32768) >> 16); } } } return dst; } }