LlamaCasty/Services/MediaCaptureFrameSource.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Runtime.InteropServices.WindowsRuntime;
using System.Threading.Tasks;
using Windows.Graphics.Imaging;
using Windows.Media.Capture;
using Windows.Media.Capture.Frames;
using Windows.Media.Devices;
using Windows.Media.MediaProperties;
using ytLive.Helpers;

namespace ytLive.Services;

/// <summary>
/// CPU-first MediaCapture source. Requests BGRA8 frames from the camera's video
/// preview source; the capture pipeline does any format conversion, so every
/// frame arrives as a normalized <see cref="VideoFrame"/>. Frames arrive on a
/// worker thread — marshal before touching WPF.
///
/// Allocation is validated, never taken on faith: after <c>InitializeAsync</c>
/// the bound device, its frame sources, and its stream properties are checked,
/// the frame reader's start status is read (not swallowed), and the live
/// state signals (<c>Failed</c>, <c>CameraStreamStateChanged</c>) are subscribed
/// so an async death surfaces as <see cref="SourceFailed"/> instead of a silent
/// empty preview.
/// </summary>
public sealed class MediaCaptureFrameSource : ICameraFrameSource
{
    private readonly string _deviceId;
    private MediaCapture? _capture;
    private MediaFrameReader? _frameReader;
    private string? _lastError;
    private bool _isFailed;

    public string DeviceId => _deviceId;

    public event Action<VideoFrame>? FrameAvailable;

    public event Action<string>? SourceFailed;

    public MediaCaptureFrameSource(string deviceId)
    {
        _deviceId = deviceId;
    }

    public async Task StartAsync()
    {
        // Fallback ladder: prefer the camera's VideoPreview stream, but if the
        // reader refuses to start there (NoVideoFrameAvailable), retry against
        // its VideoRecord stream — some devices only deliver through it.
        // (The exclusive-lock upgrade is a separate, later unit — creator
        // ruling 2026-10-05; see HANDOFF.md. It is NOT blocked by an SDK gap:
        // `MediaCaptureSharingMode.ExclusiveControl` IS projected in 19041.)
        if (!await TryStartAsync(preferVideoRecord: false))
        {
            if (!await TryStartAsync(preferVideoRecord: true))
                throw new InvalidOperationException(
                    _lastError ?? $"Camera '{_deviceId}' could not be started.");
        }
    }

    public async Task StopAsync()
    {
        var reader = _frameReader;
        _frameReader = null;
        if (reader != null)
        {
            reader.FrameArrived -= OnFrameArrived;
            try
            {
                await reader.StopAsync();
            }
            catch (Exception ex)
            {
                AppLog.Write($"MediaCaptureFrameSource: stop frame reader failed: {ex.Message}");
            }
            reader.Dispose();
        }

        var capture = _capture;
        _capture = null;
        if (capture != null)
        {
            UnsubscribeCaptureEvents(capture);
            capture.Dispose();
        }
    }

    /// <summary>
    /// Ordered reader-output-subtype candidates for a camera whose live media type
    /// is <paramref name="activeSubtype"/>. Bgra8 is the fast path (OnFrameArrived
    /// skips conversion) and works for uncompressed cameras; an MJPG-active source
    /// cannot be read as Bgra8 by a MediaFrameReader (no BGRA converter on the MJPG
    /// pipeline — StartAsync answers OutputFormatNotSupported), so MJPG leads with
    /// NV12 and falls back to the reader's default (null). Every candidate that is
    /// not Bgra8 is converted in OnFrameArrived.
    /// </summary>
    internal static string?[] ReaderSubtypeCandidates(string? activeSubtype) =>
        string.Equals(activeSubtype, "MJPG", StringComparison.OrdinalIgnoreCase)
            ? new string?[] { MediaEncodingSubtypes.Nv12, null }
            : new string?[] { MediaEncodingSubtypes.Bgra8, MediaEncodingSubtypes.Nv12, null };

    private async Task<bool> TryStartAsync(bool preferVideoRecord)
    {
        MediaCapture? capture = null;
        MediaFrameReader? reader = null;
        try
        {
            capture = new MediaCapture();
            var settings = new MediaCaptureInitializationSettings
            {
                VideoDeviceId = _deviceId,
                StreamingCaptureMode = StreamingCaptureMode.Video,
                MemoryPreference = MediaCaptureMemoryPreference.Cpu,
                SharingMode = MediaCaptureSharingMode.SharedReadOnly,
            };
            await capture.InitializeAsync(settings);

            // MediaFrameReader defaults to the camera's FIRST media type — on UVC webcams
            // (Logitech C920 et al.) that is often YUY2 at a crippled rate (720p5!), and a
            // real-time 60fps container then shows a 5-20fps face cam as "every few frames
            // cut out". OBS's mechanism, available via VideoDeviceController (VideoProfile
            // is NOT projected in the 19041 SDK): negotiate the stream property. Pick the
            // best MJPEG >=640x360 @>=30fps (largest, fastest), set it BEFORE the frame
            // reader is created, fall back to device default on refusal, and log the live
            // media type so the next diagnosis starts from facts.
            try
            {
                var controller = capture.VideoDeviceController;
                var candidates = controller == null
                    ? new System.Collections.Generic.List<Windows.Media.MediaProperties.VideoEncodingProperties>()
                    : controller
                    .GetAvailableMediaStreamProperties(MediaStreamType.VideoPreview)
                    .OfType<Windows.Media.MediaProperties.VideoEncodingProperties>()
                    .Where(v => string.Equals(v.Subtype, "MJPG", StringComparison.OrdinalIgnoreCase)
                                && v.Width >= 640 && v.Height >= 360
                                && v.FrameRate.Denominator > 0
                                && v.FrameRate.Numerator / v.FrameRate.Denominator >= 30.0)
                    .OrderByDescending(v => v.FrameRate.Numerator / (double)Math.Max(1, v.FrameRate.Denominator))
                    .ThenByDescending(v => (long)v.Width * v.Height)
                    .ToList();
                if (candidates.Count > 0)
                {
                    var pick = candidates
                        .OrderBy(v => (long)v.Width * v.Height)
                        .LastOrDefault(v => v.Width <= 1280) ?? candidates[0];
                    await controller!.SetMediaStreamPropertiesAsync(MediaStreamType.VideoPreview, pick);
                }
            }
            catch (Exception fmtEx)
            {
                AppLog.Write($"MediaCaptureFrameSource: MJPG negotiation refused ({fmtEx.Message}); " +
                             "keeping device-default media type");
            }

            // Post-init resource validation: the capture must actually be bound to
            // the device we asked for and must expose a live video controller.
            if (!string.Equals(capture.MediaCaptureSettings?.VideoDeviceId, _deviceId,
                    StringComparison.OrdinalIgnoreCase))
                throw new InvalidOperationException(
                    $"Camera '{_deviceId}' initialized but bound a different device.");

            var colorSource = capture.FrameSources.Values
                .OrderBy(s => preferVideoRecord
                    ? (s.Info.MediaStreamType == MediaStreamType.VideoRecord ? 0 : 1)
                    : (s.Info.MediaStreamType == MediaStreamType.VideoPreview ? 0 : 1))
                .FirstOrDefault(s => s.Info.MediaStreamType is MediaStreamType.VideoPreview or MediaStreamType.VideoRecord);
            if (colorSource == null)
            {
                var kinds = string.Join(", ", capture.FrameSources.Values
                    .Select(s => s.Info.MediaStreamType).Distinct());
                throw new InvalidOperationException(
                    $"Camera '{_deviceId}' exposes no video source (found: {kinds}). " +
                    "It may be locked by another app (e.g. NVIDIA Broadcast).");
            }

            // The device must answer for its preview stream — a dead, suspended, or
            // locked device returns no stream properties even after "successful" init.
            IReadOnlyList<IMediaEncodingProperties>? previewProps;
            try
            {
                previewProps = capture.VideoDeviceController?
                    .GetAvailableMediaStreamProperties(MediaStreamType.VideoPreview);
            }
            catch
            {
                previewProps = null;
            }
            if (previewProps == null || previewProps.Count == 0)
                throw new InvalidOperationException(
                    $"Camera '{_deviceId}' answered no video stream properties (offline, suspended, or locked).");

            // The OS's live state signals: async failures must surface, not vanish.
            capture.Failed += OnCaptureFailed;
            capture.CameraStreamStateChanged += OnCameraStreamStateChanged;

            // The reader must output a subtype the source can actually produce, else
            // StartAsync refuses (OutputFormatNotSupported). An uncompressed camera
            // converts to Bgra8 fine (the fast path); a camera sitting in MJPG — which
            // is what we find when ANOTHER app left it there and ours can't re-negotiate
            // under SharedReadOnly contention — cannot be read as Bgra8 by a
            // MediaFrameReader (the MJPG pipeline exposes NV12 output, not BGRA). Walk
            // the candidate ladder; anything that isn't Bgra8 is converted in
            // OnFrameArrived anyway.
            string? activeSubtype = null;
            try
            {
                var active = capture.VideoDeviceController?
                    .GetMediaStreamProperties(colorSource.Info.MediaStreamType);
                if (active is Windows.Media.MediaProperties.VideoEncodingProperties vep)
                {
                    activeSubtype = vep.Subtype;
                    AppLog.Write($"Camera '{_deviceId}' media type: {vep.Subtype} " +
                                 $"{vep.Width}x{vep.Height} @ {vep.FrameRate.Numerator}/{vep.FrameRate.Denominator}fps");
                }
            }
            catch { /* logging must never kill acquisition */ }

            // Each candidate is allocated AND started under its own catch: a WinRT
            // call answering an unsupported subtype can THROW (E_INVALIDARG / "value
            // does not fall within the expected range") instead of returning a start
            // status, so every throw must degrade to the next candidate — never let
            // one rejected format abort the whole acquisition. Individual rejections
            // are logged and collected; only the last-resort net below (the outer
            // try/catch of this method) keeps cowboying device-level errors.
            var refused = new List<string>();
            foreach (var outputSubtype in ReaderSubtypeCandidates(activeSubtype))
            {
                var attempt = outputSubtype ?? "<default>";
                try
                {
                    reader = outputSubtype is null
                        ? await capture.CreateFrameReaderAsync(colorSource)
                        : await capture.CreateFrameReaderAsync(colorSource, outputSubtype);
                    if (reader == null)
                    {
                        refused.Add($"{attempt}: no frame reader");
                        continue;
                    }

                    reader.FrameArrived += OnFrameArrived;
                    var status = await reader.StartAsync();
                    if (status == MediaFrameReaderStartStatus.Success)
                    {
                        _capture = capture;
                        _frameReader = reader;
                        _lastError = null;
                        _isFailed = false;
                        StartCadenceWindow();
                        return true;
                    }

                    reader.FrameArrived -= OnFrameArrived;
                    reader.Dispose();
                    reader = null;
                    refused.Add($"{attempt}: {status}");
                }
                catch (Exception ex)
                {
                    AppLog.Write($"MediaCaptureFrameSource: reader as '{attempt}' refused: {ex.Message}");
                    if (reader != null)
                    {
                        reader.FrameArrived -= OnFrameArrived;
                        reader.Dispose();
                        reader = null;
                    }
                    refused.Add($"{attempt}: {ex.Message}");
                }
            }

            throw new InvalidOperationException(
                $"Camera '{_deviceId}' frame reader refused to start: {string.Join(", ", refused)}.");
        }
        catch (Exception ex)
        {
            _lastError = ex.Message;
            if (reader != null)
            {
                reader.FrameArrived -= OnFrameArrived;
                reader.Dispose();
            }
            if (capture != null)
            {
                UnsubscribeCaptureEvents(capture);
                capture.Dispose();
            }
            return false;
        }
    }

    private const int CadenceWindowSeconds = 5;

    private void StartCadenceWindow()
    {
        _arrivalCadence.Reset();
        _cameraCadence.Reset();
        _cadenceNextReportTicks = System.Diagnostics.Stopwatch.GetTimestamp()
            + System.Diagnostics.Stopwatch.Frequency * CadenceWindowSeconds;
    }

    /// <summary>Logs one cadence window and opens the next. The size rides along
    /// because the whole question started as "the webcam is soft": a future format
    /// change must be readable in the SAME line that proves the frame rate.</summary>
    private void ReportCadence(int width, int height)
    {
        _cadenceNextReportTicks = System.Diagnostics.Stopwatch.GetTimestamp()
            + System.Diagnostics.Stopwatch.Frequency * CadenceWindowSeconds;
        var arrivals = _arrivalCadence.Summarize();
        var camera = _cameraCadence.Summarize();
        _arrivalCadence.Reset();
        _cameraCadence.Reset();
        AppLog.Write($"Camera cadence {width}x{height}: {arrivals.Describe("arrivals")} | {camera.Describe("camera")}");
    }

    private void UnsubscribeCaptureEvents(MediaCapture capture)
    {
        capture.Failed -= OnCaptureFailed;
        capture.CameraStreamStateChanged -= OnCameraStreamStateChanged;
    }

    private void OnCaptureFailed(MediaCapture sender, MediaCaptureFailedEventArgs args)
        => RaiseFailure($"capture failed ({args.Code}): {args.Message}");

    private void OnCameraStreamStateChanged(MediaCapture sender, object args)
    {
        // CameraStreamState enum (Windows.Media.Devices): 0=NotStreaming, 1=Streaming,
        // 2=Failed, 3=Shutdown. The 19041 SDK projection omits member names, so
        // compare by value — the enum type itself resolves via the property return.
        if ((int)sender.CameraStreamState == 2) // Failed
            RaiseFailure("camera stream state is Failed");
    }

    private void RaiseFailure(string message)
    {
        if (_isFailed) return;
        _isFailed = true;
        AppLog.Write($"MediaCaptureFrameSource: {message}");
        try
        {
            SourceFailed?.Invoke(message);
        }
        catch (Exception ex)
        {
            AppLog.Write($"MediaCaptureFrameSource: SourceFailed handler threw: {ex.Message}");
        }
    }

    // Buffer recycling (same lesson as the screen capture): a fresh ~3.7MB byte[]
    // per device frame at 30-60fps is 110-220MB/s of LOH churn — gen2 pauses land
    // on the frame pump as 35-40ms render spikes. An 8-deep ring (take-14 rule:
    // depth × source period must exceed the worst consumer hold), rotated on the
    // single frame-reader thread, with per-frame Epoch so the identity-keyed paste
    // cache can never serve a recycled array as unchanged.
    private readonly byte[]?[] _frameRing = new byte[8][];
    private int _ringNext;
    private long _epoch;

    private byte[] RentRingBuffer(int size)
    {
        for (var tries = 0; tries < _frameRing.Length; tries++)
        {
            var idx = (_ringNext + tries) % _frameRing.Length;
            var buf = _frameRing[idx];
            if (buf != null && buf.Length == size)
            {
                _ringNext = (idx + 1) % _frameRing.Length;
                return buf;
            }
        }
        var slot = _ringNext;
        _ringNext = (slot + 1) % _frameRing.Length;
        var fresh = new byte[size];
        _frameRing[slot] = fresh;
        return fresh;
    }

    // Arrival vs production cadence (UNIT 0, 2026-10-05): the recorded file cannot
    // tell a dropped camera frame from a padded repeat — rawvideo stamps frames by
    // ARRIVAL at the declared 60fps, so a webcam delivering 25 of its 30 frames still
    // muxes a full-length file. Two counters, one reader, settle it: the camera's OWN
    // SystemRelativeTime stamps (device/USB/driver cadence) against the moment the
    // reader handed each frame to us (our cadence, including coalescing drops — the
    // reader deliberately keeps only the LATEST frame, so falling behind is silent).
    // Steady camera + ragged arrivals = we are behind. Ragged in both = the device.
    private readonly FrameCadence _arrivalCadence = new();
    private readonly FrameCadence _cameraCadence = new();
    private long _cadenceNextReportTicks;

    private void OnFrameArrived(MediaFrameReader sender, MediaFrameArrivedEventArgs args)
    {
        var arrivedTicks = System.Diagnostics.Stopwatch.GetTimestamp();
        using var frame = sender.TryAcquireLatestFrame();
        var videoFrame = frame?.VideoMediaFrame?.SoftwareBitmap;
        if (videoFrame == null) return;

        // SystemRelativeTime is QPC-domain, the same clock Stopwatch reads here, so the
        // camera's own frame spacing is measurable without a second conversion. It is
        // NULLABLE in the projection (null = the device supplied no stamp), so the
        // production side of the comparison is then simply missing from the line —
        // arrivals alone still measure what reached us.
        _arrivalCadence.Add(arrivedTicks);
        if (frame!.SystemRelativeTime is { } cameraStamp)
            _cameraCadence.Add((long)(cameraStamp.TotalSeconds * System.Diagnostics.Stopwatch.Frequency));
        if (arrivedTicks >= _cadenceNextReportTicks) ReportCadence(videoFrame.PixelWidth, videoFrame.PixelHeight);

        var bitmap = videoFrame.BitmapPixelFormat == BitmapPixelFormat.Bgra8
            ? videoFrame
            : SoftwareBitmap.Convert(videoFrame, BitmapPixelFormat.Bgra8);

        try
        {
            using var buffer = bitmap.LockBuffer(BitmapBufferAccessMode.Read);
            using var reference = buffer.CreateReference();
            if (WindowsRuntimeMarshal.TryGetDataUnsafe(reference, out var pixelsPtr, out var capacity))
            {
                var pixels = RentRingBuffer((int)capacity);
                Marshal.Copy(pixelsPtr, pixels, 0, (int)capacity);

                // MediaCapture video carries no alpha plane — the OS fills 255.
                var outFrame = new VideoFrame(bitmap.PixelWidth, bitmap.PixelHeight, pixels)
                    { IsOpaque = true, Epoch = ++_epoch };

                FrameAvailable?.Invoke(outFrame);
            }
        }
        catch (Exception ex)
        {
            AppLog.Write($"MediaCaptureFrameSource: frame copy failed: {ex.Message}");
        }
        finally
        {
            if (!ReferenceEquals(bitmap, videoFrame))
                bitmap.Dispose();
        }
    }
}