using System; using System.Collections.Generic; using System.Linq; using System.Runtime.InteropServices; using System.Runtime.InteropServices.WindowsRuntime; using System.Threading.Tasks; using Windows.Graphics.Imaging; using Windows.Media.Capture; using Windows.Media.Capture.Frames; using Windows.Media.Devices; using Windows.Media.MediaProperties; using ytLive.Helpers; namespace ytLive.Services; /// /// CPU-first MediaCapture source. Requests BGRA8 frames from the camera's video /// preview source; the capture pipeline does any format conversion, so every /// frame arrives as a normalized . Frames arrive on a /// worker thread — marshal before touching WPF. /// /// Allocation is validated, never taken on faith: after InitializeAsync /// the bound device, its frame sources, and its stream properties are checked, /// the frame reader's start status is read (not swallowed), and the live /// state signals (Failed, CameraStreamStateChanged) are subscribed /// so an async death surfaces as instead of a silent /// empty preview. /// public sealed class MediaCaptureFrameSource : ICameraFrameSource { private readonly string _deviceId; private MediaCapture? _capture; private MediaFrameReader? _frameReader; private string? _lastError; private bool _isFailed; public string DeviceId => _deviceId; public event Action? FrameAvailable; public event Action? SourceFailed; public MediaCaptureFrameSource(string deviceId) { _deviceId = deviceId; } public async Task StartAsync() { // Fallback ladder: prefer the camera's VideoPreview stream, but if the // reader refuses to start there (NoVideoFrameAvailable), retry against // its VideoRecord stream — some devices only deliver through it. // (MediaCaptureSharingMode.Exclusive is not projected by the 19041 SDK.) if (!await TryStartAsync(preferVideoRecord: false)) { if (!await TryStartAsync(preferVideoRecord: true)) throw new InvalidOperationException( _lastError ?? $"Camera '{_deviceId}' could not be started."); } } public async Task StopAsync() { var reader = _frameReader; _frameReader = null; if (reader != null) { reader.FrameArrived -= OnFrameArrived; try { await reader.StopAsync(); } catch (Exception ex) { AppLog.Write($"MediaCaptureFrameSource: stop frame reader failed: {ex.Message}"); } reader.Dispose(); } var capture = _capture; _capture = null; if (capture != null) { UnsubscribeCaptureEvents(capture); capture.Dispose(); } } private async Task TryStartAsync(bool preferVideoRecord) { MediaCapture? capture = null; MediaFrameReader? reader = null; try { capture = new MediaCapture(); var settings = new MediaCaptureInitializationSettings { VideoDeviceId = _deviceId, StreamingCaptureMode = StreamingCaptureMode.Video, MemoryPreference = MediaCaptureMemoryPreference.Cpu, SharingMode = MediaCaptureSharingMode.SharedReadOnly, }; await capture.InitializeAsync(settings); // MediaFrameReader defaults to the camera's FIRST media type — on UVC webcams // (Logitech C920 et al.) that is often YUY2 at a crippled rate (720p5!), and a // real-time 60fps container then shows a 5-20fps face cam as "every few frames // cut out". OBS's mechanism, available via VideoDeviceController (VideoProfile // is NOT projected in the 19041 SDK): negotiate the stream property. Pick the // best MJPEG >=640x360 @>=30fps (largest, fastest), set it BEFORE the frame // reader is created, fall back to device default on refusal, and log the live // media type so the next diagnosis starts from facts. try { var controller = capture.VideoDeviceController; var candidates = controller == null ? new System.Collections.Generic.List() : controller .GetAvailableMediaStreamProperties(MediaStreamType.VideoPreview) .OfType() .Where(v => string.Equals(v.Subtype, "MJPG", StringComparison.OrdinalIgnoreCase) && v.Width >= 640 && v.Height >= 360 && v.FrameRate.Denominator > 0 && v.FrameRate.Numerator / v.FrameRate.Denominator >= 30.0) .OrderByDescending(v => v.FrameRate.Numerator / (double)Math.Max(1, v.FrameRate.Denominator)) .ThenByDescending(v => (long)v.Width * v.Height) .ToList(); if (candidates.Count > 0) { var pick = candidates .OrderBy(v => (long)v.Width * v.Height) .LastOrDefault(v => v.Width <= 1280) ?? candidates[0]; await controller!.SetMediaStreamPropertiesAsync(MediaStreamType.VideoPreview, pick); } } catch (Exception fmtEx) { AppLog.Write($"MediaCaptureFrameSource: MJPG negotiation refused ({fmtEx.Message}); " + "keeping device-default media type"); } // Post-init resource validation: the capture must actually be bound to // the device we asked for and must expose a live video controller. if (!string.Equals(capture.MediaCaptureSettings?.VideoDeviceId, _deviceId, StringComparison.OrdinalIgnoreCase)) throw new InvalidOperationException( $"Camera '{_deviceId}' initialized but bound a different device."); var colorSource = capture.FrameSources.Values .OrderBy(s => preferVideoRecord ? (s.Info.MediaStreamType == MediaStreamType.VideoRecord ? 0 : 1) : (s.Info.MediaStreamType == MediaStreamType.VideoPreview ? 0 : 1)) .FirstOrDefault(s => s.Info.MediaStreamType is MediaStreamType.VideoPreview or MediaStreamType.VideoRecord); if (colorSource == null) { var kinds = string.Join(", ", capture.FrameSources.Values .Select(s => s.Info.MediaStreamType).Distinct()); throw new InvalidOperationException( $"Camera '{_deviceId}' exposes no video source (found: {kinds}). " + "It may be locked by another app (e.g. NVIDIA Broadcast)."); } // The device must answer for its preview stream — a dead, suspended, or // locked device returns no stream properties even after "successful" init. IReadOnlyList? previewProps; try { previewProps = capture.VideoDeviceController? .GetAvailableMediaStreamProperties(MediaStreamType.VideoPreview); } catch { previewProps = null; } if (previewProps == null || previewProps.Count == 0) throw new InvalidOperationException( $"Camera '{_deviceId}' answered no video stream properties (offline, suspended, or locked)."); // The OS's live state signals: async failures must surface, not vanish. capture.Failed += OnCaptureFailed; capture.CameraStreamStateChanged += OnCameraStreamStateChanged; reader = await capture.CreateFrameReaderAsync(colorSource, MediaEncodingSubtypes.Bgra8); if (reader == null) throw new InvalidOperationException($"Camera '{_deviceId}' created no frame reader."); // One-line truth of what the camera actually agreed to (format/size/fps). try { var active = capture.VideoDeviceController? .GetMediaStreamProperties(colorSource.Info.MediaStreamType); if (active is Windows.Media.MediaProperties.VideoEncodingProperties vep) AppLog.Write($"Camera '{_deviceId}' media type: {vep.Subtype} " + $"{vep.Width}x{vep.Height} @ {vep.FrameRate.Numerator}/{vep.FrameRate.Denominator}fps"); } catch { /* logging must never kill acquisition */ } reader.FrameArrived += OnFrameArrived; var status = await reader.StartAsync(); if (status != MediaFrameReaderStartStatus.Success) throw new InvalidOperationException( $"Camera '{_deviceId}' frame reader refused to start: {status}."); _capture = capture; _frameReader = reader; _lastError = null; _isFailed = false; return true; } catch (Exception ex) { _lastError = ex.Message; if (reader != null) { reader.FrameArrived -= OnFrameArrived; reader.Dispose(); } if (capture != null) { UnsubscribeCaptureEvents(capture); capture.Dispose(); } return false; } } private void UnsubscribeCaptureEvents(MediaCapture capture) { capture.Failed -= OnCaptureFailed; capture.CameraStreamStateChanged -= OnCameraStreamStateChanged; } private void OnCaptureFailed(MediaCapture sender, MediaCaptureFailedEventArgs args) => RaiseFailure($"capture failed ({args.Code}): {args.Message}"); private void OnCameraStreamStateChanged(MediaCapture sender, object args) { // CameraStreamState enum (Windows.Media.Devices): 0=NotStreaming, 1=Streaming, // 2=Failed, 3=Shutdown. The 19041 SDK projection omits member names, so // compare by value — the enum type itself resolves via the property return. if ((int)sender.CameraStreamState == 2) // Failed RaiseFailure("camera stream state is Failed"); } private void RaiseFailure(string message) { if (_isFailed) return; _isFailed = true; AppLog.Write($"MediaCaptureFrameSource: {message}"); try { SourceFailed?.Invoke(message); } catch (Exception ex) { AppLog.Write($"MediaCaptureFrameSource: SourceFailed handler threw: {ex.Message}"); } } // Buffer recycling (same lesson as the screen capture): a fresh ~3.7MB byte[] // per device frame at 30-60fps is 110-220MB/s of LOH churn — gen2 pauses land // on the frame pump as 35-40ms render spikes. An 8-deep ring (take-14 rule: // depth × source period must exceed the worst consumer hold), rotated on the // single frame-reader thread, with per-frame Epoch so the identity-keyed paste // cache can never serve a recycled array as unchanged. private readonly byte[]?[] _frameRing = new byte[8][]; private int _ringNext; private long _epoch; private byte[] RentRingBuffer(int size) { for (var tries = 0; tries < _frameRing.Length; tries++) { var idx = (_ringNext + tries) % _frameRing.Length; var buf = _frameRing[idx]; if (buf != null && buf.Length == size) { _ringNext = (idx + 1) % _frameRing.Length; return buf; } } var slot = _ringNext; _ringNext = (slot + 1) % _frameRing.Length; var fresh = new byte[size]; _frameRing[slot] = fresh; return fresh; } private void OnFrameArrived(MediaFrameReader sender, MediaFrameArrivedEventArgs args) { using var frame = sender.TryAcquireLatestFrame(); var videoFrame = frame?.VideoMediaFrame?.SoftwareBitmap; if (videoFrame == null) return; var bitmap = videoFrame.BitmapPixelFormat == BitmapPixelFormat.Bgra8 ? videoFrame : SoftwareBitmap.Convert(videoFrame, BitmapPixelFormat.Bgra8); try { using var buffer = bitmap.LockBuffer(BitmapBufferAccessMode.Read); using var reference = buffer.CreateReference(); if (WindowsRuntimeMarshal.TryGetDataUnsafe(reference, out var pixelsPtr, out var capacity)) { var pixels = RentRingBuffer((int)capacity); Marshal.Copy(pixelsPtr, pixels, 0, (int)capacity); // MediaCapture video carries no alpha plane — the OS fills 255. FrameAvailable?.Invoke(new VideoFrame(bitmap.PixelWidth, bitmap.PixelHeight, pixels) { IsOpaque = true, Epoch = ++_epoch }); } } catch (Exception ex) { AppLog.Write($"MediaCaptureFrameSource: frame copy failed: {ex.Message}"); } finally { if (!ReferenceEquals(bitmap, videoFrame)) bitmap.Dispose(); } } }