using System; using System.Collections.Generic; using System.Linq; using System.Runtime.InteropServices; using System.Runtime.InteropServices.WindowsRuntime; using System.Threading.Tasks; using Windows.Graphics.Imaging; using Windows.Media.Capture; using Windows.Media.Capture.Frames; using Windows.Media.Devices; using Windows.Media.MediaProperties; using ytLive.Helpers; namespace ytLive.Services; /// /// CPU-first MediaCapture source. Requests BGRA8 frames from the camera's video /// preview source; the capture pipeline does any format conversion, so every /// frame arrives as a normalized . Frames arrive on a /// worker thread — marshal before touching WPF. /// /// Allocation is validated, never taken on faith: after InitializeAsync /// the bound device, its frame sources, and its stream properties are checked, /// the frame reader's start status is read (not swallowed), and the live /// state signals (Failed, CameraStreamStateChanged) are subscribed /// so an async death surfaces as instead of a silent /// empty preview. /// public sealed class MediaCaptureFrameSource : ICameraFrameSource { private readonly string _deviceId; private MediaCapture? _capture; private MediaFrameReader? _frameReader; private string? _lastError; private bool _isFailed; public string DeviceId => _deviceId; public event Action? FrameAvailable; public event Action? SourceFailed; public MediaCaptureFrameSource(string deviceId) { _deviceId = deviceId; } public async Task StartAsync() { // Fallback ladder: prefer the camera's VideoPreview stream, but if the // reader refuses to start there (NoVideoFrameAvailable), retry against // its VideoRecord stream — some devices only deliver through it. // (MediaCaptureSharingMode.Exclusive is not projected by the 19041 SDK.) if (!await TryStartAsync(preferVideoRecord: false)) { if (!await TryStartAsync(preferVideoRecord: true)) throw new InvalidOperationException( _lastError ?? $"Camera '{_deviceId}' could not be started."); } } public async Task StopAsync() { var reader = _frameReader; _frameReader = null; if (reader != null) { reader.FrameArrived -= OnFrameArrived; try { await reader.StopAsync(); } catch (Exception ex) { AppLog.Write($"MediaCaptureFrameSource: stop frame reader failed: {ex.Message}"); } reader.Dispose(); } var capture = _capture; _capture = null; if (capture != null) { UnsubscribeCaptureEvents(capture); capture.Dispose(); } } /// /// Ordered reader-output-subtype candidates for a camera whose live media type /// is . Bgra8 is the fast path (OnFrameArrived /// skips conversion) and works for uncompressed cameras; an MJPG-active source /// cannot be read as Bgra8 by a MediaFrameReader (no BGRA converter on the MJPG /// pipeline — StartAsync answers OutputFormatNotSupported), so MJPG leads with /// NV12 and falls back to the reader's default (null). Every candidate that is /// not Bgra8 is converted in OnFrameArrived. /// internal static string?[] ReaderSubtypeCandidates(string? activeSubtype) => string.Equals(activeSubtype, "MJPG", StringComparison.OrdinalIgnoreCase) ? new string?[] { MediaEncodingSubtypes.Nv12, null } : new string?[] { MediaEncodingSubtypes.Bgra8, MediaEncodingSubtypes.Nv12, null }; private async Task TryStartAsync(bool preferVideoRecord) { MediaCapture? capture = null; MediaFrameReader? reader = null; try { capture = new MediaCapture(); var settings = new MediaCaptureInitializationSettings { VideoDeviceId = _deviceId, StreamingCaptureMode = StreamingCaptureMode.Video, MemoryPreference = MediaCaptureMemoryPreference.Cpu, SharingMode = MediaCaptureSharingMode.SharedReadOnly, }; await capture.InitializeAsync(settings); // MediaFrameReader defaults to the camera's FIRST media type — on UVC webcams // (Logitech C920 et al.) that is often YUY2 at a crippled rate (720p5!), and a // real-time 60fps container then shows a 5-20fps face cam as "every few frames // cut out". OBS's mechanism, available via VideoDeviceController (VideoProfile // is NOT projected in the 19041 SDK): negotiate the stream property. Pick the // best MJPEG >=640x360 @>=30fps (largest, fastest), set it BEFORE the frame // reader is created, fall back to device default on refusal, and log the live // media type so the next diagnosis starts from facts. try { var controller = capture.VideoDeviceController; var candidates = controller == null ? new System.Collections.Generic.List() : controller .GetAvailableMediaStreamProperties(MediaStreamType.VideoPreview) .OfType() .Where(v => string.Equals(v.Subtype, "MJPG", StringComparison.OrdinalIgnoreCase) && v.Width >= 640 && v.Height >= 360 && v.FrameRate.Denominator > 0 && v.FrameRate.Numerator / v.FrameRate.Denominator >= 30.0) .OrderByDescending(v => v.FrameRate.Numerator / (double)Math.Max(1, v.FrameRate.Denominator)) .ThenByDescending(v => (long)v.Width * v.Height) .ToList(); if (candidates.Count > 0) { var pick = candidates .OrderBy(v => (long)v.Width * v.Height) .LastOrDefault(v => v.Width <= 1280) ?? candidates[0]; await controller!.SetMediaStreamPropertiesAsync(MediaStreamType.VideoPreview, pick); } } catch (Exception fmtEx) { AppLog.Write($"MediaCaptureFrameSource: MJPG negotiation refused ({fmtEx.Message}); " + "keeping device-default media type"); } // Post-init resource validation: the capture must actually be bound to // the device we asked for and must expose a live video controller. if (!string.Equals(capture.MediaCaptureSettings?.VideoDeviceId, _deviceId, StringComparison.OrdinalIgnoreCase)) throw new InvalidOperationException( $"Camera '{_deviceId}' initialized but bound a different device."); var colorSource = capture.FrameSources.Values .OrderBy(s => preferVideoRecord ? (s.Info.MediaStreamType == MediaStreamType.VideoRecord ? 0 : 1) : (s.Info.MediaStreamType == MediaStreamType.VideoPreview ? 0 : 1)) .FirstOrDefault(s => s.Info.MediaStreamType is MediaStreamType.VideoPreview or MediaStreamType.VideoRecord); if (colorSource == null) { var kinds = string.Join(", ", capture.FrameSources.Values .Select(s => s.Info.MediaStreamType).Distinct()); throw new InvalidOperationException( $"Camera '{_deviceId}' exposes no video source (found: {kinds}). " + "It may be locked by another app (e.g. NVIDIA Broadcast)."); } // The device must answer for its preview stream — a dead, suspended, or // locked device returns no stream properties even after "successful" init. IReadOnlyList? previewProps; try { previewProps = capture.VideoDeviceController? .GetAvailableMediaStreamProperties(MediaStreamType.VideoPreview); } catch { previewProps = null; } if (previewProps == null || previewProps.Count == 0) throw new InvalidOperationException( $"Camera '{_deviceId}' answered no video stream properties (offline, suspended, or locked)."); // The OS's live state signals: async failures must surface, not vanish. capture.Failed += OnCaptureFailed; capture.CameraStreamStateChanged += OnCameraStreamStateChanged; // The reader must output a subtype the source can actually produce, else // StartAsync refuses (OutputFormatNotSupported). An uncompressed camera // converts to Bgra8 fine (the fast path); a camera sitting in MJPG — which // is what we find when ANOTHER app left it there and ours can't re-negotiate // under SharedReadOnly contention — cannot be read as Bgra8 by a // MediaFrameReader (the MJPG pipeline exposes NV12 output, not BGRA). Walk // the candidate ladder; anything that isn't Bgra8 is converted in // OnFrameArrived anyway. string? activeSubtype = null; try { var active = capture.VideoDeviceController? .GetMediaStreamProperties(colorSource.Info.MediaStreamType); if (active is Windows.Media.MediaProperties.VideoEncodingProperties vep) { activeSubtype = vep.Subtype; AppLog.Write($"Camera '{_deviceId}' media type: {vep.Subtype} " + $"{vep.Width}x{vep.Height} @ {vep.FrameRate.Numerator}/{vep.FrameRate.Denominator}fps"); } } catch { /* logging must never kill acquisition */ } // Each candidate is allocated AND started under its own catch: a WinRT // call answering an unsupported subtype can THROW (E_INVALIDARG / "value // does not fall within the expected range") instead of returning a start // status, so every throw must degrade to the next candidate — never let // one rejected format abort the whole acquisition. Individual rejections // are logged and collected; only the last-resort net below (the outer // try/catch of this method) keeps cowboying device-level errors. var refused = new List(); foreach (var outputSubtype in ReaderSubtypeCandidates(activeSubtype)) { var attempt = outputSubtype ?? ""; try { reader = outputSubtype is null ? await capture.CreateFrameReaderAsync(colorSource) : await capture.CreateFrameReaderAsync(colorSource, outputSubtype); if (reader == null) { refused.Add($"{attempt}: no frame reader"); continue; } reader.FrameArrived += OnFrameArrived; var status = await reader.StartAsync(); if (status == MediaFrameReaderStartStatus.Success) { _capture = capture; _frameReader = reader; _lastError = null; _isFailed = false; return true; } reader.FrameArrived -= OnFrameArrived; reader.Dispose(); reader = null; refused.Add($"{attempt}: {status}"); } catch (Exception ex) { AppLog.Write($"MediaCaptureFrameSource: reader as '{attempt}' refused: {ex.Message}"); if (reader != null) { reader.FrameArrived -= OnFrameArrived; reader.Dispose(); reader = null; } refused.Add($"{attempt}: {ex.Message}"); } } throw new InvalidOperationException( $"Camera '{_deviceId}' frame reader refused to start: {string.Join(", ", refused)}."); } catch (Exception ex) { _lastError = ex.Message; if (reader != null) { reader.FrameArrived -= OnFrameArrived; reader.Dispose(); } if (capture != null) { UnsubscribeCaptureEvents(capture); capture.Dispose(); } return false; } } private void UnsubscribeCaptureEvents(MediaCapture capture) { capture.Failed -= OnCaptureFailed; capture.CameraStreamStateChanged -= OnCameraStreamStateChanged; } private void OnCaptureFailed(MediaCapture sender, MediaCaptureFailedEventArgs args) => RaiseFailure($"capture failed ({args.Code}): {args.Message}"); private void OnCameraStreamStateChanged(MediaCapture sender, object args) { // CameraStreamState enum (Windows.Media.Devices): 0=NotStreaming, 1=Streaming, // 2=Failed, 3=Shutdown. The 19041 SDK projection omits member names, so // compare by value — the enum type itself resolves via the property return. if ((int)sender.CameraStreamState == 2) // Failed RaiseFailure("camera stream state is Failed"); } private void RaiseFailure(string message) { if (_isFailed) return; _isFailed = true; AppLog.Write($"MediaCaptureFrameSource: {message}"); try { SourceFailed?.Invoke(message); } catch (Exception ex) { AppLog.Write($"MediaCaptureFrameSource: SourceFailed handler threw: {ex.Message}"); } } // Buffer recycling (same lesson as the screen capture): a fresh ~3.7MB byte[] // per device frame at 30-60fps is 110-220MB/s of LOH churn — gen2 pauses land // on the frame pump as 35-40ms render spikes. An 8-deep ring (take-14 rule: // depth × source period must exceed the worst consumer hold), rotated on the // single frame-reader thread, with per-frame Epoch so the identity-keyed paste // cache can never serve a recycled array as unchanged. private readonly byte[]?[] _frameRing = new byte[8][]; private int _ringNext; private long _epoch; private byte[] RentRingBuffer(int size) { for (var tries = 0; tries < _frameRing.Length; tries++) { var idx = (_ringNext + tries) % _frameRing.Length; var buf = _frameRing[idx]; if (buf != null && buf.Length == size) { _ringNext = (idx + 1) % _frameRing.Length; return buf; } } var slot = _ringNext; _ringNext = (slot + 1) % _frameRing.Length; var fresh = new byte[size]; _frameRing[slot] = fresh; return fresh; } private void OnFrameArrived(MediaFrameReader sender, MediaFrameArrivedEventArgs args) { using var frame = sender.TryAcquireLatestFrame(); var videoFrame = frame?.VideoMediaFrame?.SoftwareBitmap; if (videoFrame == null) return; var bitmap = videoFrame.BitmapPixelFormat == BitmapPixelFormat.Bgra8 ? videoFrame : SoftwareBitmap.Convert(videoFrame, BitmapPixelFormat.Bgra8); try { using var buffer = bitmap.LockBuffer(BitmapBufferAccessMode.Read); using var reference = buffer.CreateReference(); if (WindowsRuntimeMarshal.TryGetDataUnsafe(reference, out var pixelsPtr, out var capacity)) { var pixels = RentRingBuffer((int)capacity); Marshal.Copy(pixelsPtr, pixels, 0, (int)capacity); // MediaCapture video carries no alpha plane — the OS fills 255. var outFrame = new VideoFrame(bitmap.PixelWidth, bitmap.PixelHeight, pixels) { IsOpaque = true, Epoch = ++_epoch }; FrameAvailable?.Invoke(outFrame); } } catch (Exception ex) { AppLog.Write($"MediaCaptureFrameSource: frame copy failed: {ex.Message}"); } finally { if (!ReferenceEquals(bitmap, videoFrame)) bitmap.Dispose(); } } }