using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Runtime.InteropServices.WindowsRuntime;
using System.Threading.Tasks;
using Windows.Graphics.Imaging;
using Windows.Media.Capture;
using Windows.Media.Capture.Frames;
using Windows.Media.Devices;
using Windows.Media.MediaProperties;
using ytLive.Helpers;
namespace ytLive.Services;
///
/// CPU-first MediaCapture source. Requests BGRA8 frames from the camera's video
/// preview source; the capture pipeline does any format conversion, so every
/// frame arrives as a normalized . Frames arrive on a
/// worker thread — marshal before touching WPF.
///
/// Allocation is validated, never taken on faith: after InitializeAsync
/// the bound device, its frame sources, and its stream properties are checked,
/// the frame reader's start status is read (not swallowed), and the live
/// state signals (Failed, CameraStreamStateChanged) are subscribed
/// so an async death surfaces as instead of a silent
/// empty preview.
///
public sealed class MediaCaptureFrameSource : ICameraFrameSource
{
private readonly string _deviceId;
private MediaCapture? _capture;
private MediaFrameReader? _frameReader;
private string? _lastError;
private bool _isFailed;
public string DeviceId => _deviceId;
public event Action? FrameAvailable;
public event Action? SourceFailed;
public MediaCaptureFrameSource(string deviceId)
{
_deviceId = deviceId;
}
public async Task StartAsync()
{
// Fallback ladder: prefer the camera's VideoPreview stream, but if the
// reader refuses to start there (NoVideoFrameAvailable), retry against
// its VideoRecord stream — some devices only deliver through it.
// (MediaCaptureSharingMode.Exclusive is not projected by the 19041 SDK.)
if (!await TryStartAsync(preferVideoRecord: false))
{
if (!await TryStartAsync(preferVideoRecord: true))
throw new InvalidOperationException(
_lastError ?? $"Camera '{_deviceId}' could not be started.");
}
}
public async Task StopAsync()
{
var reader = _frameReader;
_frameReader = null;
if (reader != null)
{
reader.FrameArrived -= OnFrameArrived;
try
{
await reader.StopAsync();
}
catch (Exception ex)
{
AppLog.Write($"MediaCaptureFrameSource: stop frame reader failed: {ex.Message}");
}
reader.Dispose();
}
var capture = _capture;
_capture = null;
if (capture != null)
{
UnsubscribeCaptureEvents(capture);
capture.Dispose();
}
}
///
/// Ordered reader-output-subtype candidates for a camera whose live media type
/// is . Bgra8 is the fast path (OnFrameArrived
/// skips conversion) and works for uncompressed cameras; an MJPG-active source
/// cannot be read as Bgra8 by a MediaFrameReader (no BGRA converter on the MJPG
/// pipeline — StartAsync answers OutputFormatNotSupported), so MJPG leads with
/// NV12 and falls back to the reader's default (null). Every candidate that is
/// not Bgra8 is converted in OnFrameArrived.
///
internal static string?[] ReaderSubtypeCandidates(string? activeSubtype) =>
string.Equals(activeSubtype, "MJPG", StringComparison.OrdinalIgnoreCase)
? new string?[] { MediaEncodingSubtypes.Nv12, null }
: new string?[] { MediaEncodingSubtypes.Bgra8, MediaEncodingSubtypes.Nv12, null };
private async Task TryStartAsync(bool preferVideoRecord)
{
MediaCapture? capture = null;
MediaFrameReader? reader = null;
try
{
capture = new MediaCapture();
var settings = new MediaCaptureInitializationSettings
{
VideoDeviceId = _deviceId,
StreamingCaptureMode = StreamingCaptureMode.Video,
MemoryPreference = MediaCaptureMemoryPreference.Cpu,
SharingMode = MediaCaptureSharingMode.SharedReadOnly,
};
await capture.InitializeAsync(settings);
// MediaFrameReader defaults to the camera's FIRST media type — on UVC webcams
// (Logitech C920 et al.) that is often YUY2 at a crippled rate (720p5!), and a
// real-time 60fps container then shows a 5-20fps face cam as "every few frames
// cut out". OBS's mechanism, available via VideoDeviceController (VideoProfile
// is NOT projected in the 19041 SDK): negotiate the stream property. Pick the
// best MJPEG >=640x360 @>=30fps (largest, fastest), set it BEFORE the frame
// reader is created, fall back to device default on refusal, and log the live
// media type so the next diagnosis starts from facts.
try
{
var controller = capture.VideoDeviceController;
var candidates = controller == null
? new System.Collections.Generic.List()
: controller
.GetAvailableMediaStreamProperties(MediaStreamType.VideoPreview)
.OfType()
.Where(v => string.Equals(v.Subtype, "MJPG", StringComparison.OrdinalIgnoreCase)
&& v.Width >= 640 && v.Height >= 360
&& v.FrameRate.Denominator > 0
&& v.FrameRate.Numerator / v.FrameRate.Denominator >= 30.0)
.OrderByDescending(v => v.FrameRate.Numerator / (double)Math.Max(1, v.FrameRate.Denominator))
.ThenByDescending(v => (long)v.Width * v.Height)
.ToList();
if (candidates.Count > 0)
{
var pick = candidates
.OrderBy(v => (long)v.Width * v.Height)
.LastOrDefault(v => v.Width <= 1280) ?? candidates[0];
await controller!.SetMediaStreamPropertiesAsync(MediaStreamType.VideoPreview, pick);
}
}
catch (Exception fmtEx)
{
AppLog.Write($"MediaCaptureFrameSource: MJPG negotiation refused ({fmtEx.Message}); " +
"keeping device-default media type");
}
// Post-init resource validation: the capture must actually be bound to
// the device we asked for and must expose a live video controller.
if (!string.Equals(capture.MediaCaptureSettings?.VideoDeviceId, _deviceId,
StringComparison.OrdinalIgnoreCase))
throw new InvalidOperationException(
$"Camera '{_deviceId}' initialized but bound a different device.");
var colorSource = capture.FrameSources.Values
.OrderBy(s => preferVideoRecord
? (s.Info.MediaStreamType == MediaStreamType.VideoRecord ? 0 : 1)
: (s.Info.MediaStreamType == MediaStreamType.VideoPreview ? 0 : 1))
.FirstOrDefault(s => s.Info.MediaStreamType is MediaStreamType.VideoPreview or MediaStreamType.VideoRecord);
if (colorSource == null)
{
var kinds = string.Join(", ", capture.FrameSources.Values
.Select(s => s.Info.MediaStreamType).Distinct());
throw new InvalidOperationException(
$"Camera '{_deviceId}' exposes no video source (found: {kinds}). " +
"It may be locked by another app (e.g. NVIDIA Broadcast).");
}
// The device must answer for its preview stream — a dead, suspended, or
// locked device returns no stream properties even after "successful" init.
IReadOnlyList? previewProps;
try
{
previewProps = capture.VideoDeviceController?
.GetAvailableMediaStreamProperties(MediaStreamType.VideoPreview);
}
catch
{
previewProps = null;
}
if (previewProps == null || previewProps.Count == 0)
throw new InvalidOperationException(
$"Camera '{_deviceId}' answered no video stream properties (offline, suspended, or locked).");
// The OS's live state signals: async failures must surface, not vanish.
capture.Failed += OnCaptureFailed;
capture.CameraStreamStateChanged += OnCameraStreamStateChanged;
// The reader must output a subtype the source can actually produce, else
// StartAsync refuses (OutputFormatNotSupported). An uncompressed camera
// converts to Bgra8 fine (the fast path); a camera sitting in MJPG — which
// is what we find when ANOTHER app left it there and ours can't re-negotiate
// under SharedReadOnly contention — cannot be read as Bgra8 by a
// MediaFrameReader (the MJPG pipeline exposes NV12 output, not BGRA). Walk
// the candidate ladder; anything that isn't Bgra8 is converted in
// OnFrameArrived anyway.
string? activeSubtype = null;
try
{
var active = capture.VideoDeviceController?
.GetMediaStreamProperties(colorSource.Info.MediaStreamType);
if (active is Windows.Media.MediaProperties.VideoEncodingProperties vep)
{
activeSubtype = vep.Subtype;
AppLog.Write($"Camera '{_deviceId}' media type: {vep.Subtype} " +
$"{vep.Width}x{vep.Height} @ {vep.FrameRate.Numerator}/{vep.FrameRate.Denominator}fps");
}
}
catch { /* logging must never kill acquisition */ }
// Each candidate is allocated AND started under its own catch: a WinRT
// call answering an unsupported subtype can THROW (E_INVALIDARG / "value
// does not fall within the expected range") instead of returning a start
// status, so every throw must degrade to the next candidate — never let
// one rejected format abort the whole acquisition. Individual rejections
// are logged and collected; only the last-resort net below (the outer
// try/catch of this method) keeps cowboying device-level errors.
var refused = new List();
foreach (var outputSubtype in ReaderSubtypeCandidates(activeSubtype))
{
var attempt = outputSubtype ?? "";
try
{
reader = outputSubtype is null
? await capture.CreateFrameReaderAsync(colorSource)
: await capture.CreateFrameReaderAsync(colorSource, outputSubtype);
if (reader == null)
{
refused.Add($"{attempt}: no frame reader");
continue;
}
reader.FrameArrived += OnFrameArrived;
var status = await reader.StartAsync();
if (status == MediaFrameReaderStartStatus.Success)
{
_capture = capture;
_frameReader = reader;
_lastError = null;
_isFailed = false;
return true;
}
reader.FrameArrived -= OnFrameArrived;
reader.Dispose();
reader = null;
refused.Add($"{attempt}: {status}");
}
catch (Exception ex)
{
AppLog.Write($"MediaCaptureFrameSource: reader as '{attempt}' refused: {ex.Message}");
if (reader != null)
{
reader.FrameArrived -= OnFrameArrived;
reader.Dispose();
reader = null;
}
refused.Add($"{attempt}: {ex.Message}");
}
}
throw new InvalidOperationException(
$"Camera '{_deviceId}' frame reader refused to start: {string.Join(", ", refused)}.");
}
catch (Exception ex)
{
_lastError = ex.Message;
if (reader != null)
{
reader.FrameArrived -= OnFrameArrived;
reader.Dispose();
}
if (capture != null)
{
UnsubscribeCaptureEvents(capture);
capture.Dispose();
}
return false;
}
}
private void UnsubscribeCaptureEvents(MediaCapture capture)
{
capture.Failed -= OnCaptureFailed;
capture.CameraStreamStateChanged -= OnCameraStreamStateChanged;
}
private void OnCaptureFailed(MediaCapture sender, MediaCaptureFailedEventArgs args)
=> RaiseFailure($"capture failed ({args.Code}): {args.Message}");
private void OnCameraStreamStateChanged(MediaCapture sender, object args)
{
// CameraStreamState enum (Windows.Media.Devices): 0=NotStreaming, 1=Streaming,
// 2=Failed, 3=Shutdown. The 19041 SDK projection omits member names, so
// compare by value — the enum type itself resolves via the property return.
if ((int)sender.CameraStreamState == 2) // Failed
RaiseFailure("camera stream state is Failed");
}
private void RaiseFailure(string message)
{
if (_isFailed) return;
_isFailed = true;
AppLog.Write($"MediaCaptureFrameSource: {message}");
try
{
SourceFailed?.Invoke(message);
}
catch (Exception ex)
{
AppLog.Write($"MediaCaptureFrameSource: SourceFailed handler threw: {ex.Message}");
}
}
// Buffer recycling (same lesson as the screen capture): a fresh ~3.7MB byte[]
// per device frame at 30-60fps is 110-220MB/s of LOH churn — gen2 pauses land
// on the frame pump as 35-40ms render spikes. An 8-deep ring (take-14 rule:
// depth × source period must exceed the worst consumer hold), rotated on the
// single frame-reader thread, with per-frame Epoch so the identity-keyed paste
// cache can never serve a recycled array as unchanged.
private readonly byte[]?[] _frameRing = new byte[8][];
private int _ringNext;
private long _epoch;
private byte[] RentRingBuffer(int size)
{
for (var tries = 0; tries < _frameRing.Length; tries++)
{
var idx = (_ringNext + tries) % _frameRing.Length;
var buf = _frameRing[idx];
if (buf != null && buf.Length == size)
{
_ringNext = (idx + 1) % _frameRing.Length;
return buf;
}
}
var slot = _ringNext;
_ringNext = (slot + 1) % _frameRing.Length;
var fresh = new byte[size];
_frameRing[slot] = fresh;
return fresh;
}
private void OnFrameArrived(MediaFrameReader sender, MediaFrameArrivedEventArgs args)
{
using var frame = sender.TryAcquireLatestFrame();
var videoFrame = frame?.VideoMediaFrame?.SoftwareBitmap;
if (videoFrame == null) return;
var bitmap = videoFrame.BitmapPixelFormat == BitmapPixelFormat.Bgra8
? videoFrame
: SoftwareBitmap.Convert(videoFrame, BitmapPixelFormat.Bgra8);
try
{
using var buffer = bitmap.LockBuffer(BitmapBufferAccessMode.Read);
using var reference = buffer.CreateReference();
if (WindowsRuntimeMarshal.TryGetDataUnsafe(reference, out var pixelsPtr, out var capacity))
{
var pixels = RentRingBuffer((int)capacity);
Marshal.Copy(pixelsPtr, pixels, 0, (int)capacity);
// MediaCapture video carries no alpha plane — the OS fills 255.
var outFrame = new VideoFrame(bitmap.PixelWidth, bitmap.PixelHeight, pixels)
{ IsOpaque = true, Epoch = ++_epoch };
FrameAvailable?.Invoke(outFrame);
}
}
catch (Exception ex)
{
AppLog.Write($"MediaCaptureFrameSource: frame copy failed: {ex.Message}");
}
finally
{
if (!ReferenceEquals(bitmap, videoFrame))
bitmap.Dispose();
}
}
}