Files
LlamaCasty/Services/MediaVideoSourceManager.cs
T
gramps 2670ac022b feat(media): play the clip's soundtrack and share one decode pipe
A video file was picture-only: the decode pipe passed `-an`, so a starting-soon
or BRB clip with music played silent everywhere. Adds sound to every clip and
collapses the two decoders into one.

- `Services/VideoClipPipe.cs`: the shared pipe behind `MediaVideoSource` (TASK 21)
  and `AlertClipDecoder` (TASK 47) — video (`rawvideo bgra`, scaled) + audio
  (`f32le -ac 2 -ar 48000`, 50 ms chunks, byte-carry across reads), both paced to
  real time, loop restarts both so picture and sound stay together. Alert
  behaviour is unchanged.
- `MediaVideoSourceManager` applies the layer's Level/Mute per session and raises
  `AudioAvailable`; `MainViewModel` feeds it to the new
  `AudioMixer.EnqueueDesktopMediaAudio`, which joins the DESKTOP/GAME ring so the
  desktop fader and the mic auto-ducker apply exactly as they do for game audio
  and TRAX (creator ruling 2026-11-05: audio joins the group).
- Per-layer `MediaMuted` + existing `MediaVolume` surfaced as Mute/Level in the
  properties panel; `MediaMuted` persisted (ALTER TABLE guard + SELECT/INSERT).
- Level/mute are per SESSION because one decoder is shared per file path — the
  last layer touched wins; one decoder per layer would play the file twice.

Test: `MediaSourceAudioTests` — the pipe emits 50 ms chunks at 48k stereo, the
manager scales by level and drops when muted, and a media layer's sound moves the
desktop meter. Alert/decode suites unchanged and green.

Docs: task-21 closed on item 10, the "silent soundtrack" facts corrected in
frame-pipeline/output/overlays, HANDOFF rewritten. verify.sh green (529 passed;
the two documented pre-existing failures only).
2026-11-05 07:54:06 -08:00

346 lines
12 KiB
C#

using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Media;
using System.Windows.Media.Imaging;
using System.Windows.Threading;
using ytLive.Helpers;
using ytLive.Services.Audio;
namespace ytLive.Services;
/// <summary>
/// TASK 21: owns media-file decode sessions app-wide, refcounted by file path.
/// One file = at most one decoder, one shared WriteableBitmap; the last release
/// stops and disposes the source. Frames arrive on a worker thread and are
/// coalesced onto the UI dispatcher (at most one pending copy per session, using
/// the latest frame). Mirrors ScreenCaptureManager/CameraManager.
/// </summary>
public sealed class MediaVideoSourceManager : IDisposable
{
private sealed class MediaSession
{
public string Key { get; }
public IMediaFrameSource Source { get; }
public Action<VideoFrame>? FrameHandler;
public Action<AudioSample>? AudioHandler;
public int RefCount;
public bool Started;
public bool AudioMuted;
public double AudioVolume = 1.0;
public WriteableBitmap? PreviewBitmap;
public VideoFrame? LatestFrame;
public bool FramePending;
public MediaSession(string key, IMediaFrameSource source)
{
Key = key;
Source = source;
RefCount = 1;
}
}
private readonly Func<string, IMediaFrameSource?> _sourceFactory;
private readonly Dispatcher? _uiDispatcher;
private readonly Func<string, bool>? _loopProvider;
private readonly Dictionary<string, MediaSession> _sessions = new();
private readonly object _gate = new();
/// <summary>Raised on the UI thread when a media file's shared preview bitmap is first created.</summary>
public event Action<string, WriteableBitmap>? PreviewBitmapChanged;
/// <summary>Raised when a media decoder cannot be created (bad path).</summary>
public event Action<string, string>? MediaFailed;
/// <summary>Raised (worker thread) per audio chunk of a file's own soundtrack, already
/// scaled by the session's level and dropped when muted.</summary>
public event Action<string, AudioSample>? AudioAvailable;
public MediaVideoSourceManager(
Func<string, IMediaFrameSource?> sourceFactory,
Dispatcher? uiDispatcher = null,
Func<string, bool>? loopProvider = null)
{
_sourceFactory = sourceFactory;
_uiDispatcher = uiDispatcher;
_loopProvider = loopProvider;
}
/// <summary>Increments the refcount for a file, starting the decoder the first time.</summary>
public async Task<bool> AcquireAsync(string key)
{
if (string.IsNullOrWhiteSpace(key)) return false;
MediaSession session;
bool shouldStart;
lock (_gate)
{
if (_sessions.TryGetValue(key, out var existing))
{
existing.RefCount++;
shouldStart = false;
session = existing;
}
else
{
IMediaFrameSource? source;
try
{
source = _sourceFactory(key);
}
catch (Exception ex)
{
AppLog.Write($"MediaSource: creating decoder '{key}' failed: {ex.Message}");
MediaFailed?.Invoke(key, ex.Message);
return false;
}
if (source == null)
{
MediaFailed?.Invoke(key, "No decoder for this file");
return false;
}
session = new MediaSession(key, source);
session.FrameHandler = frame => OnFrameAvailable(session, frame);
session.Source.FrameAvailable += session.FrameHandler;
session.AudioHandler = sample => OnAudioAvailable(session, sample);
session.Source.AudioAvailable += session.AudioHandler;
// Loop intent lives on the Source (per scene), so the caller supplies the
// per-path answer ("any live reference loops"). Applied before Start so a
// zero-length or instantly-EOF file cannot slip past a later assignment.
session.Source.Looping = _loopProvider?.Invoke(key) ?? false;
_sessions[key] = session;
shouldStart = true;
}
}
if (!shouldStart) return session.Started;
try
{
await session.Source.StartAsync();
session.Started = true;
return true;
}
catch (Exception ex)
{
lock (_gate)
_sessions.Remove(key);
session.Source.FrameAvailable -= session.FrameHandler;
if (session.AudioHandler != null) session.Source.AudioAvailable -= session.AudioHandler;
AppLog.Write($"MediaSource: failed to start decoder '{key}': {ex.Message}");
MediaFailed?.Invoke(key, ex.Message);
await SafeStopAsync(session.Source);
return false;
}
}
public async Task ReleaseAsync(string key)
{
MediaSession? toStop = null;
lock (_gate)
{
if (!_sessions.TryGetValue(key, out var session)) return;
if (--session.RefCount > 0) return;
_sessions.Remove(key);
toStop = session;
}
if (toStop == null) return;
toStop.Source.FrameAvailable -= toStop.FrameHandler;
if (toStop.AudioHandler != null) toStop.Source.AudioAvailable -= toStop.AudioHandler;
await SafeStopAsync(toStop.Source);
toStop.PreviewBitmap = null;
}
/// <summary>Releases a file unconditionally (every ref) — used on layout reloads.</summary>
public async Task ReleaseAllAsync(string key)
{
MediaSession? toStop = null;
lock (_gate)
{
if (!_sessions.TryGetValue(key, out var session)) return;
session.RefCount = 0;
_sessions.Remove(key);
toStop = session;
}
if (toStop == null) return;
toStop.Source.FrameAvailable -= toStop.FrameHandler;
if (toStop.AudioHandler != null) toStop.Source.AudioAvailable -= toStop.AudioHandler;
await SafeStopAsync(toStop.Source);
toStop.PreviewBitmap = null;
}
/// <summary>The most recent decoded frame for a file, or null before the first
/// frame (or if the file has no session). The live compositor reads media from here.</summary>
public VideoFrame? GetLatestFrame(string key)
{
lock (_gate)
return _sessions.TryGetValue(key, out var session) ? session.LatestFrame : null;
}
/// <summary>True while a decode session exists for the file — the caller uses this to
/// keep one acquire per path instead of stacking refcounts on every layout pass.</summary>
public bool HasSession(string key)
{
lock (_gate)
return _sessions.ContainsKey(key);
}
/// <summary>The files with a live session, for reconciling a layout load.</summary>
public IReadOnlyList<string> GetActiveKeys()
{
lock (_gate)
return new List<string>(_sessions.Keys);
}
/// <summary>Applies a loop change to a running session (the creator flipped the
/// toggle). No-op when the file has no session.</summary>
public void SetLooping(string key, bool looping)
{
lock (_gate)
{
if (_sessions.TryGetValue(key, out var session))
session.Source.Looping = looping;
}
}
/// <summary>The layer's sound controls for this file. One decoder is shared per path,
/// so the level/mute are per SESSION: whichever layer the creator last touched wins —
/// the alternative (one decoder per layer) would play the same file twice.</summary>
public void SetAudioSettings(string key, double volume, bool muted)
{
lock (_gate)
{
if (!_sessions.TryGetValue(key, out var session)) return;
session.AudioVolume = Math.Clamp(volume, 0, 1);
session.AudioMuted = muted;
}
}
private void OnAudioAvailable(MediaSession session, AudioSample sample)
{
if (!TryGetActiveSession(session)) return;
double volume;
bool muted;
lock (_gate)
{
volume = session.AudioVolume;
muted = session.AudioMuted;
}
if (muted || volume <= 0) return;
if (volume < 0.999)
{
// Copy before scaling: the source owns its buffer and may recycle it.
var scaled = new float[sample.Samples.Length];
var gain = (float)volume;
for (var i = 0; i < scaled.Length; i++)
scaled[i] = sample.Samples[i] * gain;
AudioAvailable?.Invoke(session.Key, new AudioSample(scaled, sample.SampleRate, sample.Channels));
return;
}
AudioAvailable?.Invoke(session.Key, sample);
}
/// <summary>The shared preview bitmap for a file (so a config created after the
/// first frame can adopt the running bitmap), or null.</summary>
public WriteableBitmap? GetPreviewBitmap(string key)
{
lock (_gate)
return _sessions.TryGetValue(key, out var session) ? session.PreviewBitmap : null;
}
public void Dispose()
{
List<MediaSession> sessions;
lock (_gate)
{
sessions = new List<MediaSession>(_sessions.Values);
_sessions.Clear();
}
foreach (var session in sessions)
{
session.Source.FrameAvailable -= session.FrameHandler;
if (session.AudioHandler != null) session.Source.AudioAvailable -= session.AudioHandler;
_ = SafeStopAsync(session.Source);
}
}
private static async Task SafeStopAsync(IMediaFrameSource source)
{
try
{
await source.StopAsync();
}
catch (Exception ex)
{
AppLog.Write($"MediaSource: stopping decoder failed: {ex.Message}");
}
}
private bool TryGetActiveSession(MediaSession session)
{
lock (_gate)
return _sessions.TryGetValue(session.Key, out var current) && ReferenceEquals(current, session);
}
private void OnFrameAvailable(MediaSession session, VideoFrame frame)
{
if (!TryGetActiveSession(session)) return;
session.LatestFrame = frame;
if (session.PreviewBitmap == null)
{
if (_uiDispatcher == null) return;
if (_uiDispatcher.CheckAccess())
EnsurePreviewBitmap(session);
else
_uiDispatcher.BeginInvoke(() =>
{
if (TryGetActiveSession(session))
EnsurePreviewBitmap(session);
}, DispatcherPriority.Render);
return;
}
if (_uiDispatcher == null || _uiDispatcher.CheckAccess())
{
CopyFrame(session);
return;
}
if (session.FramePending) return;
session.FramePending = true;
_uiDispatcher.BeginInvoke(() =>
{
session.FramePending = false;
if (TryGetActiveSession(session) && session.PreviewBitmap != null)
CopyFrame(session);
}, DispatcherPriority.Render);
}
private void EnsurePreviewBitmap(MediaSession session)
{
if (session.PreviewBitmap != null || session.LatestFrame == null) return;
var frame = session.LatestFrame;
var bitmap = new WriteableBitmap(frame.Width, frame.Height, 96, 96, PixelFormats.Bgra32, null);
bitmap.WritePixels(new Int32Rect(0, 0, frame.Width, frame.Height), frame.BgraPixels, frame.Stride, 0);
session.PreviewBitmap = bitmap;
PreviewBitmapChanged?.Invoke(session.Key, bitmap);
}
private void CopyFrame(MediaSession session)
{
var bitmap = session.PreviewBitmap;
var frame = session.LatestFrame;
if (bitmap == null || frame == null) return;
if (frame.Width != bitmap.PixelWidth || frame.Height != bitmap.PixelHeight) return;
bitmap.WritePixels(new Int32Rect(0, 0, frame.Width, frame.Height), frame.BgraPixels, frame.Stride, 0);
}
}