feat(media): play the clip's soundtrack and share one decode pipe
A video file was picture-only: the decode pipe passed `-an`, so a starting-soon or BRB clip with music played silent everywhere. Adds sound to every clip and collapses the two decoders into one. - `Services/VideoClipPipe.cs`: the shared pipe behind `MediaVideoSource` (TASK 21) and `AlertClipDecoder` (TASK 47) — video (`rawvideo bgra`, scaled) + audio (`f32le -ac 2 -ar 48000`, 50 ms chunks, byte-carry across reads), both paced to real time, loop restarts both so picture and sound stay together. Alert behaviour is unchanged. - `MediaVideoSourceManager` applies the layer's Level/Mute per session and raises `AudioAvailable`; `MainViewModel` feeds it to the new `AudioMixer.EnqueueDesktopMediaAudio`, which joins the DESKTOP/GAME ring so the desktop fader and the mic auto-ducker apply exactly as they do for game audio and TRAX (creator ruling 2026-11-05: audio joins the group). - Per-layer `MediaMuted` + existing `MediaVolume` surfaced as Mute/Level in the properties panel; `MediaMuted` persisted (ALTER TABLE guard + SELECT/INSERT). - Level/mute are per SESSION because one decoder is shared per file path — the last layer touched wins; one decoder per layer would play the file twice. Test: `MediaSourceAudioTests` — the pipe emits 50 ms chunks at 48k stereo, the manager scales by level and drops when muted, and a media layer's sound moves the desktop meter. Alert/decode suites unchanged and green. Docs: task-21 closed on item 10, the "silent soundtrack" facts corrected in frame-pipeline/output/overlays, HANDOFF rewritten. verify.sh green (529 passed; the two documented pre-existing failures only).
This commit is contained in:
@@ -6,6 +6,7 @@ using System.Windows.Media;
|
||||
using System.Windows.Media.Imaging;
|
||||
using System.Windows.Threading;
|
||||
using ytLive.Helpers;
|
||||
using ytLive.Services.Audio;
|
||||
|
||||
namespace ytLive.Services;
|
||||
|
||||
@@ -23,8 +24,11 @@ public sealed class MediaVideoSourceManager : IDisposable
|
||||
public string Key { get; }
|
||||
public IMediaFrameSource Source { get; }
|
||||
public Action<VideoFrame>? FrameHandler;
|
||||
public Action<AudioSample>? AudioHandler;
|
||||
public int RefCount;
|
||||
public bool Started;
|
||||
public bool AudioMuted;
|
||||
public double AudioVolume = 1.0;
|
||||
public WriteableBitmap? PreviewBitmap;
|
||||
public VideoFrame? LatestFrame;
|
||||
public bool FramePending;
|
||||
@@ -49,6 +53,10 @@ public sealed class MediaVideoSourceManager : IDisposable
|
||||
/// <summary>Raised when a media decoder cannot be created (bad path).</summary>
|
||||
public event Action<string, string>? MediaFailed;
|
||||
|
||||
/// <summary>Raised (worker thread) per audio chunk of a file's own soundtrack, already
|
||||
/// scaled by the session's level and dropped when muted.</summary>
|
||||
public event Action<string, AudioSample>? AudioAvailable;
|
||||
|
||||
public MediaVideoSourceManager(
|
||||
Func<string, IMediaFrameSource?> sourceFactory,
|
||||
Dispatcher? uiDispatcher = null,
|
||||
@@ -95,6 +103,8 @@ public sealed class MediaVideoSourceManager : IDisposable
|
||||
session = new MediaSession(key, source);
|
||||
session.FrameHandler = frame => OnFrameAvailable(session, frame);
|
||||
session.Source.FrameAvailable += session.FrameHandler;
|
||||
session.AudioHandler = sample => OnAudioAvailable(session, sample);
|
||||
session.Source.AudioAvailable += session.AudioHandler;
|
||||
// Loop intent lives on the Source (per scene), so the caller supplies the
|
||||
// per-path answer ("any live reference loops"). Applied before Start so a
|
||||
// zero-length or instantly-EOF file cannot slip past a later assignment.
|
||||
@@ -117,6 +127,7 @@ public sealed class MediaVideoSourceManager : IDisposable
|
||||
lock (_gate)
|
||||
_sessions.Remove(key);
|
||||
session.Source.FrameAvailable -= session.FrameHandler;
|
||||
if (session.AudioHandler != null) session.Source.AudioAvailable -= session.AudioHandler;
|
||||
AppLog.Write($"MediaSource: failed to start decoder '{key}': {ex.Message}");
|
||||
MediaFailed?.Invoke(key, ex.Message);
|
||||
await SafeStopAsync(session.Source);
|
||||
@@ -137,6 +148,7 @@ public sealed class MediaVideoSourceManager : IDisposable
|
||||
|
||||
if (toStop == null) return;
|
||||
toStop.Source.FrameAvailable -= toStop.FrameHandler;
|
||||
if (toStop.AudioHandler != null) toStop.Source.AudioAvailable -= toStop.AudioHandler;
|
||||
await SafeStopAsync(toStop.Source);
|
||||
toStop.PreviewBitmap = null;
|
||||
}
|
||||
@@ -155,6 +167,7 @@ public sealed class MediaVideoSourceManager : IDisposable
|
||||
|
||||
if (toStop == null) return;
|
||||
toStop.Source.FrameAvailable -= toStop.FrameHandler;
|
||||
if (toStop.AudioHandler != null) toStop.Source.AudioAvailable -= toStop.AudioHandler;
|
||||
await SafeStopAsync(toStop.Source);
|
||||
toStop.PreviewBitmap = null;
|
||||
}
|
||||
@@ -193,6 +206,46 @@ public sealed class MediaVideoSourceManager : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The layer's sound controls for this file. One decoder is shared per path,
|
||||
/// so the level/mute are per SESSION: whichever layer the creator last touched wins —
|
||||
/// the alternative (one decoder per layer) would play the same file twice.</summary>
|
||||
public void SetAudioSettings(string key, double volume, bool muted)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (!_sessions.TryGetValue(key, out var session)) return;
|
||||
session.AudioVolume = Math.Clamp(volume, 0, 1);
|
||||
session.AudioMuted = muted;
|
||||
}
|
||||
}
|
||||
|
||||
private void OnAudioAvailable(MediaSession session, AudioSample sample)
|
||||
{
|
||||
if (!TryGetActiveSession(session)) return;
|
||||
|
||||
double volume;
|
||||
bool muted;
|
||||
lock (_gate)
|
||||
{
|
||||
volume = session.AudioVolume;
|
||||
muted = session.AudioMuted;
|
||||
}
|
||||
if (muted || volume <= 0) return;
|
||||
|
||||
if (volume < 0.999)
|
||||
{
|
||||
// Copy before scaling: the source owns its buffer and may recycle it.
|
||||
var scaled = new float[sample.Samples.Length];
|
||||
var gain = (float)volume;
|
||||
for (var i = 0; i < scaled.Length; i++)
|
||||
scaled[i] = sample.Samples[i] * gain;
|
||||
AudioAvailable?.Invoke(session.Key, new AudioSample(scaled, sample.SampleRate, sample.Channels));
|
||||
return;
|
||||
}
|
||||
|
||||
AudioAvailable?.Invoke(session.Key, sample);
|
||||
}
|
||||
|
||||
/// <summary>The shared preview bitmap for a file (so a config created after the
|
||||
/// first frame can adopt the running bitmap), or null.</summary>
|
||||
public WriteableBitmap? GetPreviewBitmap(string key)
|
||||
@@ -213,6 +266,7 @@ public sealed class MediaVideoSourceManager : IDisposable
|
||||
foreach (var session in sessions)
|
||||
{
|
||||
session.Source.FrameAvailable -= session.FrameHandler;
|
||||
if (session.AudioHandler != null) session.Source.AudioAvailable -= session.AudioHandler;
|
||||
_ = SafeStopAsync(session.Source);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user