feat(media): play the clip's soundtrack and share one decode pipe

A video file was picture-only: the decode pipe passed `-an`, so a starting-soon
or BRB clip with music played silent everywhere. Adds sound to every clip and
collapses the two decoders into one.

- `Services/VideoClipPipe.cs`: the shared pipe behind `MediaVideoSource` (TASK 21)
  and `AlertClipDecoder` (TASK 47) — video (`rawvideo bgra`, scaled) + audio
  (`f32le -ac 2 -ar 48000`, 50 ms chunks, byte-carry across reads), both paced to
  real time, loop restarts both so picture and sound stay together. Alert
  behaviour is unchanged.
- `MediaVideoSourceManager` applies the layer's Level/Mute per session and raises
  `AudioAvailable`; `MainViewModel` feeds it to the new
  `AudioMixer.EnqueueDesktopMediaAudio`, which joins the DESKTOP/GAME ring so the
  desktop fader and the mic auto-ducker apply exactly as they do for game audio
  and TRAX (creator ruling 2026-11-05: audio joins the group).
- Per-layer `MediaMuted` + existing `MediaVolume` surfaced as Mute/Level in the
  properties panel; `MediaMuted` persisted (ALTER TABLE guard + SELECT/INSERT).
- Level/mute are per SESSION because one decoder is shared per file path — the
  last layer touched wins; one decoder per layer would play the file twice.

Test: `MediaSourceAudioTests` — the pipe emits 50 ms chunks at 48k stereo, the
manager scales by level and drops when muted, and a media layer's sound moves the
desktop meter. Alert/decode suites unchanged and green.

Docs: task-21 closed on item 10, the "silent soundtrack" facts corrected in
frame-pipeline/output/overlays, HANDOFF rewritten. verify.sh green (529 passed;
the two documented pre-existing failures only).
This commit is contained in:
2026-11-05 07:54:06 -08:00
parent 776f84fb88
commit 2670ac022b
21 changed files with 669 additions and 328 deletions
+21 -115
View File
@@ -1,38 +1,32 @@
using System.Diagnostics;
using System.IO;
using ytLive.Services.Audio;
using ytLive.Services.Encoder;
namespace ytLive.Services;
/// <summary>
/// TASK 21: decodes a local media file (any format ffmpeg handles) into
/// <see cref="VideoFrame"/>s for a MediaSource. Implements
/// <see cref="IMediaFrameSource"/>: raises <see cref="FrameAvailable"/> per
/// frame and <see cref="IMediaFrameSource.Completed"/> on natural EOF (unless
/// <see cref="IMediaFrameSource.Looping"/>). Spawns ffmpeg with
/// <c>-f rawvideo -pix_fmt bgra</c> and drains the raw BGRA pipe via
/// <see cref="RawVideoFrameReader"/>. The decoder process and emitters are seams
/// so tests run without a real codec or audio device.
/// <see cref="VideoFrame"/>s for a MediaSource. This is the <see cref="IMediaFrameSource"/>
/// shape over the shared <see cref="VideoClipPipe"/> — the same pipe an alert clip uses
/// (TASK 47), so the two features cannot drift apart again in how they decode.
/// <para>Video AND the file's own soundtrack are decoded. The manager scales the audio by
/// the layer's level/mute and hands it to the mixer's desktop group.</para>
/// </summary>
public sealed class MediaVideoSource : IMediaFrameSource
{
public event Action<VideoFrame>? FrameAvailable;
public event Action<AudioSample>? AudioAvailable;
public event Action? Completed;
public string Key => _path;
public bool Looping { get; set; }
public bool Looping
{
get => _clip.Looping;
set => _clip.Looping = value;
}
private readonly string _path;
private readonly int _width;
private readonly int _height;
private readonly IFfmpegLocator _locator;
private readonly Func<IDecodeProcess> _processFactory;
private readonly IFrameRateProbe? _probe;
private readonly Func<TimeSpan, CancellationToken, Task> _delay;
private readonly CancellationTokenSource _cts = new();
private IDisposable? _run;
private IDecodeProcess? _active;
private readonly VideoClipPipe _clip;
public MediaVideoSource(
string path,
@@ -44,104 +38,16 @@ public sealed class MediaVideoSource : IMediaFrameSource
Func<TimeSpan, CancellationToken, Task>? delay = null)
{
_path = path;
_width = width;
_height = height;
_locator = locator;
_processFactory = processFactory;
_probe = frameRateProbe;
_delay = delay ?? ((t, ct) => Task.Delay(t, ct));
_clip = new VideoClipPipe(
path, width, height, locator, processFactory, frameRateProbe, delay, audio: true);
_clip.FrameAvailable += frame => FrameAvailable?.Invoke(frame);
_clip.AudioAvailable += sample => AudioAvailable?.Invoke(sample);
_clip.Completed += () => Completed?.Invoke();
}
public Task StartAsync(CancellationToken cancellationToken = default)
{
if (_run != null) return Task.CompletedTask;
_run = _cts.Token.Register(StopInternal);
_ = Task.Run(RunAsync, CancellationToken.None);
return Task.CompletedTask;
}
public Task StartAsync(CancellationToken cancellationToken = default) => _clip.StartAsync(cancellationToken);
public Task StopAsync()
{
_cts.Cancel();
_run = null;
return Task.CompletedTask;
}
public Task StopAsync() => _clip.StopAsync();
public void Dispose()
{
_cts.Cancel();
_cts.Dispose();
_active?.Dispose();
}
private void StopInternal() => KillProcess();
private async Task RunAsync()
{
try
{
double? frameRate = null;
if (_probe != null)
frameRate = await _probe.ProbeAsync(_path, _cts.Token).ConfigureAwait(false);
var frameDuration = frameRate is > 0 ? TimeSpan.FromSeconds(1.0 / frameRate.Value) : TimeSpan.Zero;
var ffmpegPath = await _locator.LocateAsync(_cts.Token).ConfigureAwait(false);
var args =
$"-hide_banner -loglevel error -i \"{_path}\" " +
$"-f rawvideo -pix_fmt bgra -vf scale={_width}:{_height} -an -";
var startInfo = new ProcessStartInfo
{
FileName = ffmpegPath,
Arguments = args,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true,
};
do
{
using (var process = _processFactory())
{
_active = process;
process.Start(startInfo);
var reader = new RawVideoFrameReader(_width, _height);
var buffer = new byte[65536];
var stream = process.StandardOutput;
while (!_cts.IsCancellationRequested && !process.HasExited)
{
var read = await stream.ReadAsync(buffer, 0, buffer.Length, _cts.Token).ConfigureAwait(false);
if (read <= 0) break;
foreach (var frame in reader.Feed(buffer.AsSpan(0, read).ToArray()))
{
FrameAvailable?.Invoke(frame);
if (frameDuration > TimeSpan.Zero)
await _delay(frameDuration, _cts.Token).ConfigureAwait(false);
}
}
}
_active = null;
}
while (Looping && !_cts.IsCancellationRequested);
}
catch (OperationCanceledException)
{
}
catch (Exception ex)
{
Fail($"Media decode failed: {ex.Message}");
return;
}
if (!_cts.IsCancellationRequested)
Completed?.Invoke();
}
private void Fail(string message) => Console.Error.WriteLine(message);
private void KillProcess()
{
try { _active?.Kill(); } catch { /* already exited */ }
}
public void Dispose() => _clip.Dispose();
}