feat(media): play the clip's soundtrack and share one decode pipe
A video file was picture-only: the decode pipe passed `-an`, so a starting-soon or BRB clip with music played silent everywhere. Adds sound to every clip and collapses the two decoders into one. - `Services/VideoClipPipe.cs`: the shared pipe behind `MediaVideoSource` (TASK 21) and `AlertClipDecoder` (TASK 47) — video (`rawvideo bgra`, scaled) + audio (`f32le -ac 2 -ar 48000`, 50 ms chunks, byte-carry across reads), both paced to real time, loop restarts both so picture and sound stay together. Alert behaviour is unchanged. - `MediaVideoSourceManager` applies the layer's Level/Mute per session and raises `AudioAvailable`; `MainViewModel` feeds it to the new `AudioMixer.EnqueueDesktopMediaAudio`, which joins the DESKTOP/GAME ring so the desktop fader and the mic auto-ducker apply exactly as they do for game audio and TRAX (creator ruling 2026-11-05: audio joins the group). - Per-layer `MediaMuted` + existing `MediaVolume` surfaced as Mute/Level in the properties panel; `MediaMuted` persisted (ALTER TABLE guard + SELECT/INSERT). - Level/mute are per SESSION because one decoder is shared per file path — the last layer touched wins; one decoder per layer would play the file twice. Test: `MediaSourceAudioTests` — the pipe emits 50 ms chunks at 48k stereo, the manager scales by level and drops when muted, and a media layer's sound moves the desktop meter. Alert/decode suites unchanged and green. Docs: task-21 closed on item 10, the "silent soundtrack" facts corrected in frame-pipeline/output/overlays, HANDOFF rewritten. verify.sh green (529 passed; the two documented pre-existing failures only).
This commit is contained in:
+21
-115
@@ -1,38 +1,32 @@
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using ytLive.Services.Audio;
|
||||
using ytLive.Services.Encoder;
|
||||
|
||||
namespace ytLive.Services;
|
||||
|
||||
/// <summary>
|
||||
/// TASK 21: decodes a local media file (any format ffmpeg handles) into
|
||||
/// <see cref="VideoFrame"/>s for a MediaSource. Implements
|
||||
/// <see cref="IMediaFrameSource"/>: raises <see cref="FrameAvailable"/> per
|
||||
/// frame and <see cref="IMediaFrameSource.Completed"/> on natural EOF (unless
|
||||
/// <see cref="IMediaFrameSource.Looping"/>). Spawns ffmpeg with
|
||||
/// <c>-f rawvideo -pix_fmt bgra</c> and drains the raw BGRA pipe via
|
||||
/// <see cref="RawVideoFrameReader"/>. The decoder process and emitters are seams
|
||||
/// so tests run without a real codec or audio device.
|
||||
/// <see cref="VideoFrame"/>s for a MediaSource. This is the <see cref="IMediaFrameSource"/>
|
||||
/// shape over the shared <see cref="VideoClipPipe"/> — the same pipe an alert clip uses
|
||||
/// (TASK 47), so the two features cannot drift apart again in how they decode.
|
||||
/// <para>Video AND the file's own soundtrack are decoded. The manager scales the audio by
|
||||
/// the layer's level/mute and hands it to the mixer's desktop group.</para>
|
||||
/// </summary>
|
||||
public sealed class MediaVideoSource : IMediaFrameSource
|
||||
{
|
||||
public event Action<VideoFrame>? FrameAvailable;
|
||||
public event Action<AudioSample>? AudioAvailable;
|
||||
public event Action? Completed;
|
||||
|
||||
public string Key => _path;
|
||||
|
||||
public bool Looping { get; set; }
|
||||
public bool Looping
|
||||
{
|
||||
get => _clip.Looping;
|
||||
set => _clip.Looping = value;
|
||||
}
|
||||
|
||||
private readonly string _path;
|
||||
private readonly int _width;
|
||||
private readonly int _height;
|
||||
private readonly IFfmpegLocator _locator;
|
||||
private readonly Func<IDecodeProcess> _processFactory;
|
||||
private readonly IFrameRateProbe? _probe;
|
||||
private readonly Func<TimeSpan, CancellationToken, Task> _delay;
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private IDisposable? _run;
|
||||
private IDecodeProcess? _active;
|
||||
private readonly VideoClipPipe _clip;
|
||||
|
||||
public MediaVideoSource(
|
||||
string path,
|
||||
@@ -44,104 +38,16 @@ public sealed class MediaVideoSource : IMediaFrameSource
|
||||
Func<TimeSpan, CancellationToken, Task>? delay = null)
|
||||
{
|
||||
_path = path;
|
||||
_width = width;
|
||||
_height = height;
|
||||
_locator = locator;
|
||||
_processFactory = processFactory;
|
||||
_probe = frameRateProbe;
|
||||
_delay = delay ?? ((t, ct) => Task.Delay(t, ct));
|
||||
_clip = new VideoClipPipe(
|
||||
path, width, height, locator, processFactory, frameRateProbe, delay, audio: true);
|
||||
_clip.FrameAvailable += frame => FrameAvailable?.Invoke(frame);
|
||||
_clip.AudioAvailable += sample => AudioAvailable?.Invoke(sample);
|
||||
_clip.Completed += () => Completed?.Invoke();
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (_run != null) return Task.CompletedTask;
|
||||
_run = _cts.Token.Register(StopInternal);
|
||||
_ = Task.Run(RunAsync, CancellationToken.None);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
public Task StartAsync(CancellationToken cancellationToken = default) => _clip.StartAsync(cancellationToken);
|
||||
|
||||
public Task StopAsync()
|
||||
{
|
||||
_cts.Cancel();
|
||||
_run = null;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
public Task StopAsync() => _clip.StopAsync();
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_cts.Cancel();
|
||||
_cts.Dispose();
|
||||
_active?.Dispose();
|
||||
}
|
||||
|
||||
private void StopInternal() => KillProcess();
|
||||
|
||||
private async Task RunAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
double? frameRate = null;
|
||||
if (_probe != null)
|
||||
frameRate = await _probe.ProbeAsync(_path, _cts.Token).ConfigureAwait(false);
|
||||
var frameDuration = frameRate is > 0 ? TimeSpan.FromSeconds(1.0 / frameRate.Value) : TimeSpan.Zero;
|
||||
|
||||
var ffmpegPath = await _locator.LocateAsync(_cts.Token).ConfigureAwait(false);
|
||||
var args =
|
||||
$"-hide_banner -loglevel error -i \"{_path}\" " +
|
||||
$"-f rawvideo -pix_fmt bgra -vf scale={_width}:{_height} -an -";
|
||||
var startInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = ffmpegPath,
|
||||
Arguments = args,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true,
|
||||
};
|
||||
|
||||
do
|
||||
{
|
||||
using (var process = _processFactory())
|
||||
{
|
||||
_active = process;
|
||||
process.Start(startInfo);
|
||||
|
||||
var reader = new RawVideoFrameReader(_width, _height);
|
||||
var buffer = new byte[65536];
|
||||
var stream = process.StandardOutput;
|
||||
while (!_cts.IsCancellationRequested && !process.HasExited)
|
||||
{
|
||||
var read = await stream.ReadAsync(buffer, 0, buffer.Length, _cts.Token).ConfigureAwait(false);
|
||||
if (read <= 0) break;
|
||||
foreach (var frame in reader.Feed(buffer.AsSpan(0, read).ToArray()))
|
||||
{
|
||||
FrameAvailable?.Invoke(frame);
|
||||
if (frameDuration > TimeSpan.Zero)
|
||||
await _delay(frameDuration, _cts.Token).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
_active = null;
|
||||
}
|
||||
while (Looping && !_cts.IsCancellationRequested);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Fail($"Media decode failed: {ex.Message}");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!_cts.IsCancellationRequested)
|
||||
Completed?.Invoke();
|
||||
}
|
||||
|
||||
private void Fail(string message) => Console.Error.WriteLine(message);
|
||||
|
||||
private void KillProcess()
|
||||
{
|
||||
try { _active?.Kill(); } catch { /* already exited */ }
|
||||
}
|
||||
public void Dispose() => _clip.Dispose();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user