feat(media): play the clip's soundtrack and share one decode pipe
A video file was picture-only: the decode pipe passed `-an`, so a starting-soon or BRB clip with music played silent everywhere. Adds sound to every clip and collapses the two decoders into one. - `Services/VideoClipPipe.cs`: the shared pipe behind `MediaVideoSource` (TASK 21) and `AlertClipDecoder` (TASK 47) — video (`rawvideo bgra`, scaled) + audio (`f32le -ac 2 -ar 48000`, 50 ms chunks, byte-carry across reads), both paced to real time, loop restarts both so picture and sound stay together. Alert behaviour is unchanged. - `MediaVideoSourceManager` applies the layer's Level/Mute per session and raises `AudioAvailable`; `MainViewModel` feeds it to the new `AudioMixer.EnqueueDesktopMediaAudio`, which joins the DESKTOP/GAME ring so the desktop fader and the mic auto-ducker apply exactly as they do for game audio and TRAX (creator ruling 2026-11-05: audio joins the group). - Per-layer `MediaMuted` + existing `MediaVolume` surfaced as Mute/Level in the properties panel; `MediaMuted` persisted (ALTER TABLE guard + SELECT/INSERT). - Level/mute are per SESSION because one decoder is shared per file path — the last layer touched wins; one decoder per layer would play the file twice. Test: `MediaSourceAudioTests` — the pipe emits 50 ms chunks at 48k stereo, the manager scales by level and drops when muted, and a media layer's sound moves the desktop meter. Alert/decode suites unchanged and green. Docs: task-21 closed on item 10, the "silent soundtrack" facts corrected in frame-pipeline/output/overlays, HANDOFF rewritten. verify.sh green (529 passed; the two documented pre-existing failures only).
This commit is contained in:
@@ -0,0 +1,191 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Xunit;
|
||||
using ytLive.Services;
|
||||
using ytLive.Services.Audio;
|
||||
using ytLive.Services.Encoder;
|
||||
using ytLive.ViewModels;
|
||||
|
||||
namespace ytLive.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// TASK 21 audio slice (2026-11-05): a video file's own soundtrack used to be thrown away
|
||||
/// (`-an` on the decode pipe), so a "starting soon" clip with music played silent. This pins
|
||||
/// the replacement: the shared clip pipe decodes the sound, the manager applies the layer's
|
||||
/// level/mute, and a media layer's audio lands in the mixer's desktop/game group.
|
||||
/// </summary>
|
||||
[Collection("RealApp")]
|
||||
public sealed class MediaSourceAudioTests
|
||||
{
|
||||
private readonly RealAppHost _app;
|
||||
|
||||
public MediaSourceAudioTests(RealAppHost app) => _app = app;
|
||||
|
||||
// ─── the pipe decodes the soundtrack ───
|
||||
|
||||
[Fact]
|
||||
public async Task ClipPipe_EmitsTheSoundtrackInPacedChunks()
|
||||
{
|
||||
// 4800 stereo floats = one 50 ms chunk.
|
||||
var audioBytes = new byte[4800 * 4];
|
||||
for (var i = 0; i < 4800; i++)
|
||||
BitConverter.TryWriteBytes(audioBytes.AsSpan(i * 4, 4), 0.5f);
|
||||
|
||||
using var pipe = new VideoClipPipe(
|
||||
"clip.mp4", 1, 1,
|
||||
new FakeLocator(),
|
||||
() => new FakeDecodeProcess(args => args.Contains("-vn") ? audioBytes : Array.Empty<byte>()),
|
||||
delay: (_, _) => Task.CompletedTask);
|
||||
|
||||
var chunks = new List<AudioSample>();
|
||||
var completed = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
pipe.AudioAvailable += sample => chunks.Add(sample);
|
||||
pipe.Completed += () => completed.TrySetResult(true);
|
||||
|
||||
await pipe.StartAsync();
|
||||
await completed.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
|
||||
Assert.Single(chunks);
|
||||
Assert.Equal(4800, chunks[0].Samples.Length);
|
||||
Assert.Equal(48000, chunks[0].SampleRate);
|
||||
Assert.Equal(2, chunks[0].Channels);
|
||||
}
|
||||
|
||||
// ─── the manager applies the layer's controls ───
|
||||
|
||||
[Fact]
|
||||
public async Task Manager_ScalesTheSoundtrackByTheSessionsLevel_AndDropsItWhenMuted()
|
||||
{
|
||||
var source = new FakeMediaSource("clip.mp4");
|
||||
var manager = new MediaVideoSourceManager(_ => source, uiDispatcher: null);
|
||||
var heard = new List<AudioSample>();
|
||||
manager.AudioAvailable += (_, sample) => heard.Add(sample);
|
||||
|
||||
Assert.True(await manager.AcquireAsync("clip.mp4"));
|
||||
|
||||
manager.SetAudioSettings("clip.mp4", 0.5, muted: false);
|
||||
source.EmitAudio(new AudioSample(new[] { 1.0f, -1.0f }, 48000, 2));
|
||||
Assert.Single(heard);
|
||||
Assert.Equal(0.5f, heard[0].Samples[0], 3);
|
||||
Assert.Equal(-0.5f, heard[0].Samples[1], 3);
|
||||
|
||||
manager.SetAudioSettings("clip.mp4", 1.0, muted: true);
|
||||
source.EmitAudio(new AudioSample(new[] { 1.0f, -1.0f }, 48000, 2));
|
||||
Assert.Single(heard); // muted: nothing reaches the mixer
|
||||
|
||||
manager.Dispose();
|
||||
}
|
||||
|
||||
// ─── the layer's audio reaches the desktop/game group ───
|
||||
|
||||
[Fact]
|
||||
public void MediaLayerSound_ReachesTheDesktopAudioGroup()
|
||||
{
|
||||
_app.Run(() =>
|
||||
{
|
||||
var tempDb = Path.Combine(Path.GetTempPath(), $"ytLlive-test-{Guid.NewGuid():N}.db");
|
||||
var videoPath = Path.Combine(Path.GetTempPath(), "brb-bed.mp4");
|
||||
var decoders = new List<FakeMediaSource>();
|
||||
MainViewModel.LayoutPathOverride = tempDb;
|
||||
MainViewModel.MediaFrameSourceFactoryOverride = key =>
|
||||
{
|
||||
var decoder = new FakeMediaSource(key);
|
||||
decoders.Add(decoder);
|
||||
return decoder;
|
||||
};
|
||||
|
||||
var window = new MainWindow();
|
||||
var vm = (MainViewModel)window.DataContext;
|
||||
try
|
||||
{
|
||||
window.Show();
|
||||
window.UpdateLayout();
|
||||
|
||||
vm.AddMediaSource(videoPath);
|
||||
var decoder = decoders.Single();
|
||||
|
||||
// The desktop/game meter is what the creator watches. It has to move for a
|
||||
// media layer's sound, or the soundtrack is in the mix but invisible.
|
||||
Assert.Equal(0.0, vm.GameAudioLevel);
|
||||
|
||||
var loud = Enumerable.Repeat(0.8f, 9600).ToArray();
|
||||
for (var i = 0; i < 6; i++)
|
||||
decoder.EmitAudio(new AudioSample(loud, 48000, 2));
|
||||
|
||||
Assert.True(vm.GameAudioLevel > 0.05,
|
||||
$"the media soundtrack never reached the desktop group (level {vm.GameAudioLevel})");
|
||||
|
||||
vm.StagedScene!.Elements.OfType<ytLive.Models.Source>()
|
||||
.Single(s => s.IsMediaSource).MediaMuted = true;
|
||||
var afterMute = vm.GameAudioLevel;
|
||||
|
||||
decoder.EmitAudio(new AudioSample(loud, 48000, 2));
|
||||
Assert.Equal(afterMute, vm.GameAudioLevel); // muted layer stops feeding the meter
|
||||
}
|
||||
finally
|
||||
{
|
||||
window.Close();
|
||||
MainViewModel.MediaFrameSourceFactoryOverride = null;
|
||||
MainViewModel.LayoutPathOverride = null;
|
||||
SqliteConnection.ClearAllPools();
|
||||
try { File.Delete(tempDb); } catch { /* best-effort cleanup */ }
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private sealed class FakeMediaSource : IMediaFrameSource
|
||||
{
|
||||
public FakeMediaSource(string key) => Key = key;
|
||||
|
||||
public string Key { get; }
|
||||
public bool Looping { get; set; }
|
||||
|
||||
public event Action<VideoFrame>? FrameAvailable { add { } remove { } }
|
||||
public event Action<AudioSample>? AudioAvailable;
|
||||
public event Action? Completed { add { } remove { } }
|
||||
|
||||
public void EmitAudio(AudioSample sample) => AudioAvailable?.Invoke(sample);
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken = default) => Task.CompletedTask;
|
||||
public Task StopAsync() => Task.CompletedTask;
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeDecodeProcess : IDecodeProcess
|
||||
{
|
||||
private readonly Func<string, byte[]> _pick;
|
||||
private readonly MemoryStream _stream = new();
|
||||
|
||||
public FakeDecodeProcess(Func<string, byte[]> pick) => _pick = pick;
|
||||
|
||||
public void Start(ProcessStartInfo startInfo)
|
||||
{
|
||||
// The pipe runs a video process and an audio process; the payload depends on
|
||||
// which one this is, so the fake reads the arguments.
|
||||
var bytes = _pick(startInfo.Arguments);
|
||||
_stream.SetLength(0);
|
||||
_stream.Write(bytes, 0, bytes.Length);
|
||||
_stream.Position = 0;
|
||||
}
|
||||
|
||||
public Stream StandardOutput => _stream;
|
||||
public bool HasExited => _stream.Position >= _stream.Length;
|
||||
public int ExitCode => 0;
|
||||
public void Kill() { }
|
||||
public Task WaitForExitAsync(CancellationToken ct = default) => Task.CompletedTask;
|
||||
public void Dispose() => _stream.Dispose();
|
||||
}
|
||||
|
||||
private sealed class FakeLocator : IFfmpegLocator
|
||||
{
|
||||
public Task<string> LocateAsync(CancellationToken ct = default) => Task.FromResult("ffmpeg.exe");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user