diff --git a/Controls/LeftPanel.xaml b/Controls/LeftPanel.xaml index e6f0ce8..30af66c 100644 --- a/Controls/LeftPanel.xaml +++ b/Controls/LeftPanel.xaml @@ -746,7 +746,20 @@ - + + + + + + + + diff --git a/HANDOFF.md b/HANDOFF.md index 3e103ce..c2c8a2f 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -1,20 +1,21 @@ -# HANDOFF — 2026-11-05 (TASK 21 — video file layer) +# HANDOFF — 2026-11-05 (TASK 21 — video file layer + its audio) ## State -- `main`; TASK 21 is complete: a video file added from the (+) menu decodes, composites into preview - and stream, loops on toggle, and stops with its last scene reference. 3·17 (text layers) completed - 2026-11-04. -- New test `MediaSourceAcquisitionTests` (add → one decoder starts + frames reach the layer + loop flip - reaches the decoder + remove → stops). App build is 0 warnings. -- **Media is picture-only:** `MediaVideoSource` decodes with `-an`, so a video file's own soundtrack - is silent on stream — the loopback does not carry it because nothing plays it. Two routes recorded - in `task-21` item 10 (alert-style mixer feed vs TRAX-style local play); no volume slider until one exists. +- `main`; TASK 21 is complete. A video file added from the (+) menu decodes, composites into preview and + stream, loops on toggle, and **plays its own soundtrack** (Mute + Level per layer, mixed into the + desktop/game group so the desktop fader and the ducker apply). Removing it stops the decoder. +- Decoding is now ONE pipe: `Services/VideoClipPipe.cs` behind both `MediaVideoSource` (TASK 21) and + `AlertClipDecoder` (TASK 47) — "a clip is a clip". Alert behaviour is unchanged and its tests stay green. +- Tests: `MediaSourceAcquisitionTests` (add → decoder starts, frames, loop, remove) and + `MediaSourceAudioTests` (chunks, level scaling, mute, sound reaching the desktop meter). App build 0 warnings. ## Next action - Next queued unit: TASK 49 (chat profanity filter). Day 3's TASK 32/33 resilience slices are still outstanding — 32·4 (fragmented MP4) is the smallest. +- Agreed next feature work (designed, not started): green/blue screen detection + keying through the clip + pipe — suggest-only by default, an opt-in "Auto Green/Blue Screen" setting, alert clips included. - Do not push absent a checkpoint request. ## Landmines @@ -23,5 +24,6 @@ mouse input. Do not run it while the desktop is in use. - Integration tests constructing `MainWindow` must set a temporary `LayoutPathOverride` and clear it in `finally`. - Media sessions are keyed per **path**, not per layer: acquire through `EnsureMediaSession`, release - through `ReleaseMediaIfUnused`, never `AcquireAsync` twice for one path. + through `ReleaseMediaIfUnused`, never `AcquireAsync` twice for one path. Level/mute are therefore + per session too. - Bundled FFmpeg stays outside the single-file exe (`ExcludeFromSingleFile="true"`). diff --git a/Models/Source.cs b/Models/Source.cs index a52d775..a08ca4a 100644 --- a/Models/Source.cs +++ b/Models/Source.cs @@ -264,6 +264,15 @@ public class Source : SceneElement set => Set(ref _mediaVolume, Math.Clamp(value, 0.0, 1.0)); } + private bool _mediaMuted; + /// Silences this layer's soundtrack (the picture keeps playing). One decoder is + /// shared per file, so the level/mute are the file's — the layer is the control. + public bool MediaMuted + { + get => _mediaMuted; + set => Set(ref _mediaMuted, value); + } + private MediaPlaybackState _mediaPlaybackState = MediaPlaybackState.Stopped; public MediaPlaybackState MediaPlaybackState { diff --git a/Services/AlertClipDecoder.cs b/Services/AlertClipDecoder.cs index 46594d9..62af420 100644 --- a/Services/AlertClipDecoder.cs +++ b/Services/AlertClipDecoder.cs @@ -1,6 +1,5 @@ -using System.Diagnostics; -using System.IO; -using ytLive.Helpers; +using System; +using System.Threading; using ytLive.Services.Audio; using ytLive.Services.Encoder; @@ -11,10 +10,10 @@ namespace ytLive.Services; /// same file, both paced to real time. The alert box owns a decoder per event /// (start at play, dispose at drain) — unlike MediaSource's app-wide refcounted /// session, an alert REPLAYS the same clip each event, so a fresh decode per -/// play is required. Video is rawvideo BGRA scaled to the box rect (the -/// shape); audio is ffmpeg f32le at the -/// mixer's native 48 kHz stereo (so the mixer ring needs no resampler). The -/// fade is applied by the alert layer, not here. +/// play is required. The fade is applied by the alert layer, not here. +/// Since TASK 21's audio slice this is a thin wrapper over the shared +/// (the same pipe a media layer uses); the alert case +/// simply never sets Looping. /// public interface IAlertClipDecoder : IDisposable { @@ -35,30 +34,14 @@ public interface IAlertClipDecoder : IDisposable event Action? Completed; } -/// Default : two ffmpeg subprocesses on -/// the same file. Video via on a rawvideo -/// bgra pipe (-an), audio via an f32le -ac 2 -ar 48000 pipe (-vn). -/// The seams (locator, process factory, frame-rate probe, delay) let tests drive -/// a fake decoder without a real codec or Windows. +/// Default : the shared clip pipe with looping off. public sealed class AlertClipDecoder : IAlertClipDecoder { public event Action? FrameAvailable; public event Action? AudioReady; public event Action? Completed; - private const int AudioSampleRate = 48000; - private const int AudioChannels = 2; - private const int AudioChunkFloats = 4800; // 50ms of stereo - - private readonly string _path; - private readonly int _width; - private readonly int _height; - private readonly IFfmpegLocator _locator; - private readonly Func _processFactory; - private readonly IFrameRateProbe? _probe; - private readonly Func _delay; - private readonly CancellationTokenSource _cts = new(); - private Task? _run; + private readonly VideoClipPipe _clip; public AlertClipDecoder( string path, @@ -69,159 +52,16 @@ public sealed class AlertClipDecoder : IAlertClipDecoder IFrameRateProbe? frameRateProbe = null, Func? delay = null) { - _path = path; - _width = width; - _height = height; - _locator = locator; - _processFactory = processFactory; - _probe = frameRateProbe; - _delay = delay ?? ((t, ct) => Task.Delay(t, ct)); + _clip = new VideoClipPipe( + path, width, height, locator, processFactory, frameRateProbe, delay, audio: true); + _clip.FrameAvailable += frame => FrameAvailable?.Invoke(frame); + _clip.AudioAvailable += sample => AudioReady?.Invoke(sample); + _clip.Completed += () => Completed?.Invoke(); } - public void Start() - { - if (_run != null) return; - _run = Task.Run(RunAsync, CancellationToken.None); - } + public void Start() => _ = _clip.StartAsync(); - public void Stop() => _cts.Cancel(); + public void Stop() => _ = _clip.StopAsync(); - public void Dispose() - { - _cts.Cancel(); - _cts.Dispose(); - } - - private async Task RunAsync() - { - var ct = _cts.Token; - var videoFrames = 0; - var audioChunks = 0; - var failed = false; - try - { - double? frameRate = null; - if (_probe != null) - frameRate = await _probe.ProbeAsync(_path, ct).ConfigureAwait(false); - var frameDuration = frameRate is > 0 ? TimeSpan.FromSeconds(1.0 / frameRate.Value) : TimeSpan.Zero; - - var ffmpegPath = await _locator.LocateAsync(ct).ConfigureAwait(false); - - var videoTask = RunVideoAsync(ffmpegPath, frameDuration, ct); - var audioTask = RunAudioAsync(ffmpegPath, ct); - videoFrames = await videoTask.ConfigureAwait(false); - audioChunks = await audioTask.ConfigureAwait(false); - } - catch (OperationCanceledException) - { - failed = true; - return; - } - catch (Exception ex) - { - // A failed decode is not fatal: the alert layer falls back to the six - // animations (or the frame it already holds) exactly as a missing file. - // Logged so a silent blank box traces back to a decode failure. - failed = true; - AppLog.Write(ex, $"Alert clip decode failed ({Path.GetFileName(_path)})", AppLogErrorClass.Class2); - } - - if (!ct.IsCancellationRequested) - { - AppLog.Write($"Alert clip end: {Path.GetFileName(_path)} frames={videoFrames} audioChunks={audioChunks} failed={failed}"); - Completed?.Invoke(); - } - } - - private async Task RunVideoAsync(string ffmpegPath, TimeSpan frameDuration, CancellationToken ct) - { - var args = - $"-hide_banner -loglevel error -i \"{_path}\" " + - $"-f rawvideo -pix_fmt bgra -vf scale={_width}:{_height} -an -"; - using var process = _processFactory(); - process.Start(new ProcessStartInfo - { - FileName = ffmpegPath, - Arguments = args, - UseShellExecute = false, - RedirectStandardOutput = true, - RedirectStandardError = true, - CreateNoWindow = true, - }); - - var reader = new RawVideoFrameReader(_width, _height); - var buffer = new byte[65536]; - var stream = process.StandardOutput; - var frames = 0; - while (!ct.IsCancellationRequested && !process.HasExited) - { - var read = await stream.ReadAsync(buffer, 0, buffer.Length, ct).ConfigureAwait(false); - if (read <= 0) break; - var chunk = new byte[read]; - Buffer.BlockCopy(buffer, 0, chunk, 0, read); - foreach (var frame in reader.Feed(chunk)) - { - frames++; - FrameAvailable?.Invoke(frame); - if (frameDuration > TimeSpan.Zero) - await _delay(frameDuration, ct).ConfigureAwait(false); - } - } - return frames; - } - - private async Task RunAudioAsync(string ffmpegPath, CancellationToken ct) - { - var args = - $"-hide_banner -loglevel error -i \"{_path}\" " + - $"-f f32le -ac {AudioChannels} -ar {AudioSampleRate} -vn -"; - using var process = _processFactory(); - process.Start(new ProcessStartInfo - { - FileName = ffmpegPath, - Arguments = args, - UseShellExecute = false, - RedirectStandardOutput = true, - RedirectStandardError = true, - CreateNoWindow = true, - }); - - var stream = process.StandardOutput; - var raw = new byte[65536]; - var carry = new byte[0]; - var floatScratch = new float[AudioChunkFloats]; - var floatIndex = 0; - var chunks = 0; - // The pipe is byte-delimited only: a read may split a float, so leftover - // bytes are carried into the next read instead of dropped (a dropped - // half-float would desync the entire stream). - while (!ct.IsCancellationRequested && !process.HasExited) - { - var read = await stream.ReadAsync(raw, 0, raw.Length, ct).ConfigureAwait(false); - if (read <= 0) break; - var total = carry.Length + read; - var bytes = total <= raw.Length ? raw : new byte[total]; - if (carry.Length > 0) - { - Buffer.BlockCopy(carry, 0, bytes, 0, carry.Length); - Buffer.BlockCopy(raw, 0, bytes, carry.Length, read); - } - var floatsTo = total - (total % 4); - for (var i = 0; i < floatsTo && floatIndex < floatScratch.Length; i += 4) - floatScratch[floatIndex++] = BitConverter.ToSingle(bytes, i); - carry = total > floatsTo ? bytes[floatsTo..total] : new byte[0]; - - while (floatIndex == floatScratch.Length) - { - chunks++; - AudioReady?.Invoke(new AudioSample(floatScratch, AudioSampleRate, AudioChannels)); - // Pace the NEXT chunk so the whole clip's audio lands at real-time - // cadence (the fade in the alert layer tracks the same wall clock). - await _delay(TimeSpan.FromSeconds(AudioChunkFloats / (double)(AudioSampleRate * AudioChannels)), ct).ConfigureAwait(false); - floatScratch = new float[AudioChunkFloats]; - floatIndex = 0; - } - } - return chunks; - } + public void Dispose() => _clip.Dispose(); } diff --git a/Services/Audio/AudioMixer.cs b/Services/Audio/AudioMixer.cs index 130bd94..1dc1328 100644 --- a/Services/Audio/AudioMixer.cs +++ b/Services/Audio/AudioMixer.cs @@ -283,6 +283,25 @@ public sealed class AudioMixer : IDisposable LoopbackLevelChanged?.Invoke(level); } + /// Drops a MEDIA clip's audio into the desktop/game ring (TASK 21's audio slice). + /// It joins that group on purpose: the desktop fader and the mic auto-ducker then apply + /// exactly as they do to game audio and TRAX, so a BRB bed ducks under the creator's + /// voice without a second audio knob. The layer's own level/mute are applied upstream + /// by . + public void EnqueueDesktopMediaAudio(AudioSample sample) + { + var data = sample.Samples; + if (data.Length == 0) return; + if (sample.SampleRate != OutputSampleRate) + data = ResampleLoopback(data, sample.SampleRate); + if (sample.Channels == 1) + data = UpmixToStereo(data); + _loopbackBuffer.Write(data); + + var level = _loopbackMeter.Push(new AudioSample(data, OutputSampleRate, sample.Channels)); + LoopbackLevelChanged?.Invoke(level); + } + /// Drops the alert clip's audio into its own ring (TASK 47 — the /// stream-alert video's soundtrack, pre-scaled by the alert layer to /// volume × fade). Mixed at unity into the live output — NO auto-duck: an diff --git a/Services/IMediaFrameSource.cs b/Services/IMediaFrameSource.cs index fb26de5..eb6695a 100644 --- a/Services/IMediaFrameSource.cs +++ b/Services/IMediaFrameSource.cs @@ -1,3 +1,5 @@ +using ytLive.Services.Audio; + namespace ytLive.Services; /// @@ -17,6 +19,11 @@ public interface IMediaFrameSource : IDisposable /// Raised (on a worker thread) for each decoded frame. event Action? FrameAvailable; + /// Raised (on a worker thread) per chunk of the file's own soundtrack + /// (48 kHz stereo float). The manager applies the layer's level/mute and hands it + /// to the mixer; a source with no audio simply never raises it. + event Action? AudioAvailable; + /// Raised when the file finishes decoding (natural EOF) — not raised /// while is true. event Action? Completed; diff --git a/Services/LayoutStore.Load.cs b/Services/LayoutStore.Load.cs index 5298b9b..4c7871e 100644 --- a/Services/LayoutStore.Load.cs +++ b/Services/LayoutStore.Load.cs @@ -113,7 +113,7 @@ public partial class LayoutStore : IDisposable AlertVideoAssetId, AlertVideoPath, AlertUseDefaultVideo, AlertShowTicker, AlertVideoVolume, SortOrder, AlertDisplayMethod, IsBrandLogo, BlurAmount, AlertAudioPath, AlertAudioLoop, TextContent, TextFontFamily, TextFontSize, TextColor, TextBold, TextItalic, - TextUnderline, TextAlignment, TextVerticalAlignment + TextUnderline, TextAlignment, TextVerticalAlignment, MediaMuted FROM Source ORDER BY SortOrder """; using var reader = cmd.ExecuteReader(); @@ -183,6 +183,7 @@ public partial class LayoutStore : IDisposable || !Enum.IsDefined((TextLayerVerticalAlignment)reader.GetInt32(49)) ? TextLayerVerticalAlignment.Top : (TextLayerVerticalAlignment)reader.GetInt32(49), + MediaMuted = reader.GetInt32(reader.GetOrdinal("MediaMuted")) != 0, }; if (!sourcesByScene.TryGetValue(sceneId, out var list)) sourcesByScene[sceneId] = list = new List<(Source, int)>(); diff --git a/Services/LayoutStore.Migrations.cs b/Services/LayoutStore.Migrations.cs index bd3fbde..ec05b85 100644 --- a/Services/LayoutStore.Migrations.cs +++ b/Services/LayoutStore.Migrations.cs @@ -378,6 +378,12 @@ public partial class LayoutStore : IDisposable cmd.CommandText = "ALTER TABLE Source ADD COLUMN MediaPlaybackState TEXT NOT NULL DEFAULT 'Stopped';"; cmd.ExecuteNonQuery(); } + if (!columns.Contains("MediaMuted")) + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = "ALTER TABLE Source ADD COLUMN MediaMuted INTEGER NOT NULL DEFAULT 0;"; + cmd.ExecuteNonQuery(); + } // TASK 47: alert video config. The default clip asset id + the creator's // replacement path (a FILE reference, never a stored BLOB) + the pill, diff --git a/Services/LayoutStore.Save.cs b/Services/LayoutStore.Save.cs index ea553f9..1933e81 100644 --- a/Services/LayoutStore.Save.cs +++ b/Services/LayoutStore.Save.cs @@ -94,7 +94,7 @@ public partial class LayoutStore : IDisposable AlertVideoAssetId, AlertVideoPath, AlertUseDefaultVideo, AlertShowTicker, AlertVideoVolume, AlertDisplayMethod, IsBrandLogo, BlurAmount, AlertAudioPath, AlertAudioLoop, TextContent, TextFontFamily, TextFontSize, TextColor, TextBold, TextItalic, - TextUnderline, TextAlignment, TextVerticalAlignment) + TextUnderline, TextAlignment, TextVerticalAlignment, MediaMuted) VALUES ($id, $sceneId, $assetId, $type, $name, $isEnabled, $x, $y, $w, $h, $opacity, $monitor, $clip, $mirrored, $isBackground, $captureKey, $showDesktop, $sort, @@ -104,7 +104,7 @@ public partial class LayoutStore : IDisposable $alertVideoAssetId, $alertVideoPath, $alertUseDefaultVideo, $alertShowTicker, $alertVideoVolume, $alertDisplayMethod, $isBrandLogo, $blurAmount, $alertAudioPath, $alertAudioLoop, $textContent, $textFontFamily, $textFontSize, $textColor, $textBold, $textItalic, - $textUnderline, $textAlignment, $textVerticalAlignment) + $textUnderline, $textAlignment, $textVerticalAlignment, $mediaMuted) """; cmd.Transaction = tx; var idP = cmd.Parameters.Add("$id", SqliteType.Text); @@ -137,6 +137,7 @@ public partial class LayoutStore : IDisposable var mediaPathP = cmd.Parameters.Add("$mediaPath", SqliteType.Text); var mediaIsLoopingP = cmd.Parameters.Add("$mediaIsLooping", SqliteType.Integer); var mediaVolumeP = cmd.Parameters.Add("$mediaVolume", SqliteType.Real); + var mediaMutedP = cmd.Parameters.Add("$mediaMuted", SqliteType.Integer); var mediaPlaybackStateP = cmd.Parameters.Add("$mediaPlaybackState", SqliteType.Text); var alertVideoAssetIdP = cmd.Parameters.Add("$alertVideoAssetId", SqliteType.Text); var alertVideoPathP = cmd.Parameters.Add("$alertVideoPath", SqliteType.Text); @@ -198,6 +199,7 @@ public partial class LayoutStore : IDisposable mediaPathP.Value = (object?)source.MediaPath ?? DBNull.Value; mediaIsLoopingP.Value = source.MediaIsLooping ? 1 : 0; mediaVolumeP.Value = source.MediaVolume; + mediaMutedP.Value = source.MediaMuted ? 1 : 0; mediaPlaybackStateP.Value = source.MediaPlaybackState.ToString(); alertVideoAssetIdP.Value = (object?)source.AlertVideoAssetId ?? DBNull.Value; alertVideoPathP.Value = (object?)source.AlertVideoPath ?? DBNull.Value; diff --git a/Services/MediaVideoSource.cs b/Services/MediaVideoSource.cs index 62912b3..24110d2 100644 --- a/Services/MediaVideoSource.cs +++ b/Services/MediaVideoSource.cs @@ -1,38 +1,32 @@ -using System.Diagnostics; -using System.IO; +using ytLive.Services.Audio; using ytLive.Services.Encoder; namespace ytLive.Services; /// /// TASK 21: decodes a local media file (any format ffmpeg handles) into -/// s for a MediaSource. Implements -/// : raises per -/// frame and on natural EOF (unless -/// ). Spawns ffmpeg with -/// -f rawvideo -pix_fmt bgra and drains the raw BGRA pipe via -/// . The decoder process and emitters are seams -/// so tests run without a real codec or audio device. +/// s for a MediaSource. This is the +/// shape over the shared — the same pipe an alert clip uses +/// (TASK 47), so the two features cannot drift apart again in how they decode. +/// Video AND the file's own soundtrack are decoded. The manager scales the audio by +/// the layer's level/mute and hands it to the mixer's desktop group. /// public sealed class MediaVideoSource : IMediaFrameSource { public event Action? FrameAvailable; + public event Action? AudioAvailable; public event Action? Completed; public string Key => _path; - public bool Looping { get; set; } + public bool Looping + { + get => _clip.Looping; + set => _clip.Looping = value; + } private readonly string _path; - private readonly int _width; - private readonly int _height; - private readonly IFfmpegLocator _locator; - private readonly Func _processFactory; - private readonly IFrameRateProbe? _probe; - private readonly Func _delay; - private readonly CancellationTokenSource _cts = new(); - private IDisposable? _run; - private IDecodeProcess? _active; + private readonly VideoClipPipe _clip; public MediaVideoSource( string path, @@ -44,104 +38,16 @@ public sealed class MediaVideoSource : IMediaFrameSource Func? delay = null) { _path = path; - _width = width; - _height = height; - _locator = locator; - _processFactory = processFactory; - _probe = frameRateProbe; - _delay = delay ?? ((t, ct) => Task.Delay(t, ct)); + _clip = new VideoClipPipe( + path, width, height, locator, processFactory, frameRateProbe, delay, audio: true); + _clip.FrameAvailable += frame => FrameAvailable?.Invoke(frame); + _clip.AudioAvailable += sample => AudioAvailable?.Invoke(sample); + _clip.Completed += () => Completed?.Invoke(); } - public Task StartAsync(CancellationToken cancellationToken = default) - { - if (_run != null) return Task.CompletedTask; - _run = _cts.Token.Register(StopInternal); - _ = Task.Run(RunAsync, CancellationToken.None); - return Task.CompletedTask; - } + public Task StartAsync(CancellationToken cancellationToken = default) => _clip.StartAsync(cancellationToken); - public Task StopAsync() - { - _cts.Cancel(); - _run = null; - return Task.CompletedTask; - } + public Task StopAsync() => _clip.StopAsync(); - public void Dispose() - { - _cts.Cancel(); - _cts.Dispose(); - _active?.Dispose(); - } - - private void StopInternal() => KillProcess(); - - private async Task RunAsync() - { - try - { - double? frameRate = null; - if (_probe != null) - frameRate = await _probe.ProbeAsync(_path, _cts.Token).ConfigureAwait(false); - var frameDuration = frameRate is > 0 ? TimeSpan.FromSeconds(1.0 / frameRate.Value) : TimeSpan.Zero; - - var ffmpegPath = await _locator.LocateAsync(_cts.Token).ConfigureAwait(false); - var args = - $"-hide_banner -loglevel error -i \"{_path}\" " + - $"-f rawvideo -pix_fmt bgra -vf scale={_width}:{_height} -an -"; - var startInfo = new ProcessStartInfo - { - FileName = ffmpegPath, - Arguments = args, - UseShellExecute = false, - RedirectStandardOutput = true, - RedirectStandardError = true, - CreateNoWindow = true, - }; - - do - { - using (var process = _processFactory()) - { - _active = process; - process.Start(startInfo); - - var reader = new RawVideoFrameReader(_width, _height); - var buffer = new byte[65536]; - var stream = process.StandardOutput; - while (!_cts.IsCancellationRequested && !process.HasExited) - { - var read = await stream.ReadAsync(buffer, 0, buffer.Length, _cts.Token).ConfigureAwait(false); - if (read <= 0) break; - foreach (var frame in reader.Feed(buffer.AsSpan(0, read).ToArray())) - { - FrameAvailable?.Invoke(frame); - if (frameDuration > TimeSpan.Zero) - await _delay(frameDuration, _cts.Token).ConfigureAwait(false); - } - } - } - _active = null; - } - while (Looping && !_cts.IsCancellationRequested); - } - catch (OperationCanceledException) - { - } - catch (Exception ex) - { - Fail($"Media decode failed: {ex.Message}"); - return; - } - - if (!_cts.IsCancellationRequested) - Completed?.Invoke(); - } - - private void Fail(string message) => Console.Error.WriteLine(message); - - private void KillProcess() - { - try { _active?.Kill(); } catch { /* already exited */ } - } + public void Dispose() => _clip.Dispose(); } diff --git a/Services/MediaVideoSourceManager.cs b/Services/MediaVideoSourceManager.cs index 7e44ccc..e3d0de1 100644 --- a/Services/MediaVideoSourceManager.cs +++ b/Services/MediaVideoSourceManager.cs @@ -6,6 +6,7 @@ using System.Windows.Media; using System.Windows.Media.Imaging; using System.Windows.Threading; using ytLive.Helpers; +using ytLive.Services.Audio; namespace ytLive.Services; @@ -23,8 +24,11 @@ public sealed class MediaVideoSourceManager : IDisposable public string Key { get; } public IMediaFrameSource Source { get; } public Action? FrameHandler; + public Action? AudioHandler; public int RefCount; public bool Started; + public bool AudioMuted; + public double AudioVolume = 1.0; public WriteableBitmap? PreviewBitmap; public VideoFrame? LatestFrame; public bool FramePending; @@ -49,6 +53,10 @@ public sealed class MediaVideoSourceManager : IDisposable /// Raised when a media decoder cannot be created (bad path). public event Action? MediaFailed; + /// Raised (worker thread) per audio chunk of a file's own soundtrack, already + /// scaled by the session's level and dropped when muted. + public event Action? AudioAvailable; + public MediaVideoSourceManager( Func sourceFactory, Dispatcher? uiDispatcher = null, @@ -95,6 +103,8 @@ public sealed class MediaVideoSourceManager : IDisposable session = new MediaSession(key, source); session.FrameHandler = frame => OnFrameAvailable(session, frame); session.Source.FrameAvailable += session.FrameHandler; + session.AudioHandler = sample => OnAudioAvailable(session, sample); + session.Source.AudioAvailable += session.AudioHandler; // Loop intent lives on the Source (per scene), so the caller supplies the // per-path answer ("any live reference loops"). Applied before Start so a // zero-length or instantly-EOF file cannot slip past a later assignment. @@ -117,6 +127,7 @@ public sealed class MediaVideoSourceManager : IDisposable lock (_gate) _sessions.Remove(key); session.Source.FrameAvailable -= session.FrameHandler; + if (session.AudioHandler != null) session.Source.AudioAvailable -= session.AudioHandler; AppLog.Write($"MediaSource: failed to start decoder '{key}': {ex.Message}"); MediaFailed?.Invoke(key, ex.Message); await SafeStopAsync(session.Source); @@ -137,6 +148,7 @@ public sealed class MediaVideoSourceManager : IDisposable if (toStop == null) return; toStop.Source.FrameAvailable -= toStop.FrameHandler; + if (toStop.AudioHandler != null) toStop.Source.AudioAvailable -= toStop.AudioHandler; await SafeStopAsync(toStop.Source); toStop.PreviewBitmap = null; } @@ -155,6 +167,7 @@ public sealed class MediaVideoSourceManager : IDisposable if (toStop == null) return; toStop.Source.FrameAvailable -= toStop.FrameHandler; + if (toStop.AudioHandler != null) toStop.Source.AudioAvailable -= toStop.AudioHandler; await SafeStopAsync(toStop.Source); toStop.PreviewBitmap = null; } @@ -193,6 +206,46 @@ public sealed class MediaVideoSourceManager : IDisposable } } + /// The layer's sound controls for this file. One decoder is shared per path, + /// so the level/mute are per SESSION: whichever layer the creator last touched wins — + /// the alternative (one decoder per layer) would play the same file twice. + public void SetAudioSettings(string key, double volume, bool muted) + { + lock (_gate) + { + if (!_sessions.TryGetValue(key, out var session)) return; + session.AudioVolume = Math.Clamp(volume, 0, 1); + session.AudioMuted = muted; + } + } + + private void OnAudioAvailable(MediaSession session, AudioSample sample) + { + if (!TryGetActiveSession(session)) return; + + double volume; + bool muted; + lock (_gate) + { + volume = session.AudioVolume; + muted = session.AudioMuted; + } + if (muted || volume <= 0) return; + + if (volume < 0.999) + { + // Copy before scaling: the source owns its buffer and may recycle it. + var scaled = new float[sample.Samples.Length]; + var gain = (float)volume; + for (var i = 0; i < scaled.Length; i++) + scaled[i] = sample.Samples[i] * gain; + AudioAvailable?.Invoke(session.Key, new AudioSample(scaled, sample.SampleRate, sample.Channels)); + return; + } + + AudioAvailable?.Invoke(session.Key, sample); + } + /// The shared preview bitmap for a file (so a config created after the /// first frame can adopt the running bitmap), or null. public WriteableBitmap? GetPreviewBitmap(string key) @@ -213,6 +266,7 @@ public sealed class MediaVideoSourceManager : IDisposable foreach (var session in sessions) { session.Source.FrameAvailable -= session.FrameHandler; + if (session.AudioHandler != null) session.Source.AudioAvailable -= session.AudioHandler; _ = SafeStopAsync(session.Source); } } diff --git a/Services/VideoClipPipe.cs b/Services/VideoClipPipe.cs new file mode 100644 index 0000000..bdfa4ca --- /dev/null +++ b/Services/VideoClipPipe.cs @@ -0,0 +1,248 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using ytLive.Helpers; +using ytLive.Services.Audio; +using ytLive.Services.Encoder; + +namespace ytLive.Services; + +/// +/// One video FILE played through ffmpeg — the shared pipe behind every clip, so a media +/// layer (TASK 21) and an alert clip (TASK 47) decode identically. Owns two subprocesses +/// on the same file: video (rawvideo bgra, scaled to the target rect) and audio +/// (f32le -ac 2 -ar 48000, matching the mixer's native format so no resampler is +/// needed). Both are paced to real time, so the picture and the sound stay together even +/// though they arrive on separate pipes. +/// Looping is the media case: on natural EOF the whole pass restarts — a fresh +/// pair of processes, because a cannot be re-started. An alert clip +/// never loops, so its wrapper simply never sets . +/// The seams (locator, process factory, frame-rate probe, delay) exist so tests drive +/// a fake decoder with no ffmpeg, no codec, and no audio device. +/// +public sealed class VideoClipPipe : IDisposable +{ + public const int AudioSampleRate = 48000; + public const int AudioChannels = 2; + private const int AudioChunkFloats = 4800; // 50 ms of stereo + + public event Action? FrameAvailable; + public event Action? AudioAvailable; + public event Action? Completed; + + /// Restart the pass on natural EOF (media/BRB loops). + public bool Looping { get; set; } + + private readonly string _path; + private readonly int _width; + private readonly int _height; + private readonly IFfmpegLocator _locator; + private readonly Func _processFactory; + private readonly IFrameRateProbe? _probe; + private readonly Func _delay; + private readonly bool _audio; + private readonly CancellationTokenSource _cts = new(); + private IDisposable? _run; + private IDecodeProcess? _activeVideo; + private IDecodeProcess? _activeAudio; + + public VideoClipPipe( + string path, + int width, + int height, + IFfmpegLocator locator, + Func processFactory, + IFrameRateProbe? frameRateProbe = null, + Func? delay = null, + bool audio = true) + { + _path = path; + _width = width; + _height = height; + _locator = locator; + _processFactory = processFactory; + _probe = frameRateProbe; + _delay = delay ?? ((t, ct) => Task.Delay(t, ct)); + _audio = audio; + } + + public Task StartAsync(CancellationToken cancellationToken = default) + { + if (_run != null) return Task.CompletedTask; + _run = _cts.Token.Register(StopInternal); + _ = Task.Run(RunAsync, CancellationToken.None); + return Task.CompletedTask; + } + + public Task StopAsync() + { + _cts.Cancel(); + _run = null; + return Task.CompletedTask; + } + + public void Dispose() + { + _cts.Cancel(); + _cts.Dispose(); + _activeVideo?.Dispose(); + _activeAudio?.Dispose(); + } + + private void StopInternal() + { + Kill(_activeVideo); + Kill(_activeAudio); + } + + private async Task RunAsync() + { + var ct = _cts.Token; + var frames = 0; + var chunks = 0; + try + { + double? frameRate = null; + if (_probe != null) + frameRate = await _probe.ProbeAsync(_path, ct).ConfigureAwait(false); + var frameDuration = frameRate is > 0 ? TimeSpan.FromSeconds(1.0 / frameRate.Value) : TimeSpan.Zero; + + var ffmpegPath = await _locator.LocateAsync(ct).ConfigureAwait(false); + + do + { + // Both pipes per pass, awaited together: a restart must begin a new + // video AND a new audio stream, or the picture drifts from the sound + // after the first loop. + var video = RunVideoAsync(ffmpegPath, frameDuration, ct); + var audio = _audio ? RunAudioAsync(ffmpegPath, ct) : Task.FromResult(0); + frames += await video.ConfigureAwait(false); + chunks += await audio.ConfigureAwait(false); + } + while (Looping && !ct.IsCancellationRequested); + } + catch (OperationCanceledException) + { + return; + } + catch (Exception ex) + { + // A failed decode is not fatal: the alert layer falls back to its animations, + // and a media layer just holds its last frame. + AppLog.Write($"Video clip decode failed ({Path.GetFileName(_path)}): {ex.Message}"); + return; + } + + if (!ct.IsCancellationRequested) + { + AppLog.Write($"Video clip end: {Path.GetFileName(_path)} frames={frames} audioChunks={chunks}"); + Completed?.Invoke(); + } + } + + private async Task RunVideoAsync(string ffmpegPath, TimeSpan frameDuration, CancellationToken ct) + { + var args = + $"-hide_banner -loglevel error -i \"{_path}\" " + + $"-f rawvideo -pix_fmt bgra -vf scale={_width}:{_height} -an -"; + using var process = _processFactory(); + _activeVideo = process; + process.Start(new ProcessStartInfo + { + FileName = ffmpegPath, + Arguments = args, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }); + + var reader = new RawVideoFrameReader(_width, _height); + var buffer = new byte[65536]; + var stream = process.StandardOutput; + var frames = 0; + while (!ct.IsCancellationRequested && !process.HasExited) + { + var read = await stream.ReadAsync(buffer, 0, buffer.Length, ct).ConfigureAwait(false); + if (read <= 0) break; + var chunk = new byte[read]; + Buffer.BlockCopy(buffer, 0, chunk, 0, read); + foreach (var frame in reader.Feed(chunk)) + { + frames++; + FrameAvailable?.Invoke(frame); + if (frameDuration > TimeSpan.Zero) + await _delay(frameDuration, ct).ConfigureAwait(false); + } + } + _activeVideo = null; + return frames; + } + + private async Task RunAudioAsync(string ffmpegPath, CancellationToken ct) + { + var args = + $"-hide_banner -loglevel error -i \"{_path}\" " + + $"-f f32le -ac {AudioChannels} -ar {AudioSampleRate} -vn -"; + using var process = _processFactory(); + _activeAudio = process; + process.Start(new ProcessStartInfo + { + FileName = ffmpegPath, + Arguments = args, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + }); + + var stream = process.StandardOutput; + var raw = new byte[65536]; + var carry = Array.Empty(); + var floatScratch = new float[AudioChunkFloats]; + var floatIndex = 0; + var chunks = 0; + // The pipe is byte-delimited only: a read may split a float, so leftover + // bytes are carried into the next read instead of dropped (a dropped + // half-float would desync the entire stream). + while (!ct.IsCancellationRequested && !process.HasExited) + { + var read = await stream.ReadAsync(raw, 0, raw.Length, ct).ConfigureAwait(false); + if (read <= 0) break; + var total = carry.Length + read; + var bytes = total <= raw.Length ? raw : new byte[total]; + if (carry.Length > 0) + { + Buffer.BlockCopy(carry, 0, bytes, 0, carry.Length); + Buffer.BlockCopy(raw, 0, bytes, carry.Length, read); + } + var floatsTo = total - (total % 4); + for (var i = 0; i < floatsTo && floatIndex < floatScratch.Length; i += 4) + floatScratch[floatIndex++] = BitConverter.ToSingle(bytes, i); + carry = total > floatsTo ? bytes[floatsTo..total] : Array.Empty(); + + while (floatIndex == floatScratch.Length) + { + chunks++; + AudioAvailable?.Invoke(new AudioSample(floatScratch, AudioSampleRate, AudioChannels)); + // Pace the NEXT chunk so the clip's audio lands at real-time cadence, + // matching how the video pipe is paced by frame rate. + await _delay( + TimeSpan.FromSeconds(AudioChunkFloats / (double)(AudioSampleRate * AudioChannels)), ct) + .ConfigureAwait(false); + floatScratch = new float[AudioChunkFloats]; + floatIndex = 0; + } + } + _activeAudio = null; + return chunks; + } + + private static void Kill(IDecodeProcess? process) + { + try { process?.Kill(); } catch { /* already exited */ } + } +} diff --git a/TASKS/task-21-media-source.md b/TASKS/task-21-media-source.md index bc43088..5ba0bce 100644 --- a/TASKS/task-21-media-source.md +++ b/TASKS/task-21-media-source.md @@ -7,8 +7,7 @@ ### Status: ✅ Complete 2026-11-05 — a video file added in the (+) menu decodes, composites into the preview and the stream, loops on the creator's toggle, and stops when the layer or its last scene -reference goes away. **Picture only: the file's own audio is not played anywhere** (the decode pipe -passes `-an`), so a video with a soundtrack is silent on stream — the audio route is open, item 10. +reference goes away, and **the file's own soundtrack plays into the stream** (item 10). 1. ✅ `MediaSourceType` enum: `Video`, `Audio` (audio-only files via media source) 2. ✅ `SourceType.MediaSource` addition to the enum @@ -25,13 +24,13 @@ passes `-an`), so a video with a soundtrack is silent on stream — the audio ro when `Looping` (fresh process per pass). `MediaIsLooping` reaches the session through the manager's per-path `loopProvider` at start and `SetLooping` on a live toggle. Rule (recorded): the path's session loops if **any** live media reference has it on, because one decoder is shared per path. -10. ⏸ File audio + volume — **the file's soundtrack is silent everywhere today**: the decode pipe - passes `-an`, so nothing plays it and the desktop loopback has nothing to carry. Two established - routes when this is taken: **alert-style** (a second `f32le` pipe into the mixer — - `AlertClipDecoder.RunAudioAsync` is the working model, independent of desktop capture) or - **TRAX-style** (`MusicPlayer` plays it to the default device, so the loopback, the desktop fader - and ducking all apply — at the cost of a second clock to hold in sync with the video). Product - call; not started, and no volume slider until one of them exists. +10. ✅ File audio + volume — the shared clip pipe decodes the soundtrack (`f32le -ac 2 -ar 48000 -vn`, + 50 ms chunks paced to real time, byte-carry across reads) and the layer carries **Mute** + **Level**. + The manager applies them per session (one decoder per path, so the last layer touched wins — the + alternative would play the same file twice) and the audio joins the DESKTOP/GAME group in + `AudioMixer.EnqueueDesktopMediaAudio`, so the desktop fader and the voice ducker behave exactly as + they do for game audio and TRAX. Test: `MediaSourceAudioTests` (chunks, level scaling, mute, and the + layer's sound reaching the desktop meter). 11. ✅ UI: `AddMediaCommand` (+ menu → "Video File", video-filtered `OpenFileDialog`), `ChangeMediaFileCommand` ("Change…" in the properties panel), loop toggle, video name display. 12. ✅ Schema migration for media source settings (file path, loop, volume) — shipped with item 3. @@ -41,10 +40,11 @@ passes `-an`), so a video with a soundtrack is silent on stream — the audio ro ### Design decisions - **FFmpeg handles all formats** — no codec-specific code. FFmpeg already in the project. -- **Audio is NOT wired, and it is not hiding behind the loopback** — `MediaVideoSource` decodes with - `-an`, so a video file's soundtrack is silent on stream today (the loopback only carries what the - machine actually plays). Item 10 records the two established routes; do not describe media as - audible until one is built. +- **Audio joins the desktop/game group** (creator ruling 2026-11-05) — the clip's sound is decoded by + the same pipe as its picture, mixed directly into the stream (not played locally and captured), and + grouped with game audio so one fader and the ducker cover it. A per-layer Mute/Level sits on top. + Chosen over TRAX-style local playback because it works regardless of the creator's desktop-capture + state, and because an alert clip needs the same direct route anyway. - **This pairs with TRAX** — TRAX is background music, media source is background video. Together they make non-Live scenes (Starting/BRB/Ending) feel polished. - **Not a full NLE** — no trimming, no multi-track, no effects. Just "play this video in the scene." - **Layers are defined in Live first** (creator 2026-11-05): this unit adds the layer and the picker in diff --git a/ViewModels/MainViewModel.Media.cs b/ViewModels/MainViewModel.Media.cs index f1aebe4..1515856 100644 --- a/ViewModels/MainViewModel.Media.cs +++ b/ViewModels/MainViewModel.Media.cs @@ -6,6 +6,7 @@ using System.Windows.Media.Imaging; using Microsoft.Win32; using ytLive.Models; using ytLive.Services; +using ytLive.Services.Audio; namespace ytLive.ViewModels; @@ -105,6 +106,11 @@ public partial class MainViewModel { _mediaManager.SetLooping(path, AnyMediaReferenceLoops(path)); } + else if (e.PropertyName is nameof(Source.MediaVolume) or nameof(Source.MediaMuted) + && source.MediaPath is { } audioPath) + { + ApplyMediaAudioSettings(audioPath); + } } _mediaHandlers[source.Id] = OnSourceChanged; @@ -121,7 +127,11 @@ public partial class MainViewModel private void EnsureMediaSession(string? path) { if (string.IsNullOrWhiteSpace(path)) return; - if (_mediaManager.HasSession(path)) return; + if (_mediaManager.HasSession(path)) + { + ApplyMediaAudioSettings(path); + return; + } _ = AcquireMediaAsync(path); } @@ -133,8 +143,25 @@ public partial class MainViewModel // case the bitmap is already published and will not be raised again for us. if (_mediaManager.GetPreviewBitmap(path) is { } bitmap) OnMediaPreviewBitmapChanged(path, bitmap); + + ApplyMediaAudioSettings(path); } + /// The file's sound controls: one decoder is shared per path, so the level and + /// mute are per session — the layer that was last touched is the one that wins. + private void ApplyMediaAudioSettings(string path) + { + var source = Scenes.SelectMany(s => s.Elements).OfType() + .FirstOrDefault(s => s.IsMediaSource && s.MediaPath == path); + if (source == null) return; + _mediaManager.SetAudioSettings(path, source.MediaVolume, source.MediaMuted); + } + + /// A clip's own soundtrack joins the desktop/game group, so the desktop fader and + /// the voice ducker treat it exactly like game audio (TASK 21 audio slice). + private void OnMediaAudioAvailable(string path, AudioSample sample) + => _audioMixer.EnqueueDesktopMediaAudio(sample); + /// Shared sessions or not, the loop state is the path's: any live media /// reference asking for a loop makes the one decoder loop (recorded rule, TASK 21). private bool AnyMediaReferenceLoops(string path) diff --git a/ViewModels/MainViewModel.cs b/ViewModels/MainViewModel.cs index 69529cc..2a894da 100644 --- a/ViewModels/MainViewModel.cs +++ b/ViewModels/MainViewModel.cs @@ -412,6 +412,7 @@ public partial class MainViewModel : ViewModelBase AnyMediaReferenceLoops); _mediaManager.PreviewBitmapChanged += OnMediaPreviewBitmapChanged; _mediaManager.MediaFailed += OnMediaFailed; + _mediaManager.AudioAvailable += OnMediaAudioAvailable; _chatLayer = new ChatOverlayLayer(new ChatBoxRenderer()); _alertLayer = new AlertOverlayLayer( diff --git a/docs/ai/frame-pipeline.md b/docs/ai/frame-pipeline.md index 1960ae2..26cef71 100644 --- a/docs/ai/frame-pipeline.md +++ b/docs/ai/frame-pipeline.md @@ -799,9 +799,12 @@ Full suite 290/291 passing, the sole failure the pre-existing compositor pixel t released (`GetActiveKeys`). Loop is a manager-level per-path answer: the ctor takes `Func? loopProvider` (production: "any live media reference loops"), applied before `StartAsync`, and `SetLooping(key, bool)` applies a live toggle. A file swap releases the old path after - the layer no longer references it. **A video file's own soundtrack is silent today** — the decode pipe - is `-an`, so nothing plays it and the loopback has nothing to carry; wiring it (alert-style mixer feed - or TRAX-style local playback) is open, and no volume slider exists until then. + the layer no longer references it. **Sound ships with it (2026-11-05):** both `MediaVideoSource` and + `AlertClipDecoder` are now thin wrappers over one `Services/VideoClipPipe.cs` (video pipe + `f32le` + audio pipe, both real-time paced, loop restarts both), so a clip is a clip. The manager applies the + layer's Level/Mute per session and raises `AudioAvailable`, and `MainViewModel` feeds it to + `AudioMixer.EnqueueDesktopMediaAudio` — the DESKTOP/GAME ring, so the desktop fader and the voice + ducker apply. The alert clip keeps its own unity-gain ring (`EnqueueAlertAudio`, never ducked). - **Social bar on the output (bar bug-fix branch):** the `FramePump` takes an optional `socialBar: Func<(VideoFrame? Frame, SocialBarPosition Position)>?` seam, re-read **every frame** (so a mid-stream position flip applies immediately). The strip is pre-rasterized by `Compositor/SocialBarRenderer.cs` diff --git a/docs/ai/output.md b/docs/ai/output.md index cdb6041..2a0c729 100644 --- a/docs/ai/output.md +++ b/docs/ai/output.md @@ -250,6 +250,13 @@ devices, no timers). `StopLive()` (closes the pipe → ffmpeg audio EOF) **BEFORE** stopping the frame pump (video EOF) — the reverse order stalls on pipe backpressure. `FfmpegEncoder` itself is untouched; the VM owns the pipe lifecycle. + **Two clip-audio rings join that mix (2026-11-05):** `EnqueueAlertAudio` (TASK 47) is pre-scaled by the + alert layer and mixed at **unity, never ducked** — an alert must be heard over the mic. + `EnqueueDesktopMediaAudio` (TASK 21's audio slice) is the opposite on purpose: a video layer's + soundtrack joins the **desktop/game ring**, so the desktop fader and the auto-ducker apply exactly as + they do for game audio and TRAX. Both are fed by the shared `Services/VideoClipPipe.cs`; the layer's + own Mute/Level are applied upstream by `MediaVideoSourceManager` (level/mute are per **session**, since + one decoder is shared per file path). - **Audio sync offset (TASK 22):** after the limiter, `FillAndMix` routes the whole interleaved stereo mix through a **`AudioSyncDelay`** (`Services/Audio/AudioSyncDelay.cs`) — a pure delay line whose offset comes from a `Func syncOffsetMs` seam (the VM's global `AudioSyncOffsetMs`, **−500..+500 diff --git a/docs/ai/overlays.md b/docs/ai/overlays.md index c37c1d2..e96b3ca 100644 --- a/docs/ai/overlays.md +++ b/docs/ai/overlays.md @@ -97,12 +97,11 @@ published decisions (recorded in `TASKS/task-43-native-alerts.md`): `AlertClipDecoder.RunVideoAsync` and `MediaVideoSource.RunAsync` are near-duplicates — same ffmpeg args (`-f rawvideo -pix_fmt bgra -vf scale=W:H -an -`), same `RawVideoFrameReader` feed loop, same `IFrameRateProbe` + delay pacing, both behind - `IDecodeProcess`. What differs is ownership and role, not decoding: alerts are per-play - and disposed at drain, media is a refcounted per-path session; alerts also run an - `f32le` audio pipe into the mixer, media runs none at all (`-an`) — so a media file's - soundtrack is silent today, and task-21 item 10 holds the two routes for changing that; - alerts play once, media loops. A shared raw-video pump - is the obvious consolidation if a third consumer appears — do not do it for two. + `IDecodeProcess`. Consolidated 2026-11-05 with the media audio slice: both are now thin wrappers over + one `Services/VideoClipPipe.cs` (video pipe + `f32le` audio pipe, real-time paced, loop restarts both), + so decode behaviour cannot drift between them again. What still differs is ownership and role: alerts + are per-play and disposed at drain, media is a refcounted per-path session; alerts play once, media + loops; alert audio is mixed at unity and never ducked, media audio joins the desktop/game group. Fades in/out via **`FadeDurationSeconds = 0.30`** (`AlertOverlayLayer` const): fade-in by `elapsed` on straight-alpha frame copies; EOF → freeze-frame → fade-out → drain (the clip branch of `Advance` **clears diff --git a/ytLive.Tests/MediaSourceAcquisitionTests.cs b/ytLive.Tests/MediaSourceAcquisitionTests.cs index d266651..025fe1c 100644 --- a/ytLive.Tests/MediaSourceAcquisitionTests.cs +++ b/ytLive.Tests/MediaSourceAcquisitionTests.cs @@ -100,6 +100,7 @@ public sealed class MediaSourceAcquisitionTests public int StopCount { get; private set; } public event Action? FrameAvailable; + public event Action? AudioAvailable { add { } remove { } } public event Action? Completed { add { } remove { } } public Task StartAsync(CancellationToken cancellationToken = default) diff --git a/ytLive.Tests/MediaSourceAudioTests.cs b/ytLive.Tests/MediaSourceAudioTests.cs new file mode 100644 index 0000000..1651347 --- /dev/null +++ b/ytLive.Tests/MediaSourceAudioTests.cs @@ -0,0 +1,191 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Data.Sqlite; +using Xunit; +using ytLive.Services; +using ytLive.Services.Audio; +using ytLive.Services.Encoder; +using ytLive.ViewModels; + +namespace ytLive.Tests; + +/// +/// TASK 21 audio slice (2026-11-05): a video file's own soundtrack used to be thrown away +/// (`-an` on the decode pipe), so a "starting soon" clip with music played silent. This pins +/// the replacement: the shared clip pipe decodes the sound, the manager applies the layer's +/// level/mute, and a media layer's audio lands in the mixer's desktop/game group. +/// +[Collection("RealApp")] +public sealed class MediaSourceAudioTests +{ + private readonly RealAppHost _app; + + public MediaSourceAudioTests(RealAppHost app) => _app = app; + + // ─── the pipe decodes the soundtrack ─── + + [Fact] + public async Task ClipPipe_EmitsTheSoundtrackInPacedChunks() + { + // 4800 stereo floats = one 50 ms chunk. + var audioBytes = new byte[4800 * 4]; + for (var i = 0; i < 4800; i++) + BitConverter.TryWriteBytes(audioBytes.AsSpan(i * 4, 4), 0.5f); + + using var pipe = new VideoClipPipe( + "clip.mp4", 1, 1, + new FakeLocator(), + () => new FakeDecodeProcess(args => args.Contains("-vn") ? audioBytes : Array.Empty()), + delay: (_, _) => Task.CompletedTask); + + var chunks = new List(); + var completed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + pipe.AudioAvailable += sample => chunks.Add(sample); + pipe.Completed += () => completed.TrySetResult(true); + + await pipe.StartAsync(); + await completed.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Single(chunks); + Assert.Equal(4800, chunks[0].Samples.Length); + Assert.Equal(48000, chunks[0].SampleRate); + Assert.Equal(2, chunks[0].Channels); + } + + // ─── the manager applies the layer's controls ─── + + [Fact] + public async Task Manager_ScalesTheSoundtrackByTheSessionsLevel_AndDropsItWhenMuted() + { + var source = new FakeMediaSource("clip.mp4"); + var manager = new MediaVideoSourceManager(_ => source, uiDispatcher: null); + var heard = new List(); + manager.AudioAvailable += (_, sample) => heard.Add(sample); + + Assert.True(await manager.AcquireAsync("clip.mp4")); + + manager.SetAudioSettings("clip.mp4", 0.5, muted: false); + source.EmitAudio(new AudioSample(new[] { 1.0f, -1.0f }, 48000, 2)); + Assert.Single(heard); + Assert.Equal(0.5f, heard[0].Samples[0], 3); + Assert.Equal(-0.5f, heard[0].Samples[1], 3); + + manager.SetAudioSettings("clip.mp4", 1.0, muted: true); + source.EmitAudio(new AudioSample(new[] { 1.0f, -1.0f }, 48000, 2)); + Assert.Single(heard); // muted: nothing reaches the mixer + + manager.Dispose(); + } + + // ─── the layer's audio reaches the desktop/game group ─── + + [Fact] + public void MediaLayerSound_ReachesTheDesktopAudioGroup() + { + _app.Run(() => + { + var tempDb = Path.Combine(Path.GetTempPath(), $"ytLlive-test-{Guid.NewGuid():N}.db"); + var videoPath = Path.Combine(Path.GetTempPath(), "brb-bed.mp4"); + var decoders = new List(); + MainViewModel.LayoutPathOverride = tempDb; + MainViewModel.MediaFrameSourceFactoryOverride = key => + { + var decoder = new FakeMediaSource(key); + decoders.Add(decoder); + return decoder; + }; + + var window = new MainWindow(); + var vm = (MainViewModel)window.DataContext; + try + { + window.Show(); + window.UpdateLayout(); + + vm.AddMediaSource(videoPath); + var decoder = decoders.Single(); + + // The desktop/game meter is what the creator watches. It has to move for a + // media layer's sound, or the soundtrack is in the mix but invisible. + Assert.Equal(0.0, vm.GameAudioLevel); + + var loud = Enumerable.Repeat(0.8f, 9600).ToArray(); + for (var i = 0; i < 6; i++) + decoder.EmitAudio(new AudioSample(loud, 48000, 2)); + + Assert.True(vm.GameAudioLevel > 0.05, + $"the media soundtrack never reached the desktop group (level {vm.GameAudioLevel})"); + + vm.StagedScene!.Elements.OfType() + .Single(s => s.IsMediaSource).MediaMuted = true; + var afterMute = vm.GameAudioLevel; + + decoder.EmitAudio(new AudioSample(loud, 48000, 2)); + Assert.Equal(afterMute, vm.GameAudioLevel); // muted layer stops feeding the meter + } + finally + { + window.Close(); + MainViewModel.MediaFrameSourceFactoryOverride = null; + MainViewModel.LayoutPathOverride = null; + SqliteConnection.ClearAllPools(); + try { File.Delete(tempDb); } catch { /* best-effort cleanup */ } + } + }); + } + + private sealed class FakeMediaSource : IMediaFrameSource + { + public FakeMediaSource(string key) => Key = key; + + public string Key { get; } + public bool Looping { get; set; } + + public event Action? FrameAvailable { add { } remove { } } + public event Action? AudioAvailable; + public event Action? Completed { add { } remove { } } + + public void EmitAudio(AudioSample sample) => AudioAvailable?.Invoke(sample); + + public Task StartAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; + public Task StopAsync() => Task.CompletedTask; + public void Dispose() + { + } + } + + private sealed class FakeDecodeProcess : IDecodeProcess + { + private readonly Func _pick; + private readonly MemoryStream _stream = new(); + + public FakeDecodeProcess(Func pick) => _pick = pick; + + public void Start(ProcessStartInfo startInfo) + { + // The pipe runs a video process and an audio process; the payload depends on + // which one this is, so the fake reads the arguments. + var bytes = _pick(startInfo.Arguments); + _stream.SetLength(0); + _stream.Write(bytes, 0, bytes.Length); + _stream.Position = 0; + } + + public Stream StandardOutput => _stream; + public bool HasExited => _stream.Position >= _stream.Length; + public int ExitCode => 0; + public void Kill() { } + public Task WaitForExitAsync(CancellationToken ct = default) => Task.CompletedTask; + public void Dispose() => _stream.Dispose(); + } + + private sealed class FakeLocator : IFfmpegLocator + { + public Task LocateAsync(CancellationToken ct = default) => Task.FromResult("ffmpeg.exe"); + } +} diff --git a/ytLive.Tests/MediaVideoSourceManagerTests.cs b/ytLive.Tests/MediaVideoSourceManagerTests.cs index e37f0af..9a73c44 100644 --- a/ytLive.Tests/MediaVideoSourceManagerTests.cs +++ b/ytLive.Tests/MediaVideoSourceManagerTests.cs @@ -3,6 +3,7 @@ using System.Windows.Media.Imaging; using System.Windows.Threading; using Xunit; using ytLive.Services; +using ytLive.Services.Audio; namespace ytLive.Tests; @@ -22,6 +23,7 @@ public class MediaVideoSourceManagerTests public string Key { get; } public bool Looping { get; set; } public event Action? FrameAvailable; + public event Action? AudioAvailable; public event Action? Completed; public FakeMediaSource(string key, List? started = null, List? stopped = null) @@ -44,6 +46,7 @@ public class MediaVideoSourceManagerTests } public void Pump(VideoFrame frame) => FrameAvailable?.Invoke(frame); + public void EmitAudio(AudioSample sample) => AudioAvailable?.Invoke(sample); public void Complete() => Completed?.Invoke(); public void Dispose() { } } @@ -53,12 +56,14 @@ public class MediaVideoSourceManagerTests public string Key { get; } public bool Looping { get; set; } public event Action? FrameAvailable; + public event Action? AudioAvailable; public event Action? Completed; public FailingMediaSource(string key) => Key = key; public Task StartAsync(CancellationToken cancellationToken = default) => throw new InvalidOperationException("access denied"); public Task StopAsync() => Task.CompletedTask; public void Pump(VideoFrame frame) => FrameAvailable?.Invoke(frame); + public void EmitAudio(AudioSample sample) => AudioAvailable?.Invoke(sample); public void Complete() => Completed?.Invoke(); public void Dispose() { } }