feat(media): play the clip's soundtrack and share one decode pipe

A video file was picture-only: the decode pipe passed `-an`, so a starting-soon
or BRB clip with music played silent everywhere. Adds sound to every clip and
collapses the two decoders into one.

- `Services/VideoClipPipe.cs`: the shared pipe behind `MediaVideoSource` (TASK 21)
  and `AlertClipDecoder` (TASK 47) — video (`rawvideo bgra`, scaled) + audio
  (`f32le -ac 2 -ar 48000`, 50 ms chunks, byte-carry across reads), both paced to
  real time, loop restarts both so picture and sound stay together. Alert
  behaviour is unchanged.
- `MediaVideoSourceManager` applies the layer's Level/Mute per session and raises
  `AudioAvailable`; `MainViewModel` feeds it to the new
  `AudioMixer.EnqueueDesktopMediaAudio`, which joins the DESKTOP/GAME ring so the
  desktop fader and the mic auto-ducker apply exactly as they do for game audio
  and TRAX (creator ruling 2026-11-05: audio joins the group).
- Per-layer `MediaMuted` + existing `MediaVolume` surfaced as Mute/Level in the
  properties panel; `MediaMuted` persisted (ALTER TABLE guard + SELECT/INSERT).
- Level/mute are per SESSION because one decoder is shared per file path — the
  last layer touched wins; one decoder per layer would play the file twice.

Test: `MediaSourceAudioTests` — the pipe emits 50 ms chunks at 48k stereo, the
manager scales by level and drops when muted, and a media layer's sound moves the
desktop meter. Alert/decode suites unchanged and green.

Docs: task-21 closed on item 10, the "silent soundtrack" facts corrected in
frame-pipeline/output/overlays, HANDOFF rewritten. verify.sh green (529 passed;
the two documented pre-existing failures only).
This commit is contained in:
2026-11-05 07:54:06 -08:00
parent 776f84fb88
commit 2670ac022b
21 changed files with 669 additions and 328 deletions
+16 -176
View File
@@ -1,6 +1,5 @@
using System.Diagnostics;
using System.IO;
using ytLive.Helpers;
using System;
using System.Threading;
using ytLive.Services.Audio;
using ytLive.Services.Encoder;
@@ -11,10 +10,10 @@ namespace ytLive.Services;
/// same file, both paced to real time. The alert box owns a decoder per event
/// (start at play, dispose at drain) — unlike MediaSource's app-wide refcounted
/// session, an alert REPLAYS the same clip each event, so a fresh decode per
/// play is required. Video is rawvideo BGRA scaled to the box rect (the
/// <see cref="MediaVideoSource"/> shape); audio is ffmpeg <c>f32le</c> at the
/// mixer's native 48 kHz stereo (so the mixer ring needs no resampler). The
/// fade is applied by the alert layer, not here.
/// play is required. The fade is applied by the alert layer, not here.
/// <para>Since TASK 21's audio slice this is a thin wrapper over the shared
/// <see cref="VideoClipPipe"/> (the same pipe a media layer uses); the alert case
/// simply never sets <c>Looping</c>.</para>
/// </summary>
public interface IAlertClipDecoder : IDisposable
{
@@ -35,30 +34,14 @@ public interface IAlertClipDecoder : IDisposable
event Action? Completed;
}
/// <summary>Default <see cref="IAlertClipDecoder"/>: two ffmpeg subprocesses on
/// the same file. Video via <see cref="RawVideoFrameReader"/> on a <c>rawvideo
/// bgra</c> pipe (-an), audio via an <c>f32le -ac 2 -ar 48000</c> pipe (-vn).
/// The seams (locator, process factory, frame-rate probe, delay) let tests drive
/// a fake decoder without a real codec or Windows.</summary>
/// <summary>Default <see cref="IAlertClipDecoder"/>: the shared clip pipe with looping off.</summary>
public sealed class AlertClipDecoder : IAlertClipDecoder
{
public event Action<VideoFrame>? FrameAvailable;
public event Action<AudioSample>? AudioReady;
public event Action? Completed;
private const int AudioSampleRate = 48000;
private const int AudioChannels = 2;
private const int AudioChunkFloats = 4800; // 50ms of stereo
private readonly string _path;
private readonly int _width;
private readonly int _height;
private readonly IFfmpegLocator _locator;
private readonly Func<IDecodeProcess> _processFactory;
private readonly IFrameRateProbe? _probe;
private readonly Func<TimeSpan, CancellationToken, Task> _delay;
private readonly CancellationTokenSource _cts = new();
private Task? _run;
private readonly VideoClipPipe _clip;
public AlertClipDecoder(
string path,
@@ -69,159 +52,16 @@ public sealed class AlertClipDecoder : IAlertClipDecoder
IFrameRateProbe? frameRateProbe = null,
Func<TimeSpan, CancellationToken, Task>? delay = null)
{
_path = path;
_width = width;
_height = height;
_locator = locator;
_processFactory = processFactory;
_probe = frameRateProbe;
_delay = delay ?? ((t, ct) => Task.Delay(t, ct));
_clip = new VideoClipPipe(
path, width, height, locator, processFactory, frameRateProbe, delay, audio: true);
_clip.FrameAvailable += frame => FrameAvailable?.Invoke(frame);
_clip.AudioAvailable += sample => AudioReady?.Invoke(sample);
_clip.Completed += () => Completed?.Invoke();
}
public void Start()
{
if (_run != null) return;
_run = Task.Run(RunAsync, CancellationToken.None);
}
public void Start() => _ = _clip.StartAsync();
public void Stop() => _cts.Cancel();
public void Stop() => _ = _clip.StopAsync();
public void Dispose()
{
_cts.Cancel();
_cts.Dispose();
}
private async Task RunAsync()
{
var ct = _cts.Token;
var videoFrames = 0;
var audioChunks = 0;
var failed = false;
try
{
double? frameRate = null;
if (_probe != null)
frameRate = await _probe.ProbeAsync(_path, ct).ConfigureAwait(false);
var frameDuration = frameRate is > 0 ? TimeSpan.FromSeconds(1.0 / frameRate.Value) : TimeSpan.Zero;
var ffmpegPath = await _locator.LocateAsync(ct).ConfigureAwait(false);
var videoTask = RunVideoAsync(ffmpegPath, frameDuration, ct);
var audioTask = RunAudioAsync(ffmpegPath, ct);
videoFrames = await videoTask.ConfigureAwait(false);
audioChunks = await audioTask.ConfigureAwait(false);
}
catch (OperationCanceledException)
{
failed = true;
return;
}
catch (Exception ex)
{
// A failed decode is not fatal: the alert layer falls back to the six
// animations (or the frame it already holds) exactly as a missing file.
// Logged so a silent blank box traces back to a decode failure.
failed = true;
AppLog.Write(ex, $"Alert clip decode failed ({Path.GetFileName(_path)})", AppLogErrorClass.Class2);
}
if (!ct.IsCancellationRequested)
{
AppLog.Write($"Alert clip end: {Path.GetFileName(_path)} frames={videoFrames} audioChunks={audioChunks} failed={failed}");
Completed?.Invoke();
}
}
private async Task<int> RunVideoAsync(string ffmpegPath, TimeSpan frameDuration, CancellationToken ct)
{
var args =
$"-hide_banner -loglevel error -i \"{_path}\" " +
$"-f rawvideo -pix_fmt bgra -vf scale={_width}:{_height} -an -";
using var process = _processFactory();
process.Start(new ProcessStartInfo
{
FileName = ffmpegPath,
Arguments = args,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true,
});
var reader = new RawVideoFrameReader(_width, _height);
var buffer = new byte[65536];
var stream = process.StandardOutput;
var frames = 0;
while (!ct.IsCancellationRequested && !process.HasExited)
{
var read = await stream.ReadAsync(buffer, 0, buffer.Length, ct).ConfigureAwait(false);
if (read <= 0) break;
var chunk = new byte[read];
Buffer.BlockCopy(buffer, 0, chunk, 0, read);
foreach (var frame in reader.Feed(chunk))
{
frames++;
FrameAvailable?.Invoke(frame);
if (frameDuration > TimeSpan.Zero)
await _delay(frameDuration, ct).ConfigureAwait(false);
}
}
return frames;
}
private async Task<int> RunAudioAsync(string ffmpegPath, CancellationToken ct)
{
var args =
$"-hide_banner -loglevel error -i \"{_path}\" " +
$"-f f32le -ac {AudioChannels} -ar {AudioSampleRate} -vn -";
using var process = _processFactory();
process.Start(new ProcessStartInfo
{
FileName = ffmpegPath,
Arguments = args,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true,
});
var stream = process.StandardOutput;
var raw = new byte[65536];
var carry = new byte[0];
var floatScratch = new float[AudioChunkFloats];
var floatIndex = 0;
var chunks = 0;
// The pipe is byte-delimited only: a read may split a float, so leftover
// bytes are carried into the next read instead of dropped (a dropped
// half-float would desync the entire stream).
while (!ct.IsCancellationRequested && !process.HasExited)
{
var read = await stream.ReadAsync(raw, 0, raw.Length, ct).ConfigureAwait(false);
if (read <= 0) break;
var total = carry.Length + read;
var bytes = total <= raw.Length ? raw : new byte[total];
if (carry.Length > 0)
{
Buffer.BlockCopy(carry, 0, bytes, 0, carry.Length);
Buffer.BlockCopy(raw, 0, bytes, carry.Length, read);
}
var floatsTo = total - (total % 4);
for (var i = 0; i < floatsTo && floatIndex < floatScratch.Length; i += 4)
floatScratch[floatIndex++] = BitConverter.ToSingle(bytes, i);
carry = total > floatsTo ? bytes[floatsTo..total] : new byte[0];
while (floatIndex == floatScratch.Length)
{
chunks++;
AudioReady?.Invoke(new AudioSample(floatScratch, AudioSampleRate, AudioChannels));
// Pace the NEXT chunk so the whole clip's audio lands at real-time
// cadence (the fade in the alert layer tracks the same wall clock).
await _delay(TimeSpan.FromSeconds(AudioChunkFloats / (double)(AudioSampleRate * AudioChannels)), ct).ConfigureAwait(false);
floatScratch = new float[AudioChunkFloats];
floatIndex = 0;
}
}
return chunks;
}
public void Dispose() => _clip.Dispose();
}
+19
View File
@@ -283,6 +283,25 @@ public sealed class AudioMixer : IDisposable
LoopbackLevelChanged?.Invoke(level);
}
/// <summary>Drops a MEDIA clip's audio into the desktop/game ring (TASK 21's audio slice).
/// It joins that group on purpose: the desktop fader and the mic auto-ducker then apply
/// exactly as they do to game audio and TRAX, so a BRB bed ducks under the creator's
/// voice without a second audio knob. The layer's own level/mute are applied upstream
/// by <see cref="MediaVideoSourceManager"/>.</summary>
public void EnqueueDesktopMediaAudio(AudioSample sample)
{
var data = sample.Samples;
if (data.Length == 0) return;
if (sample.SampleRate != OutputSampleRate)
data = ResampleLoopback(data, sample.SampleRate);
if (sample.Channels == 1)
data = UpmixToStereo(data);
_loopbackBuffer.Write(data);
var level = _loopbackMeter.Push(new AudioSample(data, OutputSampleRate, sample.Channels));
LoopbackLevelChanged?.Invoke(level);
}
/// <summary>Drops the alert clip's audio into its own ring (TASK 47 — the
/// stream-alert video's soundtrack, pre-scaled by the alert layer to
/// volume × fade). Mixed at unity into the live output — NO auto-duck: an
+7
View File
@@ -1,3 +1,5 @@
using ytLive.Services.Audio;
namespace ytLive.Services;
/// <summary>
@@ -17,6 +19,11 @@ public interface IMediaFrameSource : IDisposable
/// <summary>Raised (on a worker thread) for each decoded frame.</summary>
event Action<VideoFrame>? FrameAvailable;
/// <summary>Raised (on a worker thread) per chunk of the file's own soundtrack
/// (48 kHz stereo float). The manager applies the layer's level/mute and hands it
/// to the mixer; a source with no audio simply never raises it.</summary>
event Action<AudioSample>? AudioAvailable;
/// <summary>Raised when the file finishes decoding (natural EOF) — not raised
/// while <see cref="Looping"/> is true.</summary>
event Action? Completed;
+2 -1
View File
@@ -113,7 +113,7 @@ public partial class LayoutStore : IDisposable
AlertVideoAssetId, AlertVideoPath, AlertUseDefaultVideo, AlertShowTicker, AlertVideoVolume, SortOrder,
AlertDisplayMethod, IsBrandLogo, BlurAmount, AlertAudioPath, AlertAudioLoop,
TextContent, TextFontFamily, TextFontSize, TextColor, TextBold, TextItalic,
TextUnderline, TextAlignment, TextVerticalAlignment
TextUnderline, TextAlignment, TextVerticalAlignment, MediaMuted
FROM Source ORDER BY SortOrder
""";
using var reader = cmd.ExecuteReader();
@@ -183,6 +183,7 @@ public partial class LayoutStore : IDisposable
|| !Enum.IsDefined((TextLayerVerticalAlignment)reader.GetInt32(49))
? TextLayerVerticalAlignment.Top
: (TextLayerVerticalAlignment)reader.GetInt32(49),
MediaMuted = reader.GetInt32(reader.GetOrdinal("MediaMuted")) != 0,
};
if (!sourcesByScene.TryGetValue(sceneId, out var list))
sourcesByScene[sceneId] = list = new List<(Source, int)>();
+6
View File
@@ -378,6 +378,12 @@ public partial class LayoutStore : IDisposable
cmd.CommandText = "ALTER TABLE Source ADD COLUMN MediaPlaybackState TEXT NOT NULL DEFAULT 'Stopped';";
cmd.ExecuteNonQuery();
}
if (!columns.Contains("MediaMuted"))
{
using var cmd = _connection.CreateCommand();
cmd.CommandText = "ALTER TABLE Source ADD COLUMN MediaMuted INTEGER NOT NULL DEFAULT 0;";
cmd.ExecuteNonQuery();
}
// TASK 47: alert video config. The default clip asset id + the creator's
// replacement path (a FILE reference, never a stored BLOB) + the pill,
+4 -2
View File
@@ -94,7 +94,7 @@ public partial class LayoutStore : IDisposable
AlertVideoAssetId, AlertVideoPath, AlertUseDefaultVideo, AlertShowTicker, AlertVideoVolume,
AlertDisplayMethod, IsBrandLogo, BlurAmount, AlertAudioPath, AlertAudioLoop,
TextContent, TextFontFamily, TextFontSize, TextColor, TextBold, TextItalic,
TextUnderline, TextAlignment, TextVerticalAlignment)
TextUnderline, TextAlignment, TextVerticalAlignment, MediaMuted)
VALUES ($id, $sceneId, $assetId, $type, $name, $isEnabled,
$x, $y, $w, $h, $opacity, $monitor,
$clip, $mirrored, $isBackground, $captureKey, $showDesktop, $sort,
@@ -104,7 +104,7 @@ public partial class LayoutStore : IDisposable
$alertVideoAssetId, $alertVideoPath, $alertUseDefaultVideo, $alertShowTicker, $alertVideoVolume,
$alertDisplayMethod, $isBrandLogo, $blurAmount, $alertAudioPath, $alertAudioLoop,
$textContent, $textFontFamily, $textFontSize, $textColor, $textBold, $textItalic,
$textUnderline, $textAlignment, $textVerticalAlignment)
$textUnderline, $textAlignment, $textVerticalAlignment, $mediaMuted)
""";
cmd.Transaction = tx;
var idP = cmd.Parameters.Add("$id", SqliteType.Text);
@@ -137,6 +137,7 @@ public partial class LayoutStore : IDisposable
var mediaPathP = cmd.Parameters.Add("$mediaPath", SqliteType.Text);
var mediaIsLoopingP = cmd.Parameters.Add("$mediaIsLooping", SqliteType.Integer);
var mediaVolumeP = cmd.Parameters.Add("$mediaVolume", SqliteType.Real);
var mediaMutedP = cmd.Parameters.Add("$mediaMuted", SqliteType.Integer);
var mediaPlaybackStateP = cmd.Parameters.Add("$mediaPlaybackState", SqliteType.Text);
var alertVideoAssetIdP = cmd.Parameters.Add("$alertVideoAssetId", SqliteType.Text);
var alertVideoPathP = cmd.Parameters.Add("$alertVideoPath", SqliteType.Text);
@@ -198,6 +199,7 @@ public partial class LayoutStore : IDisposable
mediaPathP.Value = (object?)source.MediaPath ?? DBNull.Value;
mediaIsLoopingP.Value = source.MediaIsLooping ? 1 : 0;
mediaVolumeP.Value = source.MediaVolume;
mediaMutedP.Value = source.MediaMuted ? 1 : 0;
mediaPlaybackStateP.Value = source.MediaPlaybackState.ToString();
alertVideoAssetIdP.Value = (object?)source.AlertVideoAssetId ?? DBNull.Value;
alertVideoPathP.Value = (object?)source.AlertVideoPath ?? DBNull.Value;
+21 -115
View File
@@ -1,38 +1,32 @@
using System.Diagnostics;
using System.IO;
using ytLive.Services.Audio;
using ytLive.Services.Encoder;
namespace ytLive.Services;
/// <summary>
/// TASK 21: decodes a local media file (any format ffmpeg handles) into
/// <see cref="VideoFrame"/>s for a MediaSource. Implements
/// <see cref="IMediaFrameSource"/>: raises <see cref="FrameAvailable"/> per
/// frame and <see cref="IMediaFrameSource.Completed"/> on natural EOF (unless
/// <see cref="IMediaFrameSource.Looping"/>). Spawns ffmpeg with
/// <c>-f rawvideo -pix_fmt bgra</c> and drains the raw BGRA pipe via
/// <see cref="RawVideoFrameReader"/>. The decoder process and emitters are seams
/// so tests run without a real codec or audio device.
/// <see cref="VideoFrame"/>s for a MediaSource. This is the <see cref="IMediaFrameSource"/>
/// shape over the shared <see cref="VideoClipPipe"/> — the same pipe an alert clip uses
/// (TASK 47), so the two features cannot drift apart again in how they decode.
/// <para>Video AND the file's own soundtrack are decoded. The manager scales the audio by
/// the layer's level/mute and hands it to the mixer's desktop group.</para>
/// </summary>
public sealed class MediaVideoSource : IMediaFrameSource
{
public event Action<VideoFrame>? FrameAvailable;
public event Action<AudioSample>? AudioAvailable;
public event Action? Completed;
public string Key => _path;
public bool Looping { get; set; }
public bool Looping
{
get => _clip.Looping;
set => _clip.Looping = value;
}
private readonly string _path;
private readonly int _width;
private readonly int _height;
private readonly IFfmpegLocator _locator;
private readonly Func<IDecodeProcess> _processFactory;
private readonly IFrameRateProbe? _probe;
private readonly Func<TimeSpan, CancellationToken, Task> _delay;
private readonly CancellationTokenSource _cts = new();
private IDisposable? _run;
private IDecodeProcess? _active;
private readonly VideoClipPipe _clip;
public MediaVideoSource(
string path,
@@ -44,104 +38,16 @@ public sealed class MediaVideoSource : IMediaFrameSource
Func<TimeSpan, CancellationToken, Task>? delay = null)
{
_path = path;
_width = width;
_height = height;
_locator = locator;
_processFactory = processFactory;
_probe = frameRateProbe;
_delay = delay ?? ((t, ct) => Task.Delay(t, ct));
_clip = new VideoClipPipe(
path, width, height, locator, processFactory, frameRateProbe, delay, audio: true);
_clip.FrameAvailable += frame => FrameAvailable?.Invoke(frame);
_clip.AudioAvailable += sample => AudioAvailable?.Invoke(sample);
_clip.Completed += () => Completed?.Invoke();
}
public Task StartAsync(CancellationToken cancellationToken = default)
{
if (_run != null) return Task.CompletedTask;
_run = _cts.Token.Register(StopInternal);
_ = Task.Run(RunAsync, CancellationToken.None);
return Task.CompletedTask;
}
public Task StartAsync(CancellationToken cancellationToken = default) => _clip.StartAsync(cancellationToken);
public Task StopAsync()
{
_cts.Cancel();
_run = null;
return Task.CompletedTask;
}
public Task StopAsync() => _clip.StopAsync();
public void Dispose()
{
_cts.Cancel();
_cts.Dispose();
_active?.Dispose();
}
private void StopInternal() => KillProcess();
private async Task RunAsync()
{
try
{
double? frameRate = null;
if (_probe != null)
frameRate = await _probe.ProbeAsync(_path, _cts.Token).ConfigureAwait(false);
var frameDuration = frameRate is > 0 ? TimeSpan.FromSeconds(1.0 / frameRate.Value) : TimeSpan.Zero;
var ffmpegPath = await _locator.LocateAsync(_cts.Token).ConfigureAwait(false);
var args =
$"-hide_banner -loglevel error -i \"{_path}\" " +
$"-f rawvideo -pix_fmt bgra -vf scale={_width}:{_height} -an -";
var startInfo = new ProcessStartInfo
{
FileName = ffmpegPath,
Arguments = args,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true,
};
do
{
using (var process = _processFactory())
{
_active = process;
process.Start(startInfo);
var reader = new RawVideoFrameReader(_width, _height);
var buffer = new byte[65536];
var stream = process.StandardOutput;
while (!_cts.IsCancellationRequested && !process.HasExited)
{
var read = await stream.ReadAsync(buffer, 0, buffer.Length, _cts.Token).ConfigureAwait(false);
if (read <= 0) break;
foreach (var frame in reader.Feed(buffer.AsSpan(0, read).ToArray()))
{
FrameAvailable?.Invoke(frame);
if (frameDuration > TimeSpan.Zero)
await _delay(frameDuration, _cts.Token).ConfigureAwait(false);
}
}
}
_active = null;
}
while (Looping && !_cts.IsCancellationRequested);
}
catch (OperationCanceledException)
{
}
catch (Exception ex)
{
Fail($"Media decode failed: {ex.Message}");
return;
}
if (!_cts.IsCancellationRequested)
Completed?.Invoke();
}
private void Fail(string message) => Console.Error.WriteLine(message);
private void KillProcess()
{
try { _active?.Kill(); } catch { /* already exited */ }
}
public void Dispose() => _clip.Dispose();
}
+54
View File
@@ -6,6 +6,7 @@ using System.Windows.Media;
using System.Windows.Media.Imaging;
using System.Windows.Threading;
using ytLive.Helpers;
using ytLive.Services.Audio;
namespace ytLive.Services;
@@ -23,8 +24,11 @@ public sealed class MediaVideoSourceManager : IDisposable
public string Key { get; }
public IMediaFrameSource Source { get; }
public Action<VideoFrame>? FrameHandler;
public Action<AudioSample>? AudioHandler;
public int RefCount;
public bool Started;
public bool AudioMuted;
public double AudioVolume = 1.0;
public WriteableBitmap? PreviewBitmap;
public VideoFrame? LatestFrame;
public bool FramePending;
@@ -49,6 +53,10 @@ public sealed class MediaVideoSourceManager : IDisposable
/// <summary>Raised when a media decoder cannot be created (bad path).</summary>
public event Action<string, string>? MediaFailed;
/// <summary>Raised (worker thread) per audio chunk of a file's own soundtrack, already
/// scaled by the session's level and dropped when muted.</summary>
public event Action<string, AudioSample>? AudioAvailable;
public MediaVideoSourceManager(
Func<string, IMediaFrameSource?> sourceFactory,
Dispatcher? uiDispatcher = null,
@@ -95,6 +103,8 @@ public sealed class MediaVideoSourceManager : IDisposable
session = new MediaSession(key, source);
session.FrameHandler = frame => OnFrameAvailable(session, frame);
session.Source.FrameAvailable += session.FrameHandler;
session.AudioHandler = sample => OnAudioAvailable(session, sample);
session.Source.AudioAvailable += session.AudioHandler;
// Loop intent lives on the Source (per scene), so the caller supplies the
// per-path answer ("any live reference loops"). Applied before Start so a
// zero-length or instantly-EOF file cannot slip past a later assignment.
@@ -117,6 +127,7 @@ public sealed class MediaVideoSourceManager : IDisposable
lock (_gate)
_sessions.Remove(key);
session.Source.FrameAvailable -= session.FrameHandler;
if (session.AudioHandler != null) session.Source.AudioAvailable -= session.AudioHandler;
AppLog.Write($"MediaSource: failed to start decoder '{key}': {ex.Message}");
MediaFailed?.Invoke(key, ex.Message);
await SafeStopAsync(session.Source);
@@ -137,6 +148,7 @@ public sealed class MediaVideoSourceManager : IDisposable
if (toStop == null) return;
toStop.Source.FrameAvailable -= toStop.FrameHandler;
if (toStop.AudioHandler != null) toStop.Source.AudioAvailable -= toStop.AudioHandler;
await SafeStopAsync(toStop.Source);
toStop.PreviewBitmap = null;
}
@@ -155,6 +167,7 @@ public sealed class MediaVideoSourceManager : IDisposable
if (toStop == null) return;
toStop.Source.FrameAvailable -= toStop.FrameHandler;
if (toStop.AudioHandler != null) toStop.Source.AudioAvailable -= toStop.AudioHandler;
await SafeStopAsync(toStop.Source);
toStop.PreviewBitmap = null;
}
@@ -193,6 +206,46 @@ public sealed class MediaVideoSourceManager : IDisposable
}
}
/// <summary>The layer's sound controls for this file. One decoder is shared per path,
/// so the level/mute are per SESSION: whichever layer the creator last touched wins —
/// the alternative (one decoder per layer) would play the same file twice.</summary>
public void SetAudioSettings(string key, double volume, bool muted)
{
lock (_gate)
{
if (!_sessions.TryGetValue(key, out var session)) return;
session.AudioVolume = Math.Clamp(volume, 0, 1);
session.AudioMuted = muted;
}
}
private void OnAudioAvailable(MediaSession session, AudioSample sample)
{
if (!TryGetActiveSession(session)) return;
double volume;
bool muted;
lock (_gate)
{
volume = session.AudioVolume;
muted = session.AudioMuted;
}
if (muted || volume <= 0) return;
if (volume < 0.999)
{
// Copy before scaling: the source owns its buffer and may recycle it.
var scaled = new float[sample.Samples.Length];
var gain = (float)volume;
for (var i = 0; i < scaled.Length; i++)
scaled[i] = sample.Samples[i] * gain;
AudioAvailable?.Invoke(session.Key, new AudioSample(scaled, sample.SampleRate, sample.Channels));
return;
}
AudioAvailable?.Invoke(session.Key, sample);
}
/// <summary>The shared preview bitmap for a file (so a config created after the
/// first frame can adopt the running bitmap), or null.</summary>
public WriteableBitmap? GetPreviewBitmap(string key)
@@ -213,6 +266,7 @@ public sealed class MediaVideoSourceManager : IDisposable
foreach (var session in sessions)
{
session.Source.FrameAvailable -= session.FrameHandler;
if (session.AudioHandler != null) session.Source.AudioAvailable -= session.AudioHandler;
_ = SafeStopAsync(session.Source);
}
}
+248
View File
@@ -0,0 +1,248 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using ytLive.Helpers;
using ytLive.Services.Audio;
using ytLive.Services.Encoder;
namespace ytLive.Services;
/// <summary>
/// One video FILE played through ffmpeg — the shared pipe behind every clip, so a media
/// layer (TASK 21) and an alert clip (TASK 47) decode identically. Owns two subprocesses
/// on the same file: video (<c>rawvideo bgra</c>, scaled to the target rect) and audio
/// (<c>f32le -ac 2 -ar 48000</c>, matching the mixer's native format so no resampler is
/// needed). Both are paced to real time, so the picture and the sound stay together even
/// though they arrive on separate pipes.
/// <para><b>Looping is the media case</b>: on natural EOF the whole pass restarts — a fresh
/// pair of processes, because a <see cref="Process"/> cannot be re-started. An alert clip
/// never loops, so its wrapper simply never sets <see cref="Looping"/>.</para>
/// <para>The seams (locator, process factory, frame-rate probe, delay) exist so tests drive
/// a fake decoder with no ffmpeg, no codec, and no audio device.</para>
/// </summary>
public sealed class VideoClipPipe : IDisposable
{
public const int AudioSampleRate = 48000;
public const int AudioChannels = 2;
private const int AudioChunkFloats = 4800; // 50 ms of stereo
public event Action<VideoFrame>? FrameAvailable;
public event Action<AudioSample>? AudioAvailable;
public event Action? Completed;
/// <summary>Restart the pass on natural EOF (media/BRB loops).</summary>
public bool Looping { get; set; }
private readonly string _path;
private readonly int _width;
private readonly int _height;
private readonly IFfmpegLocator _locator;
private readonly Func<IDecodeProcess> _processFactory;
private readonly IFrameRateProbe? _probe;
private readonly Func<TimeSpan, CancellationToken, Task> _delay;
private readonly bool _audio;
private readonly CancellationTokenSource _cts = new();
private IDisposable? _run;
private IDecodeProcess? _activeVideo;
private IDecodeProcess? _activeAudio;
public VideoClipPipe(
string path,
int width,
int height,
IFfmpegLocator locator,
Func<IDecodeProcess> processFactory,
IFrameRateProbe? frameRateProbe = null,
Func<TimeSpan, CancellationToken, Task>? delay = null,
bool audio = true)
{
_path = path;
_width = width;
_height = height;
_locator = locator;
_processFactory = processFactory;
_probe = frameRateProbe;
_delay = delay ?? ((t, ct) => Task.Delay(t, ct));
_audio = audio;
}
public Task StartAsync(CancellationToken cancellationToken = default)
{
if (_run != null) return Task.CompletedTask;
_run = _cts.Token.Register(StopInternal);
_ = Task.Run(RunAsync, CancellationToken.None);
return Task.CompletedTask;
}
public Task StopAsync()
{
_cts.Cancel();
_run = null;
return Task.CompletedTask;
}
public void Dispose()
{
_cts.Cancel();
_cts.Dispose();
_activeVideo?.Dispose();
_activeAudio?.Dispose();
}
private void StopInternal()
{
Kill(_activeVideo);
Kill(_activeAudio);
}
private async Task RunAsync()
{
var ct = _cts.Token;
var frames = 0;
var chunks = 0;
try
{
double? frameRate = null;
if (_probe != null)
frameRate = await _probe.ProbeAsync(_path, ct).ConfigureAwait(false);
var frameDuration = frameRate is > 0 ? TimeSpan.FromSeconds(1.0 / frameRate.Value) : TimeSpan.Zero;
var ffmpegPath = await _locator.LocateAsync(ct).ConfigureAwait(false);
do
{
// Both pipes per pass, awaited together: a restart must begin a new
// video AND a new audio stream, or the picture drifts from the sound
// after the first loop.
var video = RunVideoAsync(ffmpegPath, frameDuration, ct);
var audio = _audio ? RunAudioAsync(ffmpegPath, ct) : Task.FromResult(0);
frames += await video.ConfigureAwait(false);
chunks += await audio.ConfigureAwait(false);
}
while (Looping && !ct.IsCancellationRequested);
}
catch (OperationCanceledException)
{
return;
}
catch (Exception ex)
{
// A failed decode is not fatal: the alert layer falls back to its animations,
// and a media layer just holds its last frame.
AppLog.Write($"Video clip decode failed ({Path.GetFileName(_path)}): {ex.Message}");
return;
}
if (!ct.IsCancellationRequested)
{
AppLog.Write($"Video clip end: {Path.GetFileName(_path)} frames={frames} audioChunks={chunks}");
Completed?.Invoke();
}
}
private async Task<int> RunVideoAsync(string ffmpegPath, TimeSpan frameDuration, CancellationToken ct)
{
var args =
$"-hide_banner -loglevel error -i \"{_path}\" " +
$"-f rawvideo -pix_fmt bgra -vf scale={_width}:{_height} -an -";
using var process = _processFactory();
_activeVideo = process;
process.Start(new ProcessStartInfo
{
FileName = ffmpegPath,
Arguments = args,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true,
});
var reader = new RawVideoFrameReader(_width, _height);
var buffer = new byte[65536];
var stream = process.StandardOutput;
var frames = 0;
while (!ct.IsCancellationRequested && !process.HasExited)
{
var read = await stream.ReadAsync(buffer, 0, buffer.Length, ct).ConfigureAwait(false);
if (read <= 0) break;
var chunk = new byte[read];
Buffer.BlockCopy(buffer, 0, chunk, 0, read);
foreach (var frame in reader.Feed(chunk))
{
frames++;
FrameAvailable?.Invoke(frame);
if (frameDuration > TimeSpan.Zero)
await _delay(frameDuration, ct).ConfigureAwait(false);
}
}
_activeVideo = null;
return frames;
}
private async Task<int> RunAudioAsync(string ffmpegPath, CancellationToken ct)
{
var args =
$"-hide_banner -loglevel error -i \"{_path}\" " +
$"-f f32le -ac {AudioChannels} -ar {AudioSampleRate} -vn -";
using var process = _processFactory();
_activeAudio = process;
process.Start(new ProcessStartInfo
{
FileName = ffmpegPath,
Arguments = args,
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true,
});
var stream = process.StandardOutput;
var raw = new byte[65536];
var carry = Array.Empty<byte>();
var floatScratch = new float[AudioChunkFloats];
var floatIndex = 0;
var chunks = 0;
// The pipe is byte-delimited only: a read may split a float, so leftover
// bytes are carried into the next read instead of dropped (a dropped
// half-float would desync the entire stream).
while (!ct.IsCancellationRequested && !process.HasExited)
{
var read = await stream.ReadAsync(raw, 0, raw.Length, ct).ConfigureAwait(false);
if (read <= 0) break;
var total = carry.Length + read;
var bytes = total <= raw.Length ? raw : new byte[total];
if (carry.Length > 0)
{
Buffer.BlockCopy(carry, 0, bytes, 0, carry.Length);
Buffer.BlockCopy(raw, 0, bytes, carry.Length, read);
}
var floatsTo = total - (total % 4);
for (var i = 0; i < floatsTo && floatIndex < floatScratch.Length; i += 4)
floatScratch[floatIndex++] = BitConverter.ToSingle(bytes, i);
carry = total > floatsTo ? bytes[floatsTo..total] : Array.Empty<byte>();
while (floatIndex == floatScratch.Length)
{
chunks++;
AudioAvailable?.Invoke(new AudioSample(floatScratch, AudioSampleRate, AudioChannels));
// Pace the NEXT chunk so the clip's audio lands at real-time cadence,
// matching how the video pipe is paced by frame rate.
await _delay(
TimeSpan.FromSeconds(AudioChunkFloats / (double)(AudioSampleRate * AudioChannels)), ct)
.ConfigureAwait(false);
floatScratch = new float[AudioChunkFloats];
floatIndex = 0;
}
}
_activeAudio = null;
return chunks;
}
private static void Kill(IDecodeProcess? process)
{
try { process?.Kill(); } catch { /* already exited */ }
}
}