feat(media): play the clip's soundtrack and share one decode pipe
A video file was picture-only: the decode pipe passed `-an`, so a starting-soon or BRB clip with music played silent everywhere. Adds sound to every clip and collapses the two decoders into one. - `Services/VideoClipPipe.cs`: the shared pipe behind `MediaVideoSource` (TASK 21) and `AlertClipDecoder` (TASK 47) — video (`rawvideo bgra`, scaled) + audio (`f32le -ac 2 -ar 48000`, 50 ms chunks, byte-carry across reads), both paced to real time, loop restarts both so picture and sound stay together. Alert behaviour is unchanged. - `MediaVideoSourceManager` applies the layer's Level/Mute per session and raises `AudioAvailable`; `MainViewModel` feeds it to the new `AudioMixer.EnqueueDesktopMediaAudio`, which joins the DESKTOP/GAME ring so the desktop fader and the mic auto-ducker apply exactly as they do for game audio and TRAX (creator ruling 2026-11-05: audio joins the group). - Per-layer `MediaMuted` + existing `MediaVolume` surfaced as Mute/Level in the properties panel; `MediaMuted` persisted (ALTER TABLE guard + SELECT/INSERT). - Level/mute are per SESSION because one decoder is shared per file path — the last layer touched wins; one decoder per layer would play the file twice. Test: `MediaSourceAudioTests` — the pipe emits 50 ms chunks at 48k stereo, the manager scales by level and drops when muted, and a media layer's sound moves the desktop meter. Alert/decode suites unchanged and green. Docs: task-21 closed on item 10, the "silent soundtrack" facts corrected in frame-pipeline/output/overlays, HANDOFF rewritten. verify.sh green (529 passed; the two documented pre-existing failures only).
This commit is contained in:
+16
-176
@@ -1,6 +1,5 @@
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using ytLive.Helpers;
|
||||
using System;
|
||||
using System.Threading;
|
||||
using ytLive.Services.Audio;
|
||||
using ytLive.Services.Encoder;
|
||||
|
||||
@@ -11,10 +10,10 @@ namespace ytLive.Services;
|
||||
/// same file, both paced to real time. The alert box owns a decoder per event
|
||||
/// (start at play, dispose at drain) — unlike MediaSource's app-wide refcounted
|
||||
/// session, an alert REPLAYS the same clip each event, so a fresh decode per
|
||||
/// play is required. Video is rawvideo BGRA scaled to the box rect (the
|
||||
/// <see cref="MediaVideoSource"/> shape); audio is ffmpeg <c>f32le</c> at the
|
||||
/// mixer's native 48 kHz stereo (so the mixer ring needs no resampler). The
|
||||
/// fade is applied by the alert layer, not here.
|
||||
/// play is required. The fade is applied by the alert layer, not here.
|
||||
/// <para>Since TASK 21's audio slice this is a thin wrapper over the shared
|
||||
/// <see cref="VideoClipPipe"/> (the same pipe a media layer uses); the alert case
|
||||
/// simply never sets <c>Looping</c>.</para>
|
||||
/// </summary>
|
||||
public interface IAlertClipDecoder : IDisposable
|
||||
{
|
||||
@@ -35,30 +34,14 @@ public interface IAlertClipDecoder : IDisposable
|
||||
event Action? Completed;
|
||||
}
|
||||
|
||||
/// <summary>Default <see cref="IAlertClipDecoder"/>: two ffmpeg subprocesses on
|
||||
/// the same file. Video via <see cref="RawVideoFrameReader"/> on a <c>rawvideo
|
||||
/// bgra</c> pipe (-an), audio via an <c>f32le -ac 2 -ar 48000</c> pipe (-vn).
|
||||
/// The seams (locator, process factory, frame-rate probe, delay) let tests drive
|
||||
/// a fake decoder without a real codec or Windows.</summary>
|
||||
/// <summary>Default <see cref="IAlertClipDecoder"/>: the shared clip pipe with looping off.</summary>
|
||||
public sealed class AlertClipDecoder : IAlertClipDecoder
|
||||
{
|
||||
public event Action<VideoFrame>? FrameAvailable;
|
||||
public event Action<AudioSample>? AudioReady;
|
||||
public event Action? Completed;
|
||||
|
||||
private const int AudioSampleRate = 48000;
|
||||
private const int AudioChannels = 2;
|
||||
private const int AudioChunkFloats = 4800; // 50ms of stereo
|
||||
|
||||
private readonly string _path;
|
||||
private readonly int _width;
|
||||
private readonly int _height;
|
||||
private readonly IFfmpegLocator _locator;
|
||||
private readonly Func<IDecodeProcess> _processFactory;
|
||||
private readonly IFrameRateProbe? _probe;
|
||||
private readonly Func<TimeSpan, CancellationToken, Task> _delay;
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private Task? _run;
|
||||
private readonly VideoClipPipe _clip;
|
||||
|
||||
public AlertClipDecoder(
|
||||
string path,
|
||||
@@ -69,159 +52,16 @@ public sealed class AlertClipDecoder : IAlertClipDecoder
|
||||
IFrameRateProbe? frameRateProbe = null,
|
||||
Func<TimeSpan, CancellationToken, Task>? delay = null)
|
||||
{
|
||||
_path = path;
|
||||
_width = width;
|
||||
_height = height;
|
||||
_locator = locator;
|
||||
_processFactory = processFactory;
|
||||
_probe = frameRateProbe;
|
||||
_delay = delay ?? ((t, ct) => Task.Delay(t, ct));
|
||||
_clip = new VideoClipPipe(
|
||||
path, width, height, locator, processFactory, frameRateProbe, delay, audio: true);
|
||||
_clip.FrameAvailable += frame => FrameAvailable?.Invoke(frame);
|
||||
_clip.AudioAvailable += sample => AudioReady?.Invoke(sample);
|
||||
_clip.Completed += () => Completed?.Invoke();
|
||||
}
|
||||
|
||||
public void Start()
|
||||
{
|
||||
if (_run != null) return;
|
||||
_run = Task.Run(RunAsync, CancellationToken.None);
|
||||
}
|
||||
public void Start() => _ = _clip.StartAsync();
|
||||
|
||||
public void Stop() => _cts.Cancel();
|
||||
public void Stop() => _ = _clip.StopAsync();
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_cts.Cancel();
|
||||
_cts.Dispose();
|
||||
}
|
||||
|
||||
private async Task RunAsync()
|
||||
{
|
||||
var ct = _cts.Token;
|
||||
var videoFrames = 0;
|
||||
var audioChunks = 0;
|
||||
var failed = false;
|
||||
try
|
||||
{
|
||||
double? frameRate = null;
|
||||
if (_probe != null)
|
||||
frameRate = await _probe.ProbeAsync(_path, ct).ConfigureAwait(false);
|
||||
var frameDuration = frameRate is > 0 ? TimeSpan.FromSeconds(1.0 / frameRate.Value) : TimeSpan.Zero;
|
||||
|
||||
var ffmpegPath = await _locator.LocateAsync(ct).ConfigureAwait(false);
|
||||
|
||||
var videoTask = RunVideoAsync(ffmpegPath, frameDuration, ct);
|
||||
var audioTask = RunAudioAsync(ffmpegPath, ct);
|
||||
videoFrames = await videoTask.ConfigureAwait(false);
|
||||
audioChunks = await audioTask.ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
failed = true;
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// A failed decode is not fatal: the alert layer falls back to the six
|
||||
// animations (or the frame it already holds) exactly as a missing file.
|
||||
// Logged so a silent blank box traces back to a decode failure.
|
||||
failed = true;
|
||||
AppLog.Write(ex, $"Alert clip decode failed ({Path.GetFileName(_path)})", AppLogErrorClass.Class2);
|
||||
}
|
||||
|
||||
if (!ct.IsCancellationRequested)
|
||||
{
|
||||
AppLog.Write($"Alert clip end: {Path.GetFileName(_path)} frames={videoFrames} audioChunks={audioChunks} failed={failed}");
|
||||
Completed?.Invoke();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<int> RunVideoAsync(string ffmpegPath, TimeSpan frameDuration, CancellationToken ct)
|
||||
{
|
||||
var args =
|
||||
$"-hide_banner -loglevel error -i \"{_path}\" " +
|
||||
$"-f rawvideo -pix_fmt bgra -vf scale={_width}:{_height} -an -";
|
||||
using var process = _processFactory();
|
||||
process.Start(new ProcessStartInfo
|
||||
{
|
||||
FileName = ffmpegPath,
|
||||
Arguments = args,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true,
|
||||
});
|
||||
|
||||
var reader = new RawVideoFrameReader(_width, _height);
|
||||
var buffer = new byte[65536];
|
||||
var stream = process.StandardOutput;
|
||||
var frames = 0;
|
||||
while (!ct.IsCancellationRequested && !process.HasExited)
|
||||
{
|
||||
var read = await stream.ReadAsync(buffer, 0, buffer.Length, ct).ConfigureAwait(false);
|
||||
if (read <= 0) break;
|
||||
var chunk = new byte[read];
|
||||
Buffer.BlockCopy(buffer, 0, chunk, 0, read);
|
||||
foreach (var frame in reader.Feed(chunk))
|
||||
{
|
||||
frames++;
|
||||
FrameAvailable?.Invoke(frame);
|
||||
if (frameDuration > TimeSpan.Zero)
|
||||
await _delay(frameDuration, ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
return frames;
|
||||
}
|
||||
|
||||
private async Task<int> RunAudioAsync(string ffmpegPath, CancellationToken ct)
|
||||
{
|
||||
var args =
|
||||
$"-hide_banner -loglevel error -i \"{_path}\" " +
|
||||
$"-f f32le -ac {AudioChannels} -ar {AudioSampleRate} -vn -";
|
||||
using var process = _processFactory();
|
||||
process.Start(new ProcessStartInfo
|
||||
{
|
||||
FileName = ffmpegPath,
|
||||
Arguments = args,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true,
|
||||
});
|
||||
|
||||
var stream = process.StandardOutput;
|
||||
var raw = new byte[65536];
|
||||
var carry = new byte[0];
|
||||
var floatScratch = new float[AudioChunkFloats];
|
||||
var floatIndex = 0;
|
||||
var chunks = 0;
|
||||
// The pipe is byte-delimited only: a read may split a float, so leftover
|
||||
// bytes are carried into the next read instead of dropped (a dropped
|
||||
// half-float would desync the entire stream).
|
||||
while (!ct.IsCancellationRequested && !process.HasExited)
|
||||
{
|
||||
var read = await stream.ReadAsync(raw, 0, raw.Length, ct).ConfigureAwait(false);
|
||||
if (read <= 0) break;
|
||||
var total = carry.Length + read;
|
||||
var bytes = total <= raw.Length ? raw : new byte[total];
|
||||
if (carry.Length > 0)
|
||||
{
|
||||
Buffer.BlockCopy(carry, 0, bytes, 0, carry.Length);
|
||||
Buffer.BlockCopy(raw, 0, bytes, carry.Length, read);
|
||||
}
|
||||
var floatsTo = total - (total % 4);
|
||||
for (var i = 0; i < floatsTo && floatIndex < floatScratch.Length; i += 4)
|
||||
floatScratch[floatIndex++] = BitConverter.ToSingle(bytes, i);
|
||||
carry = total > floatsTo ? bytes[floatsTo..total] : new byte[0];
|
||||
|
||||
while (floatIndex == floatScratch.Length)
|
||||
{
|
||||
chunks++;
|
||||
AudioReady?.Invoke(new AudioSample(floatScratch, AudioSampleRate, AudioChannels));
|
||||
// Pace the NEXT chunk so the whole clip's audio lands at real-time
|
||||
// cadence (the fade in the alert layer tracks the same wall clock).
|
||||
await _delay(TimeSpan.FromSeconds(AudioChunkFloats / (double)(AudioSampleRate * AudioChannels)), ct).ConfigureAwait(false);
|
||||
floatScratch = new float[AudioChunkFloats];
|
||||
floatIndex = 0;
|
||||
}
|
||||
}
|
||||
return chunks;
|
||||
}
|
||||
public void Dispose() => _clip.Dispose();
|
||||
}
|
||||
|
||||
@@ -283,6 +283,25 @@ public sealed class AudioMixer : IDisposable
|
||||
LoopbackLevelChanged?.Invoke(level);
|
||||
}
|
||||
|
||||
/// <summary>Drops a MEDIA clip's audio into the desktop/game ring (TASK 21's audio slice).
|
||||
/// It joins that group on purpose: the desktop fader and the mic auto-ducker then apply
|
||||
/// exactly as they do to game audio and TRAX, so a BRB bed ducks under the creator's
|
||||
/// voice without a second audio knob. The layer's own level/mute are applied upstream
|
||||
/// by <see cref="MediaVideoSourceManager"/>.</summary>
|
||||
public void EnqueueDesktopMediaAudio(AudioSample sample)
|
||||
{
|
||||
var data = sample.Samples;
|
||||
if (data.Length == 0) return;
|
||||
if (sample.SampleRate != OutputSampleRate)
|
||||
data = ResampleLoopback(data, sample.SampleRate);
|
||||
if (sample.Channels == 1)
|
||||
data = UpmixToStereo(data);
|
||||
_loopbackBuffer.Write(data);
|
||||
|
||||
var level = _loopbackMeter.Push(new AudioSample(data, OutputSampleRate, sample.Channels));
|
||||
LoopbackLevelChanged?.Invoke(level);
|
||||
}
|
||||
|
||||
/// <summary>Drops the alert clip's audio into its own ring (TASK 47 — the
|
||||
/// stream-alert video's soundtrack, pre-scaled by the alert layer to
|
||||
/// volume × fade). Mixed at unity into the live output — NO auto-duck: an
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
using ytLive.Services.Audio;
|
||||
|
||||
namespace ytLive.Services;
|
||||
|
||||
/// <summary>
|
||||
@@ -17,6 +19,11 @@ public interface IMediaFrameSource : IDisposable
|
||||
/// <summary>Raised (on a worker thread) for each decoded frame.</summary>
|
||||
event Action<VideoFrame>? FrameAvailable;
|
||||
|
||||
/// <summary>Raised (on a worker thread) per chunk of the file's own soundtrack
|
||||
/// (48 kHz stereo float). The manager applies the layer's level/mute and hands it
|
||||
/// to the mixer; a source with no audio simply never raises it.</summary>
|
||||
event Action<AudioSample>? AudioAvailable;
|
||||
|
||||
/// <summary>Raised when the file finishes decoding (natural EOF) — not raised
|
||||
/// while <see cref="Looping"/> is true.</summary>
|
||||
event Action? Completed;
|
||||
|
||||
@@ -113,7 +113,7 @@ public partial class LayoutStore : IDisposable
|
||||
AlertVideoAssetId, AlertVideoPath, AlertUseDefaultVideo, AlertShowTicker, AlertVideoVolume, SortOrder,
|
||||
AlertDisplayMethod, IsBrandLogo, BlurAmount, AlertAudioPath, AlertAudioLoop,
|
||||
TextContent, TextFontFamily, TextFontSize, TextColor, TextBold, TextItalic,
|
||||
TextUnderline, TextAlignment, TextVerticalAlignment
|
||||
TextUnderline, TextAlignment, TextVerticalAlignment, MediaMuted
|
||||
FROM Source ORDER BY SortOrder
|
||||
""";
|
||||
using var reader = cmd.ExecuteReader();
|
||||
@@ -183,6 +183,7 @@ public partial class LayoutStore : IDisposable
|
||||
|| !Enum.IsDefined((TextLayerVerticalAlignment)reader.GetInt32(49))
|
||||
? TextLayerVerticalAlignment.Top
|
||||
: (TextLayerVerticalAlignment)reader.GetInt32(49),
|
||||
MediaMuted = reader.GetInt32(reader.GetOrdinal("MediaMuted")) != 0,
|
||||
};
|
||||
if (!sourcesByScene.TryGetValue(sceneId, out var list))
|
||||
sourcesByScene[sceneId] = list = new List<(Source, int)>();
|
||||
|
||||
@@ -378,6 +378,12 @@ public partial class LayoutStore : IDisposable
|
||||
cmd.CommandText = "ALTER TABLE Source ADD COLUMN MediaPlaybackState TEXT NOT NULL DEFAULT 'Stopped';";
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
if (!columns.Contains("MediaMuted"))
|
||||
{
|
||||
using var cmd = _connection.CreateCommand();
|
||||
cmd.CommandText = "ALTER TABLE Source ADD COLUMN MediaMuted INTEGER NOT NULL DEFAULT 0;";
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
// TASK 47: alert video config. The default clip asset id + the creator's
|
||||
// replacement path (a FILE reference, never a stored BLOB) + the pill,
|
||||
|
||||
@@ -94,7 +94,7 @@ public partial class LayoutStore : IDisposable
|
||||
AlertVideoAssetId, AlertVideoPath, AlertUseDefaultVideo, AlertShowTicker, AlertVideoVolume,
|
||||
AlertDisplayMethod, IsBrandLogo, BlurAmount, AlertAudioPath, AlertAudioLoop,
|
||||
TextContent, TextFontFamily, TextFontSize, TextColor, TextBold, TextItalic,
|
||||
TextUnderline, TextAlignment, TextVerticalAlignment)
|
||||
TextUnderline, TextAlignment, TextVerticalAlignment, MediaMuted)
|
||||
VALUES ($id, $sceneId, $assetId, $type, $name, $isEnabled,
|
||||
$x, $y, $w, $h, $opacity, $monitor,
|
||||
$clip, $mirrored, $isBackground, $captureKey, $showDesktop, $sort,
|
||||
@@ -104,7 +104,7 @@ public partial class LayoutStore : IDisposable
|
||||
$alertVideoAssetId, $alertVideoPath, $alertUseDefaultVideo, $alertShowTicker, $alertVideoVolume,
|
||||
$alertDisplayMethod, $isBrandLogo, $blurAmount, $alertAudioPath, $alertAudioLoop,
|
||||
$textContent, $textFontFamily, $textFontSize, $textColor, $textBold, $textItalic,
|
||||
$textUnderline, $textAlignment, $textVerticalAlignment)
|
||||
$textUnderline, $textAlignment, $textVerticalAlignment, $mediaMuted)
|
||||
""";
|
||||
cmd.Transaction = tx;
|
||||
var idP = cmd.Parameters.Add("$id", SqliteType.Text);
|
||||
@@ -137,6 +137,7 @@ public partial class LayoutStore : IDisposable
|
||||
var mediaPathP = cmd.Parameters.Add("$mediaPath", SqliteType.Text);
|
||||
var mediaIsLoopingP = cmd.Parameters.Add("$mediaIsLooping", SqliteType.Integer);
|
||||
var mediaVolumeP = cmd.Parameters.Add("$mediaVolume", SqliteType.Real);
|
||||
var mediaMutedP = cmd.Parameters.Add("$mediaMuted", SqliteType.Integer);
|
||||
var mediaPlaybackStateP = cmd.Parameters.Add("$mediaPlaybackState", SqliteType.Text);
|
||||
var alertVideoAssetIdP = cmd.Parameters.Add("$alertVideoAssetId", SqliteType.Text);
|
||||
var alertVideoPathP = cmd.Parameters.Add("$alertVideoPath", SqliteType.Text);
|
||||
@@ -198,6 +199,7 @@ public partial class LayoutStore : IDisposable
|
||||
mediaPathP.Value = (object?)source.MediaPath ?? DBNull.Value;
|
||||
mediaIsLoopingP.Value = source.MediaIsLooping ? 1 : 0;
|
||||
mediaVolumeP.Value = source.MediaVolume;
|
||||
mediaMutedP.Value = source.MediaMuted ? 1 : 0;
|
||||
mediaPlaybackStateP.Value = source.MediaPlaybackState.ToString();
|
||||
alertVideoAssetIdP.Value = (object?)source.AlertVideoAssetId ?? DBNull.Value;
|
||||
alertVideoPathP.Value = (object?)source.AlertVideoPath ?? DBNull.Value;
|
||||
|
||||
+21
-115
@@ -1,38 +1,32 @@
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using ytLive.Services.Audio;
|
||||
using ytLive.Services.Encoder;
|
||||
|
||||
namespace ytLive.Services;
|
||||
|
||||
/// <summary>
|
||||
/// TASK 21: decodes a local media file (any format ffmpeg handles) into
|
||||
/// <see cref="VideoFrame"/>s for a MediaSource. Implements
|
||||
/// <see cref="IMediaFrameSource"/>: raises <see cref="FrameAvailable"/> per
|
||||
/// frame and <see cref="IMediaFrameSource.Completed"/> on natural EOF (unless
|
||||
/// <see cref="IMediaFrameSource.Looping"/>). Spawns ffmpeg with
|
||||
/// <c>-f rawvideo -pix_fmt bgra</c> and drains the raw BGRA pipe via
|
||||
/// <see cref="RawVideoFrameReader"/>. The decoder process and emitters are seams
|
||||
/// so tests run without a real codec or audio device.
|
||||
/// <see cref="VideoFrame"/>s for a MediaSource. This is the <see cref="IMediaFrameSource"/>
|
||||
/// shape over the shared <see cref="VideoClipPipe"/> — the same pipe an alert clip uses
|
||||
/// (TASK 47), so the two features cannot drift apart again in how they decode.
|
||||
/// <para>Video AND the file's own soundtrack are decoded. The manager scales the audio by
|
||||
/// the layer's level/mute and hands it to the mixer's desktop group.</para>
|
||||
/// </summary>
|
||||
public sealed class MediaVideoSource : IMediaFrameSource
|
||||
{
|
||||
public event Action<VideoFrame>? FrameAvailable;
|
||||
public event Action<AudioSample>? AudioAvailable;
|
||||
public event Action? Completed;
|
||||
|
||||
public string Key => _path;
|
||||
|
||||
public bool Looping { get; set; }
|
||||
public bool Looping
|
||||
{
|
||||
get => _clip.Looping;
|
||||
set => _clip.Looping = value;
|
||||
}
|
||||
|
||||
private readonly string _path;
|
||||
private readonly int _width;
|
||||
private readonly int _height;
|
||||
private readonly IFfmpegLocator _locator;
|
||||
private readonly Func<IDecodeProcess> _processFactory;
|
||||
private readonly IFrameRateProbe? _probe;
|
||||
private readonly Func<TimeSpan, CancellationToken, Task> _delay;
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private IDisposable? _run;
|
||||
private IDecodeProcess? _active;
|
||||
private readonly VideoClipPipe _clip;
|
||||
|
||||
public MediaVideoSource(
|
||||
string path,
|
||||
@@ -44,104 +38,16 @@ public sealed class MediaVideoSource : IMediaFrameSource
|
||||
Func<TimeSpan, CancellationToken, Task>? delay = null)
|
||||
{
|
||||
_path = path;
|
||||
_width = width;
|
||||
_height = height;
|
||||
_locator = locator;
|
||||
_processFactory = processFactory;
|
||||
_probe = frameRateProbe;
|
||||
_delay = delay ?? ((t, ct) => Task.Delay(t, ct));
|
||||
_clip = new VideoClipPipe(
|
||||
path, width, height, locator, processFactory, frameRateProbe, delay, audio: true);
|
||||
_clip.FrameAvailable += frame => FrameAvailable?.Invoke(frame);
|
||||
_clip.AudioAvailable += sample => AudioAvailable?.Invoke(sample);
|
||||
_clip.Completed += () => Completed?.Invoke();
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (_run != null) return Task.CompletedTask;
|
||||
_run = _cts.Token.Register(StopInternal);
|
||||
_ = Task.Run(RunAsync, CancellationToken.None);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
public Task StartAsync(CancellationToken cancellationToken = default) => _clip.StartAsync(cancellationToken);
|
||||
|
||||
public Task StopAsync()
|
||||
{
|
||||
_cts.Cancel();
|
||||
_run = null;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
public Task StopAsync() => _clip.StopAsync();
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_cts.Cancel();
|
||||
_cts.Dispose();
|
||||
_active?.Dispose();
|
||||
}
|
||||
|
||||
private void StopInternal() => KillProcess();
|
||||
|
||||
private async Task RunAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
double? frameRate = null;
|
||||
if (_probe != null)
|
||||
frameRate = await _probe.ProbeAsync(_path, _cts.Token).ConfigureAwait(false);
|
||||
var frameDuration = frameRate is > 0 ? TimeSpan.FromSeconds(1.0 / frameRate.Value) : TimeSpan.Zero;
|
||||
|
||||
var ffmpegPath = await _locator.LocateAsync(_cts.Token).ConfigureAwait(false);
|
||||
var args =
|
||||
$"-hide_banner -loglevel error -i \"{_path}\" " +
|
||||
$"-f rawvideo -pix_fmt bgra -vf scale={_width}:{_height} -an -";
|
||||
var startInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = ffmpegPath,
|
||||
Arguments = args,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true,
|
||||
};
|
||||
|
||||
do
|
||||
{
|
||||
using (var process = _processFactory())
|
||||
{
|
||||
_active = process;
|
||||
process.Start(startInfo);
|
||||
|
||||
var reader = new RawVideoFrameReader(_width, _height);
|
||||
var buffer = new byte[65536];
|
||||
var stream = process.StandardOutput;
|
||||
while (!_cts.IsCancellationRequested && !process.HasExited)
|
||||
{
|
||||
var read = await stream.ReadAsync(buffer, 0, buffer.Length, _cts.Token).ConfigureAwait(false);
|
||||
if (read <= 0) break;
|
||||
foreach (var frame in reader.Feed(buffer.AsSpan(0, read).ToArray()))
|
||||
{
|
||||
FrameAvailable?.Invoke(frame);
|
||||
if (frameDuration > TimeSpan.Zero)
|
||||
await _delay(frameDuration, _cts.Token).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
_active = null;
|
||||
}
|
||||
while (Looping && !_cts.IsCancellationRequested);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Fail($"Media decode failed: {ex.Message}");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!_cts.IsCancellationRequested)
|
||||
Completed?.Invoke();
|
||||
}
|
||||
|
||||
private void Fail(string message) => Console.Error.WriteLine(message);
|
||||
|
||||
private void KillProcess()
|
||||
{
|
||||
try { _active?.Kill(); } catch { /* already exited */ }
|
||||
}
|
||||
public void Dispose() => _clip.Dispose();
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ using System.Windows.Media;
|
||||
using System.Windows.Media.Imaging;
|
||||
using System.Windows.Threading;
|
||||
using ytLive.Helpers;
|
||||
using ytLive.Services.Audio;
|
||||
|
||||
namespace ytLive.Services;
|
||||
|
||||
@@ -23,8 +24,11 @@ public sealed class MediaVideoSourceManager : IDisposable
|
||||
public string Key { get; }
|
||||
public IMediaFrameSource Source { get; }
|
||||
public Action<VideoFrame>? FrameHandler;
|
||||
public Action<AudioSample>? AudioHandler;
|
||||
public int RefCount;
|
||||
public bool Started;
|
||||
public bool AudioMuted;
|
||||
public double AudioVolume = 1.0;
|
||||
public WriteableBitmap? PreviewBitmap;
|
||||
public VideoFrame? LatestFrame;
|
||||
public bool FramePending;
|
||||
@@ -49,6 +53,10 @@ public sealed class MediaVideoSourceManager : IDisposable
|
||||
/// <summary>Raised when a media decoder cannot be created (bad path).</summary>
|
||||
public event Action<string, string>? MediaFailed;
|
||||
|
||||
/// <summary>Raised (worker thread) per audio chunk of a file's own soundtrack, already
|
||||
/// scaled by the session's level and dropped when muted.</summary>
|
||||
public event Action<string, AudioSample>? AudioAvailable;
|
||||
|
||||
public MediaVideoSourceManager(
|
||||
Func<string, IMediaFrameSource?> sourceFactory,
|
||||
Dispatcher? uiDispatcher = null,
|
||||
@@ -95,6 +103,8 @@ public sealed class MediaVideoSourceManager : IDisposable
|
||||
session = new MediaSession(key, source);
|
||||
session.FrameHandler = frame => OnFrameAvailable(session, frame);
|
||||
session.Source.FrameAvailable += session.FrameHandler;
|
||||
session.AudioHandler = sample => OnAudioAvailable(session, sample);
|
||||
session.Source.AudioAvailable += session.AudioHandler;
|
||||
// Loop intent lives on the Source (per scene), so the caller supplies the
|
||||
// per-path answer ("any live reference loops"). Applied before Start so a
|
||||
// zero-length or instantly-EOF file cannot slip past a later assignment.
|
||||
@@ -117,6 +127,7 @@ public sealed class MediaVideoSourceManager : IDisposable
|
||||
lock (_gate)
|
||||
_sessions.Remove(key);
|
||||
session.Source.FrameAvailable -= session.FrameHandler;
|
||||
if (session.AudioHandler != null) session.Source.AudioAvailable -= session.AudioHandler;
|
||||
AppLog.Write($"MediaSource: failed to start decoder '{key}': {ex.Message}");
|
||||
MediaFailed?.Invoke(key, ex.Message);
|
||||
await SafeStopAsync(session.Source);
|
||||
@@ -137,6 +148,7 @@ public sealed class MediaVideoSourceManager : IDisposable
|
||||
|
||||
if (toStop == null) return;
|
||||
toStop.Source.FrameAvailable -= toStop.FrameHandler;
|
||||
if (toStop.AudioHandler != null) toStop.Source.AudioAvailable -= toStop.AudioHandler;
|
||||
await SafeStopAsync(toStop.Source);
|
||||
toStop.PreviewBitmap = null;
|
||||
}
|
||||
@@ -155,6 +167,7 @@ public sealed class MediaVideoSourceManager : IDisposable
|
||||
|
||||
if (toStop == null) return;
|
||||
toStop.Source.FrameAvailable -= toStop.FrameHandler;
|
||||
if (toStop.AudioHandler != null) toStop.Source.AudioAvailable -= toStop.AudioHandler;
|
||||
await SafeStopAsync(toStop.Source);
|
||||
toStop.PreviewBitmap = null;
|
||||
}
|
||||
@@ -193,6 +206,46 @@ public sealed class MediaVideoSourceManager : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The layer's sound controls for this file. One decoder is shared per path,
|
||||
/// so the level/mute are per SESSION: whichever layer the creator last touched wins —
|
||||
/// the alternative (one decoder per layer) would play the same file twice.</summary>
|
||||
public void SetAudioSettings(string key, double volume, bool muted)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (!_sessions.TryGetValue(key, out var session)) return;
|
||||
session.AudioVolume = Math.Clamp(volume, 0, 1);
|
||||
session.AudioMuted = muted;
|
||||
}
|
||||
}
|
||||
|
||||
private void OnAudioAvailable(MediaSession session, AudioSample sample)
|
||||
{
|
||||
if (!TryGetActiveSession(session)) return;
|
||||
|
||||
double volume;
|
||||
bool muted;
|
||||
lock (_gate)
|
||||
{
|
||||
volume = session.AudioVolume;
|
||||
muted = session.AudioMuted;
|
||||
}
|
||||
if (muted || volume <= 0) return;
|
||||
|
||||
if (volume < 0.999)
|
||||
{
|
||||
// Copy before scaling: the source owns its buffer and may recycle it.
|
||||
var scaled = new float[sample.Samples.Length];
|
||||
var gain = (float)volume;
|
||||
for (var i = 0; i < scaled.Length; i++)
|
||||
scaled[i] = sample.Samples[i] * gain;
|
||||
AudioAvailable?.Invoke(session.Key, new AudioSample(scaled, sample.SampleRate, sample.Channels));
|
||||
return;
|
||||
}
|
||||
|
||||
AudioAvailable?.Invoke(session.Key, sample);
|
||||
}
|
||||
|
||||
/// <summary>The shared preview bitmap for a file (so a config created after the
|
||||
/// first frame can adopt the running bitmap), or null.</summary>
|
||||
public WriteableBitmap? GetPreviewBitmap(string key)
|
||||
@@ -213,6 +266,7 @@ public sealed class MediaVideoSourceManager : IDisposable
|
||||
foreach (var session in sessions)
|
||||
{
|
||||
session.Source.FrameAvailable -= session.FrameHandler;
|
||||
if (session.AudioHandler != null) session.Source.AudioAvailable -= session.AudioHandler;
|
||||
_ = SafeStopAsync(session.Source);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using ytLive.Helpers;
|
||||
using ytLive.Services.Audio;
|
||||
using ytLive.Services.Encoder;
|
||||
|
||||
namespace ytLive.Services;
|
||||
|
||||
/// <summary>
|
||||
/// One video FILE played through ffmpeg — the shared pipe behind every clip, so a media
|
||||
/// layer (TASK 21) and an alert clip (TASK 47) decode identically. Owns two subprocesses
|
||||
/// on the same file: video (<c>rawvideo bgra</c>, scaled to the target rect) and audio
|
||||
/// (<c>f32le -ac 2 -ar 48000</c>, matching the mixer's native format so no resampler is
|
||||
/// needed). Both are paced to real time, so the picture and the sound stay together even
|
||||
/// though they arrive on separate pipes.
|
||||
/// <para><b>Looping is the media case</b>: on natural EOF the whole pass restarts — a fresh
|
||||
/// pair of processes, because a <see cref="Process"/> cannot be re-started. An alert clip
|
||||
/// never loops, so its wrapper simply never sets <see cref="Looping"/>.</para>
|
||||
/// <para>The seams (locator, process factory, frame-rate probe, delay) exist so tests drive
|
||||
/// a fake decoder with no ffmpeg, no codec, and no audio device.</para>
|
||||
/// </summary>
|
||||
public sealed class VideoClipPipe : IDisposable
|
||||
{
|
||||
public const int AudioSampleRate = 48000;
|
||||
public const int AudioChannels = 2;
|
||||
private const int AudioChunkFloats = 4800; // 50 ms of stereo
|
||||
|
||||
public event Action<VideoFrame>? FrameAvailable;
|
||||
public event Action<AudioSample>? AudioAvailable;
|
||||
public event Action? Completed;
|
||||
|
||||
/// <summary>Restart the pass on natural EOF (media/BRB loops).</summary>
|
||||
public bool Looping { get; set; }
|
||||
|
||||
private readonly string _path;
|
||||
private readonly int _width;
|
||||
private readonly int _height;
|
||||
private readonly IFfmpegLocator _locator;
|
||||
private readonly Func<IDecodeProcess> _processFactory;
|
||||
private readonly IFrameRateProbe? _probe;
|
||||
private readonly Func<TimeSpan, CancellationToken, Task> _delay;
|
||||
private readonly bool _audio;
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private IDisposable? _run;
|
||||
private IDecodeProcess? _activeVideo;
|
||||
private IDecodeProcess? _activeAudio;
|
||||
|
||||
public VideoClipPipe(
|
||||
string path,
|
||||
int width,
|
||||
int height,
|
||||
IFfmpegLocator locator,
|
||||
Func<IDecodeProcess> processFactory,
|
||||
IFrameRateProbe? frameRateProbe = null,
|
||||
Func<TimeSpan, CancellationToken, Task>? delay = null,
|
||||
bool audio = true)
|
||||
{
|
||||
_path = path;
|
||||
_width = width;
|
||||
_height = height;
|
||||
_locator = locator;
|
||||
_processFactory = processFactory;
|
||||
_probe = frameRateProbe;
|
||||
_delay = delay ?? ((t, ct) => Task.Delay(t, ct));
|
||||
_audio = audio;
|
||||
}
|
||||
|
||||
public Task StartAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (_run != null) return Task.CompletedTask;
|
||||
_run = _cts.Token.Register(StopInternal);
|
||||
_ = Task.Run(RunAsync, CancellationToken.None);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task StopAsync()
|
||||
{
|
||||
_cts.Cancel();
|
||||
_run = null;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_cts.Cancel();
|
||||
_cts.Dispose();
|
||||
_activeVideo?.Dispose();
|
||||
_activeAudio?.Dispose();
|
||||
}
|
||||
|
||||
private void StopInternal()
|
||||
{
|
||||
Kill(_activeVideo);
|
||||
Kill(_activeAudio);
|
||||
}
|
||||
|
||||
private async Task RunAsync()
|
||||
{
|
||||
var ct = _cts.Token;
|
||||
var frames = 0;
|
||||
var chunks = 0;
|
||||
try
|
||||
{
|
||||
double? frameRate = null;
|
||||
if (_probe != null)
|
||||
frameRate = await _probe.ProbeAsync(_path, ct).ConfigureAwait(false);
|
||||
var frameDuration = frameRate is > 0 ? TimeSpan.FromSeconds(1.0 / frameRate.Value) : TimeSpan.Zero;
|
||||
|
||||
var ffmpegPath = await _locator.LocateAsync(ct).ConfigureAwait(false);
|
||||
|
||||
do
|
||||
{
|
||||
// Both pipes per pass, awaited together: a restart must begin a new
|
||||
// video AND a new audio stream, or the picture drifts from the sound
|
||||
// after the first loop.
|
||||
var video = RunVideoAsync(ffmpegPath, frameDuration, ct);
|
||||
var audio = _audio ? RunAudioAsync(ffmpegPath, ct) : Task.FromResult(0);
|
||||
frames += await video.ConfigureAwait(false);
|
||||
chunks += await audio.ConfigureAwait(false);
|
||||
}
|
||||
while (Looping && !ct.IsCancellationRequested);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// A failed decode is not fatal: the alert layer falls back to its animations,
|
||||
// and a media layer just holds its last frame.
|
||||
AppLog.Write($"Video clip decode failed ({Path.GetFileName(_path)}): {ex.Message}");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!ct.IsCancellationRequested)
|
||||
{
|
||||
AppLog.Write($"Video clip end: {Path.GetFileName(_path)} frames={frames} audioChunks={chunks}");
|
||||
Completed?.Invoke();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<int> RunVideoAsync(string ffmpegPath, TimeSpan frameDuration, CancellationToken ct)
|
||||
{
|
||||
var args =
|
||||
$"-hide_banner -loglevel error -i \"{_path}\" " +
|
||||
$"-f rawvideo -pix_fmt bgra -vf scale={_width}:{_height} -an -";
|
||||
using var process = _processFactory();
|
||||
_activeVideo = process;
|
||||
process.Start(new ProcessStartInfo
|
||||
{
|
||||
FileName = ffmpegPath,
|
||||
Arguments = args,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true,
|
||||
});
|
||||
|
||||
var reader = new RawVideoFrameReader(_width, _height);
|
||||
var buffer = new byte[65536];
|
||||
var stream = process.StandardOutput;
|
||||
var frames = 0;
|
||||
while (!ct.IsCancellationRequested && !process.HasExited)
|
||||
{
|
||||
var read = await stream.ReadAsync(buffer, 0, buffer.Length, ct).ConfigureAwait(false);
|
||||
if (read <= 0) break;
|
||||
var chunk = new byte[read];
|
||||
Buffer.BlockCopy(buffer, 0, chunk, 0, read);
|
||||
foreach (var frame in reader.Feed(chunk))
|
||||
{
|
||||
frames++;
|
||||
FrameAvailable?.Invoke(frame);
|
||||
if (frameDuration > TimeSpan.Zero)
|
||||
await _delay(frameDuration, ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
_activeVideo = null;
|
||||
return frames;
|
||||
}
|
||||
|
||||
private async Task<int> RunAudioAsync(string ffmpegPath, CancellationToken ct)
|
||||
{
|
||||
var args =
|
||||
$"-hide_banner -loglevel error -i \"{_path}\" " +
|
||||
$"-f f32le -ac {AudioChannels} -ar {AudioSampleRate} -vn -";
|
||||
using var process = _processFactory();
|
||||
_activeAudio = process;
|
||||
process.Start(new ProcessStartInfo
|
||||
{
|
||||
FileName = ffmpegPath,
|
||||
Arguments = args,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true,
|
||||
});
|
||||
|
||||
var stream = process.StandardOutput;
|
||||
var raw = new byte[65536];
|
||||
var carry = Array.Empty<byte>();
|
||||
var floatScratch = new float[AudioChunkFloats];
|
||||
var floatIndex = 0;
|
||||
var chunks = 0;
|
||||
// The pipe is byte-delimited only: a read may split a float, so leftover
|
||||
// bytes are carried into the next read instead of dropped (a dropped
|
||||
// half-float would desync the entire stream).
|
||||
while (!ct.IsCancellationRequested && !process.HasExited)
|
||||
{
|
||||
var read = await stream.ReadAsync(raw, 0, raw.Length, ct).ConfigureAwait(false);
|
||||
if (read <= 0) break;
|
||||
var total = carry.Length + read;
|
||||
var bytes = total <= raw.Length ? raw : new byte[total];
|
||||
if (carry.Length > 0)
|
||||
{
|
||||
Buffer.BlockCopy(carry, 0, bytes, 0, carry.Length);
|
||||
Buffer.BlockCopy(raw, 0, bytes, carry.Length, read);
|
||||
}
|
||||
var floatsTo = total - (total % 4);
|
||||
for (var i = 0; i < floatsTo && floatIndex < floatScratch.Length; i += 4)
|
||||
floatScratch[floatIndex++] = BitConverter.ToSingle(bytes, i);
|
||||
carry = total > floatsTo ? bytes[floatsTo..total] : Array.Empty<byte>();
|
||||
|
||||
while (floatIndex == floatScratch.Length)
|
||||
{
|
||||
chunks++;
|
||||
AudioAvailable?.Invoke(new AudioSample(floatScratch, AudioSampleRate, AudioChannels));
|
||||
// Pace the NEXT chunk so the clip's audio lands at real-time cadence,
|
||||
// matching how the video pipe is paced by frame rate.
|
||||
await _delay(
|
||||
TimeSpan.FromSeconds(AudioChunkFloats / (double)(AudioSampleRate * AudioChannels)), ct)
|
||||
.ConfigureAwait(false);
|
||||
floatScratch = new float[AudioChunkFloats];
|
||||
floatIndex = 0;
|
||||
}
|
||||
}
|
||||
_activeAudio = null;
|
||||
return chunks;
|
||||
}
|
||||
|
||||
private static void Kill(IDecodeProcess? process)
|
||||
{
|
||||
try { process?.Kill(); } catch { /* already exited */ }
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user