gramps/ytLlive
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

TASK 9 audio milestone: real stream audio + voice filters + auto-duck + free TRAX music — the mixer now feeds the encoder real audio (ffmpeg reads a Windows named pipe, -f f32le -ar 48000 -ac 2 -i \.\pipe\ytllive_audio, replacing anullsrc silence; explicit -map 0:v -map 1:a via EncoderOptions.AudioPipeName), with honest gains reaching the stream (MicVolume scales mic, GameAudioVolume + mute scale loopback, Func<double> seams on AudioMixer), an always-on pure-C# voice chain on the mic BEFORE the meter and mix (LowShelfFilter 120Hz +4dB → HighShelfFilter 8kHz +3dB → NoiseGate 0.005/hysteresis 0.5 → Compressor 0.5 4:1, all TDF2), AutoDucker (mic RMS>0.02 → loopback ×0.25, attack 0.05/release 0.005), and TRAX free background music (MusicPlayer = MediaFoundationReader → VolumeWaveProvider16 at fixed 0.20 → WaveOutEvent via the new sibling NAudio.WinMM 2.2.1 package; plays to the default device so the existing loopback carries it, ducked with game; footer TRAX button with red/yellow/green status dot, left-click toggles/picks, right-click opens the OpenFileDialog picker, tooltip shows the track name; persisted via schema v9 single-row Music). Build-time deviations from the plan (recorded in ai.md + TASKS.md): WasapiCapture in NAudio 2.2.1 exposes no overridable GetDefaultMixFormat so sources run device mix format and the mixer's TinyResampler normalizes any rate to 48kHz (resampler IS the design); FfmpegEncoder.cs untouched — MainViewModel.StopStream calls _audioMixer.StopLive() (pipe EOF) before the frame pump stops; sound-bar relabelled 'Desktop Audio', IsGameAudioBarVisible = game sound OR music playing. Tests: new AudioPipelineTests (DSP/ring-buffer/ducker/resampler units + the ONE integration test Mix_WithFiltersDuckAndGain_Lands_On_AudioPipe reading real pipe bytes; ring-buffer overwrite bug found + fixed), FfmpegEncoderTests/LayoutStorePersistenceTests updated — 196 tests passing, 0 warnings

Browse Source
This commit is contained in:
Llama Chile Shop
2026-08-14 18:09:26 -07:00
parent 3d92bd0225
commit 6d71acef8c
22 changed files with 1605 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Services/Audio/AudioMixer.cs
+212 -10
View File
@@ -4,28 +4,72 @@ namespace ytLive.Services.Audio;
/// <summary>
/// Owns the mic + desktop/game capture sources (TASK 4 ship step 4) and drives
/// the footer meters. Capture runs for the app's lifetime (started once at
/// startup, stopped on shutdown) so both bars stay live in preview: mic samples
/// are level-metered and forwarded, loopback samples feed the game bar's meter.
/// The mixer surfaces mic connection state (Connected/Failed) for the status
/// dot and can restart the mic source mid-session when a device is re-picked.
/// the footer meters; since TASK 9 it is also the live audio path into the
/// encoder. Capture runs for the app's lifetime so both bars stay live in
/// preview: mic samples are downmixed to mono, resampled to 48 kHz, put
/// through the voice chain (bass → treble → noise gate → compressor), THEN
/// level-metered and queued; loopback samples are resampled to 48 kHz stereo
/// and queued. While live (<see cref="StartLive"/>), a 10 ms loop drains both
/// queues, applies the honest gains (mic = MicVolume, loopback = GameAudioVolume
/// × auto-duck when the mic is hot), mixes them to stereo float and writes the
/// chunk to the encoder's audio pipe.
/// </summary>
public sealed class AudioMixer : IDisposable
{
public const int OutputSampleRate = 48000;
private static readonly TimeSpan DefaultMixInterval = TimeSpan.FromMilliseconds(10);
private readonly IAudioSource _mic;
private readonly IAudioSource _loopback;
private readonly AudioLevelMeter _meter;
private readonly AudioLevelMeter _loopbackMeter;
private readonly Action<string>? _log;
private readonly Func<double>? _micGain;
private readonly Func<double>? _loopbackGain;
private readonly TimeSpan _mixInterval;
private bool _started;
public AudioMixer(IAudioSource mic, IAudioSource loopback, Action<string>? log = null)
// Live path (TASK 9): per-source resampling → voice chain on the mic →
// thread-safe queues → the ducker + gain/mix loop → the encoder's pipe.
private readonly AudioRingBuffer _micBuffer;
private readonly AudioRingBuffer _loopbackBuffer;
private readonly VoiceFilterChain _voiceChain;
private readonly AutoDucker _ducker;
private TinyResampler? _micResampler;
private TinyResampler? _loopbackResampler;
private CancellationTokenSource? _liveCts;
private NamedPipeAudioWriter _pipe = new();
private float[]? _micChunk;
private float[]? _loopbackChunk;
private float[]? _mixBuffer;
public AudioMixer(
IAudioSource mic,
IAudioSource loopback,
Action<string>? log = null,
Func<double>? micGain = null,
Func<double>? loopbackGain = null,
TimeSpan? mixInterval = null)
{
_mic = mic;
_loopback = loopback;
_meter = new AudioLevelMeter();
_loopbackMeter = new AudioLevelMeter();
_log = log;
_micGain = micGain;
_loopbackGain = loopbackGain;
_mixInterval = mixInterval ?? DefaultMixInterval;
var seconds = _mixInterval.TotalSeconds;
var framesPerTick = Math.Max(1, (int)Math.Round(OutputSampleRate * seconds));
_micBuffer = new AudioRingBuffer(OutputSampleRate * 2);
_loopbackBuffer = new AudioRingBuffer(OutputSampleRate * 2 * 2);
_voiceChain = new VoiceFilterChain(OutputSampleRate);
_ducker = new AutoDucker();
_micChunk = new float[framesPerTick];
_loopbackChunk = new float[framesPerTick * 2];
_mixBuffer = new float[framesPerTick * 2];
_mic.Started += OnMicStarted;
_mic.SampleReady += OnMicSample;
@@ -34,7 +78,7 @@ public sealed class AudioMixer : IDisposable
_loopback.Failed += OnLoopbackFailed;
}
/// <summary>Current smoothed mic level (0..1).</summary>
/// <summary>Current smoothed mic level (0..1), post voice chain.</summary>
public float MicLevel => _meter.Level;
/// <summary>Current smoothed desktop/game level (0..1).</summary>
@@ -70,6 +114,9 @@ public sealed class AudioMixer : IDisposable
{
_mic.Stop();
_meter.Reset();
_micBuffer.Clear();
_micResampler?.Reset();
_voiceChain.Reset();
MicLevelChanged?.Invoke(0);
_mic.Start();
}
@@ -80,6 +127,7 @@ public sealed class AudioMixer : IDisposable
return;
_started = false;
StopLive();
_mic.Stop();
_loopback.Stop();
_meter.Reset();
@@ -88,6 +136,30 @@ public sealed class AudioMixer : IDisposable
LoopbackLevelChanged?.Invoke(0);
}
/// <summary>Begins the live mix loop: drains the capture queues, applies the
/// honest gains + auto-duck, and streams stereo float into the named audio
/// pipe. Idempotent — safe to call once per go-live.</summary>
public void StartLive(string pipeName)
{
if (_liveCts != null)
return;
var cts = new CancellationTokenSource();
_liveCts = cts;
_pipe = new NamedPipeAudioWriter();
_pipe.Start(pipeName);
_ = Task.Run(() => LiveLoopAsync(_pipe, cts.Token));
}
/// <summary>Ends the live mix loop and closes the audio pipe — ffmpeg sees
/// EOF on the audio input. Safe when not live.</summary>
public void StopLive()
{
_liveCts?.Cancel();
_liveCts = null;
_pipe.Stop();
}
public void Dispose()
{
Stop();
@@ -107,15 +179,26 @@ public sealed class AudioMixer : IDisposable
private void OnMicSample(AudioSample sample)
{
var mono = DownmixToMono(sample);
mono = ResampleMic(mono, sample.SampleRate);
for (var i = 0; i < mono.Length; i++)
mono[i] = _voiceChain.Process(mono[i]);
_micBuffer.Write(mono);
// Push unconditionally: the ?. on the event would otherwise skip the
// argument (and the meter update) when nothing is subscribed yet.
var level = _meter.Push(sample);
// argument (and the meter update) when nothing is subscribed yet. The
// level is the POST-filtered signal — the meter shows what the stream
// will carry.
var level = _meter.Push(new AudioSample(mono, OutputSampleRate, 1));
MicLevelChanged?.Invoke(level);
}
private void OnLoopbackSample(AudioSample sample)
{
var level = _loopbackMeter.Push(sample);
var data = ResampleLoopback(sample.Samples, sample.SampleRate);
_loopbackBuffer.Write(data);
var level = _loopbackMeter.Push(new AudioSample(data, OutputSampleRate, sample.Channels));
LoopbackLevelChanged?.Invoke(level);
}
@@ -130,4 +213,123 @@ public sealed class AudioMixer : IDisposable
{
_log?.Invoke($"Desktop audio capture failed: {ex.Message}");
}
private static float[] DownmixToMono(AudioSample sample)
{
if (sample.Channels <= 1)
return sample.Samples;
var frames = sample.Samples.Length / sample.Channels;
var mono = new float[frames];
for (var i = 0; i < frames; i++)
{
var sum = 0f;
for (var c = 0; c < sample.Channels; c++)
sum += sample.Samples[i * sample.Channels + c];
mono[i] = sum / sample.Channels;
}
return mono;
}
private float[] ResampleMic(float[] mono, int inputRate)
{
if (inputRate == OutputSampleRate)
return mono;
_micResampler ??= new TinyResampler(inputRate, OutputSampleRate);
if (_micResampler.NeedsResampling)
return _micResampler.Process(mono);
return mono;
}
private float[] ResampleLoopback(float[] interleaved, int inputRate)
{
if (inputRate == OutputSampleRate)
return interleaved;
_loopbackResampler ??= new TinyResampler(inputRate, OutputSampleRate);
if (_loopbackResampler.NeedsResampling)
return _loopbackResampler.Process(interleaved);
return interleaved;
}
private async Task LiveLoopAsync(IAudioPipeWriter pipe, CancellationToken cancellationToken)
{
var mixBuffer = _mixBuffer!;
var micChunk = _micChunk!;
var loopbackChunk = _loopbackChunk!;
while (!cancellationToken.IsCancellationRequested)
{
var nextTick = DateTime.UtcNow + _mixInterval;
try
{
FillAndMix(micChunk, loopbackChunk, mixBuffer);
await pipe.WriteAsync(mixBuffer, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
break;
}
catch (Exception ex)
{
_log?.Invoke($"Audio live loop error: {ex.Message}");
}
var delay = nextTick - DateTime.UtcNow;
if (delay > TimeSpan.Zero)
{
try
{
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
break;
}
}
}
}
/// <summary>Drains one tick's worth of mic + loopback, silence-fills any
/// underrun, applies the ducker and the honest gains, and mixes to stereo.</summary>
private float FillAndMix(float[] micChunk, float[] loopbackChunk, float[] mix)
{
var micCount = _micBuffer.Read(micChunk, micChunk.Length);
var loopCount = _loopbackBuffer.Read(loopbackChunk, loopbackChunk.Length);
float micRms = 0;
if (micCount == 0)
{
Array.Clear(micChunk, 0, micChunk.Length);
}
else
{
double sumSquares = 0;
for (var i = 0; i < micCount; i++)
sumSquares += micChunk[i] * micChunk[i];
micRms = (float)Math.Sqrt(sumSquares / micCount);
if (micCount < micChunk.Length)
Array.Clear(micChunk, micCount, micChunk.Length - micCount);
}
if (loopCount == 0)
{
Array.Clear(loopbackChunk, 0, loopbackChunk.Length);
}
else if (loopCount < loopbackChunk.Length)
{
Array.Clear(loopbackChunk, loopCount, loopbackChunk.Length - loopCount);
}
var duck = _ducker.Update(micRms);
var micGain = (float)(_micGain?.Invoke() ?? 1.0);
var loopGain = (float)(_loopbackGain?.Invoke() ?? 1.0) * duck;
var frames = micChunk.Length;
for (var i = 0; i < frames; i++)
{
var m = micChunk[i] * micGain;
mix[i * 2] = m + loopbackChunk[i * 2] * loopGain;
mix[i * 2 + 1] = m + loopbackChunk[i * 2 + 1] * loopGain;
}
return micRms;
}
}