From 58c0f8e8c4fd283fbb0f54f3eefbde26f85d213d Mon Sep 17 00:00:00 2001 From: gramps Date: Wed, 12 Aug 2026 21:18:07 -0700 Subject: [PATCH] =?UTF-8?q?TASK=204=20ship=20step=204:=20WASAPI=20audio=20?= =?UTF-8?q?capture=20=E2=80=94=20NAudio=20loopback=20+=20mic=20behind=20an?= =?UTF-8?q?=20IAudioSource=20seam,=20AudioMixer=20driving=20AudioLevel=20w?= =?UTF-8?q?hile=20live,=20pure=20level=20meter=20+=20WaveToFloat,=20NAudio?= =?UTF-8?q?.Wasapi=202.2.1=20(MIT,=20notices=20item=209)=20=E2=80=94=20139?= =?UTF-8?q?=20tests=20passing,=200=20warnings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- HANDOFF.md | 73 ++--- Services/Audio/AudioLevelMeter.cs | 35 +++ Services/Audio/AudioMixer.cs | 92 +++++++ Services/Audio/AudioSample.cs | 20 ++ Services/Audio/IAudioSource.cs | 22 ++ Services/Audio/WasapiLoopbackAudioSource.cs | 69 +++++ Services/Audio/WasapiMicAudioSource.cs | 102 +++++++ Services/Audio/WaveToFloat.cs | 41 +++ Services/Encoder/FfmpegEncoder.cs | 2 +- Services/index.md | 7 + TASKS.md | 38 ++- THIRD-PARTY-NOTICES.txt | 8 + ViewModels/MainViewModel.cs | 26 ++ ai.md | 32 ++- ytLive.Tests/AudioMixerTests.cs | 280 ++++++++++++++++++++ ytLive.csproj | 1 + 16 files changed, 814 insertions(+), 34 deletions(-) create mode 100644 Services/Audio/AudioLevelMeter.cs create mode 100644 Services/Audio/AudioMixer.cs create mode 100644 Services/Audio/AudioSample.cs create mode 100644 Services/Audio/IAudioSource.cs create mode 100644 Services/Audio/WasapiLoopbackAudioSource.cs create mode 100644 Services/Audio/WasapiMicAudioSource.cs create mode 100644 Services/Audio/WaveToFloat.cs create mode 100644 ytLive.Tests/AudioMixerTests.cs diff --git a/HANDOFF.md b/HANDOFF.md index df7d4ab..4ddab80 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -7,37 +7,50 @@ ## Session state (last updated: 2026-08-12) -- **Branch:** `main`. TASK 4 ship step 3 (encoder + RTMP push) is **built and - tested** but **NOT committed** — the working tree is dirty with the encoder - (10 new files in `Services/Encoder/` + `ytLive.Tests/FfmpegEncoderTests.cs`) and - its memory updates (TASKS.md, ai.md, Services/index.md). Ready to commit when - the user says so. -- **Finished this session:** TASK 4 ship step 3 — the FFmpeg subprocess encoder: - `EncoderOptions` + `IFfmpegEncoder`/`FfmpegEncoder` + `IEncoderProcess`/ - `FfmpegEncoderProcess` + pure `FfmpegArgs`/`FfmpegProgressParser`/ - `FfmpegEncoderPicker` in `Services/Encoder/`. StartAsync (locate → probe - `-encoders` → spawn → stderr loop), SubmitFrameAsync (serialized BGRA stdin), - StopAsync (stdin EOF → ffmpeg finalizes; 10s kill watchdog), ProcessFailed on - unexpected non-zero exit. Not yet constructed by the app (ship step 5 wiring). - Build: **0 warnings**. Tests: **122 passing** (was 112; +10 new). +- **Branch:** `main`. TASK 4 ship step 4 (WASAPI audio capture) is **built and + tested** but **NOT committed** — the working tree is dirty with the audio + layer (6 new files in `Services/Audio/` + `ytLive.Tests/AudioMixerTests.cs`, + the `NAudio.Wasapi` 2.2.1 package reference, the THIRD-PARTY-NOTICES entry, + the `MainViewModel` wiring, the `FfmpegEncoder.cs:139` CS8602 fix) and its + memory updates (TASKS.md, ai.md, Services/index.md, HANDOFF.md). Ready to + commit when the user says so. +- **Finished this session:** TASK 4 ship step 4 — the live audio capture layer: + `IAudioSource`/`AudioSample` seam, `WasapiLoopbackAudioSource` + (`WasapiLoopbackCapture` on the default render device), `WasapiMicAudioSource` + (`WasapiCapture`, NAudio device resolved by `FriendlyName` matching + `MicSourceName` via a re-read `Func` provider, default-endpoint + fallback), `AudioMixer` (owns both sources, starts/stops with go-live, feeds + the pure `AudioLevelMeter` → `MicLevelChanged`), `WaveToFloat` (IEEE-float / + PCM16 / extensible). `MainViewModel` constructs the mixer, `BeginGoLive` + success → `Start()`, `StopStream` → `Stop()`, `MicLevelChanged` marshalled to + the UI thread → `AudioLevel`. Loopback samples are currently dropped (the + encoder's AAC mix, ship step 5, consumes them). Build: **0 warnings** (a + pre-existing CS8602 in `FfmpegEncoder.cs:139` surfaced during the rebuild and + was fixed with `process!`). Tests: **139 passing** (was 122; +17 new audio). - **Landmines:** - - `ChannelReader.ReadAsync` on a completed channel **throws** - `ChannelClosedException` — it does NOT return `null` like a StreamReader EOF. - The test fake (`QueuedReader` in `FfmpegEncoderTests.cs`) catches it and - returns `null`, or the encoder's stderr loop treats it as a fault and - `ProcessFailed` never fires (that's exactly what happened on the first run — - see commit history). - - The probe process (`FakeEncoderProcess`) must be a *separate* `IEncoderProcess` - instance from the encoder process in `Start_*` tests — `StartAsync` calls the - factory twice (probe → encoder), and the fake can't simulate both roles at - once. - - `StopAsync` waits the full `ExitTimeout` if the fake's process doesn't signal - exit — fakes must call `SignalExit()` inside `StopAsync`'s stdin-EOF path. - - Windows-only: `FfmpegEncoderProcess` sets `UseShellExecute=false` + - `RedirectStandardXxx=true` — never spawn with a shell. -- **Next step:** TASK 4 ship step 4 — WASAPI audio capture (loopback + mic) - feeding `AudioLevel` (req 7). Nothing else queued — do not expand the task - queue on your own. + - `WaveFormatExtensible.SubFormat` is a **`Guid`** in NAudio 2.x, not a + `WaveFormatEncoding` — compare it to + `NAudio.Dmo.AudioMediaSubtypes.MEDIASUBTYPE_IEEE_FLOAT` (the IEEE-float + subtype constant; it lives in the `NAudio.Dmo` namespace, not CoreAudioApi). + - Use the **`NAudio.Wasapi` 2.2.1 feature package**, not the `NAudio` + meta-package — the wasapi types come from that package (`WasapiCapture` in + `NAudio.CoreAudioApi`, `WasapiLoopbackCapture` in `NAudio.Wave`); `NAudio.Core` + comes in transitively. + - The audio meter is an exponential smoother (0.2 factor) — a single pushed + sample only moves 20% toward its RMS. Tests must push repeatedly before + asserting a converged level. + - Tests never instantiate `MainViewModel` directly except the round-clip + integration test (a real `MainWindow`), which never goes live — so the mixer + is constructed but never started there; NAudio types are only constructed, + never touching devices. Keep it that way. + - Mic device resolution must re-read `MicSourceName` at each `Start` (the app + persists only the DisplayName, not a device ID). + - Windows-only: all capture runs only while live (privacy indicator otherwise). + +- **Next step:** TASK 4 ship step 5 — the frame-pipeline wiring + (`CameraManager`/`ScreenCaptureManager` → compositor resolver → encoder + construction, plus the loopback → encoder AAC mix). Nothing else queued — do + not expand the task queue on your own. - **Secret/DB/port facts live:** OAuth client id/secret in `Helpers/OAuthCredentials.cs`; OAuth session token in `Helpers/TokenStore.cs` (DPAPI → `%APPDATA%\ytLlive\ytLlive.auth`); diff --git a/Services/Audio/AudioLevelMeter.cs b/Services/Audio/AudioLevelMeter.cs new file mode 100644 index 0000000..00d2993 --- /dev/null +++ b/Services/Audio/AudioLevelMeter.cs @@ -0,0 +1,35 @@ +namespace ytLive.Services.Audio; + +/// +/// Computes the smoothed mic level (0..1) from captured samples. Pure — +/// unit-tested; the only feeds it and forwards the +/// result. RMS-based so it tracks perceived loudness, smoothed so the meter +/// does not flicker. +/// +public sealed class AudioLevelMeter +{ + private const float Smoothing = 0.2f; + private float _level; + + public float Level => _level; + + public float Push(AudioSample sample) + { + if (sample.Samples.Length == 0) + return _level; + + double sumSquares = 0; + var count = 0; + foreach (var value in sample.Samples) + { + sumSquares += value * value; + count++; + } + + var rms = (float)Math.Sqrt(sumSquares / count); + _level = _level * (1 - Smoothing) + rms * Smoothing; + return _level; + } + + public void Reset() => _level = 0; +} diff --git a/Services/Audio/AudioMixer.cs b/Services/Audio/AudioMixer.cs new file mode 100644 index 0000000..61fc0d7 --- /dev/null +++ b/Services/Audio/AudioMixer.cs @@ -0,0 +1,92 @@ +using ytLive.Services; + +namespace ytLive.Services.Audio; + +/// +/// Owns the mic + desktop/game capture sources (TASK 4 ship step 4) and drives +/// the live mic meter. Runs only while live: started when go-live succeeds, +/// stopped on end-stream. Mic samples are level-metered and forwarded; loopback +/// samples are currently dropped (consumed by the encoder mix in a later step). +/// +public sealed class AudioMixer : IDisposable +{ + private readonly IAudioSource _mic; + private readonly IAudioSource _loopback; + private readonly AudioLevelMeter _meter; + private readonly Action? _log; + private bool _started; + + public AudioMixer(IAudioSource mic, IAudioSource loopback, Action? log = null) + { + _mic = mic; + _loopback = loopback; + _meter = new AudioLevelMeter(); + _log = log; + + _mic.SampleReady += OnMicSample; + _loopback.SampleReady += OnLoopbackSample; + _mic.Failed += OnMicFailed; + _loopback.Failed += OnLoopbackFailed; + } + + /// Current smoothed mic level (0..1). + public float MicLevel => _meter.Level; + + /// Raised whenever the smoothed mic level changes. + public event Action? MicLevelChanged; + + public void Start() + { + if (_started) + return; + + _started = true; + _meter.Reset(); + _loopback.Start(); + _mic.Start(); + } + + public void Stop() + { + if (!_started) + return; + + _started = false; + _mic.Stop(); + _loopback.Stop(); + _meter.Reset(); + MicLevelChanged?.Invoke(0); + } + + public void Dispose() + { + Stop(); + _mic.SampleReady -= OnMicSample; + _loopback.SampleReady -= OnLoopbackSample; + _mic.Failed -= OnMicFailed; + _loopback.Failed -= OnLoopbackFailed; + _mic.Dispose(); + _loopback.Dispose(); + } + + private void OnMicSample(AudioSample sample) + { + MicLevelChanged?.Invoke(_meter.Push(sample)); + } + + private void OnLoopbackSample(AudioSample sample) + { + // Desktop/game audio: captured for the future encoder mix; no UI yet. + } + + private void OnMicFailed(Exception ex) + { + _log?.Invoke($"Mic capture failed: {ex.Message}"); + MicLevelChanged?.Invoke(0); + } + + private void OnLoopbackFailed(Exception ex) + { + _log?.Invoke($"Desktop audio capture failed: {ex.Message}"); + } +} diff --git a/Services/Audio/AudioSample.cs b/Services/Audio/AudioSample.cs new file mode 100644 index 0000000..693c331 --- /dev/null +++ b/Services/Audio/AudioSample.cs @@ -0,0 +1,20 @@ +namespace ytLive.Services.Audio; + +/// +/// A chunk of interleaved PCM float samples (-1..1) with its format. This is +/// the unit every produces and the +/// consumes (TASK 4 ship step 4). +/// +public sealed class AudioSample +{ + public float[] Samples { get; } + public int SampleRate { get; } + public int Channels { get; } + + public AudioSample(float[] samples, int sampleRate, int channels) + { + Samples = samples; + SampleRate = sampleRate; + Channels = channels; + } +} diff --git a/Services/Audio/IAudioSource.cs b/Services/Audio/IAudioSource.cs new file mode 100644 index 0000000..c2257e4 --- /dev/null +++ b/Services/Audio/IAudioSource.cs @@ -0,0 +1,22 @@ +namespace ytLive.Services.Audio; + +/// +/// A live audio capture source (TASK 4 ship step 4 seam): produces PCM float +/// chunks, runs only while live. The default implementations wrap NAudio's +/// WASAPI capture (mic) / loopback (desktop/game); the mixer and the tests +/// consume this interface, never NAudio directly. +/// +public interface IAudioSource : IDisposable +{ + /// Starts capturing. Safe to call only once per Stop. + void Start(); + + /// Stops capturing; a later Start begins a fresh session. + void Stop(); + + /// Raises each captured chunk (interleaved PCM float, -1..1). + event Action? SampleReady; + + /// Raises when capture dies or fails to start (e.g. no device). + event Action? Failed; +} diff --git a/Services/Audio/WasapiLoopbackAudioSource.cs b/Services/Audio/WasapiLoopbackAudioSource.cs new file mode 100644 index 0000000..620ef5c --- /dev/null +++ b/Services/Audio/WasapiLoopbackAudioSource.cs @@ -0,0 +1,69 @@ +using NAudio.Wave; + +namespace ytLive.Services.Audio; + +/// +/// Captures the user's desktop/game audio (TASK 4 ship step 4) via WASAPI +/// loopback on the default render device. Starts/stops with go-live only. +/// +public sealed class WasapiLoopbackAudioSource : IAudioSource +{ + private WasapiLoopbackCapture? _capture; + + public event Action? SampleReady; + public event Action? Failed; + + public void Start() + { + if (_capture != null) + throw new InvalidOperationException("Already started."); + + try + { + _capture = new WasapiLoopbackCapture(); + _capture.DataAvailable += OnDataAvailable; + _capture.RecordingStopped += OnRecordingStopped; + _capture.StartRecording(); + } + catch (Exception ex) + { + Stop(); + Failed?.Invoke(ex); + } + } + + public void Stop() + { + if (_capture == null) + return; + + try + { + _capture.StopRecording(); + } + catch + { + } + _capture.Dispose(); + _capture = null; + } + + public void Dispose() => Stop(); + + private void OnDataAvailable(object? sender, WaveInEventArgs e) + { + if (e.BytesRecorded <= 0) + return; + + var format = _capture?.WaveFormat ?? WaveFormat.CreateIeeeFloatWaveFormat(48000, 2); + var samples = WaveToFloat.Convert(e.Buffer, e.BytesRecorded, format); + if (samples.Length > 0) + SampleReady?.Invoke(new AudioSample(samples, format.SampleRate, format.Channels)); + } + + private void OnRecordingStopped(object? sender, StoppedEventArgs e) + { + if (e.Exception != null) + Failed?.Invoke(e.Exception); + } +} diff --git a/Services/Audio/WasapiMicAudioSource.cs b/Services/Audio/WasapiMicAudioSource.cs new file mode 100644 index 0000000..313b14a --- /dev/null +++ b/Services/Audio/WasapiMicAudioSource.cs @@ -0,0 +1,102 @@ +using NAudio.CoreAudioApi; +using NAudio.Wave; + +namespace ytLive.Services.Audio; + +/// +/// Captures the selected microphone (TASK 4 ship step 4) via WASAPI. Resolves +/// the NAudio device by FriendlyName matching MicSourceName (the app only +/// persists DisplayName), falling back to the default capture endpoint. +/// +public sealed class WasapiMicAudioSource : IAudioSource +{ + private readonly Func _micNameProvider; + private WasapiCapture? _capture; + + /// Returns the current mic DisplayName; read + /// at each Start so a device picked mid-session takes effect next go-live. + public WasapiMicAudioSource(Func micNameProvider) + { + _micNameProvider = micNameProvider; + } + + public event Action? SampleReady; + public event Action? Failed; + + public void Start() + { + if (_capture != null) + throw new InvalidOperationException("Already started."); + + try + { + var device = ResolveDevice(); + _capture = device != null ? new WasapiCapture(device) : new WasapiCapture(); + _capture.DataAvailable += OnDataAvailable; + _capture.RecordingStopped += OnRecordingStopped; + _capture.StartRecording(); + } + catch (Exception ex) + { + Stop(); + Failed?.Invoke(ex); + } + } + + public void Stop() + { + if (_capture == null) + return; + + try + { + _capture.StopRecording(); + } + catch + { + } + _capture.Dispose(); + _capture = null; + } + + public void Dispose() => Stop(); + + private MMDevice? ResolveDevice() + { + var micName = _micNameProvider(); + if (string.IsNullOrWhiteSpace(micName)) + return null; + + try + { + using var enumerator = new MMDeviceEnumerator(); + foreach (var endpoint in enumerator.EnumerateAudioEndPoints(DataFlow.Capture, DeviceState.Active)) + { + if (string.Equals(endpoint.FriendlyName, micName, StringComparison.OrdinalIgnoreCase)) + return endpoint; + } + } + catch + { + } + + return null; + } + + private void OnDataAvailable(object? sender, WaveInEventArgs e) + { + if (e.BytesRecorded <= 0) + return; + + var format = _capture?.WaveFormat ?? WaveFormat.CreateIeeeFloatWaveFormat(48000, 1); + var samples = WaveToFloat.Convert(e.Buffer, e.BytesRecorded, format); + if (samples.Length > 0) + SampleReady?.Invoke(new AudioSample(samples, format.SampleRate, format.Channels)); + } + + private void OnRecordingStopped(object? sender, StoppedEventArgs e) + { + if (e.Exception != null) + Failed?.Invoke(e.Exception); + } +} diff --git a/Services/Audio/WaveToFloat.cs b/Services/Audio/WaveToFloat.cs new file mode 100644 index 0000000..9503839 --- /dev/null +++ b/Services/Audio/WaveToFloat.cs @@ -0,0 +1,41 @@ +using NAudio.Dmo; +using NAudio.Wave; + +namespace ytLive.Services.Audio; + +/// +/// Converts NAudio's raw WASAPI capture buffers (byte[], whatever bit depth the +/// device's mix format reports) into interleaved PCM float samples. Pure — +/// unit-tested; the two WASAPI sources share it. +/// +public static class WaveToFloat +{ + public static float[] Convert(byte[] buffer, int bytesRecorded, WaveFormat format) + { + if (format.Encoding == WaveFormatEncoding.IeeeFloat && format.BitsPerSample == 32) + return ConvertIeeeFloat(buffer, bytesRecorded); + if (format.Encoding == WaveFormatEncoding.Pcm && format.BitsPerSample == 16) + return ConvertPcm16(buffer, bytesRecorded); + if (format is WaveFormatExtensible extensible + && extensible.SubFormat == AudioMediaSubtypes.MEDIASUBTYPE_IEEE_FLOAT) + return ConvertIeeeFloat(buffer, bytesRecorded); + return ConvertPcm16(buffer, bytesRecorded); + } + + private static float[] ConvertIeeeFloat(byte[] buffer, int bytesRecorded) + { + var count = bytesRecorded / 4; + var result = new float[count]; + Buffer.BlockCopy(buffer, 0, result, 0, count * 4); + return result; + } + + private static float[] ConvertPcm16(byte[] buffer, int bytesRecorded) + { + var count = bytesRecorded / 2; + var result = new float[count]; + for (var i = 0; i < count; i++) + result[i] = BitConverter.ToInt16(buffer, i * 2) / 32768f; + return result; + } +} diff --git a/Services/Encoder/FfmpegEncoder.cs b/Services/Encoder/FfmpegEncoder.cs index 489e448..4b062f2 100644 --- a/Services/Encoder/FfmpegEncoder.cs +++ b/Services/Encoder/FfmpegEncoder.cs @@ -136,7 +136,7 @@ public sealed class FfmpegEncoder : IFfmpegEncoder { using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); timeout.CancelAfter(TimeSpan.FromSeconds(10)); - await process.WaitForExitAsync(timeout.Token).ConfigureAwait(false); + await process!.WaitForExitAsync(timeout.Token).ConfigureAwait(false); } catch (OperationCanceledException) { diff --git a/Services/index.md b/Services/index.md index a4429f6..2e78bee 100644 --- a/Services/index.md +++ b/Services/index.md @@ -42,6 +42,13 @@ External-facing logic: YouTube API, persistence. See | `Encoder/FfmpegProgressParser.cs` | Pure parser for `frame=/fps=/size=/time=/bitrate=` stats lines → `FfmpegProgress` | | `Encoder/FfmpegEncoderPicker.cs` | Pure H.264 encoder picker from `-encoders` output: NVENC → QSV → AMF → OpenH264; **never returns libx264** (GPL) | | `Encoder/FfmpegLocator.cs` | The default locator: PATH probe wins, then the cache, then pull+extract (`ffmpeg.exe` **+ the `libav*.dll` family** via a staging dir, failures via `AppLog`). Pinned to BtbN `autobuild-2026-08-09-13-03` **lgpl-shared** (dynamic linking = LGPL compliance without static-relink §6 material; no GPL libx264; NVENC/QSV/AMF + libopenh264 + native AAC — see `ai.md` → Licensing). Search dirs / tools dir / downloader constructor-injected for hermetic tests. Not yet constructed by the app (the encoder step wires it) | +| `Audio/IAudioSource.cs` | **TASK 4 ship step 4 seam**: live capture source — `Start`/`Stop`/`SampleReady(AudioSample)`/`Failed(Exception)`, `IDisposable`; runs only while live. The app consumes this seam; tests inject hermetic fakes | +| `Audio/AudioSample.cs` | One captured chunk: interleaved PCM float (-1..1) + sample rate + channels | +| `Audio/WasapiLoopbackAudioSource.cs` | Desktop/game capture: NAudio `WasapiLoopbackCapture` on the default render device — automatic at unity, zero UI | +| `Audio/WasapiMicAudioSource.cs` | Mic capture: NAudio `WasapiCapture` with the device resolved by `FriendlyName` matching `MicSourceName` (DisplayName only), default capture endpoint fallback; name provider `Func` re-read at each `Start` so a mic picked mid-session takes effect next go-live | +| `Audio/AudioMixer.cs` | Owns both sources; `Start`/`Stop` follow go-live (`MainViewModel.BeginGoLive`/`StopStream`). Mic samples → `AudioLevelMeter` → `MicLevelChanged`; loopback samples currently dropped (the future encoder AAC mix consumes them). Failures log via `AppLog`; mic failure zeroes the meter, loopback failure doesn't kill the mic | +| `Audio/AudioLevelMeter.cs` | Pure smoothed RMS level (0..1): `Push(AudioSample)` + `Reset` — the unit-tested math behind `AudioLevel` | +| `Audio/WaveToFloat.cs` | Pure WASAPI buffer → float conversion: IEEE float 32-bit direct, PCM 16-bit normalized, `WaveFormatExtensible` IEEE-float subformat GUID, trailing partial samples ignored | Related: constructed in [`ViewModels/MainViewModel.cs`](../ViewModels/MainViewModel.cs) (no DI container yet). Models in [`Models/index.md`](../Models/index.md). diff --git a/TASKS.md b/TASKS.md index 64e80c9..5cf4f8a 100644 --- a/TASKS.md +++ b/TASKS.md @@ -200,7 +200,7 @@ Preview shows the transition too (WYSIWYG). No wipes/slides/LUTs beyond the four 1. ✅ **Ship step 1 — the output compositor SHIPPED** (2026-08-10) 2. ✅ **Ship step 2 — the FFmpeg locator SHIPPED** (2026-08-10) 3. ✅ **Encoder + RTMP push SHIPPED** (2026-08-12) — the FFmpeg subprocess: raw BGRA frames via stdin, stderr health parsing, FLV mux + push to the ingestion URL (see the ship step 3 plan below) -4. ☐ **WASAPI audio capture** — loopback (desktop/game) + the picked mic feeding `AudioLevel` so the realtime meter comes alive (req 7) +4. ✅ **WASAPI audio capture SHIPPED** (2026-08-12) — NAudio loopback (desktop/game) + the picked mic feeding `AudioLevel`, so the realtime meter comes alive (see the ship step 4 plan below) 5. ☐ **Frame-pipeline wiring** — `CameraManager`/`ScreenCaptureManager` → compositor resolver → encoder 6. ☐ **Health stats** — bitrate, FPS, dropped frames reported live in the bottom bar (req 5) 7. ☐ **One-click go live + private-only enforcement** — Go Live always creates/updates the broadcast with `privacyStatus = "private"` + PRIVATE badge (req 8, test-verifiable) @@ -390,6 +390,42 @@ Tests: `FfmpegEncoderTests` integration (probe → spawn with NVENC preferred progress parsed → graceful stop, no kill) + units (args compliance/GOP, progress parser, picker preference + GPL guard, no-URL/not-running/noop stops, process-death `ProcessFailed`) — **122 passing**. +#### Ship step 4 — WASAPI audio capture (the meter comes alive) + +**Goal:** capture desktop/game audio (loopback) and the picked mic, feed the mic level into `AudioLevel` +so the realtime meter reads something other than 0, run capture **only while live** (req 7). + +**Decisions (locked):** +1. **NAudio `NAudio.Wasapi` 2.2.1** — the wasapi feature package (not the `NAudio` meta-package): it + carries the capture types (`WasapiCapture`/`WasapiLoopbackCapture` + the MMDevice enumeration) with + `NAudio.Core`/`NAudio.Asio` pulled in transitively. MIT — recorded in `THIRD-PARTY-NOTICES.txt` (item 9). +2. **`IAudioSource` seam** (`Start`/`Stop`/`SampleReady`/`Failed`, IDisposable) — the app consumes the + seam; the two WASAPI implementations wrap NAudio; tests inject hermetic fakes (no real audio devices, + no timers). Loopback = `WasapiLoopbackCapture` on the default render device; mic = `WasapiCapture` + with the NAudio device resolved by `FriendlyName` matching `MicSourceName` (the app only persists the + DisplayName), falling back to the default capture endpoint. Mic device resolution is re-read at each + `Start` via a name provider so a mic picked mid-session takes effect next go-live. +3. **`AudioMixer` owns both sources** — starts/stops both with go-live (`BeginGoLive` success → `Start`, + `StopStream` → `Stop`). Mic samples feed a pure `AudioLevelMeter` (RMS, exponential smoothing) and + raise `MicLevelChanged`, marshalled to the UI thread into `AudioLevel`; desktop samples are currently + dropped (consumed by the encoder's AAC mix in a later step). Capture failures are logged via `AppLog` + (mic failure also zeroes the meter); loopback failure doesn't kill the mic. +4. **Byte→float** — pure `WaveToFloat.Convert` handles the WASAPI mix formats: IEEE float 32-bit (direct) + and PCM 16-bit (normalized to -1..1), including `WaveFormatExtensible` with the IEEE-float subformat + GUID. Trailing partial samples are ignored. + +**Built (2026-08-12):** `Services/Audio/` ships `IAudioSource` + `AudioSample`, `WasapiLoopbackAudioSource`, +`WasapiMicAudioSource`, `AudioMixer`, `AudioLevelMeter`, `WaveToFloat`; `MainViewModel` constructs the +mixer (mic source fed `() => MicSourceName`), starts it on go-live and stops it on end-stream, and maps +`MicLevelChanged` → `AudioLevel`. A pre-existing CS8602 in `FfmpegEncoder.cs:139` surfaced during this +step's rebuild and was fixed (`process!`) — build **0 warnings**. Tests: `AudioMixerTests` (mixer +lifecycle/forwarding/failure against fakes, meter RMS/smoothing/reset, `WaveToFloat` float/PCM16/ +extensible/truncation) — **139 passing**. + +**Deferred (later ship steps):** wiring the desktop-capture samples into the encoder's AAC mix (replaces +the `-f lavfi -i anullsrc` placeholder; ship step 5 owns the encoder construction), WASAPI capture while +not live, and any audio UI beyond the existing mic controls. + --- ## TASK 5 — YouTube Live Stream Management diff --git a/THIRD-PARTY-NOTICES.txt b/THIRD-PARTY-NOTICES.txt index dcd6233..37f78bc 100644 --- a/THIRD-PARTY-NOTICES.txt +++ b/THIRD-PARTY-NOTICES.txt @@ -79,6 +79,14 @@ the guardrails — the "never do" list is there on purpose. obligations; it is listed here per this file's "list everything" policy. +9. NAudio (WASAPI audio capture — loopback + mic) + Copyright (c) Mark Heath and contributors + License: MIT + Home: https://github.com/naudio/NAudio + Used as: the WASAPI loopback (desktop/game audio) and mic capture sources + behind the `IAudioSource` seam (TASK 4 ship step 4). MIT imposes + no source offer; this notice is kept per this file's policy. + -------------------------------------------------------------------------------- MISCELLANEOUS diff --git a/ViewModels/MainViewModel.cs b/ViewModels/MainViewModel.cs index a75d0d1..c0fb583 100644 --- a/ViewModels/MainViewModel.cs +++ b/ViewModels/MainViewModel.cs @@ -14,6 +14,7 @@ using Microsoft.Win32; using ytLive.Helpers; using ytLive.Models; using ytLive.Services; +using ytLive.Services.Audio; namespace ytLive.ViewModels; @@ -79,6 +80,12 @@ public class MainViewModel : ViewModelBase private SocialsConfig? _socials; private readonly IMicrophoneEnumerator _microphoneEnumerator; + // Live audio capture (TASK 4 ship step 4): desktop/game via WASAPI loopback, + // mic via WASAPI capture, both owned by the mixer and running only while + // live. Mic level feeds AudioLevel (the meter); loopback is for the future + // encoder mix. Private by design — no UI beyond the existing mic controls. + private readonly AudioMixer _audioMixer; + // Screen backdrop: a permanent live capture (desktop/game) that every scene // shows at the bottom layer. One shared capture session per key — the // ScreenCaptureManager refcounts by key, mirroring CameraManager. @@ -832,6 +839,12 @@ public class MainViewModel : ViewModelBase _microphoneEnumerator = new WinRtMicrophoneEnumerator(); + _audioMixer = new AudioMixer( + new WasapiMicAudioSource(() => MicSourceName), + new WasapiLoopbackAudioSource(), + message => AppLog.Write(message)); + _audioMixer.MicLevelChanged += OnMicLevelChanged; + _fullScreenDetector = new Win32FullScreenDetector(); foreach (var display in _fullScreenDetector.GetDisplays()) Displays.Add(display); @@ -1725,6 +1738,7 @@ public class MainViewModel : ViewModelBase ? "ytLlive" : $"{dialog.StreamTitle} — ytLlive"; StreamStatus = StreamStatus.Streaming; + _audioMixer.Start(); } } @@ -1732,6 +1746,7 @@ public class MainViewModel : ViewModelBase { StreamStatus = StreamStatus.Offline; WindowTitle = "ytLlive"; + _audioMixer.Stop(); // Graceful end completes the session = signs out (the DPAPI token is // cleared so the next Start Stream requires a fresh sign-in). A crash // never runs this, so the token survives and the creator stays signed in. @@ -1742,6 +1757,17 @@ public class MainViewModel : ViewModelBase AppLog.Write("Stream ended; session signed out"); } + private void OnMicLevelChanged(float level) + { + // NAudio raises on its capture thread; marshal to the UI thread so the + // meter binding updates safely. + var dispatcher = System.Windows.Application.Current?.Dispatcher; + if (dispatcher != null && !dispatcher.CheckAccess()) + dispatcher.BeginInvoke(() => AudioLevel = level); + else + AudioLevel = level; + } + private void UpdateLiveVisuals() { var live = IsLive; diff --git a/ai.md b/ai.md index f2d77a2..054fd28 100644 --- a/ai.md +++ b/ai.md @@ -91,7 +91,7 @@ C# / WPF (.NET 8) following MVVM: |------|------| | `Models/` | Plain data types — Scene, Source (incl. `ClipShape`, `IsMirrored`, `VideoImageSource`), QualityOption, StreamConfig, StreamHealth, YouTubeChannel, ChatMessage, **Socials (`SocialService` enum + `SocialEntry`/`SocialsConfig` + `SocialServiceIcons`) — the social bar** | | `ViewModels/` | MainViewModel — exposes collections + commands for the UI; GoLiveViewModel, ReuseImageViewModel, CameraPickerViewModel, **SocialsDialogViewModel** | -| `Services/` | YouTube OAuth2, stream/broadcast management, live chat polling, LayoutStore (SQLite), **SocialValidator (`ISocialValidator` seam + `HttpSocialValidator` default)**, **webcam: `VideoFrame` seam + `CameraDeviceInfo`/`ICameraEnumerator`/`ICameraFrameSource` interfaces + `MediaCaptureCameraEnumerator`/`MediaCaptureFrameSource` (WinRT) + `CameraManager`**, **screen capture: `IFullScreenDetector`/`Win32FullScreenDetector` + `IScreenCaptureSource`/`ScreenCaptureFrameSource` (WinRT GraphicsCapture) + `ScreenCaptureManager` + `ScreenCaptureSourceFactory` + `Direct3D11Helper`/`CaptureInterop` (COM bridges)** | +| `Services/` | YouTube OAuth2, stream/broadcast management, live chat polling, LayoutStore (SQLite), **SocialValidator (`ISocialValidator` seam + `HttpSocialValidator` default)**, **webcam: `VideoFrame` seam + `CameraDeviceInfo`/`ICameraEnumerator`/`ICameraFrameSource` interfaces + `MediaCaptureCameraEnumerator`/`MediaCaptureFrameSource` (WinRT) + `CameraManager`**, **screen capture: `IFullScreenDetector`/`Win32FullScreenDetector` + `IScreenCaptureSource`/`ScreenCaptureFrameSource` (WinRT GraphicsCapture) + `ScreenCaptureManager` + `ScreenCaptureSourceFactory` + `Direct3D11Helper`/`CaptureInterop` (COM bridges)**, **audio: `IAudioSource` seam + `WasapiLoopbackAudioSource`/`WasapiMicAudioSource` (NAudio WASAPI) + `AudioMixer` + pure `AudioLevelMeter`/`WaveToFloat` (see "Live audio capture")** | | `Helpers/` | ViewModelBase (INotifyPropertyChanged), RelayCommand, ImageCache, AppLog (file logger), FocusPreservingListBox, OAuthCredentials, **TokenStore (DPAPI session persistence)**, visibility converters | | `Themes/` | `Controls.xaml` — the single dark-theme source, merged once in `App.xaml` (see `Themes/index.md`) | | `MainWindow.xaml` | Dark theme; layout: top bar (controls), center (preview), left (scenes/sources), right (chat), bottom (health) | @@ -114,7 +114,7 @@ C# / WPF (.NET 8) following MVVM: - `Helpers/OAuthCredentials.cs` contains the real ClientId/ClientSecret. Auth is complete and the session **persists via Windows DPAPI** (`Helpers/TokenStore.cs` → `%APPDATA%\ytLlive\ytLlive.auth`, CurrentUser scope), reloaded best-effort at startup with a proactive refresh of a near-expiry access token. Sign-in/Change Account lives **inside the Start Stream dialog** (two-state flow — no separate Connect button). A **graceful End Livestream signs out**: `StopStream()` clears the session + token, so the next go-live needs a fresh sign-in; a crash never runs End, so the token survives and the creator stays signed in. `YouTubeAuthService` takes an optional `HttpClient` + `sessionChanged` callback (test seam + save hook; services are still constructed in `MainViewModel`) - Scene/source/asset layout + the social bar persist (SQLite, schema v8); the OAuth session persists (DPAPI); the paid-unlock state does not (yet — itch.io key verification pending) - `YouTubeStreamService` uses hardcoded `1080p`/`60fps` and per-broadcast streams — must switch to the v3 `variable` reusable stream -- Webcam capture is shipped (milestone 1); the live desktop/game backdrop is shipped (ship task #1); **the output compositor (TASK 4 ship step 1) is SHIPPED**, **the FFmpeg locator (TASK 4 ship step 2) is SHIPPED**, **the encoder + RTMP push (TASK 4 ship step 3) is SHIPPED** — full plan in `TASKS.md`; WASAPI audio capture and the frame-pipeline wiring follow (each its own PR) +- Webcam capture is shipped (milestone 1); the live desktop/game backdrop is shipped (ship task #1); **the output compositor (TASK 4 ship step 1) is SHIPPED**, **the FFmpeg locator (TASK 4 ship step 2) is SHIPPED**, **the encoder + RTMP push (TASK 4 ship step 3) is SHIPPED**, **WASAPI audio capture (TASK 4 ship step 4) is SHIPPED** — full plan in `TASKS.md`; the frame-pipeline wiring follows (its own PR) - `StreamConfig` defaults (`TargetBitrate=6000`, `Resolution="1920x1080"`) are stale — the live dropdown drives `StreamHealth.CurrentBitrate`/`FPS` instead ### Screen backdrop capture (TASK 3 ship task #1) @@ -354,6 +354,34 @@ NVENC → QSV → AMF → OpenH264 fallback, **never libx264** (GPL; see Licensi forces one and skips the probe. `EncoderOptions` also carries W×H/FPS/bitrate from the quality tier and `GopSize` = FPS×4. Not yet constructed by the app — the frame-pipeline wiring (ship step 5) owns it. +### Live audio capture (TASK 4 ship step 4 — shipped 2026-08-12, plan in TASKS.md) + +**Capture runs only while live and is KISS by rule**: desktop/game audio is automatic (WASAPI loopback, +zero UI), the mic is the creator's only audio control (meter/mute/volume already shipped). The whole +layer sits behind an **`IAudioSource` seam** (`Services/Audio/`: `Start`/`Stop`/`SampleReady`/`Failed`, +IDisposable) so the app never touches NAudio directly and tests inject hermetic fakes (no devices, no +timers). + +- **Sources (NAudio `NAudio.Wasapi` 2.2.1, MIT — item 9 in `THIRD-PARTY-NOTICES.txt`):** + `WasapiLoopbackAudioSource` = `WasapiLoopbackCapture` on the default render device; + `WasapiMicAudioSource` = `WasapiCapture` with the device resolved by `FriendlyName` matching + `MicSourceName` (the app only persists the **DisplayName**), falling back to the default capture + endpoint. Mic device resolution re-reads the name provider `Func` at each `Start`, so a mic + picked mid-session takes effect next go-live. +- **`AudioMixer`** owns both sources; `Start`/`Stop` follow go-live (`MainViewModel.BeginGoLive` success + → `_audioMixer.Start()`, `StopStream` → `Stop()`). Mic samples feed a pure **`AudioLevelMeter`** (RMS + with 0.2 exponential smoothing) → `MicLevelChanged` → marshalled to the UI thread → `AudioLevel`. + Desktop samples are currently **dropped** — the future encoder's AAC mix (ship step 5) consumes them, + replacing the `-f lavfi -i anullsrc` placeholder. Failures log via `AppLog`; a mic failure zeroes the + meter, a loopback failure never kills the mic. +- **`WaveToFloat`** (pure, shared): WASAPI mix formats → interleaved float — IEEE float 32-bit direct, + PCM 16-bit normalized to -1..1, `WaveFormatExtensible` with the IEEE-float subformat GUID + (`NAudio.Dmo.AudioMediaSubtypes.MEDIASUBTYPE_IEEE_FLOAT`), trailing partial samples ignored. +- `FfmpegEncoder.cs:139` pre-existing CS8602 fixed (`process!`) — build **0 warnings**; **139 passing**. + +**Deferred:** loopback→encoder mix wiring and the encoder construction (ship step 5); capture while not +live is deliberately not shipped (privacy indicator otherwise). + ### Social bar (TASK 14 — shipped 2026-08-12, plan in TASKS.md) A **global bar layer** (never a Source, no sources-list row) that sits over the bottom or top of the diff --git a/ytLive.Tests/AudioMixerTests.cs b/ytLive.Tests/AudioMixerTests.cs new file mode 100644 index 0000000..d613b90 --- /dev/null +++ b/ytLive.Tests/AudioMixerTests.cs @@ -0,0 +1,280 @@ +using NAudio.Wave; +using Xunit; +using ytLive.Services.Audio; + +namespace ytLive.Tests; + +/// +/// TASK 4 ship step 4: WASAPI audio capture behind the +/// seam. The units pin down the pure pieces — byte→float conversion, the level +/// meter math, and the mixer lifecycle/forwarding against fakes. No real audio +/// devices (NAudio device resolution is a thin wrapper left to a manual smoke +/// test), no timers. +/// +public class AudioMixerTests +{ + private sealed class FakeSource : IAudioSource + { + public int StartCount { get; private set; } + public int StopCount { get; private set; } + public bool Disposed { get; private set; } + public event Action? SampleReady; + public event Action? Failed; + + public void Start() => StartCount++; + public void Stop() => StopCount++; + public void Dispose() => Disposed = true; + + public void Emit(AudioSample sample) => SampleReady?.Invoke(sample); + public void Fail(Exception ex) => Failed?.Invoke(ex); + } + + [Fact] + public void Start_StartsBothSources() + { + var mic = new FakeSource(); + var loopback = new FakeSource(); + var mixer = new AudioMixer(mic, loopback); + + mixer.Start(); + + Assert.Equal(1, mic.StartCount); + Assert.Equal(1, loopback.StartCount); + } + + [Fact] + public void Start_IsIdempotent() + { + var mic = new FakeSource(); + var mixer = new AudioMixer(mic, new FakeSource()); + + mixer.Start(); + mixer.Start(); + + Assert.Equal(1, mic.StartCount); + } + + [Fact] + public void Stop_StopsBothAndResetsLevel() + { + var mic = new FakeSource(); + var loopback = new FakeSource(); + var mixer = new AudioMixer(mic, loopback); + mixer.Start(); + mic.Emit(new AudioSample(new[] { 0.8f }, 48000, 1)); + + var last = -1f; + mixer.MicLevelChanged += l => last = l; + mixer.Stop(); + + Assert.Equal(1, mic.StopCount); + Assert.Equal(1, loopback.StopCount); + Assert.Equal(0f, last); + Assert.Equal(0f, mixer.MicLevel); + } + + [Fact] + public void Stop_WithNoStart_DoesNothing() + { + var mic = new FakeSource(); + var mixer = new AudioMixer(mic, new FakeSource()); + + mixer.Stop(); + + Assert.Equal(0, mic.StopCount); + } + + [Fact] + public void MicSamples_DriveMicLevelChanged() + { + var mic = new FakeSource(); + var mixer = new AudioMixer(mic, new FakeSource()); + var levels = new List(); + mixer.MicLevelChanged += l => levels.Add(l); + + mixer.Start(); + mic.Emit(new AudioSample(new[] { 1f, 1f, 1f, 1f }, 48000, 1)); + mic.Emit(new AudioSample(new[] { -1f, -1f, -1f, -1f }, 48000, 1)); + + Assert.NotEmpty(levels); + Assert.All(levels, l => Assert.InRange(l, 0f, 1f)); + } + + [Fact] + public void LoopbackSamples_DoNotChangeMicLevel() + { + var loopback = new FakeSource(); + var mixer = new AudioMixer(new FakeSource(), loopback); + var levels = new List(); + mixer.MicLevelChanged += l => levels.Add(l); + + mixer.Start(); + loopback.Emit(new AudioSample(new[] { 1f, 1f, 1f, 1f }, 48000, 2)); + + Assert.Empty(levels); + Assert.Equal(0f, mixer.MicLevel); + } + + [Fact] + public void MicFailure_LogsAndResetsLevel() + { + var mic = new FakeSource(); + var logs = new List(); + var mixer = new AudioMixer(mic, new FakeSource(), m => logs.Add(m)); + mixer.Start(); + mic.Emit(new AudioSample(new[] { 0.5f }, 48000, 1)); + + var last = -1f; + mixer.MicLevelChanged += l => last = l; + mic.Fail(new InvalidOperationException("boom")); + + Assert.Contains(logs, l => l.Contains("boom")); + Assert.Equal(0f, last); + } + + [Fact] + public void LoopbackFailure_LogsButKeepsMic() + { + var loopback = new FakeSource(); + var logs = new List(); + var mixer = new AudioMixer(new FakeSource(), loopback, m => logs.Add(m)); + mixer.Start(); + + var last = -1f; + mixer.MicLevelChanged += l => last = l; + loopback.Fail(new InvalidOperationException("boom")); + + Assert.Contains(logs, l => l.Contains("boom")); + Assert.Equal(-1f, last); + } + + [Fact] + public void Dispose_StopsAndDisposesSources() + { + var mic = new FakeSource(); + var loopback = new FakeSource(); + var mixer = new AudioMixer(mic, loopback); + mixer.Start(); + + mixer.Dispose(); + + Assert.True(mic.Disposed); + Assert.True(loopback.Disposed); + Assert.Equal(1, mic.StopCount); + } + + [Fact] + public void Dispose_UnsubscribesEvents() + { + var mic = new FakeSource(); + var loopback = new FakeSource(); + var mixer = new AudioMixer(mic, loopback); + mixer.Dispose(); + + var levels = new List(); + mixer.MicLevelChanged += l => levels.Add(l); + mic.Emit(new AudioSample(new[] { 1f }, 48000, 1)); + + Assert.Empty(levels); + } +} + +public class AudioLevelMeterTests +{ + [Fact] + public void ConstantSine_ConvergesToRms() + { + var meter = new AudioLevelMeter(); + var sample = new AudioSample(new[] { 0.5f, -0.5f, 0.5f, -0.5f }, 48000, 1); + + float level = 0; + for (var i = 0; i < 30; i++) + level = meter.Push(sample); + + Assert.InRange(level, 0.45f, 0.55f); + } + + [Fact] + public void Silence_DrivesTowardZero() + { + var meter = new AudioLevelMeter(); + meter.Push(new AudioSample(new[] { 1f, 1f, 1f, 1f }, 48000, 1)); + var samples = new float[1024]; + var sample = new AudioSample(samples, 48000, 1); + + for (var i = 0; i < 50; i++) + meter.Push(sample); + + Assert.Equal(0f, meter.Level, 3); + } + + [Fact] + public void EmptySample_KeepsLevel() + { + var meter = new AudioLevelMeter(); + meter.Push(new AudioSample(new[] { 1f, 1f, 1f, 1f }, 48000, 1)); + var before = meter.Level; + + var level = meter.Push(new AudioSample(Array.Empty(), 48000, 1)); + + Assert.Equal(before, level); + } + + [Fact] + public void Reset_ZerosLevel() + { + var meter = new AudioLevelMeter(); + meter.Push(new AudioSample(new[] { 1f, 1f, 1f, 1f }, 48000, 1)); + + meter.Reset(); + + Assert.Equal(0f, meter.Level); + } +} + +public class WaveToFloatTests +{ + private static byte[] FloatSamples(params float[] values) + { + var bytes = new byte[values.Length * 4]; + Buffer.BlockCopy(values, 0, bytes, 0, bytes.Length); + return bytes; + } + + [Fact] + public void IeeeFloat32_PreservesValues() + { + var format = WaveFormat.CreateIeeeFloatWaveFormat(48000, 2); + var samples = WaveToFloat.Convert(FloatSamples(0.25f, -0.5f, 1f), 12, format); + + Assert.Equal(3, samples.Length); + Assert.Equal(0.25f, samples[0], 4); + Assert.Equal(-0.5f, samples[1], 4); + Assert.Equal(1f, samples[2], 4); + } + + [Fact] + public void Pcm16_NormalizesToUnitRange() + { + var format = WaveFormat.CreateCustomFormat(WaveFormatEncoding.Pcm, 48000, 1, 48000 * 2, 2, 16); + var bytes = new byte[] { 0x00, 0x00, 0xFF, 0x7F, 0x00, 0x80 }; + var samples = WaveToFloat.Convert(bytes, 6, format); + + Assert.Equal(3, samples.Length); + Assert.Equal(0f, samples[0], 4); + Assert.Equal(1f, samples[1], 4); + Assert.Equal(-1f, samples[2], 4); + } + + [Fact] + public void TruncatedTrailingBytes_AreIgnored() + { + var format = WaveFormat.CreateIeeeFloatWaveFormat(48000, 2); + var bytes = new byte[] { 0, 0, 0x80, 0x3F, 1, 2, 3 }; // 1 float + 3 stray bytes + + var samples = WaveToFloat.Convert(bytes, bytes.Length, format); + + Assert.Single(samples); + Assert.Equal(1f, samples[0], 4); + } +} diff --git a/ytLive.csproj b/ytLive.csproj index 0141d54..8af3795 100644 --- a/ytLive.csproj +++ b/ytLive.csproj @@ -36,6 +36,7 @@ +