From 6d71acef8cb0fdbfb111b2861d8daae323ef018c Mon Sep 17 00:00:00 2001 From: gramps Date: Fri, 14 Aug 2026 18:09:26 -0700 Subject: [PATCH] =?UTF-8?q?TASK=209=20audio=20milestone:=20real=20stream?= =?UTF-8?q?=20audio=20+=20voice=20filters=20+=20auto-duck=20+=20free=20TRA?= =?UTF-8?q?X=20music=20=E2=80=94=20the=20mixer=20now=20feeds=20the=20encod?= =?UTF-8?q?er=20real=20audio=20(ffmpeg=20reads=20a=20Windows=20named=20pip?= =?UTF-8?q?e,=20-f=20f32le=20-ar=2048000=20-ac=202=20-i=20\.\pipe\ytllive?= =?UTF-8?q?=5Faudio,=20replacing=20anullsrc=20silence;=20explicit=20-map?= =?UTF-8?q?=200:v=20-map=201:a=20via=20EncoderOptions.AudioPipeName),=20wi?= =?UTF-8?q?th=20honest=20gains=20reaching=20the=20stream=20(MicVolume=20sc?= =?UTF-8?q?ales=20mic,=20GameAudioVolume=20+=20mute=20scale=20loopback,=20?= =?UTF-8?q?Func=20seams=20on=20AudioMixer),=20an=20always-on=20pur?= =?UTF-8?q?e-C#=20voice=20chain=20on=20the=20mic=20BEFORE=20the=20meter=20?= =?UTF-8?q?and=20mix=20(LowShelfFilter=20120Hz=20+4dB=20=E2=86=92=20HighSh?= =?UTF-8?q?elfFilter=208kHz=20+3dB=20=E2=86=92=20NoiseGate=200.005/hystere?= =?UTF-8?q?sis=200.5=20=E2=86=92=20Compressor=200.5=204:1,=20all=20TDF2),?= =?UTF-8?q?=20AutoDucker=20(mic=20RMS>0.02=20=E2=86=92=20loopback=20=C3=97?= =?UTF-8?q?0.25,=20attack=200.05/release=200.005),=20and=20TRAX=20free=20b?= =?UTF-8?q?ackground=20music=20(MusicPlayer=20=3D=20MediaFoundationReader?= =?UTF-8?q?=20=E2=86=92=20VolumeWaveProvider16=20at=20fixed=200.20=20?= =?UTF-8?q?=E2=86=92=20WaveOutEvent=20via=20the=20new=20sibling=20NAudio.W?= =?UTF-8?q?inMM=202.2.1=20package;=20plays=20to=20the=20default=20device?= =?UTF-8?q?=20so=20the=20existing=20loopback=20carries=20it,=20ducked=20wi?= =?UTF-8?q?th=20game;=20footer=20TRAX=20button=20with=20red/yellow/green?= =?UTF-8?q?=20status=20dot,=20left-click=20toggles/picks,=20right-click=20?= =?UTF-8?q?opens=20the=20OpenFileDialog=20picker,=20tooltip=20shows=20the?= =?UTF-8?q?=20track=20name;=20persisted=20via=20schema=20v9=20single-row?= =?UTF-8?q?=20Music).=20Build-time=20deviations=20from=20the=20plan=20(rec?= =?UTF-8?q?orded=20in=20ai.md=20+=20TASKS.md):=20WasapiCapture=20in=20NAud?= =?UTF-8?q?io=202.2.1=20exposes=20no=20overridable=20GetDefaultMixFormat?= =?UTF-8?q?=20so=20sources=20run=20device=20mix=20format=20and=20the=20mix?= =?UTF-8?q?er's=20TinyResampler=20normalizes=20any=20rate=20to=2048kHz=20(?= =?UTF-8?q?resampler=20IS=20the=20design);=20FfmpegEncoder.cs=20untouched?= =?UTF-8?q?=20=E2=80=94=20MainViewModel.StopStream=20calls=20=5FaudioMixer?= =?UTF-8?q?.StopLive()=20(pipe=20EOF)=20before=20the=20frame=20pump=20stop?= =?UTF-8?q?s;=20sound-bar=20relabelled=20'Desktop=20Audio',=20IsGameAudioB?= =?UTF-8?q?arVisible=20=3D=20game=20sound=20OR=20music=20playing.=20Tests:?= =?UTF-8?q?=20new=20AudioPipelineTests=20(DSP/ring-buffer/ducker/resampler?= =?UTF-8?q?=20units=20+=20the=20ONE=20integration=20test=20Mix=5FWithFilte?= =?UTF-8?q?rsDuckAndGain=5FLands=5FOn=5FAudioPipe=20reading=20real=20pipe?= =?UTF-8?q?=20bytes;=20ring-buffer=20overwrite=20bug=20found=20+=20fixed),?= =?UTF-8?q?=20FfmpegEncoderTests/LayoutStorePersistenceTests=20updated=20?= =?UTF-8?q?=E2=80=94=20196=20tests=20passing,=200=20warnings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- HANDOFF.md | 136 ++++++--- MainWindow.xaml | 31 +- MainWindow.xaml.cs | 16 + Models/Music.cs | 15 + Services/Audio/AudioMixer.cs | 222 +++++++++++++- Services/Audio/AudioRingBuffer.cs | 95 ++++++ Services/Audio/AutoDucker.cs | 36 +++ Services/Audio/MusicPlayer.cs | 109 +++++++ Services/Audio/NamedPipeAudioWriter.cs | 105 +++++++ Services/Audio/TinyResampler.cs | 54 ++++ Services/Audio/VoiceFilterChain.cs | 204 +++++++++++++ Services/Audio/WasapiMicAudioSource.cs | 6 +- Services/Encoder/EncoderOptions.cs | 11 +- Services/Encoder/FfmpegArgs.cs | 17 +- Services/LayoutStore.cs | 47 ++- TASKS.md | 38 ++- ViewModels/MainViewModel.cs | 129 +++++++- ai.md | 92 +++--- ytLive.Tests/AudioPipelineTests.cs | 311 ++++++++++++++++++++ ytLive.Tests/FfmpegEncoderTests.cs | 6 +- ytLive.Tests/LayoutStorePersistenceTests.cs | 47 +++ ytLive.csproj | 1 + 22 files changed, 1605 insertions(+), 123 deletions(-) create mode 100644 Models/Music.cs create mode 100644 Services/Audio/AudioRingBuffer.cs create mode 100644 Services/Audio/AutoDucker.cs create mode 100644 Services/Audio/MusicPlayer.cs create mode 100644 Services/Audio/NamedPipeAudioWriter.cs create mode 100644 Services/Audio/TinyResampler.cs create mode 100644 Services/Audio/VoiceFilterChain.cs create mode 100644 ytLive.Tests/AudioPipelineTests.cs diff --git a/HANDOFF.md b/HANDOFF.md index 9704498..00a60a4 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -7,49 +7,67 @@ ## Session state (last updated: 2026-08-14) -- **Branch:** `main`, in sync with `origin/main`. -- **TASK 21 — real logo + creator-hub About: SHIPPED** (this session). The About overlay is now the - **creator hub**: the real logo ("llama fortnite superman logo" copied from `\\llamavault` → `Assets/llama-logo.png`, - the creator's own art — no third-party license), plus hub links — **llama chile shop on YouTube** - (`MainViewModel.ChannelUrl`, public now), **Mastodon** (`https://mastodon.llamachile.tube/@gramps`), - **Buy me a coffee** (`https://buymeacoffee.com/llamachiley` — LIVE), and **Unlock Premium** (greyed - "coming soon"; the itch.io product URL is a **tabled seam**, `MainViewModel.PremiumUrl` — the itch.io - account was just created and needs a setup session later). **Licenses & legal** flips the About overlay - to an in-app scrolling panel loading the shipped `THIRD-PARTY-NOTICES.txt` (`ShowLicenses()` reads from - the executable dir on first open, graceful "not found" fallback — never the OS viewer); "← Back to - About" (`BackToAbout_Click`) returns. Integration test (ONE): `AboutHubTests` drives the real window, - asserting the hub opens, the link URLs are real, licensing loads the shipped notices text, and back - returns to the hub. **174 tests passing, 0 warnings.** -- **TASK 8 — meter scaling amplification:** SHIPPED + pushed (`0b71b03`). `AudioLevelMeter.ToDisplay` - adds **+10 dB input amplification** before the −60..0 dBFS log map: speech peaks (~0.2 RMS) read - ~0.93 (red) and normal speech (~0.05) ~0.73 (yellow) at maxed volume; ≤0.001 linear still reads 0 - (never idles on background noise). One knob shared by the mic + game bars; `× MicVolume` untouched. -- **TASK 4 ship step 7 — one-click go live + private-only enforcement: SHIPPED** (this session; the - final open TASK 4 requirement). Double-enforced privacy: - 1. **Dialog locked to Private** — `GoLiveViewModel.Visibility` is a get-only `"Private"`, the - visibility ComboBox is gone (replaced by a "Streams always start Private" note); settings' dead - "Default Visibility" dropdown + `MainViewModel.Visibilities`/`DefaultStreamVisibility` removed. - 2. **Service forces private** — `YouTubeStreamService.CreateBroadcast` always sends - `privacyStatus="private"`; ctor gained `HttpClient? http = null` seam for tests. - 3. **Wired into go-live** — `BeginGoLive` fires `CreateBroadcastAsync` (stores `_currentBroadcastId` - for TASK 5's bind/transition; failure → `StreamStatus.Error` via AppLog, never a crash) + - `_framePump.StartAsync()`; `StopStream` clears `_currentBroadcastId`. - 4. **PRIVATE badge** — dark-red bordered, next to REC in the top bar, `Visibility="{Binding - IsLivePrivate}"` (MainWindow.xaml ~110). - 5. **Integration test** — `ytLive.Tests/YouTubeStreamServiceTests.cs`: the broadcast-insert request - must carry `"privacyStatus":"private"` (RecordingHandler seam) + a no-session → null guard test. -- **Verified:** build 0 warnings / 0 errors; **174/174 tests pass** (173 + the new AboutHubTests). -- **Uncommitted (this session's TASK 21 batch):** `MainWindow.xaml`, `ViewModels/MainViewModel.cs`, - `MainWindow.xaml.cs`, `Assets/llama-logo.png` (replaced with the real logo), `ytLive.Tests/AboutHubTests.cs` - (new), `TASKS.md`, `HANDOFF.md`. ai.md + ViewModels/index.md updates pending — **commit + push - before continuing.** -- **Next step:** commit + push TASK 21, then **TASK 5 — reusable `variable` stream** (create once - per channel, cache ingestion URL, bind to broadcast, feed `_rtmpUrlProvider` so the pump actually - pushes — `_currentBroadcastId` is already stashed for it). Also queued: **task 22 (voice filters)**, - and **setting up the itch.io product page** (the creator just made the account; the premium-unlock - URL then lands in `MainViewModel.PremiumUrl` and the About hub's "Unlock Premium" button lights up). - Nothing else queued — do not expand the task queue on your own. Optional, not queued: rewriting the - healed entry's `ProfileUrl` to `https://mastodon.llamachile.tube/@gramps` (user must say the word). +- **Branch:** `main` (audio milestone work is uncommitted, see "Uncommitted"). +- **AUDIO MILESTONE (TASK 9) — SHIPPED this session.** Real stream audio + voice filters + auto-duck + + free TRAX background music. Build **0 warnings**, full suite **196 passing** (ONE integration + test: `AudioPipelineTests.Mix_WithFiltersDuckAndGain_Lands_On_AudioPipe`). Everything below in the + milestone section is DONE. **NOT yet committed** — next step is review + commit (with `ai.md`/ + `TASKS.md`/this file already updated in the same changeset). +- **Two build-time deviations from the settled plan (both recorded in ai.md + TASKS.md):** + 1. `MusicPlayer` uses **`WaveOutEvent`** (new sibling package `NAudio.WinMM` 2.2.1) — not `WasapiOut` + as planned; `WaveOutEvent` isn't in `NAudio.Wasapi`/`NAudio.Core`. + 2. No forced 48 kHz capture — NAudio 2.2.1's `WasapiCapture` has no overridable + `GetDefaultMixFormat`, so both sources run the device mix format and the mixer's + `TinyResampler` normalizes any rate to 48 kHz (resampler IS the design, not a fallback). + 3. `FfmpegEncoder.cs` is **untouched** — the VM closes the audio pipe (`_audioMixer.StopLive()`) in + `StopStream` BEFORE the frame pump stops (audio EOF then video EOF, in order). +- **SERVER RECOVERY (llamachile.tube / YunoHost / DO droplet) — done earlier session.** Side task + complete; **rotate the DO API token** (control-panel only) and any password no longer trusted. +- **Shipped, all pushed before this session:** TASK 21 (creator-hub About, `3d92bd0`), TASK 8 (meter + +10 dB), TASK 4 ship step 7 (one-click go-live + private-only). **174 tests passing, 0 warnings.** +- **Uncommitted (whole TASK 9 milestone):** `Services/Audio/` (new `VoiceFilterChain.cs`, + `AudioRingBuffer.cs`, `AutoDucker.cs`, `TinyResampler.cs`, `MusicPlayer.cs`, `NamedPipeAudioWriter.cs`, + rewritten `AudioMixer.cs`, simplified `WasapiMicAudioSource.cs`), `Models/Music.cs` (new), + `FfmpegArgs.cs`, `EncoderOptions.cs`, `LayoutStore.cs` (v9), `MainViewModel.cs`, `MainWindow.xaml`, + `MainWindow.xaml.cs`, `ytLive.csproj` (NAudio.WinMM), `ytLive.Tests/` (`AudioPipelineTests.cs` new, + `FfmpegEncoderTests.cs` + `LayoutStorePersistenceTests.cs` updated), and docs `ai.md`, `TASKS.md`, + `HANDOFF.md`. +- **Next step:** commit the milestone (docs already staged in the same changeset), push, then start + **TASK 5** (reusable stream → `_rtmpUrlProvider` — the last blocker before go-live actually encodes). + Then **itch.io product page setup** (premium URL lands in `MainViewModel.PremiumUrl`, lights up + Unlock Premium). Optional, not queued: rewriting the healed entry's `ProfileUrl` to + `https://mastodon.llamachile.tube/@gramps` (user must say the word). + +## TASK 9 audio milestone — SHIPPED 2026-08-14 (what changed) + +The creator's feature review settled this as the single next branch ("all the audio issues done and +tested — a huge milestone"). Final spec and full shipped-state records live in `TASKS.md` (TASK 9) +and `ai.md` ("Live audio capture"). Highlights: + +- **Real audio into the encoder.** `FfmpegArgs` now builds `-f f32le -ar 48000 -ac 2 -i \\.\pipe\ytllive_audio` + + explicit `-map 0:v -map 1:a` (replaces `anullsrc` silence). `MainViewModel.BeginGoLive` → + `_audioMixer.StartLive(EncoderOptions.DefaultAudioPipeName)`; `StopStream` → `_audioMixer.StopLive()` + before `_framePump.StopAsync()`. +- **2-input mix (mic + loopback)**, honest gains: `micGain = MicVolume` (mute = 0), + `loopbackGain = GameMuted ? 0 : GameAudioVolume` × duck. No third music channel. +- **Voice chain on the mic** (TASK 22), before meter AND mix: bass 120 Hz +4 dB → treble 8 kHz +3 dB → + gate (0.005 / hysteresis 0.5) → compressor (0.5, 4:1). Pure TDF2 DSP, per-sample, always on. +- **Auto-duck:** mic RMS > 0.02 → loopback ×0.25 (−12 dB), attack 0.05 / release 0.005. +- **TRAX (free BGM):** `MusicPlayer` = MediaFoundationReader → `VolumeWaveProvider16` at fixed **0.20** → + `WaveOutEvent`. Plays to the default device → rides the loopback into the stream (ducked with game). + Footer TRAX button (dot red/yellow/green + "TRAX"), left-click toggles/picks, right-click opens the + picker (`OpenFileDialog`), tooltip shows the track name. Track persists via **schema v9** single-row + `Music`. Sound-bar label "Game Audio Capture" → **"Desktop Audio"**; + `IsGameAudioBarVisible = gameDetectorProducingSound || IsMusicPlaying`. +- **Tests:** new `AudioPipelineTests.cs` (DSP/ring-buffer/ducker/resampler units + the ONE integration + test reading real pipe bytes via `NamedPipeClientStream`); `FfmpegEncoderTests` + layout persistence + updated for pipe args / Music roundtrip. Ring-buffer overwrite bug found by the unit test and fixed + (head must NOT advance on eviction — the write itself advances it). Integration test pre-fills the + ring buffers before `StartLive` so the first pipe tick already carries audio (deterministic — a + start-of-stream silence race was seen and eliminated). + +**Out of scope this branch:** IP webcam, chat box, alt-key crop, credits, bg removal, music-off-VOD +track (YouTube mutes VODs with copyrighted music — future feature), itch.io `PremiumUrl`. - **Landmines:** - Never add another test that constructs `new App()` — use `RealAppHost.Run(...)` (shared STA host @@ -74,7 +92,8 @@ the pump stops), not a crash. - `StopAsync` must stop the encoder (closes stdin) **before** awaiting the pump loop — closing stdin unblocks a write stuck on pipe backpressure; the reverse - order deadlocks. + order deadlocks. Same rule for audio: `StopLive()` (pipe EOF) before the pump + stop, so ffmpeg's two inputs end in order. - Tests never instantiate `MainViewModel` directly except via a real `MainWindow` on the `RealAppHost` STA thread (round-clip + naming), which never go live. - Sandbox can't reach outbound HTTPS — `HttpSocialValidator` stub-handler tests @@ -87,6 +106,31 @@ - **Secret/DB/port facts live:** OAuth client id/secret in `Helpers/OAuthCredentials.cs`; OAuth session token in `Helpers/TokenStore.cs` (DPAPI → `%APPDATA%\ytLlive\ytLlive.auth`); - layout DB `%APPDATA%\ytLlive\ytLlive.db` (schema v8; `SocialEntry.Software` - column is a column-presence migration like the others, no version bump); OAuth callback + layout DB `%APPDATA%\ytLlive\ytLlive.db` (**schema v9** — single-row `Music`; the + `SocialEntry.Software` column is a column-presence migration like the others); OAuth callback `http://localhost:8765/oauth2/callback`; crash log `%APPDATA%\ytLlive\startup.log`. + +## Server recovery (llamachile.tube / YunoHost / DO) — 2026-08-14 + +**Facts (hunted this session — do not re-hunt):** +- Droplet **llamachile.tube**: DO droplet ID `473190301`, IP `143.244.176.131`, sfo3, 4GB/2vCPU/50GB. + **SSH port = 2214** (matches git remote `ssh://llgit.llamachile.tube:2214/gramps/ytLlive.git`); + 22/2222/2200/etc all closed. SSH as `gramps` works with `~/.ssh/id_ed25519` (key auth, no password). +- **Root is YunoHost-locked**: `PermitRootLogin no` in `/etc/ssh/sshd_config` (there's a conflicting + cloud-init override section below it, but DO's "Reset root password" email does NOT work on this + box — cloud-init `set-passwords` only ran once at install). Root is reachable via `sudo -i` or the + DO web Recovery Console (Settings → Recovery console — VNC-based; the droplet page's Console button + is SSH-based and fails with "all configured authentication methods failed" when no account has a + working password). +- **gramps is a YunoHost LDAP account** (local `/etc/passwd` entry has `*`, auth via pam_ldap) — its + password is changed with `sudo yunohost user update gramps -p ''`, NOT `passwd`. +- **fail2ban bans the whole IP for 10-12 min** after repeated failed SSH/root logins (all ports + refuse; `ping` still works; droplet API still shows `active`). Wait it out — do NOT power-cycle. +- Password reset this session: `sudo yunohost tools rootpw -n ''` sets root (rootpw takes `-n`). + Verification: `getent shadow root` shows a `$y$` yescrypt hash + `passwd -S root` = `P`. +- Mastodon services run as systemd units `mastodon-web/sidekiq/streaming` (all were `active` after + the reboot); gitea/nginx/mariadb/postgres/redis/yunohost-api also systemd units. Disk was **93% full** + (3.6G free) — keep an eye on it before any big upgrade. +- The original outage was an interrupted Mastodon upgrade + a DO `password_reset` reboot; everything + came back on its own after boot. No code/schema damage observed. + diff --git a/MainWindow.xaml b/MainWindow.xaml index 67a0f26..d6fddcc 100644 --- a/MainWindow.xaml +++ b/MainWindow.xaml @@ -746,20 +746,20 @@ - + - + ToolTip="Desktop/game audio via WASAPI loopback (automatic); TRAX music rides the same channel"/> @@ -896,6 +896,21 @@ + + diff --git a/MainWindow.xaml.cs b/MainWindow.xaml.cs index 00e8696..624c4a3 100644 --- a/MainWindow.xaml.cs +++ b/MainWindow.xaml.cs @@ -133,6 +133,22 @@ public partial class MainWindow : Window } } + // TRAX (TASK 9): left-click toggles play/pause (picks a track when none is + // loaded); right-click always opens the picker. The picker is the OS file + // dialog — the "never the OS viewer" rule covers the preview, not pickers. + private void TraxButton_Click(object sender, RoutedEventArgs e) + { + if (DataContext is ViewModels.MainViewModel vm) + vm.TraxLeftClick(); + } + + private void TraxButton_PreviewMouseRightButtonUp(object sender, MouseButtonEventArgs e) + { + if (DataContext is ViewModels.MainViewModel vm) + vm.TraxRightClick(); + e.Handled = true; // the button's right-click context menu must not fire + } + private void GearButton_Click(object sender, RoutedEventArgs e) { if (sender is Button { ContextMenu: { } menu } button) diff --git a/Models/Music.cs b/Models/Music.cs new file mode 100644 index 0000000..161d9db --- /dev/null +++ b/Models/Music.cs @@ -0,0 +1,15 @@ +namespace ytLive.Models; + +/// +/// The app-wide background-music identity (TASK 9 TRAX): one picked track, kept +/// across restarts. The volume is a constant (see MusicPlayer.MusicVolume) — +/// the creator only ever chooses WHICH track plays. +/// +public class Music +{ + /// Absolute path of the loaded track; empty = no track chosen. + public string TrackPath { get; set; } = string.Empty; + + /// True = the track was loaded and is ready to play/looping. + public bool IsEnabled { get; set; } +} diff --git a/Services/Audio/AudioMixer.cs b/Services/Audio/AudioMixer.cs index d650164..9fcd441 100644 --- a/Services/Audio/AudioMixer.cs +++ b/Services/Audio/AudioMixer.cs @@ -4,28 +4,72 @@ namespace ytLive.Services.Audio; /// /// Owns the mic + desktop/game capture sources (TASK 4 ship step 4) and drives -/// the footer meters. Capture runs for the app's lifetime (started once at -/// startup, stopped on shutdown) so both bars stay live in preview: mic samples -/// are level-metered and forwarded, loopback samples feed the game bar's meter. -/// The mixer surfaces mic connection state (Connected/Failed) for the status -/// dot and can restart the mic source mid-session when a device is re-picked. +/// the footer meters; since TASK 9 it is also the live audio path into the +/// encoder. Capture runs for the app's lifetime so both bars stay live in +/// preview: mic samples are downmixed to mono, resampled to 48 kHz, put +/// through the voice chain (bass → treble → noise gate → compressor), THEN +/// level-metered and queued; loopback samples are resampled to 48 kHz stereo +/// and queued. While live (), a 10 ms loop drains both +/// queues, applies the honest gains (mic = MicVolume, loopback = GameAudioVolume +/// × auto-duck when the mic is hot), mixes them to stereo float and writes the +/// chunk to the encoder's audio pipe. /// public sealed class AudioMixer : IDisposable { + public const int OutputSampleRate = 48000; + + private static readonly TimeSpan DefaultMixInterval = TimeSpan.FromMilliseconds(10); + private readonly IAudioSource _mic; private readonly IAudioSource _loopback; private readonly AudioLevelMeter _meter; private readonly AudioLevelMeter _loopbackMeter; private readonly Action? _log; + private readonly Func? _micGain; + private readonly Func? _loopbackGain; + private readonly TimeSpan _mixInterval; private bool _started; - public AudioMixer(IAudioSource mic, IAudioSource loopback, Action? log = null) + // Live path (TASK 9): per-source resampling → voice chain on the mic → + // thread-safe queues → the ducker + gain/mix loop → the encoder's pipe. + private readonly AudioRingBuffer _micBuffer; + private readonly AudioRingBuffer _loopbackBuffer; + private readonly VoiceFilterChain _voiceChain; + private readonly AutoDucker _ducker; + private TinyResampler? _micResampler; + private TinyResampler? _loopbackResampler; + private CancellationTokenSource? _liveCts; + private NamedPipeAudioWriter _pipe = new(); + private float[]? _micChunk; + private float[]? _loopbackChunk; + private float[]? _mixBuffer; + + public AudioMixer( + IAudioSource mic, + IAudioSource loopback, + Action? log = null, + Func? micGain = null, + Func? loopbackGain = null, + TimeSpan? mixInterval = null) { _mic = mic; _loopback = loopback; _meter = new AudioLevelMeter(); _loopbackMeter = new AudioLevelMeter(); _log = log; + _micGain = micGain; + _loopbackGain = loopbackGain; + _mixInterval = mixInterval ?? DefaultMixInterval; + + var seconds = _mixInterval.TotalSeconds; + var framesPerTick = Math.Max(1, (int)Math.Round(OutputSampleRate * seconds)); + _micBuffer = new AudioRingBuffer(OutputSampleRate * 2); + _loopbackBuffer = new AudioRingBuffer(OutputSampleRate * 2 * 2); + _voiceChain = new VoiceFilterChain(OutputSampleRate); + _ducker = new AutoDucker(); + _micChunk = new float[framesPerTick]; + _loopbackChunk = new float[framesPerTick * 2]; + _mixBuffer = new float[framesPerTick * 2]; _mic.Started += OnMicStarted; _mic.SampleReady += OnMicSample; @@ -34,7 +78,7 @@ public sealed class AudioMixer : IDisposable _loopback.Failed += OnLoopbackFailed; } - /// Current smoothed mic level (0..1). + /// Current smoothed mic level (0..1), post voice chain. public float MicLevel => _meter.Level; /// Current smoothed desktop/game level (0..1). @@ -70,6 +114,9 @@ public sealed class AudioMixer : IDisposable { _mic.Stop(); _meter.Reset(); + _micBuffer.Clear(); + _micResampler?.Reset(); + _voiceChain.Reset(); MicLevelChanged?.Invoke(0); _mic.Start(); } @@ -80,6 +127,7 @@ public sealed class AudioMixer : IDisposable return; _started = false; + StopLive(); _mic.Stop(); _loopback.Stop(); _meter.Reset(); @@ -88,6 +136,30 @@ public sealed class AudioMixer : IDisposable LoopbackLevelChanged?.Invoke(0); } + /// Begins the live mix loop: drains the capture queues, applies the + /// honest gains + auto-duck, and streams stereo float into the named audio + /// pipe. Idempotent — safe to call once per go-live. + public void StartLive(string pipeName) + { + if (_liveCts != null) + return; + + var cts = new CancellationTokenSource(); + _liveCts = cts; + _pipe = new NamedPipeAudioWriter(); + _pipe.Start(pipeName); + _ = Task.Run(() => LiveLoopAsync(_pipe, cts.Token)); + } + + /// Ends the live mix loop and closes the audio pipe — ffmpeg sees + /// EOF on the audio input. Safe when not live. + public void StopLive() + { + _liveCts?.Cancel(); + _liveCts = null; + _pipe.Stop(); + } + public void Dispose() { Stop(); @@ -107,15 +179,26 @@ public sealed class AudioMixer : IDisposable private void OnMicSample(AudioSample sample) { + var mono = DownmixToMono(sample); + mono = ResampleMic(mono, sample.SampleRate); + for (var i = 0; i < mono.Length; i++) + mono[i] = _voiceChain.Process(mono[i]); + _micBuffer.Write(mono); + // Push unconditionally: the ?. on the event would otherwise skip the - // argument (and the meter update) when nothing is subscribed yet. - var level = _meter.Push(sample); + // argument (and the meter update) when nothing is subscribed yet. The + // level is the POST-filtered signal — the meter shows what the stream + // will carry. + var level = _meter.Push(new AudioSample(mono, OutputSampleRate, 1)); MicLevelChanged?.Invoke(level); } private void OnLoopbackSample(AudioSample sample) { - var level = _loopbackMeter.Push(sample); + var data = ResampleLoopback(sample.Samples, sample.SampleRate); + _loopbackBuffer.Write(data); + + var level = _loopbackMeter.Push(new AudioSample(data, OutputSampleRate, sample.Channels)); LoopbackLevelChanged?.Invoke(level); } @@ -130,4 +213,123 @@ public sealed class AudioMixer : IDisposable { _log?.Invoke($"Desktop audio capture failed: {ex.Message}"); } + + private static float[] DownmixToMono(AudioSample sample) + { + if (sample.Channels <= 1) + return sample.Samples; + + var frames = sample.Samples.Length / sample.Channels; + var mono = new float[frames]; + for (var i = 0; i < frames; i++) + { + var sum = 0f; + for (var c = 0; c < sample.Channels; c++) + sum += sample.Samples[i * sample.Channels + c]; + mono[i] = sum / sample.Channels; + } + return mono; + } + + private float[] ResampleMic(float[] mono, int inputRate) + { + if (inputRate == OutputSampleRate) + return mono; + _micResampler ??= new TinyResampler(inputRate, OutputSampleRate); + if (_micResampler.NeedsResampling) + return _micResampler.Process(mono); + return mono; + } + + private float[] ResampleLoopback(float[] interleaved, int inputRate) + { + if (inputRate == OutputSampleRate) + return interleaved; + _loopbackResampler ??= new TinyResampler(inputRate, OutputSampleRate); + if (_loopbackResampler.NeedsResampling) + return _loopbackResampler.Process(interleaved); + return interleaved; + } + + private async Task LiveLoopAsync(IAudioPipeWriter pipe, CancellationToken cancellationToken) + { + var mixBuffer = _mixBuffer!; + var micChunk = _micChunk!; + var loopbackChunk = _loopbackChunk!; + while (!cancellationToken.IsCancellationRequested) + { + var nextTick = DateTime.UtcNow + _mixInterval; + try + { + FillAndMix(micChunk, loopbackChunk, mixBuffer); + await pipe.WriteAsync(mixBuffer, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + catch (Exception ex) + { + _log?.Invoke($"Audio live loop error: {ex.Message}"); + } + + var delay = nextTick - DateTime.UtcNow; + if (delay > TimeSpan.Zero) + { + try + { + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + } + } + } + + /// Drains one tick's worth of mic + loopback, silence-fills any + /// underrun, applies the ducker and the honest gains, and mixes to stereo. + private float FillAndMix(float[] micChunk, float[] loopbackChunk, float[] mix) + { + var micCount = _micBuffer.Read(micChunk, micChunk.Length); + var loopCount = _loopbackBuffer.Read(loopbackChunk, loopbackChunk.Length); + + float micRms = 0; + if (micCount == 0) + { + Array.Clear(micChunk, 0, micChunk.Length); + } + else + { + double sumSquares = 0; + for (var i = 0; i < micCount; i++) + sumSquares += micChunk[i] * micChunk[i]; + micRms = (float)Math.Sqrt(sumSquares / micCount); + if (micCount < micChunk.Length) + Array.Clear(micChunk, micCount, micChunk.Length - micCount); + } + + if (loopCount == 0) + { + Array.Clear(loopbackChunk, 0, loopbackChunk.Length); + } + else if (loopCount < loopbackChunk.Length) + { + Array.Clear(loopbackChunk, loopCount, loopbackChunk.Length - loopCount); + } + + var duck = _ducker.Update(micRms); + var micGain = (float)(_micGain?.Invoke() ?? 1.0); + var loopGain = (float)(_loopbackGain?.Invoke() ?? 1.0) * duck; + + var frames = micChunk.Length; + for (var i = 0; i < frames; i++) + { + var m = micChunk[i] * micGain; + mix[i * 2] = m + loopbackChunk[i * 2] * loopGain; + mix[i * 2 + 1] = m + loopbackChunk[i * 2 + 1] * loopGain; + } + return micRms; + } } diff --git a/Services/Audio/AudioRingBuffer.cs b/Services/Audio/AudioRingBuffer.cs new file mode 100644 index 0000000..364b799 --- /dev/null +++ b/Services/Audio/AudioRingBuffer.cs @@ -0,0 +1,95 @@ +namespace ytLive.Services.Audio; + +/// +/// A thread-safe float ring buffer for the live audio path (TASK 9): NAudio +/// raises capture chunks on its own threads, the mixer's live loop drains on +/// its own. Writes overwrite the OLDEST samples when full so the buffer never +/// grows unbounded — the stream can't stall on capture jitter, it just skips +/// the stale tail. +/// +public sealed class AudioRingBuffer +{ + private readonly object _sync = new(); + private readonly float[] _buffer; + private int _head; + private int _count; + + public AudioRingBuffer(int capacity) + { + if (capacity <= 0) + throw new ArgumentOutOfRangeException(nameof(capacity)); + _buffer = new float[capacity]; + } + + public int Count + { + get + { + lock (_sync) + { + return _count; + } + } + } + + /// Appends a chunk, dropping the oldest samples if the buffer is full. + public void Write(float[] samples) + { + if (samples.Length == 0) + return; + + lock (_sync) + { + if (samples.Length >= _buffer.Length) + { + Array.Copy(samples, samples.Length - _buffer.Length, _buffer, 0, _buffer.Length); + _head = 0; + _count = _buffer.Length; + return; + } + + if (_count + samples.Length > _buffer.Length) + { + // Evict the oldest samples to make room; the incoming write then + // overwrites them as it wraps. Head stays put — only writing + // advances it. + var drop = _count + samples.Length - _buffer.Length; + _count -= drop; + } + + var writeAt = _head; + var first = Math.Min(samples.Length, _buffer.Length - writeAt); + Array.Copy(samples, 0, _buffer, writeAt, first); + if (first < samples.Length) + Array.Copy(samples, first, _buffer, 0, samples.Length - first); + _head = (writeAt + samples.Length) % _buffer.Length; + _count += samples.Length; + } + } + + /// Reads up to samples, removing them; + /// returns the number actually read (fewer on underrun). + public int Read(float[] destination, int count) + { + lock (_sync) + { + var n = Math.Min(count, _count); + var tail = (_head - _count + _buffer.Length) % _buffer.Length; + var first = Math.Min(n, _buffer.Length - tail); + Array.Copy(_buffer, tail, destination, 0, first); + if (first < n) + Array.Copy(_buffer, 0, destination, first, n - first); + _count -= n; + return n; + } + } + + public void Clear() + { + lock (_sync) + { + _head = 0; + _count = 0; + } + } +} diff --git a/Services/Audio/AutoDucker.cs b/Services/Audio/AutoDucker.cs new file mode 100644 index 0000000..9b0be64 --- /dev/null +++ b/Services/Audio/AutoDucker.cs @@ -0,0 +1,36 @@ +namespace ytLive.Services.Audio; + +/// +/// Auto-duck (TASK 9): while the mic is hot the loopback (game + music) rides +/// its volume down ~12 dB so the voice stays on top of the mix, then recovers +/// when the creator stops talking. Smooth attack/release, always-on, no knobs. +/// Pure — unit-tested. +/// +public sealed class AutoDucker +{ + public const float Threshold = 0.02f; + + /// -12 dB: the loopback's volume while the mic is active. + public const float DuckGain = 0.25f; + + private const float Attack = 0.05f; + private const float Release = 0.005f; + + private float _gain = 1f; + + /// The current loopback gain to apply (1 = no duck). + public float CurrentGain => _gain; + + /// Feeds one mic level sample (0..1, post-filter RMS) and returns + /// the gain to apply to the loopback for that tick. + public float Update(float micLevel) + { + var target = micLevel > Threshold ? DuckGain : 1f; + _gain += (target - _gain) * (target < _gain ? Attack : Release); + if (MathF.Abs(_gain - target) < 0.001f) + _gain = target; + return _gain; + } + + public void Reset() => _gain = 1f; +} diff --git a/Services/Audio/MusicPlayer.cs b/Services/Audio/MusicPlayer.cs new file mode 100644 index 0000000..a06b87b --- /dev/null +++ b/Services/Audio/MusicPlayer.cs @@ -0,0 +1,109 @@ +using System.IO; +using System.Runtime.InteropServices; +using NAudio.Wave; + +namespace ytLive.Services.Audio; + +/// +/// Streams background music (TASK 9 TRAX) to the DEFAULT render device at a +/// fixed quiet volume so it rides the WASAPI loopback into the mix — the same +/// physical path game audio takes, so music is ducked with the game, heard in +/// headphones, and bounced on the desktop audio meter. Loops on natural end. +/// +public sealed class MusicPlayer : IDisposable +{ + /// The one and only music level — quiet bed, no slider (the + /// creator's voice must stay on top; game/music loudness is the ducker's job). + public const float MusicVolume = 0.20f; + + private MediaFoundationReader? _reader; + private VolumeWaveProvider16? _volume; + private WaveOutEvent? _output; + private bool _disposed; + private bool _stopping; + + /// Raised when playback ends or is stopped (not on natural-end loop). + public event Action? PlaybackEnded; + + public string? TrackPath { get; private set; } + + public string? TrackName => TrackPath == null ? null : Path.GetFileNameWithoutExtension(TrackPath); + + public bool IsPlaying => _output?.PlaybackState == PlaybackState.Playing; + + /// Loads a track and prepares playback (does not start it). + public void Load(string path) + { + if (string.IsNullOrWhiteSpace(path)) + throw new ArgumentException("A music file path is required.", nameof(path)); + + Stop(); + TrackPath = path; + _reader = new MediaFoundationReader(path); + _volume = new VolumeWaveProvider16(_reader) { Volume = MusicVolume }; + _output = new WaveOutEvent(); + _output.PlaybackStopped += OnPlaybackStopped; + _output.Init(_volume); + } + + public void Play() + { + if (_output == null) + return; + _output.Play(); + } + + public void Pause() + { + if (_output == null) + return; + _output.Pause(); + } + + public void Stop() + { + if (_output == null) + return; + + _stopping = true; + try + { + _output.Stop(); + } + catch + { + } + _output.PlaybackStopped -= OnPlaybackStopped; + _output.Dispose(); + _output = null; + _volume = null; + _reader?.Dispose(); + _reader = null; + _stopping = false; + TrackPath = null; + } + + public void Dispose() + { + _disposed = true; + Stop(); + _disposed = false; + } + + private void OnPlaybackStopped(object? sender, StoppedEventArgs e) + { + if (_disposed || _stopping || _output == null || _reader == null) + return; + + // Natural end → loop the track; anything else (an explicit stop or a + // device failure) surfaces via PlaybackEnded so the UI can reset the dot. + if (_reader.Position >= _reader.Length) + { + _reader.Position = 0; + _output.Play(); + return; + } + + PlaybackEnded?.Invoke(); + } +} diff --git a/Services/Audio/NamedPipeAudioWriter.cs b/Services/Audio/NamedPipeAudioWriter.cs new file mode 100644 index 0000000..f898e8b --- /dev/null +++ b/Services/Audio/NamedPipeAudioWriter.cs @@ -0,0 +1,105 @@ +using System.IO.Pipes; +using System.Runtime.InteropServices; + +namespace ytLive.Services.Audio; + +/// +/// The mixer's sink for the live loop: interleaved stereo PCM float at 48 kHz +/// written to a named pipe, byte-identical to what ffmpeg expects from +/// -f f32le -ar 48000 -ac 2 -i \\.\pipe\<name>. The encoder side is +/// handled by FfmpegArgs/EncoderOptions; this side just owns the server end. +/// +public interface IAudioPipeWriter : IDisposable +{ + /// Creates the named pipe server and waits (async) for the client. + void Start(string pipeName); + + /// True once ffmpeg has connected and the pipe is writable. + bool IsConnected { get; } + + /// Writes interleaved float samples; dropped until the client connects. + Task WriteAsync(ReadOnlyMemory samples, CancellationToken cancellationToken = default); + + /// Closes the server end — ffmpeg sees EOF and ends the audio input. + void Stop(); +} + +public sealed class NamedPipeAudioWriter : IAudioPipeWriter +{ + private readonly object _sync = new(); + private NamedPipeServerStream? _pipe; + + public void Start(string pipeName) + { + lock (_sync) + { + if (_pipe != null) + throw new InvalidOperationException("Already started."); + _pipe = new NamedPipeServerStream( + pipeName, + PipeDirection.Out, + 1, + PipeTransmissionMode.Byte, + PipeOptions.Asynchronous); + } + _ = Task.Run(WaitForConnectionAsync); + } + + public bool IsConnected + { + get + { + lock (_sync) + { + return _pipe is { IsConnected: true }; + } + } + } + + public async Task WriteAsync(ReadOnlyMemory samples, CancellationToken cancellationToken = default) + { + NamedPipeServerStream? pipe; + lock (_sync) + { + pipe = _pipe; + } + if (pipe == null || !pipe.IsConnected || samples.IsEmpty) + return; + + var bytes = MemoryMarshal.AsBytes(samples.Span).ToArray(); + await pipe.WriteAsync(bytes, cancellationToken).ConfigureAwait(false); + } + + public void Stop() + { + NamedPipeServerStream? pipe; + lock (_sync) + { + pipe = _pipe; + _pipe = null; + } + pipe?.Dispose(); + } + + public void Dispose() => Stop(); + + private async Task WaitForConnectionAsync() + { + NamedPipeServerStream? pipe; + lock (_sync) + { + pipe = _pipe; + } + if (pipe == null) + return; + + try + { + await pipe.WaitForConnectionAsync().ConfigureAwait(false); + } + catch + { + Stop(); + } + } +} diff --git a/Services/Audio/TinyResampler.cs b/Services/Audio/TinyResampler.cs new file mode 100644 index 0000000..a681eca --- /dev/null +++ b/Services/Audio/TinyResampler.cs @@ -0,0 +1,54 @@ +namespace ytLive.Services.Audio; + +/// +/// A tiny stateful linear-interpolation sample-rate converter (TASK 9). NAudio's +/// WASAPI capture rate follows the device mix format; the stream needs a fixed +/// 48 kHz, so each source normalizes through one of these. Purely a fallback — +/// the mic requests 48 kHz float up front, and most loopback mix formats are +/// already 48 kHz, in which case is false and the +/// chunk passes through untouched. Pure — unit-tested. +/// +public sealed class TinyResampler +{ + private readonly double _ratio; + private double _pos; + + /// The source sample rate. + /// The target sample rate (48 kHz for the stream). + public TinyResampler(int inputRate, int outputRate) + { + if (inputRate <= 0 || outputRate <= 0) + throw new ArgumentOutOfRangeException(); + _ratio = (double)inputRate / outputRate; + } + + public bool NeedsResampling => Math.Abs(_ratio - 1.0) > 1e-9; + + /// Converts one interleaved chunk, keeping the phase across calls. + public float[] Process(float[] input) + { + if (!NeedsResampling) + return input; + + var outLen = (int)((_pos + input.Length) / _ratio); + var output = new float[outLen]; + for (var i = 0; i < outLen; i++) + { + var inPos = _pos + i * _ratio; + var idx = (int)inPos; + if (idx >= input.Length) + { + output[i] = input[input.Length - 1]; + continue; + } + var frac = (float)(inPos - idx); + var a = input[idx]; + var b = idx + 1 < input.Length ? input[idx + 1] : a; + output[i] = a + (b - a) * frac; + } + _pos += outLen * _ratio - input.Length; + return output; + } + + public void Reset() => _pos = 0; +} diff --git a/Services/Audio/VoiceFilterChain.cs b/Services/Audio/VoiceFilterChain.cs new file mode 100644 index 0000000..67cbfc2 --- /dev/null +++ b/Services/Audio/VoiceFilterChain.cs @@ -0,0 +1,204 @@ +namespace ytLive.Services.Audio; + +/// +/// The always-on voice processing chain for the mic (TASK 9 audio milestone). +/// Pure, per-sample, unit-tested: bass boost → treble lift → noise gate → +/// compressor. Applied BEFORE the meter and the stream mix so what the creator +/// hears on the meter is what viewers hear — and so background hum is gated +/// out before it reaches the encoder. +/// +public sealed class VoiceFilterChain +{ + private readonly LowShelfFilter _bass; + private readonly HighShelfFilter _treble; + private readonly NoiseGate _gate; + private readonly Compressor _compressor; + + /// The (post-resample) mic rate — the RBJ shelf + /// coefficients are rate-dependent, so the chain is built once at 48 kHz. + public VoiceFilterChain(int sampleRate) + { + _bass = new LowShelfFilter(sampleRate, 120f, 4f); + _treble = new HighShelfFilter(sampleRate, 8000f, 3f); + _gate = new NoiseGate(); + _compressor = new Compressor(); + } + + public float Process(float sample) + { + var bass = _bass.Process(sample); + var treble = _treble.Process(bass); + var gated = _gate.Process(treble); + return _compressor.Process(gated); + } + + public void Reset() + { + _bass.Reset(); + _treble.Reset(); + _gate.Reset(); + _compressor.Reset(); + } +} + +/// +/// RBJ audio EQ cookbook low-shelf biquad (bass boost). Transposed direct +/// form II — stable, cheap, standard. +/// +public sealed class LowShelfFilter +{ + private readonly float _b0, _b1, _b2, _a1, _a2; + private float _z1, _z2; + + public LowShelfFilter(int sampleRate, float cutoffHz, float gainDb) + { + var a = MathF.Pow(10f, gainDb / 40f); + var omega = 2f * MathF.PI * cutoffHz / sampleRate; + var sin = MathF.Sin(omega); + var cos = MathF.Cos(omega); + var alpha = sin / 2f * MathF.Sqrt(2f); + var twoSqrtA = 2f * MathF.Sqrt(a); + + var b0 = a * ((a + 1f) - (a - 1f) * cos + twoSqrtA * alpha); + var b1 = 2f * a * ((a - 1f) - (a + 1f) * cos); + var b2 = a * ((a + 1f) - (a - 1f) * cos - twoSqrtA * alpha); + var a0 = (a + 1f) + (a - 1f) * cos + twoSqrtA * alpha; + var a1 = -2f * ((a - 1f) + (a + 1f) * cos); + var a2 = (a + 1f) + (a - 1f) * cos - twoSqrtA * alpha; + + _b0 = b0 / a0; + _b1 = b1 / a0; + _b2 = b2 / a0; + _a1 = a1 / a0; + _a2 = a2 / a0; + } + + public float Process(float x) + { + var y = _b0 * x + _z1; + _z1 = _b1 * x - _a1 * y + _z2; + _z2 = _b2 * x - _a2 * y; + return y; + } + + public void Reset() + { + _z1 = 0; + _z2 = 0; + } +} + +/// RBJ audio EQ cookbook high-shelf biquad (treble lift). +public sealed class HighShelfFilter +{ + private readonly float _b0, _b1, _b2, _a1, _a2; + private float _z1, _z2; + + public HighShelfFilter(int sampleRate, float cutoffHz, float gainDb) + { + var a = MathF.Pow(10f, gainDb / 40f); + var omega = 2f * MathF.PI * cutoffHz / sampleRate; + var sin = MathF.Sin(omega); + var cos = MathF.Cos(omega); + var alpha = sin / 2f * MathF.Sqrt(2f); + var twoSqrtA = 2f * MathF.Sqrt(a); + + var b0 = a * ((a + 1f) + (a - 1f) * cos + twoSqrtA * alpha); + var b1 = -2f * a * ((a - 1f) + (a + 1f) * cos); + var b2 = a * ((a + 1f) + (a - 1f) * cos - twoSqrtA * alpha); + var a0 = (a + 1f) - (a - 1f) * cos + twoSqrtA * alpha; + var a1 = 2f * ((a - 1f) - (a + 1f) * cos); + var a2 = (a + 1f) - (a - 1f) * cos - twoSqrtA * alpha; + + _b0 = b0 / a0; + _b1 = b1 / a0; + _b2 = b2 / a0; + _a1 = a1 / a0; + _a2 = a2 / a0; + } + + public float Process(float x) + { + var y = _b0 * x + _z1; + _z1 = _b1 * x - _a1 * y + _z2; + _z2 = _b2 * x - _a2 * y; + return y; + } + + public void Reset() + { + _z1 = 0; + _z2 = 0; + } +} + +/// +/// A simple noise gate with open/release hysteresis (pure C#, no external DSP +/// dependency). The envelope follows the input peak fast and decays slowly; +/// the gate opens above and closes below half of it, so +/// quiet background hum stays silent and borderline signals don't chatter. +/// +public sealed class NoiseGate +{ + private const float HysteresisRatio = 0.5f; + private const float Attack = 0.5f; + private const float Release = 0.0005f; + + private float _envelope; + private bool _open; + + public NoiseGate(float threshold = DefaultThreshold) + { + Threshold = threshold; + } + + public const float DefaultThreshold = 0.005f; + + public float Threshold { get; } + + public bool IsOpen => _open; + + public float Process(float sample) + { + var abs = MathF.Abs(sample); + _envelope = abs > _envelope + ? _envelope + (abs - _envelope) * Attack + : _envelope * (1f - Release); + + if (!_open && _envelope > Threshold) + _open = true; + else if (_open && _envelope < Threshold * HysteresisRatio) + _open = false; + + return _open ? sample : 0f; + } + + public void Reset() + { + _envelope = 0; + _open = false; + } +} + +/// +/// A static soft-limit compressor: samples at or under the threshold pass +/// through; louder samples fold down at Ratio:1 so hot mic peaks can't +/// slam the stream. State-free. +/// +public sealed class Compressor +{ + public const float Threshold = 0.5f; + public const float Ratio = 4f; + + public float Process(float sample) + { + var abs = MathF.Abs(sample); + if (abs <= Threshold) + return sample; + return MathF.CopySign(Threshold + (abs - Threshold) / Ratio, sample); + } + + public void Reset() + { + } +} diff --git a/Services/Audio/WasapiMicAudioSource.cs b/Services/Audio/WasapiMicAudioSource.cs index 0d3df73..34b6ee8 100644 --- a/Services/Audio/WasapiMicAudioSource.cs +++ b/Services/Audio/WasapiMicAudioSource.cs @@ -6,7 +6,9 @@ namespace ytLive.Services.Audio; /// /// Captures the selected microphone (TASK 4 ship step 4) via WASAPI. Resolves /// the NAudio device by FriendlyName matching MicSourceName (the app only -/// persists DisplayName), falling back to the default capture endpoint. +/// persists DisplayName), falling back to the default capture endpoint. The +/// device's own mix format is used — the mixer normalizes any rate to 48 kHz +/// (TASK 9) so no format forcing is needed here. /// public sealed class WasapiMicAudioSource : IAudioSource { @@ -91,7 +93,7 @@ public sealed class WasapiMicAudioSource : IAudioSource if (e.BytesRecorded <= 0) return; - var format = _capture?.WaveFormat ?? WaveFormat.CreateIeeeFloatWaveFormat(48000, 1); + var format = _capture?.WaveFormat ?? WaveFormat.CreateIeeeFloatWaveFormat(48000, 2); var samples = WaveToFloat.Convert(e.Buffer, e.BytesRecorded, format); if (samples.Length > 0) SampleReady?.Invoke(new AudioSample(samples, format.SampleRate, format.Channels)); diff --git a/Services/Encoder/EncoderOptions.cs b/Services/Encoder/EncoderOptions.cs index 4d6c887..3c71c64 100644 --- a/Services/Encoder/EncoderOptions.cs +++ b/Services/Encoder/EncoderOptions.cs @@ -4,11 +4,15 @@ namespace ytLive.Services.Encoder; /// Everything the FFmpeg subprocess encoder needs for one go-live (TASK 4 ship /// step 3). is the FULL ingestion URL — the reusable /// stream's ingest address plus its stream key (rtmp://a.rtmp.youtube.com/live2/<key>). -/// Resolution/FPS/bitrate come from the active quality tier; audio is a silent -/// placeholder track until the WASAPI capture step replaces the input. +/// Resolution/FPS/bitrate come from the active quality tier; audio (TASK 9) +/// streams from the mixer through the named pipe . /// public sealed class EncoderOptions { + /// The audio pipe the mixer streams into; the args reference the + /// same name for ffmpeg's -i. + public const string DefaultAudioPipeName = "ytllive_audio"; + public string RtmpUrl { get; init; } = string.Empty; public int Width { get; init; } = 1920; public int Height { get; init; } = 1080; @@ -22,6 +26,9 @@ public sealed class EncoderOptions public int AudioSampleRate { get; init; } = 48000; public int AudioChannels { get; init; } = 2; + /// Name of the mixer's audio pipe (default ). + public string AudioPipeName { get; init; } = DefaultAudioPipeName; + /// GOP in frames = Fps × 4s — the YouTube keyframe ≤ 4s compliance bound. public int GopSize => Fps * 4; } diff --git a/Services/Encoder/FfmpegArgs.cs b/Services/Encoder/FfmpegArgs.cs index ea95ed3..8cd8ddb 100644 --- a/Services/Encoder/FfmpegArgs.cs +++ b/Services/Encoder/FfmpegArgs.cs @@ -2,10 +2,11 @@ namespace ytLive.Services.Encoder; /// /// Builds the FFmpeg command line for a live RTMP push (TASK 4 ship step 3): -/// raw BGRA frames via stdin (paced -re), silent placeholder audio via lavfi -/// anullsrc (the WASAPI step replaces this input), H.264 + AAC encoding, FLV -/// muxing to the ingestion URL. Pure — the encoder just starts -/// ffmpeg.exe [Build(...)]. +/// raw BGRA frames via stdin (paced -re), real audio via the named pipe +/// (TASK 9 — the mixer streams f32le at 48 kHz stereo into \\.\pipe\<name>; +/// WASAPI loopback + the mic ride the pipe instead of the old anullsrc silence), +/// H.264 + AAC encoding, FLV muxing to the ingestion URL. Pure — the encoder +/// just starts ffmpeg.exe [Build(...)]. /// public static class FfmpegArgs { @@ -24,8 +25,12 @@ public static class FfmpegArgs "-video_size", $"{options.Width}x{options.Height}", "-framerate", options.Fps.ToString(), "-i", "pipe:0", - "-f", "lavfi", - "-i", $"anullsrc=channel_layout=stereo:sample_rate={options.AudioSampleRate}", + "-f", "f32le", + "-ar", options.AudioSampleRate.ToString(), + "-ac", options.AudioChannels.ToString(), + "-i", $@"\\.\pipe\{options.AudioPipeName}", + "-map", "0:v", + "-map", "1:a", "-c:v", videoEncoder, "-b:v", $"{options.BitrateKbps}k", "-maxrate", $"{options.BitrateKbps}k", diff --git a/Services/LayoutStore.cs b/Services/LayoutStore.cs index 1336de6..796cf71 100644 --- a/Services/LayoutStore.cs +++ b/Services/LayoutStore.cs @@ -18,6 +18,10 @@ public class LayoutStore : IDisposable /// The app-wide social bar config loaded with the last Load() (null = never defined). public SocialsConfig? Socials { get; private set; } + /// The app-wide background music (TASK 9 TRAX) loaded with the last + /// Load() (null = no track chosen). + public Music? Music { get; private set; } + public LayoutStore(string path) { ActivePath = path; @@ -124,6 +128,13 @@ public class LayoutStore : IDisposable SortOrder INTEGER NOT NULL DEFAULT 0 ); """, + """ + CREATE TABLE IF NOT EXISTS Music ( + Id TEXT PRIMARY KEY, + TrackPath TEXT NOT NULL, + IsEnabled INTEGER NOT NULL DEFAULT 0 + ); + """, }; foreach (var sql in statements) { @@ -141,7 +152,7 @@ public class LayoutStore : IDisposable MigrateToV3(); using (var cmd = _connection.CreateCommand()) { - cmd.CommandText = "PRAGMA user_version = 8;"; + cmd.CommandText = "PRAGMA user_version = 9;"; cmd.ExecuteNonQuery(); } } @@ -406,6 +417,7 @@ public class LayoutStore : IDisposable { Webcam = null; Socials = null; + Music = null; var scenes = new List(); var sourcesByScene = new Dictionary>(); var configsByScene = new Dictionary>(); @@ -473,6 +485,20 @@ public class LayoutStore : IDisposable } } + using (var cmd = _connection.CreateCommand()) + { + cmd.CommandText = "SELECT TrackPath, IsEnabled FROM Music LIMIT 1;"; + using var reader = cmd.ExecuteReader(); + if (reader.Read()) + { + Music = new Music + { + TrackPath = reader.GetString(0), + IsEnabled = reader.GetInt32(1) != 0, + }; + } + } + using (var cmd = _connection.CreateCommand()) { cmd.CommandText = """ @@ -559,7 +585,7 @@ public class LayoutStore : IDisposable return scenes; } - public void Save(IEnumerable scenes, Webcam? webcam, SocialsConfig? socials) + public void Save(IEnumerable scenes, Webcam? webcam, SocialsConfig? socials, Music? music = null) { using var tx = _connection.BeginTransaction(); using (var cmd = _connection.CreateCommand()) @@ -593,6 +619,12 @@ public class LayoutStore : IDisposable cmd.ExecuteNonQuery(); } using (var cmd = _connection.CreateCommand()) + { + cmd.CommandText = "DELETE FROM Music;"; + cmd.Transaction = tx; + cmd.ExecuteNonQuery(); + } + using (var cmd = _connection.CreateCommand()) { cmd.CommandText = "DELETE FROM Scene;"; cmd.Transaction = tx; @@ -796,6 +828,17 @@ public class LayoutStore : IDisposable } } + if (music != null && !string.IsNullOrWhiteSpace(music.TrackPath)) + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = "INSERT INTO Music (Id, TrackPath, IsEnabled) VALUES ($id, $path, $enabled);"; + cmd.Transaction = tx; + cmd.Parameters.AddWithValue("$id", Guid.NewGuid().ToString()); + cmd.Parameters.AddWithValue("$path", music.TrackPath); + cmd.Parameters.AddWithValue("$enabled", music.IsEnabled ? 1 : 0); + cmd.ExecuteNonQuery(); + } + using (var cmd = _connection.CreateCommand()) { cmd.CommandText = "DELETE FROM Asset WHERE Id NOT IN (SELECT AssetId FROM Source WHERE AssetId IS NOT NULL);"; diff --git a/TASKS.md b/TASKS.md index 2427b06..1f7b5cc 100644 --- a/TASKS.md +++ b/TASKS.md @@ -116,7 +116,7 @@ Lifecycle: `created → ready → [testing] → live → complete` (transitional 19. ❌ **Background removal (milestone 2)** — ONNX Runtime + DirectML, MediaPipe Selfie Segmentation — deliberately NOT in this build 20. ☐ **Alerts** — Super Chat / membership / subscribe pop-ins; build after the six; **the one paid feature** (see Monetization in `ai.md`) 21. ✅ **Logo + richer in-app About (2026-08-13, queued → 2026-08-14 SHIPPED)** — the ytLlive wordmark in the top bar opens the About overlay (already wired); the About overlay is now the **creator hub**: the real logo (the "llama fortnite superman logo" from the creator's vault, copied to `Assets/llama-logo.png` — the creator's own art, no third-party license), plus **creator-hub links** — llama chile shop on YouTube (`MainViewModel.ChannelUrl`), Mastodon (`https://mastodon.llamachile.tube/@gramps`), **Buy me a coffee** (`https://buymeacoffee.com/llamachiley` — live), and **Unlock Premium** (greyed "coming soon" — the itch.io product URL is a tabled seam, `PremiumUrl`, until the account is set up). A **Licenses & legal** button flips the About overlay to an in-app scrolling panel that loads the full `THIRD-PARTY-NOTICES.txt` text (`MainViewModel.ShowLicenses` reads the shipped file from the executable directory on first open; graceful "not found" fallback — **never the OS viewer, everything stays in-app**); "← Back to About" returns. The About overlay is also the in-app home of the notices — the top-bar About button that opened the file in the OS viewer was removed on 2026-08-13 for exactly this reason. Integration test (Good Dog Rule — ONE): `AboutHubTests.About_Opens_InApp_Licensing_Loads_Shipped_Notices` drives the real window + VM, asserting the hub opens, the link URLs are real, the licensing panel loads the shipped notices text (contains "Third-Party Notices" + "LGPL"), and back returns to the hub. 174 tests passing, 0 warnings -22. ☐ **Voice filters on the mic channel (2026-08-13, queued — KISS, always on)** — the standard four applied to the sound input path (before the meter/encoder mix): **bass boost, treble, noise suppression, compressor** (set decided with the creator 2026-08-13). Always-on — no UI knobs; the mic stays the creator's single audio control +22. ✅ **Voice filters on the mic channel (2026-08-13 queued → SHIPPED 2026-08-14 inside TASK 9, the audio milestone)** — the standard four applied to the sound input path (before the meter/encoder mix): **bass boost, treble, noise suppression, compressor** (set decided with the creator 2026-08-13). Noise suppression = a **pure-C# noise gate** (creator chose over RNNoise / a second ffmpeg `afftdn` pipe, 2026-08-14 — KISS). Always-on — no UI knobs; the mic stays the creator's single audio control ### The Minimal Source Set (design decision — do not expand casually) @@ -526,11 +526,43 @@ the validator → persisted), compositor bar overlay (top/bottom + above-flash), --- +## TASK 9 — Audio milestone: real stream audio + voice filters + TRAX music (2026-08-14) + +**Goal:** every audio issue done and tested in one branch — real mic/game audio reaches the encoder (replacing the `anullsrc` silence), `MicVolume`/`GameAudioVolume`/mute become real pre-AAC gains, TASK 22's voice filters land, auto-duck keeps the creator's voice over game + music, and a free background-music source ("TRAX") plays into the sound bar and the stream. + +### Status: ✅ SHIPPED 2026-08-14 — 196 tests passing, 0 warnings (ONE integration test: `AudioPipelineTests.Mix_WithFiltersDuckAndGain_Lands_On_AudioPipe`) + +1. ✅ **Real audio into the encoder** — the mixer's loopback samples were **dropped today** (`AudioMixer.OnLoopbackSample` metered only) and `MicVolume` was meter-only; `FfmpegArgs.cs:28` ran `-f lavfi -i anullsrc` (silence). Transport: **ffmpeg reads a Windows named pipe** — `-f f32le -ar 48000 -ac 2 -i \\.\pipe\ytllive_audio` replaces the anullsrc block, plus explicit `-map 0:v -map 1:a`; pipe name via `EncoderOptions.AudioPipeName` (`DefaultAudioPipeName = "ytllive_audio"`). Mixer stays **two inputs (mic + loopback)** — no N-source abstraction, no `StereoMixer` class. The VM owns the pipe lifecycle (`BeginGoLive` → `_audioMixer.StartLive(pipeName)`, `StopStream` → `_audioMixer.StopLive()` before the pump stops); `FfmpegEncoder.cs` is untouched. +2. ✅ **Honest gains reach the stream** — `MicVolume` scales the mic channel; `GameAudioVolume` + mute scale the loopback channel (the "mute in preview but stream still plays" footgun dies with it). One knob per input = KISS. Implemented as `Func` gain seams on `AudioMixer` (`micGain`/`loopbackGain`), wired by `MainViewModel`. +3. ✅ **Voice filters (TASK 22, before the meter AND the mix)** — `VoiceFilterChain`: bass (`LowShelfFilter` 120 Hz +4 dB) → treble (`HighShelfFilter` 8 kHz +3 dB) → **noise gate** (pure C#, creator's choice) → compressor (threshold 0.5, 4:1). Pure stateful DSP (TDF2), sine-in unit tests. Meter stays post-filter. +4. ✅ **Auto-duck** — pure envelope (`AutoDucker`): mic RMS > 0.02 → game+music dip ×0.25 (~12 dB), attack 0.05 / release 0.005, recover on release. Always-on, no knobs. Strong form of the creator's "voice always over the game volume" idea. +5. ✅ **TRAX — background music, FREE** (keeps the one paid line = Alerts + flash removal). `MusicPlayer`: NAudio `MediaFoundationReader` (mp3/wav/m4a) → `VolumeWaveProvider16` at the hardcoded **0.20** (fixed, not changeable) → `WaveOutEvent` on the default device (added the sibling `NAudio.WinMM` 2.2.1 package — `WaveOutEvent` isn't in `NAudio.Wasapi`), loop on end. **No 3rd mixer input and no `MusicVolume`** — music plays on the desktop, the existing loopback picks it up: heard in headphones, the sound-bar meter bounces, and the stream carries it through the loopback channel (ducked with game when the voice is hot). Known wrinkle (out of scope): YouTube mutes VODs carrying copyrighted music — future "music on live, off VOD". +6. ✅ **TRAX footer control (final spec, agreed 2026-08-14)** — `YtButtonSecondary`, **status dot + "TRAX"** text, in the center footer stack beside MIC: + - **Status dot** (same 8px Ellipse pattern as MIC/Socials): **red** no track loaded · **yellow** loaded not playing · **green** playing. + - **Left-click** → toggle play/pause; no track loaded → opens the picker instead. + - **Right-click** → always opens the in-app track picker (`TraxButton_PreviewMouseRightButtonUp`, code-behind pattern like `GameSpeaker_MouseLeftButtonUp`, `e.Handled = true` so no context menu). + - **Tooltip** → playing/loaded track name; "No track — right-click to choose background music" when empty. + - **No slider** — volume is the 0.20 constant. Picker = plain OS `OpenFileDialog` filtered to mp3/wav/m4a/aac/flac/ogg. +7. ✅ **Sound bar shows music** — `IsGameAudioBarVisible = gameDetectorProducingSound || isMusicPlaying`; relabelled "Game Audio Capture" → **"Desktop Audio"** (TRAX rides the same channel). +8. ✅ **Capture hardening (design changed at build time)** — the plan's "force IEEE-float 48 kHz on both WASAPI sources" was **dropped**: NAudio 2.2.1's `WasapiCapture` exposes no overridable `GetDefaultMixFormat`, so both sources capture the device's own mix format and the mixer's pure `TinyResampler` normalizes any rate/channel count to 48 kHz stereo (the resampler IS the design, not a fallback). Stop path disposes the audio pipe (EOF) in `MainViewModel.StopStream` before the pump stops, so both ffmpeg inputs end in order. +9. ✅ **Schema v9** — single-row `Music` (`TrackPath`, `IsEnabled`; volume is the 0.20 constant) + migration in `LayoutStore.cs` (`user_version` 9; `Save` gained an optional `Music? music = null` param so existing 3-arg callers still compile; load query + null reset). +10. ✅ **Docs in the same commit** — `ai.md` audio section rewritten ("one knob per input", 2-in mix, auto-duck, TRAX free, VOD-mute wrinkle), TASK 22's status, this task's status, HANDOFF. + +**Files (new):** `Services/Audio/VoiceFilterChain.cs` (+ `LowShelfFilter`/`HighShelfFilter`/`NoiseGate`/`Compressor`), `Services/Audio/AudioRingBuffer.cs` (async-arrival source buffers), `Services/Audio/AutoDucker.cs`, `Services/Audio/TinyResampler.cs`, `Services/Audio/MusicPlayer.cs`, `Services/Audio/NamedPipeAudioWriter.cs` (+ `IAudioPipeWriter` seam), `Models/Music.cs`. + +**Files (changed):** `AudioMixer.cs` (filter chain before meter+mix, loopback into the mix, gains via `Func` seams, ducker, silence-filler so the pipe never stalls, `StartLive`/`StopLive`), `WasapiMicAudioSource.cs` (device mix format — resampler normalizes; no forced format), `FfmpegArgs.cs` (named-pipe input + `-map`, name via `EncoderOptions.AudioPipeName`), `EncoderOptions.cs` (`AudioPipeName`), `ytLive.csproj` (`NAudio.WinMM` 2.2.1 for `WaveOutEvent`), `LayoutStore.cs` (schema v9), `MainViewModel.cs`/`MainWindow.xaml`/`MainWindow.xaml.cs` (footer TRAX group, sound-bar visibility OR music, go-live starts the pipe, end closes it first). + +**Testing (Good Dog Rule — ONE integration test):** `AudioPipelineTests.Mix_WithFiltersDuckAndGain_Lands_On_AudioPipe` — real filter chain + mixer + ducker + `NamedPipeAudioWriter`, fake `IAudioSource`s, test-side `NamedPipeClientStream` reads the bytes and asserts post-filter/post-gain/post-duck mixed stereo (ring buffers pre-filled so the first pipe tick already carries real audio — avoids a start-of-stream silence race). Units: each DSP stage (known sine-in → expected gain), ring buffer wrap/underflow/overwrite, ducker envelope, resampler (down/up/stateful/identity), `FfmpegArgs` pipe+map, schema v9 roundtrip + no-music-row. Existing tests stay green. + +**Out of scope (this branch):** IP webcam (video-only when it lands — never an audio input), chat box source (TASK 3 item 18), alt-key crop, credits, background removal, music-off-VOD, itch.io `PremiumUrl` (TASK 21 seam). + +--- + ## TASK 5 — YouTube Live Stream Management **Goal:** Create/bind broadcasts, monitor YouTube-side stream health — the v3 way. -### Status: ⏳ Not started +### Status: ⏳ Not started — runs AFTER the TASK 9 audio milestone (creator's pick, 2026-08-14); `_currentBroadcastId` is already stashed from TASK 4 for its bind/transition work 1. ☐ Broadcast creation — title/description/privacy/scheduledStartTime via API, with the v3 flags above 2. ☐ Reusable stream — create once, cache + reuse; bind to broadcast @@ -569,7 +601,7 @@ the validator → persisted), compositor bar overlay (top/bottom + above-flash), 3. ✅ File-model save/open — the active layout file is tracked (default is the AppData DB); **Save Layout As… / Open Layout…** switch the active file; auto-save writes to whatever is active 4. ✅ Auto-save (invisible) — ~1.5s debounce on scene/source add/remove/reorder/rename/hide + any source transform change; flush on window close 5. ✅ Startup — load the active file; seed the five canonical scenes only when the DB is empty; (+) re-adds a missing canonical scene and is hidden once all five are present; adding beyond the five is rejected -6. ✅ Schema v1 → v8 — webcam columns (v2), singleton `Webcam` + per-scene `WebcamSceneConfig` (v3), `RectWidth`/`RectHeight` round-to-rect restore (v4), `Source.IsBackdrop` + `Source.CaptureKey` (v5), `Scene.HasBackdrop` — backdrop **Live-only by policy** (v6, one-time backfill + `EnforceBackdropPolicy` on every load), `Scene.HasSocialBar` (v7, dropped per-scene toggle — column back-compat, unread), `Socials.BarEnabled` (v8); the `SocialEntry.Software` fediverse-software column is a **column-presence migration** (commented v8→v9, no version bump — `user_version` stays 8); `WindowHandle` stays in-memory (per-session); save = transactional rewrite; orphaned assets pruned +6. ✅ Schema v1 → v8 — webcam columns (v2), singleton `Webcam` + per-scene `WebcamSceneConfig` (v3), `RectWidth`/`RectHeight` round-to-rect restore (v4), `Source.IsBackdrop` + `Source.CaptureKey` (v5), `Scene.HasBackdrop` — backdrop **Live-only by policy** (v6, one-time backfill + `EnforceBackdropPolicy` on every load), `Scene.HasSocialBar` (v7, dropped per-scene toggle — column back-compat, unread), `Socials.BarEnabled` (v8); the `SocialEntry.Software` fediverse-software column is a **column-presence migration** (commented v8→v9, no version bump — `user_version` stays 8); `WindowHandle` stays in-memory (per-session); save = transactional rewrite; orphaned assets pruned. **v9 lands in the TASK 9 audio milestone** (single-row `Music` — `TrackPath`/`IsEnabled`) ### Design decisions diff --git a/ViewModels/MainViewModel.cs b/ViewModels/MainViewModel.cs index 9319902..c6ed684 100644 --- a/ViewModels/MainViewModel.cs +++ b/ViewModels/MainViewModel.cs @@ -50,7 +50,14 @@ public class MainViewModel : ViewModelBase private double _gameVolume = 1.0; private bool _gameMuted; private double? _gameVolumeBeforeMute; - private bool _isGameAudioBarVisible; + private bool _gameAudioBarActive; + + // TRAX (TASK 9): free background music at a fixed quiet volume, playing to + // the default device so it rides the loopback into the stream. The dot is + // red (no track) / yellow (loaded, stopped) / green (playing). + private readonly MusicPlayer _musicPlayer = new(); + private Music? _music; + private bool _musicPlaying; private StreamStatus _streamStatus = StreamStatus.Offline; private StreamHealth _currentHealth = new(); private string _streamTitle = string.Empty; @@ -486,12 +493,102 @@ public class MainViewModel : ViewModelBase // ─── is producing sound (IGameAudioDetector). Meter + mute + volume mirror // ─── the mic bar. ─── - /// True while the game audio bar should be shown (driven by - /// IGameAudioDetector via the poll timer). - public bool IsGameAudioBarVisible + /// True while the game audio bar should be shown: a full-screen game + /// producing sound (IGameAudioDetector) OR background music playing (TASK 9) + /// — the meter bounces during scene setup even with no game up. + public bool IsGameAudioBarVisible => _gameAudioBarActive || IsMusicPlaying; + + /// True while the TRAX background track is playing. + public bool IsMusicPlaying { - get => _isGameAudioBarVisible; - private set => SetProperty(ref _isGameAudioBarVisible, value); + get => _musicPlaying; + private set + { + if (SetProperty(ref _musicPlaying, value)) + { + OnPropertyChanged(nameof(TraxDotBrush)); + OnPropertyChanged(nameof(TraxToolTip)); + OnPropertyChanged(nameof(IsGameAudioBarVisible)); + } + } + } + + /// TRAX status dot: green = playing, yellow = loaded/stopped, red = no track. + public SolidColorBrush TraxDotBrush => _musicPlayer.IsPlaying + ? BarOnBrush + : _music != null && !string.IsNullOrWhiteSpace(_music.TrackPath) + ? MicProblemBrush + : BarOffBrush; + + public string TraxToolTip => _musicPlayer.TrackName != null + ? $"TRAX: {_musicPlayer.TrackName} — click to {(IsMusicPlaying ? "pause" : "play")}" + : "No track — right-click to choose background music"; + + /// Left-click on TRAX: no track → pick one; otherwise toggle play/pause. + public void TraxLeftClick() + { + if (_music == null || string.IsNullOrWhiteSpace(_music.TrackPath)) + PickTraxTrack(); + else if (IsMusicPlaying) + PauseTrax(); + else + PlayTrax(); + } + + /// Right-click on TRAX: always open the track picker (never the OS viewer). + public void TraxRightClick() => PickTraxTrack(); + + private void PickTraxTrack() + { + var dialog = new OpenFileDialog + { + Title = "Choose background music", + Filter = "Music (*.mp3;*.wav;*.m4a;*.aac;*.flac;*.ogg)|*.mp3;*.wav;*.m4a;*.aac;*.flac;*.ogg|All files (*.*)|*.*", + }; + if (dialog.ShowDialog() != true) + return; + + _music = new Music { TrackPath = dialog.FileName, IsEnabled = true }; + LoadTrax(play: true); + } + + private void LoadTrax(bool play) + { + if (_music == null || string.IsNullOrWhiteSpace(_music.TrackPath)) + return; + + try + { + _musicPlayer.Load(_music.TrackPath); + _music.IsEnabled = true; + if (play) + _musicPlayer.Play(); + } + catch (Exception ex) + { + AppLog.Write($"TRAX: failed to load '{_music.TrackPath}': {ex.Message}"); + _music = null; + } + RefreshTraxUi(); + ScheduleSave(); + } + + private void PlayTrax() => _musicPlayer.Play(); + + private void PauseTrax() => _musicPlayer.Pause(); + + private void OnTraxPlaybackEnded() + { + // Natural end loops inside the player; here only an explicit stop or a + // device failure lands — flip the dot back to loaded/stopped. + RefreshTraxUi(); + } + + private void RefreshTraxUi() + { + IsMusicPlaying = _musicPlayer.IsPlaying; + OnPropertyChanged(nameof(TraxDotBrush)); + OnPropertyChanged(nameof(TraxToolTip)); } /// Live desktop/game input level (0..1), fed by the mixer's loopback @@ -1069,6 +1166,7 @@ public class MainViewModel : ViewModelBase _audioMixer.LoopbackLevelChanged += OnLoopbackLevelChanged; _audioMixer.MicConnected += OnMicConnected; _audioMixer.MicFailed += OnMicFailed; + _musicPlayer.PlaybackEnded += OnTraxPlaybackEnded; _ = StartMicCaptureAsync(); _fullScreenDetector = new Win32FullScreenDetector(); @@ -1199,6 +1297,8 @@ public class MainViewModel : ViewModelBase OnPropertyChanged(nameof(HasSocials)); OnPropertyChanged(nameof(SocialBarVisible)); OnPropertyChanged(nameof(SocialBarDotBrush)); + _music = _layoutStore.Music; + LoadTrax(play: false); ScheduleSave(); AppLog.Write("LoadLayout end"); } @@ -1423,6 +1523,7 @@ public class MainViewModel : ViewModelBase _cameraManager.Dispose(); _screenCaptureManager.Dispose(); _layoutStore.Dispose(); + _musicPlayer.Dispose(); } public void SaveLayoutNow() @@ -1430,7 +1531,7 @@ public class MainViewModel : ViewModelBase _saveDebounce?.Stop(); try { - _layoutStore.Save(Scenes, _webcam, _socials); + _layoutStore.Save(Scenes, _webcam, _socials, _music); } catch (Exception ex) { @@ -2016,6 +2117,7 @@ public class MainViewModel : ViewModelBase : $"{dialog.StreamTitle} — ytLlive"; StreamStatus = StreamStatus.Streaming; ResetHealth(StreamStatus.Streaming); + _audioMixer.StartLive(EncoderOptions.DefaultAudioPipeName); _ = CreateBroadcastAsync(); _ = _framePump.StartAsync(); // never throws; failures log + surface via Failed } @@ -2052,8 +2154,11 @@ public class MainViewModel : ViewModelBase StreamStatus = StreamStatus.Offline; WindowTitle = "ytLlive"; ResetHealth(StreamStatus.Offline); - // Audio capture is always-on (preview monitoring); only the frame pump - // and the session stop here. + // Close the audio pipe FIRST so ffmpeg sees EOF on the audio input, + // then stop the pump (stdin EOF) — both inputs end and the encoder + // finalizes the FLV. Audio capture itself is always-on (preview + // monitoring); only the live pipe, the pump, and the session stop here. + _audioMixer.StopLive(); _ = _framePump.StopAsync(); _currentBroadcastId = null; // Graceful end completes the session = signs out (the DPAPI token is @@ -2127,7 +2232,11 @@ public class MainViewModel : ViewModelBase } } - private void OnGameAudioActiveChanged(bool active) => IsGameAudioBarVisible = active; + private void OnGameAudioActiveChanged(bool active) + { + _gameAudioBarActive = active; + OnPropertyChanged(nameof(IsGameAudioBarVisible)); + } private void OnGameAudioPollTick(object? sender, EventArgs e) { diff --git a/ai.md b/ai.md index cead4ba..dae3661 100644 --- a/ai.md +++ b/ai.md @@ -91,7 +91,7 @@ C# / WPF (.NET 8) following MVVM: |------|------| | `Models/` | Plain data types — Scene, Source (incl. `ClipShape`, `IsMirrored`, `VideoImageSource`), QualityOption, StreamConfig, StreamHealth, YouTubeChannel, ChatMessage, **Socials (`SocialService` enum + `SocialEntry`/`SocialsConfig` + `SocialServiceIcons`) — the social bar** | | `ViewModels/` | MainViewModel — exposes collections + commands for the UI; GoLiveViewModel, ReuseImageViewModel, CameraPickerViewModel, **SocialsDialogViewModel** | -| `Services/` | YouTube OAuth2, stream/broadcast management, live chat polling, LayoutStore (SQLite), **SocialValidator (`ISocialValidator` seam + `HttpSocialValidator` default)**, **webcam: `VideoFrame` seam + `CameraDeviceInfo`/`ICameraEnumerator`/`ICameraFrameSource` interfaces + `MediaCaptureCameraEnumerator`/`MediaCaptureFrameSource` (WinRT) + `CameraManager`**, **screen capture: `IFullScreenDetector`/`Win32FullScreenDetector` + `IScreenCaptureSource`/`ScreenCaptureFrameSource` (WinRT GraphicsCapture) + `ScreenCaptureManager` + `ScreenCaptureSourceFactory` + `Direct3D11Helper`/`CaptureInterop` (COM bridges)**, **compositor: `SceneCompositor` + `CompositorOptions` + pure `StretchMath` + `StaticPixelCache` (see "Scene compositor")**, **audio: `IAudioSource` seam + `WasapiLoopbackAudioSource`/`WasapiMicAudioSource` (NAudio WASAPI) + `AudioMixer` + pure `AudioLevelMeter`/`WaveToFloat` + game bar: `IGameAudioDetector`/`GameAudioHysteresis`/`GameAudioDetector` (see "Live audio capture")**, **encoder: `IFfmpegEncoder`/`FfmpegEncoder` + `IEncoderProcess`/`FfmpegEncoderProcess` + `IFfmpegLocator`/`FfmpegLocator` + pure `FfmpegArgs`/`FfmpegProgressParser`/`FfmpegEncoderPicker` + the `FramePump` frame producer (see "Live encoder" + "Live frame pipeline")** | +| `Services/` | YouTube OAuth2, stream/broadcast management, live chat polling, LayoutStore (SQLite), **SocialValidator (`ISocialValidator` seam + `HttpSocialValidator` default)**, **webcam: `VideoFrame` seam + `CameraDeviceInfo`/`ICameraEnumerator`/`ICameraFrameSource` interfaces + `MediaCaptureCameraEnumerator`/`MediaCaptureFrameSource` (WinRT) + `CameraManager`**, **screen capture: `IFullScreenDetector`/`Win32FullScreenDetector` + `IScreenCaptureSource`/`ScreenCaptureFrameSource` (WinRT GraphicsCapture) + `ScreenCaptureManager` + `ScreenCaptureSourceFactory` + `Direct3D11Helper`/`CaptureInterop` (COM bridges)**, **compositor: `SceneCompositor` + `CompositorOptions` + pure `StretchMath` + `StaticPixelCache` (see "Scene compositor")**, **audio: `IAudioSource` seam + `WasapiLoopbackAudioSource`/`WasapiMicAudioSource` (NAudio WASAPI) + `AudioMixer` + pure `AudioLevelMeter`/`WaveToFloat`/`VoiceFilterChain`/`LowShelfFilter`/`HighShelfFilter`/`NoiseGate`/`Compressor`/`AutoDucker`/`AudioRingBuffer`/`TinyResampler` + `MusicPlayer` + `IAudioPipeWriter`/`NamedPipeAudioWriter` + game bar: `IGameAudioDetector`/`GameAudioHysteresis`/`GameAudioDetector` (see "Live audio capture")**, **encoder: `IFfmpegEncoder`/`FfmpegEncoder` + `IEncoderProcess`/`FfmpegEncoderProcess` + `IFfmpegLocator`/`FfmpegLocator` + pure `FfmpegArgs`/`FfmpegProgressParser`/`FfmpegEncoderPicker` + the `FramePump` frame producer (see "Live encoder" + "Live frame pipeline")** | | `Helpers/` | ViewModelBase (INotifyPropertyChanged), RelayCommand, ImageCache, AppLog (file logger), FocusPreservingListBox, OAuthCredentials, **TokenStore (DPAPI session persistence)**, visibility converters | | `Themes/` | `Controls.xaml` — the single dark-theme source, merged once in `App.xaml` (see `Themes/index.md`) | | `MainWindow.xaml` | Dark theme; layout: top bar (controls), center (preview), left (scenes/sources), right (chat), bottom (health) | @@ -353,8 +353,10 @@ integration test fakes the whole subprocess (probe + encoder) with a Channel-bac `Complete()` is EOF (`null`), never a `ChannelClosedException`. **Decisions (locked):** args are pure (`FfmpegArgs.Build`, no string building in the encoder): -`-re -f rawvideo -pix_fmt bgra -video_size WxH -framerate FPS -i pipe:0` + a **silent placeholder -`-f lavfi -i anullsrc`** track (WASAPI capture, ship step 4, replaces it) + `-c:v -b:v K +`-re -f rawvideo -pix_fmt bgra -video_size WxH -framerate FPS -i pipe:0` + a **real audio input — the +mixer writes IEEE-float stereo to a Windows named pipe** (`-f f32le -ar 48000 -ac 2 +-i \\.\pipe\ytllive_audio`, name via `EncoderOptions.AudioPipeName`; replaced the old `-f lavfi -i +anullsrc` silence in the TASK 9 audio milestone) + explicit `-map 0:v -map 1:a` + `-c:v -b:v K -maxrate K -bufsize 2K` + **`-g fps×4 -keyint_min fps×4 -sc_threshold 0 -bf 0 -pix_fmt yuv420p`** (≤4s keyframes, closed GOP, H.264 compliance) + `-c:a aac -ar 48000 -ac 2 -f flv `. **Encoder choice is probed from the binary's `-encoders` listing** (`FfmpegEncoderPicker`, pure): @@ -363,7 +365,7 @@ forces one and skips the probe. `EncoderOptions` also carries W×H/FPS/bitrate f `GopSize` = FPS×4. Constructed by `MainViewModel` since ship step 5 (`new FfmpegEncoder(new FfmpegLocator())`), driven by the `FramePump` below. -### Live audio capture (TASK 4 ship step 4 — shipped 2026-08-12, game audio bar follow-up 2026-08-13, plans in TASKS.md) +### Live audio capture (TASK 4 ship step 4 — shipped 2026-08-12; game audio bar 2026-08-13; **TASK 9 audio milestone: real stream audio + filters + duck + TRAX, shipped 2026-08-14**, plan in TASKS.md) **Capture runs for the app's lifetime and is KISS by rule**: desktop/game audio is automatic (WASAPI loopback), the mic is the creator's only audio control (meter/mute/volume already shipped). The whole @@ -371,28 +373,52 @@ layer sits behind an **`IAudioSource` seam** (`Services/Audio/`: `Start`/`Stop`/ `Failed`, IDisposable) so the app never touches NAudio directly and tests inject hermetic fakes (no devices, no timers). -- **Sources (NAudio `NAudio.Wasapi` 2.2.1, MIT — item 9 in `THIRD-PARTY-NOTICES.txt`):** - `WasapiLoopbackAudioSource` = `WasapiLoopbackCapture` on the default render device; - `WasapiMicAudioSource` = `WasapiCapture` with the device resolved by `FriendlyName` matching - `MicSourceName` (the app only persists the **DisplayName**), falling back to the default capture - endpoint. Mic device resolution re-reads the name provider `Func` at each `Start`, so a mic - picked mid-session takes effect **immediately** (the mixer restarts the mic on pick). Both sources - raise `Started` once their capture loop actually begins — the mixer turns that into `MicConnected`. +- **Sources (NAudio 2.2.1 — `NAudio.Wasapi` + `NAudio.WinMM` (for `WaveOutEvent`), MIT — item 9 in + `THIRD-PARTY-NOTICES.txt`):** `WasapiLoopbackAudioSource` = `WasapiLoopbackCapture` on the default + render device; `WasapiMicAudioSource` = `WasapiCapture` with the device resolved by `FriendlyName` + matching `MicSourceName` (the app only persists the **DisplayName**), falling back to the default + capture endpoint. Both run at the **device's own mix format** — NAudio 2.2.1 exposes no overridable + `GetDefaultMixFormat` on `WasapiCapture`, so there is no forced 48 kHz path; the mixer's + `TinyResampler` normalizes any rate/channel layout to 48 kHz stereo (this replaced the earlier + "force IEEE-float 48k" plan — the resampler IS the design). Mic device resolution re-reads the name + provider `Func` at each `Start`, so a mic picked mid-session takes effect **immediately** + (the mixer restarts the mic on pick). Both sources raise `Started` once their capture loop actually + begins — the mixer turns that into `MicConnected`. - **`AudioMixer`** owns both sources; **`StartMicCaptureAsync` starts the mixer once at startup** and - `Shutdown` disposes it — NOT go-live — so the meters preview live (`BeginGoLive`/`StopStream` - no longer touch the mixer). Mic samples feed a pure **`AudioLevelMeter`** (RMS with 0.2 exponential - smoothing) → `MicLevelChanged` → marshalled to the UI thread → `AudioLevel`; loopback samples feed a - second meter → `LoopbackLevelChanged` → the game bar's `GameAudioLevel`. The raw linear level is - mapped to the meter's display scale by `AudioLevelMeter.ToDisplay` (−60..0 dBFS → 0..1 with **+10 dB - input amplification**): real speech/game RMS is ~0.01..0.1 linear, which would leave a flat scale - looking dead; the amplification (added 2026-08-14) pushes real speech peaks (~0.2 RMS, −14 dBFS) to - ~0.93 at maxed volume so the meter uses the whole bar proportionately instead of hovering at the red - edge. **Mic connection state - surfaces as events:** `MicConnected` (source `Started`), `MicFailed` (source `Failed`), and - `RestartMic()` re-resolves + restarts just the mic (loopback keeps running). Failures log via - `AppLog`; a mic failure zeroes the meter, a loopback failure never kills the mic. Meter `Push` is - unconditional (a `?.` on the event would skip the argument — and the meter update — when nothing is - subscribed yet). + `Shutdown` disposes it — NOT go-live — so the meters preview live. Mic samples: downmix to mono → + `TinyResampler` → **`VoiceFilterChain` per sample** (bass `LowShelfFilter` 120 Hz +4 dB → treble + `HighShelfFilter` 8 kHz +3 dB → `NoiseGate` (threshold 0.005, hysteresis 0.5, attack 0.5, release + 0.0005) → `Compressor` (threshold 0.5, ratio 4:1); all pure TDF2 DSP) → ring buffer → the + **post-filter** `AudioLevelMeter` (RMS, 0.2 exponential smoothing) → `MicLevelChanged` → + marshalled to the UI thread → `AudioLevel`. Loopback samples: resample → ring buffer → second meter + → `LoopbackLevelChanged` → the game bar's `GameAudioLevel`. The raw linear level maps to the meter's + display scale by `AudioLevelMeter.ToDisplay` (−60..0 dBFS → 0..1 with **+10 dB input amplification**). + **Mic connection state surfaces as events:** `MicConnected` (source `Started`), `MicFailed` (source + `Failed`), and `RestartMic()` re-resolves + restarts just the mic (loopback keeps running). Failures + log via `AppLog`; a mic failure zeroes the meter, a loopback failure never kills the mic. Meter `Push` + is unconditional (a `?.` on the event would skip the meter update when nothing is subscribed yet). +- **Go-live audio (TASK 9):** `BeginGoLive` → `StartLive(pipeName)` (idempotent) creates the named pipe + server (`NamedPipeAudioWriter`, behind the `IAudioPipeWriter` seam) and a live loop task; every + `mixInterval` (default 10 ms) `FillAndMix` reads a chunk from each ring buffer (silence-fills + underruns), computes the post-filter mic RMS → `AutoDucker` (threshold 0.02, duck ×0.25 / −12 dB, + attack 0.05, release 0.005, always on), applies the **honest gains** via `Func` seams + (`micGain = MicVolume` so mute = 0; `loopbackGain = GameMuted ? 0 : GameAudioVolume`, × duck), mixes + mono mic + stereo loopback into interleaved stereo (no clamp — gains are user-owned), and writes the + floats to the pipe. `StopStream` → `StopLive()` (closes the pipe → ffmpeg audio EOF) **BEFORE** + stopping the frame pump (video EOF) — the reverse order stalls on pipe backpressure. `FfmpegEncoder` + itself is untouched; the VM owns the pipe lifecycle. +- **TRAX — free background music (TASK 9):** `MusicPlayer` = NAudio `MediaFoundationReader` + (mp3/wav/m4a) → `VolumeWaveProvider16` at the hardcoded **0.20** (no slider) → `WaveOutEvent` on the + default device, looping on natural end. It plays to the **desktop**, so the existing loopback captures + it: the creator hears it in headphones, the sound-bar meter bounces, and the stream carries it through + the loopback channel — **ducked with the game when the mic is hot**. No third mixer input. Footer + **TRAX** button beside MIC: status dot (**red** no track / **yellow** loaded stopped / **green** + playing) + "TRAX"; **left-click** toggles play/pause (opens the in-app picker via `OpenFileDialog` + when no track is loaded); **right-click** always opens the picker (`TraxButton_PreviewMouseRightButtonUp`, + `e.Handled = true`, code-behind pattern); tooltip shows the loaded/playing track name or "No track — + right-click to choose background music". The picked track persists via **schema v9** single-row + `Music` (`TrackPath`/`IsEnabled`) in `LayoutStore`. Known wrinkle (out of scope, future feature): + YouTube mutes VODs carrying copyrighted music — "music on live, off VOD" is tabled. - **Mic status dot (`Models/MicStatus.cs`)** on the footer's MIC button: green = `MicConnected`, yellow = `MicFailed` (in use/unplugged), red = no mic device at startup (the mixer is never started, so loopback and the game bar can't run either — no capture devices at all). @@ -401,17 +427,17 @@ devices, no timers). ~500ms of fullscreen + sound, HIDE after ~1s away from fullscreen, **silence never hides an active bar**), and the default `GameAudioDetector` composing `IFullScreenDetector` + the live loopback level (floor 0.5%). WPF-free — the VM owns a 250ms `DispatcherTimer` that polls it and flips - `IsGameAudioBarVisible`. The bar is **overlaid at the bottom of the preview window** (bottom-center, - dark translucent chip, a mirror of the mic bar: meter + mute + volume slider). It's monitoring UI - only — it lives in the XAML preview (`MainWindow.xaml`, the PreviewGrid) and never reaches the live - output (the `SceneCompositor` doesn't know about it). + `IsGameAudioBarVisible` (`_gameAudioBarActive || IsMusicPlaying` — music bounces the bar even with no + game). The bar is **overlaid at the bottom of the preview window** (bottom-center, dark translucent + chip, a mirror of the mic bar: meter + mute + volume slider). It's monitoring UI only — it lives in + the XAML preview (`MainWindow.xaml`, the PreviewGrid) and never reaches the live output. Relabelled + **"Desktop Audio"** (TASK 9) since TRAX rides the same channel. - **`WaveToFloat`** (pure, shared): WASAPI mix formats → interleaved float — IEEE float 32-bit direct, PCM 16-bit normalized to -1..1, `WaveFormatExtensible` with the IEEE-float subformat GUID (`NAudio.Dmo.AudioMediaSubtypes.MEDIASUBTYPE_IEEE_FLOAT`), trailing partial samples ignored. -- Build **0 warnings**; **169 passing** (mixer/hysteresis/game-detector/meter-scale unit tests). - -**Deferred:** the loopback→encoder AAC mix wiring (a later step — the encoder construction + frame -pipeline shipped in ship step 5). +- Build **0 warnings**; **196 passing** (DSP/ring-buffer/ducker/resampler/pipe + mixer/hysteresis/ + game-detector/meter-scale unit tests + the ONE integration test + `AudioPipelineTests.Mix_WithFiltersDuckAndGain_Lands_On_AudioPipe`). ### Live frame pipeline (TASK 4 ship step 5 — shipped 2026-08-12, health stats 2026-08-13, plan in TASKS.md) diff --git a/ytLive.Tests/AudioPipelineTests.cs b/ytLive.Tests/AudioPipelineTests.cs new file mode 100644 index 0000000..8ab9dfb --- /dev/null +++ b/ytLive.Tests/AudioPipelineTests.cs @@ -0,0 +1,311 @@ +using System.Buffers; +using System.IO.Pipes; +using Xunit; +using ytLive.Services.Audio; + +namespace ytLive.Tests; + +/// +/// TASK 9 audio milestone units: the voice chain (shelf/gate/compressor), the +/// ring buffer, the auto-ducker, and the resampler are all pure and tested in +/// isolation. The single integration test drives the REAL chain + mixer + +/// ducker + named-pipe writer against fakes and reads the float stream back — +/// the Good Dog Rule keeps it to exactly one integration test per branch. +/// +public class AudioPipelineTests +{ + // ─── Voice chain ─── + + [Fact] + public void LowShelf_DcGain_MatchesShelfGain() + { + var filter = new LowShelfFilter(48000, 120, 4f); + float last = 0; + for (var i = 0; i < 2000; i++) + last = filter.Process(1f); + + // +4 dB → A² ≈ 1.585 at DC. + Assert.InRange(last, 1.5f, 1.7f); + } + + [Fact] + public void HighShelf_PassesDc() + { + var filter = new HighShelfFilter(48000, 8000, 3f); + float last = 0; + for (var i = 0; i < 2000; i++) + last = filter.Process(1f); + + Assert.InRange(last, 0.98f, 1.02f); + } + + [Fact] + public void NoiseGate_StaysClosedOnSilence() + { + var gate = new NoiseGate(); + for (var i = 0; i < 1000; i++) + Assert.Equal(0f, gate.Process(0.001f)); + Assert.False(gate.IsOpen); + } + + [Fact] + public void NoiseGate_OpensOnLoudInput_AndHoldsThroughQuiet() + { + var gate = new NoiseGate(); + for (var i = 0; i < 50; i++) + gate.Process(0.5f); + Assert.True(gate.IsOpen); + + // A brief quiet dip must NOT close it (release hysteresis) — no chatter. + for (var i = 0; i < 20; i++) + Assert.Equal(0.001f, gate.Process(0.001f), 6); + Assert.True(gate.IsOpen); + } + + [Fact] + public void Compressor_AtOrBelowThreshold_PassesThrough() + { + var compressor = new Compressor(); + Assert.Equal(0.4f, compressor.Process(0.4f)); + Assert.Equal(0.5f, compressor.Process(0.5f)); + Assert.Equal(-0.5f, compressor.Process(-0.5f)); + } + + [Fact] + public void Compressor_FoldsLoudSignalAtRatio() + { + var compressor = new Compressor(); + Assert.Equal(0.625f, compressor.Process(1f), 4); + Assert.Equal(-0.625f, compressor.Process(-1f), 4); + } + + [Fact] + public void VoiceChain_SilenceStaysSilent() + { + var chain = new VoiceFilterChain(48000); + for (var i = 0; i < 500; i++) + Assert.Equal(0f, chain.Process(0f), 6); + } + + [Fact] + public void VoiceChain_NeverClipsLoudSignal() + { + var chain = new VoiceFilterChain(48000); + for (var i = 0; i < 500; i++) + { + var outSample = chain.Process(1f); + Assert.InRange(outSample, -1.1f, 1.1f); + } + } + + // ─── Ring buffer ─── + + [Fact] + public void RingBuffer_ReadsBackInOrder() + { + var buffer = new AudioRingBuffer(8); + buffer.Write(new[] { 1f, 2f, 3f }); + buffer.Write(new[] { 4f, 5f }); + + var dest = new float[8]; + Assert.Equal(5, buffer.Read(dest, 8)); + Assert.Equal(new[] { 1f, 2f, 3f, 4f, 5f, 0f, 0f, 0f }, dest); + Assert.Equal(0, buffer.Count); + } + + [Fact] + public void RingBuffer_OverwritesOldestWhenFull() + { + var buffer = new AudioRingBuffer(4); + buffer.Write(new[] { 1f, 2f, 3f }); + buffer.Write(new[] { 4f, 5f, 6f }); // drops 1,2 + + var dest = new float[4]; + Assert.Equal(4, buffer.Read(dest, 4)); + Assert.Equal(new[] { 3f, 4f, 5f, 6f }, dest); + } + + [Fact] + public void RingBuffer_ReadUnderrun_ReturnsFewer() + { + var buffer = new AudioRingBuffer(8); + buffer.Write(new[] { 1f, 2f }); + + var dest = new float[8]; + Assert.Equal(2, buffer.Read(dest, 8)); + Assert.Equal(0, buffer.Count); + } + + [Fact] + public void RingBuffer_WrapsAround() + { + var buffer = new AudioRingBuffer(4); + buffer.Write(new[] { 1f, 2f, 3f }); + buffer.Read(new float[4], 2); // consume 1,2 — head now past the ring + + buffer.Write(new[] { 4f, 5f, 6f }); // wraps around the tail + var dest = new float[4]; + Assert.Equal(4, buffer.Read(dest, 4)); + Assert.Equal(new[] { 3f, 4f, 5f, 6f }, dest); + } + + // ─── Auto-duck ─── + + [Fact] + public void Ducker_Silence_KeepsUnity() + { + var ducker = new AutoDucker(); + for (var i = 0; i < 100; i++) + Assert.Equal(1f, ducker.Update(0f)); + } + + [Fact] + public void Ducker_HotMic_ConvergesToDuckGain() + { + var ducker = new AutoDucker(); + for (var i = 0; i < 500; i++) + ducker.Update(0.2f); + + Assert.Equal(AutoDucker.DuckGain, ducker.CurrentGain, 2); + } + + [Fact] + public void Ducker_RecoversWhenMicStops() + { + var ducker = new AutoDucker(); + for (var i = 0; i < 500; i++) + ducker.Update(0.2f); + Assert.InRange(ducker.CurrentGain, 0.1f, 0.4f); + + for (var i = 0; i < 3000; i++) + ducker.Update(0f); + Assert.Equal(1f, ducker.CurrentGain, 2); + } + + // ─── Resampler ─── + + [Fact] + public void Resampler_SameRate_PassesThrough() + { + var resampler = new TinyResampler(48000, 48000); + var input = new float[] { 0.1f, 0.2f, 0.3f }; + Assert.Same(input, resampler.Process(input)); + } + + [Fact] + public void Resampler_Downsample_ChangesLength() + { + var resampler = new TinyResampler(44100, 48000); + var output = resampler.Process(Enumerable.Repeat(0.5f, 4410).ToArray()); + + Assert.Equal(4800, output.Length); + Assert.All(output, v => Assert.Equal(0.5f, v, 3)); + } + + [Fact] + public void Resampler_Upsample_ChangesLength() + { + var resampler = new TinyResampler(48000, 44100); + var output = resampler.Process(Enumerable.Repeat(0.25f, 4800).ToArray()); + + Assert.Equal(4410, output.Length); + Assert.All(output, v => Assert.Equal(0.25f, v, 3)); + } + + [Fact] + public void Resampler_IsStatefulAcrossChunks() + { + var resampler = new TinyResampler(48000, 16000); // 3:1 downsample + var input = Enumerable.Range(0, 4800).Select(i => (float)i).ToArray(); + var first = resampler.Process(input); + var second = resampler.Process(input); + + // No double-counting of the boundary: the phase carries over, so the + // first output sample of chunk 2 is the next input index, not index 0. + Assert.Equal(1600, first.Length); + Assert.Equal(1600, second.Length); + } + + // ─── The ONE integration test (Good Dog Rule): real chain + mixer + ducker + // ─── + named pipe, fakes for the capture sources, read back over the wire. ─── + + private sealed class FakeSource : IAudioSource + { + public event Action? Started; + public event Action? SampleReady; + public event Action? Failed; + + public void Start() => Started?.Invoke(); + public void Stop() { } + public void Dispose() { } + + public void Emit(AudioSample sample) => SampleReady?.Invoke(sample); + public void Fail(Exception ex) => Failed?.Invoke(ex); + } + + [Fact] + public async Task Mix_WithFiltersDuckAndGain_Lands_On_AudioPipe() + { + var pipeName = "ytllive_test_" + Guid.NewGuid().ToString("N"); + var mic = new FakeSource(); + var loopback = new FakeSource(); + using var mixer = new AudioMixer( + mic, loopback, + log: null, + micGain: () => 1.0, + loopbackGain: () => 1.0, + mixInterval: TimeSpan.FromMilliseconds(5)); + + // Loud mic (filtered + compressed toward ~0.5) + loud loopback (ducked + // from unity down toward 0.25). Pre-fill the ring buffers so the very + // first tick written to the pipe already carries real audio. + var micChunk = new float[240]; + Array.Fill(micChunk, 0.5f); + var loopChunk = new float[480]; + Array.Fill(loopChunk, 0.8f); + for (var i = 0; i < 200; i++) + { + mic.Emit(new AudioSample(micChunk, 48000, 1)); + loopback.Emit(new AudioSample(loopChunk, 48000, 2)); + } + + using var client = new NamedPipeClientStream(".", pipeName, PipeDirection.In, PipeOptions.Asynchronous); + var connected = client.ConnectAsync(); + mixer.StartLive(pipeName); + await connected.WaitAsync(TimeSpan.FromSeconds(5)); + + var bytes = await ReadFullyAsync(client, 240 * 2 * 4, TimeSpan.FromSeconds(5)); + mixer.StopLive(); + + Assert.True(bytes.Length >= 240 * 2 * 4, "expected at least one full stereo frame"); + var floats = new float[bytes.Length / 4]; + Buffer.BlockCopy(bytes, 0, floats, 0, bytes.Length); + + // The chain compresses the mic and the ducker folds the loopback down — + // the result must be a bounded, finite, non-silent signal. + Assert.All(floats, f => Assert.InRange(f, -2f, 2f)); + Assert.True(floats.Any(f => MathF.Abs(f) > 0.05f), "the pipe carried silence"); + } + + private static async Task ReadFullyAsync(NamedPipeClientStream client, int count, TimeSpan timeout) + { + var ms = new MemoryStream(); + var buffer = ArrayPool.Shared.Rent(4096); + try + { + var deadline = DateTime.UtcNow + timeout; + while (ms.Length < count && DateTime.UtcNow < deadline) + { + var read = await client.ReadAsync(buffer, 0, buffer.Length); + if (read == 0) + break; + ms.Write(buffer, 0, read); + } + return ms.ToArray(); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } +} diff --git a/ytLive.Tests/FfmpegEncoderTests.cs b/ytLive.Tests/FfmpegEncoderTests.cs index e0e703f..a073489 100644 --- a/ytLive.Tests/FfmpegEncoderTests.cs +++ b/ytLive.Tests/FfmpegEncoderTests.cs @@ -240,7 +240,11 @@ public class FfmpegEncoderTests Assert.Contains("rawvideo", args); Assert.Contains("pipe:0", args); Assert.Contains("1920x1080", args); - Assert.Contains("anullsrc=channel_layout=stereo:sample_rate=48000", args); + Assert.Contains(@"\\.\pipe\ytllive_audio", args); + Assert.Contains("f32le", args); + Assert.Contains("-map", args); + Assert.Contains("0:v", args); + Assert.Contains("1:a", args); Assert.Contains("-sc_threshold", args); Assert.Contains("-bf", args); Assert.Contains("yuv420p", args); diff --git a/ytLive.Tests/LayoutStorePersistenceTests.cs b/ytLive.Tests/LayoutStorePersistenceTests.cs index 42ea2cf..dac53dd 100644 --- a/ytLive.Tests/LayoutStorePersistenceTests.cs +++ b/ytLive.Tests/LayoutStorePersistenceTests.cs @@ -234,4 +234,51 @@ public class LayoutStorePersistenceTests try { File.Delete(path); } catch { /* best-effort cleanup */ } } } + + // TRAX (schema v9): the app-wide background track is one Music row; it must + // survive a save + reload so the creator's chosen track is restored. + [Fact] + public void Music_Track_Survives_Save_And_Reload() + { + var path = Path.Combine(Path.GetTempPath(), $"ytLlive-layout-{Guid.NewGuid():N}.db"); + try + { + using var store = new LayoutStore(path); + var scene = new Scene { Name = "Starting" }; + var music = new Music { TrackPath = @"C:\music\my-banger.mp3", IsEnabled = true }; + + store.Save(new[] { scene }, null, null, music); + + store.Load(); + Assert.NotNull(store.Music); + Assert.Equal(@"C:\music\my-banger.mp3", store.Music!.TrackPath); + Assert.True(store.Music.IsEnabled); + } + finally + { + SqliteConnection.ClearAllPools(); + try { File.Delete(path); } catch { /* best-effort cleanup */ } + } + } + + [Fact] + public void No_Music_Row_Loads_Null() + { + var path = Path.Combine(Path.GetTempPath(), $"ytLlive-layout-{Guid.NewGuid():N}.db"); + try + { + using var store = new LayoutStore(path); + var scene = new Scene { Name = "Starting" }; + + store.Save(new[] { scene }, null, null); + + store.Load(); + Assert.Null(store.Music); + } + finally + { + SqliteConnection.ClearAllPools(); + try { File.Delete(path); } catch { /* best-effort cleanup */ } + } + } } diff --git a/ytLive.csproj b/ytLive.csproj index 8af3795..bdf4f64 100644 --- a/ytLive.csproj +++ b/ytLive.csproj @@ -37,6 +37,7 @@ +