Files
LlamaCasty/Services/AlertOverlayLayer.cs
T
gramps b00bf48b9d feat(alerts): sound byte under Built-in video, and alert audio that reaches the device
Creator request: a sound-byte browse + label under Built-in video, with an option to loop the
sound while the clip plays. The video itself is always one pass to completion.

- Source gains AlertAudioPath and AlertAudioLoop, persisted (column + migration + INSERT +
  SELECT + reader, the whole chain — that chain is where the blur crashed once).
- The sound byte plays through MediaPlayer on the default output device; MediaEnded re-seeks to
  zero while looping, and StopClip kills it so a tail cannot bleed into the next alert.
- The clip's OWN soundtrack moved off the mixer and onto the device as well
  (Services/Audio/AlertAudioOutput: one WaveOutEvent + buffered float provider).

Why the move: the creator asked that alert audio — the clip's track or the added sound byte —
appear in the desktop sound bar. Only audio that reaches the default device is picked up by the
WASAPI loopback that feeds both the meter and the stream. The old EnqueueAlertAudio route put the
sound in the stream but left the meter flat and the room silent.

Tests: the sound byte and its loop flag round-trip through SQLite, and a fresh alert layer starts
with no sound byte and no loop. Release 0 warnings.
2026-11-01 14:13:57 -08:00

579 lines
22 KiB
C#

using System.Collections.Generic;
using System.Linq;
using System.Windows;
using System.Windows.Media;
using System.Windows.Media.Imaging;
using System.Windows.Threading;
using ytLive.Helpers;
using ytLive.Models;
using ytLive.Services.Audio;
using ytLive.Services.Compositor;
namespace ytLive.Services;
/// <summary>
/// The native alert box (OBS-style celebration zone, no third party). Owns the
/// event queue and the animation clock: <see cref="ChatEventKind"/> events from
/// the chat feed are enqueued one-at-a-time and played through a celebration
/// animation (<see cref="AlertRenderer"/>) OR — when the alert-box video is
/// configured and resolves — through a decoded clip. Frames are served to the
/// stream compositor via <see cref="RenderFrame"/> and mirrored to the preview
/// via <see cref="UpdatePreview"/>. Idle = transparent (null frame); playback is
/// advanced by a 33ms dispatch ticker. The clock is exposed as
/// <see cref="Advance"/> so tests drive playback deterministically without the
/// timer. Ordinary chat rows (<see cref="ChatEventKind.None"/>) never enqueue —
/// a subscriber mention is a chat row, not an alert.
///
/// TASK 47 (Stream Alerts video): a video-backed alert replaces the animation
/// while a clip decodes (<see cref="IAlertClipDecoder"/>, one per play). The
/// clip plays at its own media pace; the layer applies a ~300ms fade-in/out
/// alpha envelope to the frames AND the audio samples (freeze-frame fade-out on
/// the tail), clips the alert to the clip length, and forwards the scaled audio
/// to the mixer's alert ring via <paramref name="audioSink"/> (no duck). The
/// per-source config (path/default/ticker/volume) is resolved through
/// <paramref name="clipPathResolver"/>; the ticker strip is served via
/// <see cref="AlertTickerFrame"/>.
/// </summary>
public sealed class AlertOverlayLayer : IDisposable
{
/// <summary>The fade-in/out ramp length (video alpha AND audio gain). Measured
/// against a real clip in the AlertLayerVideoTests unit; keep the ramp short
/// enough to read as a "pop-in" on stream (see ai.md).</summary>
internal const double FadeDurationSeconds = 0.30;
/// <summary>How long an alert waits for the first decoded frame or audio slab
/// before treating the clip as unplayable and falling back to the animations
/// (a silent decoder must never blank the alert box).</summary>
internal const double NoFrameFallbackSeconds = 1.0;
private static readonly TimeSpan TickInterval = TimeSpan.FromMilliseconds(33);
private readonly AlertRenderer _renderer;
private readonly Func<string, int, int, IAlertClipDecoder>? _clipDecoderFactory;
private readonly Func<Source, string?>? _clipPathResolver;
private readonly Action<AudioSample>? _audioSink;
private readonly Action<VideoFrame?>? _tickerPreviewSink;
private readonly Queue<ChatMessage> _queue = new();
private readonly DispatcherTimer? _ticker;
private ChatMessage? _current;
private Source? _currentBox;
private IAlertClipDecoder? _clip;
private VideoFrame? _latestClipFrame;
private bool _clipEof;
private double _clipEofElapsed;
private int _clipWidth;
private int _clipHeight;
private bool _clipSawFrame;
private bool _clipAudioHeard;
private string? _tickerText;
private AlertDisplayMethod _tickerMethod = AlertDisplayMethod.TickerScroll;
private double _elapsed;
private IEnumerable<Scene>? _scenes;
private int _contentVersion;
private int _frameVersion = -1;
private string? _frameKey;
private VideoFrame? _cachedFrame;
private DateTime _lastPreviewLog = DateTime.MinValue;
public AlertOverlayLayer(
AlertRenderer renderer,
Func<string, int, int, IAlertClipDecoder>? clipDecoderFactory = null,
Func<Source, string?>? clipPathResolver = null,
Action<AudioSample>? audioSink = null,
Action<VideoFrame?>? tickerPreviewSink = null)
{
_renderer = renderer;
_clipDecoderFactory = clipDecoderFactory;
_clipPathResolver = clipPathResolver;
_audioSink = audioSink;
_tickerPreviewSink = tickerPreviewSink;
if (Application.Current != null)
{
_ticker = new DispatcherTimer { Interval = TickInterval };
_ticker.Tick += (_, _) =>
{
Advance(TickInterval.TotalSeconds);
RefreshAlertPreviews();
};
}
}
/// <summary>Production entry from the chat feed: enqueue one event. Messages
/// arrive on the chat poller thread, so this marshals to the UI thread (same
/// rule as <see cref="ChatOverlayLayer.OnMessageReceived"/>) — the preview
/// writes create WPF <see cref="WriteableBitmap"/> instances, and creating a
/// DependencySource off the UI thread crashes the WPF binding engine
/// (ArgumentException "Must create DependencySource on same Thread").</summary>
public void OnMessageReceived(ChatMessage message, IEnumerable<Scene> scenes)
{
var app = Application.Current;
if (app != null && !app.Dispatcher.CheckAccess())
{
app.Dispatcher.Invoke(() => Ingest(message, scenes));
return;
}
Ingest(message, scenes);
}
private void Ingest(ChatMessage message, IEnumerable<Scene> scenes)
{
_scenes = scenes;
Enqueue(message);
_ticker?.Start();
RefreshAlertPreviews();
}
/// <summary>Queue an event (pure — also the test seam). Chat rows carry
/// <see cref="ChatEventKind.None"/> and are ignored: no alert for a mention.</summary>
public void Enqueue(ChatMessage message)
{
if (message.Kind == ChatEventKind.None) return;
if (_queue.Count >= 10) return; // back-to-back floods drop the tail, never stall playback
_queue.Enqueue(message);
if (_current == null)
{
_current = _queue.Dequeue();
_elapsed = 0;
BeginClip();
}
_contentVersion++;
}
/// <summary>Advance the playback clock (the ticker's beat — and the test seam;
/// tests call this directly and never start the timer). Draining the queue to
/// nothing stops the ticker and leaves the box transparent.</summary>
public void Advance(double deltaSeconds)
{
// Video-backed alert: duration is clip-driven. The clip plays at media
// pace; once it EOFs the last frame sweeps through the fade-out before
// the alert moves on (a hard cut at EOF would pop, not fade).
if (_clip != null)
{
_elapsed += deltaSeconds;
// A decoder that starts but produces not a single frame (or slab of
// audio) is dead weight — the box would sit transparent for the whole
// alert. Give it a short grace, then fall back to the animations so an
// alert ALWAYS renders (the live decode failure at 2026-09-26 11:44 was
// this exact void, invisible in the log).
if (!_clipSawFrame && !_clipAudioHeard && _elapsed >= NoFrameFallbackSeconds)
{
System.Diagnostics.Debug.WriteLine("Alert clip yielded no frames in 1s — falling back to the animation");
StopClip();
_elapsed = 0;
_contentVersion++;
return;
}
if (_clipEof && _elapsed - _clipEofElapsed >= FadeDurationSeconds)
{
_clipEof = false;
StopClip();
_current = null;
_elapsed = 0;
AdvanceToNext();
}
_contentVersion++;
return;
}
if (_current == null)
{
if (_queue.Count == 0) return;
_current = _queue.Dequeue();
_elapsed = 0;
BeginClip();
_contentVersion++;
return;
}
_elapsed += deltaSeconds;
if (_elapsed >= DurationOf(_current))
{
_current = null;
_elapsed = 0;
AdvanceToNext();
}
_contentVersion++;
}
/// <summary>Whether an alert is currently playing or queued.</summary>
public bool IsPlaying => _current != null || _queue.Count > 0;
/// <summary>The live-output frame for one AlertBox element: the decoded clip
/// frame (faded) while a clip is active, else the animation frame, null
/// (transparent) when idle.</summary>
public VideoFrame? RenderFrame(Source alertBox)
{
if (_current == null)
{
_frameKey = null;
return null;
}
if (_clip != null)
return RenderClipFrame(alertBox);
var width = (int)Math.Max(1, alertBox.Width);
var height = (int)Math.Max(1, alertBox.Height);
var key = $"{width}x{height}|{_current.Id}|{Progress()}";
if (_frameKey == key && _frameVersion == _contentVersion)
return _cachedFrame;
var app = Application.Current;
if (app != null && !app.Dispatcher.CheckAccess())
{
var capturedKey = key;
var capturedVersion = _contentVersion;
return app.Dispatcher.Invoke(() =>
{
if (_frameKey == capturedKey && _frameVersion == capturedVersion)
return _cachedFrame;
return RenderCore(alertBox, width, height, capturedKey, capturedVersion);
});
}
return RenderCore(alertBox, width, height, key, _contentVersion);
}
/// <summary>The ticker strip as the alert box shows it: a marquee, a pulse or a
/// static pill naming the event while an alert plays, else null. Pure byte-math
/// per call so the frame pump can read it on any thread (the text strip is
/// rasterized once on the UI thread and cached).
/// <para><b>Creator ruling 2026-09-26:</b> the strip is composited INSIDE the
/// Stream Alerts box, not as a full-width bar at the top edge of the frame — "the
/// ticker should appear over the stream alerts video, not over the entire preview
/// window". So the frame is rendered at the box's own size and carries the box's
/// origin on <see cref="VideoFrame.Placement"/>; every blit site reads
/// <see cref="VideoFrame.OriginX"/>/<see cref="VideoFrame.OriginY"/> instead of
/// a hardcoded 0. <b>No alert box in the scene means no ticker at all</b> — there
/// is no longer a global position for it to live in.</para></summary>
public VideoFrame? AlertTickerFrame
{
get
{
if (_current == null || string.IsNullOrWhiteSpace(_tickerText)) return null;
var box = _currentBox;
if (box == null) return null;
var w = (int)Math.Max(1, Math.Round(box.Width));
var h = (int)Math.Max(1, Math.Min(
Math.Round(box.Height), AlertTickerRenderer.Height));
var frame = AlertTickerRenderer.Render(
_tickerText!, _elapsed, _tickerMethod, AlertSeconds(),
TickerReadsPerAlert, w, h);
return frame == null
? null
: new VideoFrame(frame.Width, frame.Height, frame.BgraPixels)
{
Placement = ((int)Math.Round(box.X), (int)Math.Round(box.Y)),
};
}
}
/// <summary>How many times the marquee must cross the frame per alert, so the
/// announcement is actually readable inside a 10s clip (2026-09-26: the old
/// fixed 140px/s took ~17s per pass, so a 10s alert showed it barely once).</summary>
internal const int TickerReadsPerAlert = 3;
/// <summary>Assumed alert length for the marquee rhythm. The clip's real length
/// isn't known until it EOFs, and the shipped default clip is 10s — the number
/// the creator's "three reads" target is stated against.</summary>
internal const double AssumedAlertSeconds = 10.0;
private double AlertSeconds() => _clip != null ? AssumedAlertSeconds : DurationOf(_current!);
private VideoFrame? RenderClipFrame(Source alertBox)
{
var frame = _latestClipFrame;
if (frame == null) return null;
var w = (int)Math.Max(1, alertBox.Width);
var h = (int)Math.Max(1, alertBox.Height);
if (frame.Width != w || frame.Height != h)
frame = StretchMath.BilinearScale(frame, w, h);
var fade = FadeAt();
if (fade >= 1.0) return frame;
// Straight-alpha source-over on the compositor side: scale the source
// alpha by the envelope so the clip blends out (a 680x200 copy per fade
// tick is trivial; outside the ramps fade==1 returns the raw frame).
var copy = new byte[frame.BgraPixels.Length];
Buffer.BlockCopy(frame.BgraPixels, 0, copy, 0, copy.Length);
var a = (byte)(fade * 255 + 0.5);
for (var i = 3; i < copy.Length; i += 4)
copy[i] = (byte)((copy[i] * a) >> 8);
return new VideoFrame(frame.Width, frame.Height, copy);
}
private double FadeAt()
{
if (_clip == null) return 1.0;
if (_clipEof)
return Math.Clamp(1.0 - (_elapsed - _clipEofElapsed) / FadeDurationSeconds, 0.0, 1.0);
return Math.Clamp(_elapsed / FadeDurationSeconds, 0.0, 1.0);
}
// Sound byte (creator ruling 2026-11-01): a file the creator picks to play WITH the clip. It
// goes to the default device, not into the mixer, so it is audible and visible in the desktop
// sound bar. MediaPlayer handles the decode; the clip itself still stops playback.
private System.Windows.Media.MediaPlayer? _soundByte;
private void StartSoundByte(Source box)
{
StopSoundByte();
var path = box.AlertAudioPath;
if (string.IsNullOrWhiteSpace(path)) return;
try
{
var player = new System.Windows.Media.MediaPlayer { Volume = Math.Clamp(box.AlertVideoVolume, 0, 1) };
if (box.AlertAudioLoop) player.MediaEnded += (_, _) => { player.Position = TimeSpan.Zero; player.Play(); };
player.Open(new Uri(path, UriKind.Absolute));
player.Play();
_soundByte = player;
AppLog.Write($"Alert sound byte: '{path}' loop={box.AlertAudioLoop}");
}
catch (Exception ex)
{
AppLog.Write(ex, "Alert sound byte failed");
StopSoundByte();
}
}
private void StopSoundByte()
{
var player = _soundByte;
_soundByte = null;
if (player == null) return;
try { player.Stop(); player.Close(); } catch { /* already closed */ }
}
private void BeginClip()
{
StopClip();
_tickerText = null;
if (_current == null) return;
var box = _scenes?.SelectMany(s => s.Elements).OfType<Source>()
.FirstOrDefault(s => s.Type == SourceType.AlertBox);
_currentBox = box;
if (box != null && box.AlertShowTicker)
{
_tickerText = ComposeTicker(_current);
_tickerMethod = box.AlertDisplayMethod;
}
if (box == null) return;
StartSoundByte(box);
if (_clipDecoderFactory == null) return;
var path = _clipPathResolver?.Invoke(box);
if (string.IsNullOrWhiteSpace(path)) return;
_clipWidth = (int)Math.Max(1, box.Width);
_clipHeight = (int)Math.Max(1, box.Height);
try
{
_clipSawFrame = false;
_clipAudioHeard = false;
_clip = _clipDecoderFactory(path, _clipWidth, _clipHeight);
_clip.FrameAvailable += OnClipFrame;
_clip.AudioReady += OnClipAudio;
_clip.Completed += OnClipCompleted;
_clip.Start();
AppLog.Write($"Alert clip start: box {box.Id} path '{path}' {_clipWidth}x{_clipHeight}");
}
catch (Exception ex)
{
AppLog.Write(ex, "Alert clip setup failed");
StopClip(); // decode setup failed (bad path) — fall back to animations
}
}
private void StopClip()
{
StopSoundByte();
var clip = _clip;
_clip = null;
_latestClipFrame = null;
_clipEof = false;
_clipSawFrame = false;
_clipAudioHeard = false;
if (clip == null) return;
clip.FrameAvailable -= OnClipFrame;
clip.AudioReady -= OnClipAudio;
clip.Completed -= OnClipCompleted;
try { clip.Stop(); } catch { /* already stopped */ }
clip.Dispose();
}
private void OnClipFrame(VideoFrame frame)
{
_clipSawFrame = true;
_latestClipFrame = frame;
}
private void OnClipCompleted()
{
_clipEof = true;
_clipEofElapsed = _elapsed;
}
private void OnClipAudio(AudioSample sample)
{
if (_audioSink == null) return;
_clipAudioHeard = true;
var volume = (float)Math.Clamp(_currentBox?.AlertVideoVolume ?? 1.0, 0.0, 1.0);
if (volume <= 0f) return;
var fade = FadeAt();
if (fade <= 0.0) return;
var data = sample.Samples;
if (fade < 1.0 || Math.Abs(volume - 1f) > 0.001f)
{
data = new float[data.Length];
var gain = volume * (float)fade;
for (var i = 0; i < sample.Samples.Length; i++)
data[i] = sample.Samples[i] * gain;
}
_audioSink(new AudioSample(data, sample.SampleRate, sample.Channels));
}
private void AdvanceToNext()
{
if (_queue.Count > 0)
{
_current = _queue.Dequeue();
_elapsed = 0;
BeginClip();
}
else
{
_ticker?.Stop();
}
}
private VideoFrame? RenderCore(Source alertBox, int width, int height, string key, int version)
{
_frameKey = key;
_frameVersion = version;
if (_current == null) return _cachedFrame = null;
return _cachedFrame = _renderer.Render(_current, Progress(), width, height);
}
private double Progress()
{
if (_current == null) return 0;
return Math.Clamp(_elapsed / DurationOf(_current), 0.0, 1.0);
}
private static double DurationOf(ChatMessage m) => m.Kind switch
{
ChatEventKind.SuperChat => 4.6,
ChatEventKind.SuperSticker => 4.2,
ChatEventKind.NewMember => 4.0,
ChatEventKind.MemberGift => 5.0,
ChatEventKind.GiftReceived => 4.4,
ChatEventKind.MemberMilestone => 4.8,
_ => 4.0,
};
private static string ComposeTicker(ChatMessage m)
{
var label = m.Kind switch
{
ChatEventKind.SuperChat => "Super Chat",
ChatEventKind.SuperSticker => "Super Sticker",
ChatEventKind.NewMember => "New Member",
ChatEventKind.MemberGift => "Membership Gift",
ChatEventKind.GiftReceived => "Gifted Membership",
ChatEventKind.MemberMilestone => "Member Milestone",
_ => "Stream Alert",
};
var amount = m.Kind is ChatEventKind.SuperChat or ChatEventKind.SuperSticker
&& m.SuperChatDisplayString is { Length: > 0 } s
? $" · {s}"
: "";
return $"{m.AuthorName} — {label}{amount}";
}
/// <summary>Render the current state onto one AlertBox preview (mirrors chat):
/// the playing frame, or null when idle so the source shows nothing until an
/// event actually fires.</summary>
public void UpdatePreview(Source alertBox)
{
try
{
var frame = RenderFrame(alertBox);
// Preview eyes (TASK 47, 2026-09-26): a healthy decode painted nothing on
// the canvas. UpdatePreview swallows its failures to Debug, so say once a
// second whether the box is being handed a frame at all.
if (IsPlaying)
{
var now = DateTime.UtcNow;
if (now - _lastPreviewLog >= TimeSpan.FromSeconds(1))
{
_lastPreviewLog = now;
AppLog.Write($"Alert preview: box {alertBox.Id} frame="
+ (frame == null ? "null" : $"{frame.Width}x{frame.Height} a{frame.BgraPixels[3]}"));
}
}
if (frame == null)
{
alertBox.VideoImageSource = null;
return;
}
var width = (int)Math.Max(1, alertBox.Width);
var height = (int)Math.Max(1, alertBox.Height);
var bitmap = new WriteableBitmap(width, height, 96, 96, PixelFormats.Bgra32, null);
bitmap.WritePixels(new Int32Rect(0, 0, width, height), frame.BgraPixels, frame.Stride, 0);
alertBox.VideoImageSource = bitmap;
}
catch (Exception ex)
{
System.Diagnostics.Debug.WriteLine($"Alert preview failed: {ex.Message}");
}
}
/// <summary>Wire property-change subscriptions for all AlertBox sources across
/// every scene (geometry changes re-render), and clear/reset their previews.</summary>
public void UpdateConfiguredBoxes(IEnumerable<Scene> scenes)
{
_scenes = scenes;
foreach (var scene in scenes)
foreach (var alertBox in scene.Elements.OfType<Source>().Where(s => s.Type == SourceType.AlertBox))
{
alertBox.PropertyChanged += (_, e) =>
{
if (e.PropertyName is nameof(Source.Width) or nameof(Source.Height))
UpdatePreview(alertBox);
};
UpdatePreview(alertBox);
}
}
private void RefreshAlertPreviews()
{
// The announcement strip is a GLOBAL 1920px overlay, not a per-box frame, so
// it gets its own sink: until this existed the strip went only to the frame
// pump and was therefore invisible in the preview pane the creator watches
// (2026-09-26 — "I do not see the scrolling text").
_tickerPreviewSink?.Invoke(AlertTickerFrame);
if (_scenes == null) return;
foreach (var alertBox in _scenes.SelectMany(s => s.Elements).OfType<Source>()
.Where(s => s.Type == SourceType.AlertBox))
UpdatePreview(alertBox);
}
public void Dispose()
{
StopSoundByte();
_ticker?.Stop();
_queue.Clear();
_current = null;
StopClip();
}
}