Files
LlamaCasty/Services/AlertOverlayLayer.cs
T
gramps e2ecc248e0 TASK 47: draw the alert clip — the preview Image was Collapsed for AlertBox
Four builds (89/90 + two) burned proving the alert video was PERFECT: real h264
1280x720, 240 frames decoded, alert audio in the live mix, and the shipped asset
byte-identical (md5 0ee1f496…) to the creator's llamacasty-dancingLlama-thankyou.mp4
with every sampled frame full bright content. Build 90's new diagnostics then showed
the frame reaching BOTH consumers every second for the whole clip —
`Alert preview: frame=680x200 a255` and `Output resolver: frame 680x200
playing=True` — an opaque, correctly-sized frame, handed over and never seen.

Root cause: Controls/PreviewPane.xaml keeps the per-element Image Collapsed unless a
DataTrigger fires, and there was no IsAlertBox trigger (only IsImageSource / IsChatBox /
IsWebSource / IsWebcam). The alert box's Image was therefore Collapsed forever. Chat
boxes rendered because they HAVE a trigger — that asymmetry is the whole clue, and it
is why removing/re-adding the layer could never fix it.

Fix: an IsAlertBox DataTrigger beside the IsChatBox one.

Two Good Dog tests, because the existing fakes had been hiding this:
- AlertBoxPreviewVisibilityTests (red `Expected: Visible / Actual: Collapsed`, now
  green) drives the real MainWindow + PreviewPane and asserts the bound Image is
  Visible — the first alert test that crosses the XAML at all.
- AlertClipOutputTests is the first test in the repo to run a REAL codec: pinned ffmpeg
  generates a clip, the real AlertClipDecoder + AlertOverlayLayer + SceneCompositor
  composite it, and the box rect must be a colourful picture that DIFFERS from idle. It
  passed while the feature was broken in the app — which is exactly why it was needed:
  it exonerated decode+layer+compositor and pointed the hunt at the last hop.

Also keeps the two once-per-second alert diagnostics (resolver + preview) that settled
it, pending the creator's call on whether to keep them.

Notes: the pinned BtbN ffmpeg has no libx264, so the test clip is -c:v mpeg4.
SceneCompositor.Render fills its base with opaque black, so the test compares against an
idle render rather than counting non-zero bytes. MediaSource has the same missing trigger
in PreviewPane.xaml — left unfixed as out of scope, noted in HANDOFF.

Full suite 324/324. Lesson recorded in MyMistakes.md rule (5): a producer that hands over
a correct frame has still delivered nothing — when output arrives and the user still sees
nothing, stop auditing the producer and audit the consumer's VISIBILITY.
2026-09-26 14:44:57 -07:00

494 lines
18 KiB
C#

using System.Collections.Generic;
using System.Linq;
using System.Windows;
using System.Windows.Media;
using System.Windows.Media.Imaging;
using System.Windows.Threading;
using ytLive.Helpers;
using ytLive.Models;
using ytLive.Services.Audio;
using ytLive.Services.Compositor;
namespace ytLive.Services;
/// <summary>
/// The native alert box (OBS-style celebration zone, no third party). Owns the
/// event queue and the animation clock: <see cref="ChatEventKind"/> events from
/// the chat feed are enqueued one-at-a-time and played through a celebration
/// animation (<see cref="AlertRenderer"/>) OR — when the alert-box video is
/// configured and resolves — through a decoded clip. Frames are served to the
/// stream compositor via <see cref="RenderFrame"/> and mirrored to the preview
/// via <see cref="UpdatePreview"/>. Idle = transparent (null frame); playback is
/// advanced by a 33ms dispatch ticker. The clock is exposed as
/// <see cref="Advance"/> so tests drive playback deterministically without the
/// timer. Ordinary chat rows (<see cref="ChatEventKind.None"/>) never enqueue —
/// a subscriber mention is a chat row, not an alert.
///
/// TASK 47 (Stream Alerts video): a video-backed alert replaces the animation
/// while a clip decodes (<see cref="IAlertClipDecoder"/>, one per play). The
/// clip plays at its own media pace; the layer applies a ~300ms fade-in/out
/// alpha envelope to the frames AND the audio samples (freeze-frame fade-out on
/// the tail), clips the alert to the clip length, and forwards the scaled audio
/// to the mixer's alert ring via <paramref name="audioSink"/> (no duck). The
/// per-source config (path/default/ticker/volume) is resolved through
/// <paramref name="clipPathResolver"/>; the ticker strip is served via
/// <see cref="AlertTickerFrame"/>.
/// </summary>
public sealed class AlertOverlayLayer : IDisposable
{
/// <summary>The fade-in/out ramp length (video alpha AND audio gain). Measured
/// against a real clip in the AlertLayerVideoTests unit; keep the ramp short
/// enough to read as a "pop-in" on stream (see ai.md).</summary>
internal const double FadeDurationSeconds = 0.30;
/// <summary>How long an alert waits for the first decoded frame or audio slab
/// before treating the clip as unplayable and falling back to the animations
/// (a silent decoder must never blank the alert box).</summary>
internal const double NoFrameFallbackSeconds = 1.0;
private static readonly TimeSpan TickInterval = TimeSpan.FromMilliseconds(33);
private readonly AlertRenderer _renderer;
private readonly Func<string, int, int, IAlertClipDecoder>? _clipDecoderFactory;
private readonly Func<Source, string?>? _clipPathResolver;
private readonly Action<AudioSample>? _audioSink;
private readonly Queue<ChatMessage> _queue = new();
private readonly DispatcherTimer? _ticker;
private ChatMessage? _current;
private Source? _currentBox;
private IAlertClipDecoder? _clip;
private VideoFrame? _latestClipFrame;
private bool _clipEof;
private double _clipEofElapsed;
private int _clipWidth;
private int _clipHeight;
private bool _clipSawFrame;
private bool _clipAudioHeard;
private string? _tickerText;
private double _elapsed;
private IEnumerable<Scene>? _scenes;
private int _contentVersion;
private int _frameVersion = -1;
private string? _frameKey;
private VideoFrame? _cachedFrame;
private DateTime _lastPreviewLog = DateTime.MinValue;
public AlertOverlayLayer(
AlertRenderer renderer,
Func<string, int, int, IAlertClipDecoder>? clipDecoderFactory = null,
Func<Source, string?>? clipPathResolver = null,
Action<AudioSample>? audioSink = null)
{
_renderer = renderer;
_clipDecoderFactory = clipDecoderFactory;
_clipPathResolver = clipPathResolver;
_audioSink = audioSink;
if (Application.Current != null)
{
_ticker = new DispatcherTimer { Interval = TickInterval };
_ticker.Tick += (_, _) =>
{
Advance(TickInterval.TotalSeconds);
RefreshAlertPreviews();
};
}
}
/// <summary>Production entry from the chat feed: enqueue one event. Messages
/// arrive on the chat poller thread, so this marshals to the UI thread (same
/// rule as <see cref="ChatOverlayLayer.OnMessageReceived"/>) — the preview
/// writes create WPF <see cref="WriteableBitmap"/> instances, and creating a
/// DependencySource off the UI thread crashes the WPF binding engine
/// (ArgumentException "Must create DependencySource on same Thread").</summary>
public void OnMessageReceived(ChatMessage message, IEnumerable<Scene> scenes)
{
var app = Application.Current;
if (app != null && !app.Dispatcher.CheckAccess())
{
app.Dispatcher.Invoke(() => Ingest(message, scenes));
return;
}
Ingest(message, scenes);
}
private void Ingest(ChatMessage message, IEnumerable<Scene> scenes)
{
_scenes = scenes;
Enqueue(message);
_ticker?.Start();
RefreshAlertPreviews();
}
/// <summary>Queue an event (pure — also the test seam). Chat rows carry
/// <see cref="ChatEventKind.None"/> and are ignored: no alert for a mention.</summary>
public void Enqueue(ChatMessage message)
{
if (message.Kind == ChatEventKind.None) return;
if (_queue.Count >= 10) return; // back-to-back floods drop the tail, never stall playback
_queue.Enqueue(message);
if (_current == null)
{
_current = _queue.Dequeue();
_elapsed = 0;
BeginClip();
}
_contentVersion++;
}
/// <summary>Advance the playback clock (the ticker's beat — and the test seam;
/// tests call this directly and never start the timer). Draining the queue to
/// nothing stops the ticker and leaves the box transparent.</summary>
public void Advance(double deltaSeconds)
{
// Video-backed alert: duration is clip-driven. The clip plays at media
// pace; once it EOFs the last frame sweeps through the fade-out before
// the alert moves on (a hard cut at EOF would pop, not fade).
if (_clip != null)
{
_elapsed += deltaSeconds;
// A decoder that starts but produces not a single frame (or slab of
// audio) is dead weight — the box would sit transparent for the whole
// alert. Give it a short grace, then fall back to the animations so an
// alert ALWAYS renders (the live decode failure at 2026-09-26 11:44 was
// this exact void, invisible in the log).
if (!_clipSawFrame && !_clipAudioHeard && _elapsed >= NoFrameFallbackSeconds)
{
System.Diagnostics.Debug.WriteLine("Alert clip yielded no frames in 1s — falling back to the animation");
StopClip();
_elapsed = 0;
_contentVersion++;
return;
}
if (_clipEof && _elapsed - _clipEofElapsed >= FadeDurationSeconds)
{
_clipEof = false;
StopClip();
_current = null;
_elapsed = 0;
AdvanceToNext();
}
_contentVersion++;
return;
}
if (_current == null)
{
if (_queue.Count == 0) return;
_current = _queue.Dequeue();
_elapsed = 0;
BeginClip();
_contentVersion++;
return;
}
_elapsed += deltaSeconds;
if (_elapsed >= DurationOf(_current))
{
_current = null;
_elapsed = 0;
AdvanceToNext();
}
_contentVersion++;
}
/// <summary>Whether an alert is currently playing or queued.</summary>
public bool IsPlaying => _current != null || _queue.Count > 0;
/// <summary>The live-output frame for one AlertBox element: the decoded clip
/// frame (faded) while a clip is active, else the animation frame, null
/// (transparent) when idle.</summary>
public VideoFrame? RenderFrame(Source alertBox)
{
if (_current == null)
{
_frameKey = null;
return null;
}
if (_clip != null)
return RenderClipFrame(alertBox);
var width = (int)Math.Max(1, alertBox.Width);
var height = (int)Math.Max(1, alertBox.Height);
var key = $"{width}x{height}|{_current.Id}|{Progress()}";
if (_frameKey == key && _frameVersion == _contentVersion)
return _cachedFrame;
var app = Application.Current;
if (app != null && !app.Dispatcher.CheckAccess())
{
var capturedKey = key;
var capturedVersion = _contentVersion;
return app.Dispatcher.Invoke(() =>
{
if (_frameKey == capturedKey && _frameVersion == capturedVersion)
return _cachedFrame;
return RenderCore(alertBox, width, height, capturedKey, capturedVersion);
});
}
return RenderCore(alertBox, width, height, key, _contentVersion);
}
/// <summary>The global ticker strip for the stream output (top edge, full
/// master width): a marquee of "viewer — event" while an alert is playing
/// and the config has the ticker on, else null. Pure byte-math per call so
/// the frame pump can read it on any thread (the text strip is rasterized
/// once on the UI thread and cached).</summary>
public VideoFrame? AlertTickerFrame
{
get
{
if (_current == null || string.IsNullOrWhiteSpace(_tickerText)) return null;
return AlertTickerRenderer.Render(_tickerText!, _elapsed);
}
}
private VideoFrame? RenderClipFrame(Source alertBox)
{
var frame = _latestClipFrame;
if (frame == null) return null;
var w = (int)Math.Max(1, alertBox.Width);
var h = (int)Math.Max(1, alertBox.Height);
if (frame.Width != w || frame.Height != h)
frame = StretchMath.BilinearScale(frame, w, h);
var fade = FadeAt();
if (fade >= 1.0) return frame;
// Straight-alpha source-over on the compositor side: scale the source
// alpha by the envelope so the clip blends out (a 680x200 copy per fade
// tick is trivial; outside the ramps fade==1 returns the raw frame).
var copy = new byte[frame.BgraPixels.Length];
Buffer.BlockCopy(frame.BgraPixels, 0, copy, 0, copy.Length);
var a = (byte)(fade * 255 + 0.5);
for (var i = 3; i < copy.Length; i += 4)
copy[i] = (byte)((copy[i] * a) >> 8);
return new VideoFrame(frame.Width, frame.Height, copy);
}
private double FadeAt()
{
if (_clip == null) return 1.0;
if (_clipEof)
return Math.Clamp(1.0 - (_elapsed - _clipEofElapsed) / FadeDurationSeconds, 0.0, 1.0);
return Math.Clamp(_elapsed / FadeDurationSeconds, 0.0, 1.0);
}
private void BeginClip()
{
StopClip();
_tickerText = null;
if (_current == null) return;
var box = _scenes?.SelectMany(s => s.Elements).OfType<Source>()
.FirstOrDefault(s => s.Type == SourceType.AlertBox);
_currentBox = box;
if (box != null && box.AlertShowTicker)
_tickerText = ComposeTicker(_current);
if (box == null || _clipDecoderFactory == null) return;
var path = _clipPathResolver?.Invoke(box);
if (string.IsNullOrWhiteSpace(path)) return;
_clipWidth = (int)Math.Max(1, box.Width);
_clipHeight = (int)Math.Max(1, box.Height);
try
{
_clipSawFrame = false;
_clipAudioHeard = false;
_clip = _clipDecoderFactory(path, _clipWidth, _clipHeight);
_clip.FrameAvailable += OnClipFrame;
_clip.AudioReady += OnClipAudio;
_clip.Completed += OnClipCompleted;
_clip.Start();
AppLog.Write($"Alert clip start: box {box.Id} path '{path}' {_clipWidth}x{_clipHeight}");
}
catch (Exception ex)
{
AppLog.Write(ex, "Alert clip setup failed");
StopClip(); // decode setup failed (bad path) — fall back to animations
}
}
private void StopClip()
{
var clip = _clip;
_clip = null;
_latestClipFrame = null;
_clipEof = false;
_clipSawFrame = false;
_clipAudioHeard = false;
if (clip == null) return;
clip.FrameAvailable -= OnClipFrame;
clip.AudioReady -= OnClipAudio;
clip.Completed -= OnClipCompleted;
try { clip.Stop(); } catch { /* already stopped */ }
clip.Dispose();
}
private void OnClipFrame(VideoFrame frame)
{
_clipSawFrame = true;
_latestClipFrame = frame;
}
private void OnClipCompleted()
{
_clipEof = true;
_clipEofElapsed = _elapsed;
}
private void OnClipAudio(AudioSample sample)
{
if (_audioSink == null) return;
_clipAudioHeard = true;
var volume = (float)Math.Clamp(_currentBox?.AlertVideoVolume ?? 1.0, 0.0, 1.0);
if (volume <= 0f) return;
var fade = FadeAt();
if (fade <= 0.0) return;
var data = sample.Samples;
if (fade < 1.0 || Math.Abs(volume - 1f) > 0.001f)
{
data = new float[data.Length];
var gain = volume * (float)fade;
for (var i = 0; i < sample.Samples.Length; i++)
data[i] = sample.Samples[i] * gain;
}
_audioSink(new AudioSample(data, sample.SampleRate, sample.Channels));
}
private void AdvanceToNext()
{
if (_queue.Count > 0)
{
_current = _queue.Dequeue();
_elapsed = 0;
BeginClip();
}
else
{
_ticker?.Stop();
}
}
private VideoFrame? RenderCore(Source alertBox, int width, int height, string key, int version)
{
_frameKey = key;
_frameVersion = version;
if (_current == null) return _cachedFrame = null;
return _cachedFrame = _renderer.Render(_current, Progress(), width, height);
}
private double Progress()
{
if (_current == null) return 0;
return Math.Clamp(_elapsed / DurationOf(_current), 0.0, 1.0);
}
private static double DurationOf(ChatMessage m) => m.Kind switch
{
ChatEventKind.SuperChat => 4.6,
ChatEventKind.SuperSticker => 4.2,
ChatEventKind.NewMember => 4.0,
ChatEventKind.MemberGift => 5.0,
ChatEventKind.GiftReceived => 4.4,
ChatEventKind.MemberMilestone => 4.8,
_ => 4.0,
};
private static string ComposeTicker(ChatMessage m)
{
var label = m.Kind switch
{
ChatEventKind.SuperChat => "Super Chat",
ChatEventKind.SuperSticker => "Super Sticker",
ChatEventKind.NewMember => "New Member",
ChatEventKind.MemberGift => "Membership Gift",
ChatEventKind.GiftReceived => "Gifted Membership",
ChatEventKind.MemberMilestone => "Member Milestone",
_ => "Stream Alert",
};
var amount = m.Kind is ChatEventKind.SuperChat or ChatEventKind.SuperSticker
&& m.SuperChatDisplayString is { Length: > 0 } s
? $" · {s}"
: "";
return $"{m.AuthorName} — {label}{amount}";
}
/// <summary>Render the current state onto one AlertBox preview (mirrors chat):
/// the playing frame, or null when idle so the source shows nothing until an
/// event actually fires.</summary>
public void UpdatePreview(Source alertBox)
{
try
{
var frame = RenderFrame(alertBox);
// Preview eyes (TASK 47, 2026-09-26): a healthy decode painted nothing on
// the canvas. UpdatePreview swallows its failures to Debug, so say once a
// second whether the box is being handed a frame at all.
if (IsPlaying)
{
var now = DateTime.UtcNow;
if (now - _lastPreviewLog >= TimeSpan.FromSeconds(1))
{
_lastPreviewLog = now;
AppLog.Write($"Alert preview: box {alertBox.Id} frame="
+ (frame == null ? "null" : $"{frame.Width}x{frame.Height} a{frame.BgraPixels[3]}"));
}
}
if (frame == null)
{
alertBox.VideoImageSource = null;
return;
}
var width = (int)Math.Max(1, alertBox.Width);
var height = (int)Math.Max(1, alertBox.Height);
var bitmap = new WriteableBitmap(width, height, 96, 96, PixelFormats.Bgra32, null);
bitmap.WritePixels(new Int32Rect(0, 0, width, height), frame.BgraPixels, frame.Stride, 0);
alertBox.VideoImageSource = bitmap;
}
catch (Exception ex)
{
System.Diagnostics.Debug.WriteLine($"Alert preview failed: {ex.Message}");
}
}
/// <summary>Wire property-change subscriptions for all AlertBox sources across
/// every scene (geometry changes re-render), and clear/reset their previews.</summary>
public void UpdateConfiguredBoxes(IEnumerable<Scene> scenes)
{
_scenes = scenes;
foreach (var scene in scenes)
foreach (var alertBox in scene.Elements.OfType<Source>().Where(s => s.Type == SourceType.AlertBox))
{
alertBox.PropertyChanged += (_, e) =>
{
if (e.PropertyName is nameof(Source.Width) or nameof(Source.Height))
UpdatePreview(alertBox);
};
UpdatePreview(alertBox);
}
}
private void RefreshAlertPreviews()
{
if (_scenes == null) return;
foreach (var alertBox in _scenes.SelectMany(s => s.Elements).OfType<Source>()
.Where(s => s.Type == SourceType.AlertBox))
UpdatePreview(alertBox);
}
public void Dispose()
{
_ticker?.Stop();
_queue.Clear();
_current = null;
StopClip();
}
}