e2ecc248e0
Four builds (89/90 + two) burned proving the alert video was PERFECT: real h264 1280x720, 240 frames decoded, alert audio in the live mix, and the shipped asset byte-identical (md5 0ee1f496…) to the creator's llamacasty-dancingLlama-thankyou.mp4 with every sampled frame full bright content. Build 90's new diagnostics then showed the frame reaching BOTH consumers every second for the whole clip — `Alert preview: frame=680x200 a255` and `Output resolver: frame 680x200 playing=True` — an opaque, correctly-sized frame, handed over and never seen. Root cause: Controls/PreviewPane.xaml keeps the per-element Image Collapsed unless a DataTrigger fires, and there was no IsAlertBox trigger (only IsImageSource / IsChatBox / IsWebSource / IsWebcam). The alert box's Image was therefore Collapsed forever. Chat boxes rendered because they HAVE a trigger — that asymmetry is the whole clue, and it is why removing/re-adding the layer could never fix it. Fix: an IsAlertBox DataTrigger beside the IsChatBox one. Two Good Dog tests, because the existing fakes had been hiding this: - AlertBoxPreviewVisibilityTests (red `Expected: Visible / Actual: Collapsed`, now green) drives the real MainWindow + PreviewPane and asserts the bound Image is Visible — the first alert test that crosses the XAML at all. - AlertClipOutputTests is the first test in the repo to run a REAL codec: pinned ffmpeg generates a clip, the real AlertClipDecoder + AlertOverlayLayer + SceneCompositor composite it, and the box rect must be a colourful picture that DIFFERS from idle. It passed while the feature was broken in the app — which is exactly why it was needed: it exonerated decode+layer+compositor and pointed the hunt at the last hop. Also keeps the two once-per-second alert diagnostics (resolver + preview) that settled it, pending the creator's call on whether to keep them. Notes: the pinned BtbN ffmpeg has no libx264, so the test clip is -c:v mpeg4. SceneCompositor.Render fills its base with opaque black, so the test compares against an idle render rather than counting non-zero bytes. MediaSource has the same missing trigger in PreviewPane.xaml — left unfixed as out of scope, noted in HANDOFF. Full suite 324/324. Lesson recorded in MyMistakes.md rule (5): a producer that hands over a correct frame has still delivered nothing — when output arrives and the user still sees nothing, stop auditing the producer and audit the consumer's VISIBILITY.
494 lines
18 KiB
C#
494 lines
18 KiB
C#
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using System.Windows;
|
|
using System.Windows.Media;
|
|
using System.Windows.Media.Imaging;
|
|
using System.Windows.Threading;
|
|
using ytLive.Helpers;
|
|
using ytLive.Models;
|
|
using ytLive.Services.Audio;
|
|
using ytLive.Services.Compositor;
|
|
|
|
namespace ytLive.Services;
|
|
|
|
/// <summary>
|
|
/// The native alert box (OBS-style celebration zone, no third party). Owns the
|
|
/// event queue and the animation clock: <see cref="ChatEventKind"/> events from
|
|
/// the chat feed are enqueued one-at-a-time and played through a celebration
|
|
/// animation (<see cref="AlertRenderer"/>) OR — when the alert-box video is
|
|
/// configured and resolves — through a decoded clip. Frames are served to the
|
|
/// stream compositor via <see cref="RenderFrame"/> and mirrored to the preview
|
|
/// via <see cref="UpdatePreview"/>. Idle = transparent (null frame); playback is
|
|
/// advanced by a 33ms dispatch ticker. The clock is exposed as
|
|
/// <see cref="Advance"/> so tests drive playback deterministically without the
|
|
/// timer. Ordinary chat rows (<see cref="ChatEventKind.None"/>) never enqueue —
|
|
/// a subscriber mention is a chat row, not an alert.
|
|
///
|
|
/// TASK 47 (Stream Alerts video): a video-backed alert replaces the animation
|
|
/// while a clip decodes (<see cref="IAlertClipDecoder"/>, one per play). The
|
|
/// clip plays at its own media pace; the layer applies a ~300ms fade-in/out
|
|
/// alpha envelope to the frames AND the audio samples (freeze-frame fade-out on
|
|
/// the tail), clips the alert to the clip length, and forwards the scaled audio
|
|
/// to the mixer's alert ring via <paramref name="audioSink"/> (no duck). The
|
|
/// per-source config (path/default/ticker/volume) is resolved through
|
|
/// <paramref name="clipPathResolver"/>; the ticker strip is served via
|
|
/// <see cref="AlertTickerFrame"/>.
|
|
/// </summary>
|
|
public sealed class AlertOverlayLayer : IDisposable
|
|
{
|
|
/// <summary>The fade-in/out ramp length (video alpha AND audio gain). Measured
|
|
/// against a real clip in the AlertLayerVideoTests unit; keep the ramp short
|
|
/// enough to read as a "pop-in" on stream (see ai.md).</summary>
|
|
internal const double FadeDurationSeconds = 0.30;
|
|
|
|
/// <summary>How long an alert waits for the first decoded frame or audio slab
|
|
/// before treating the clip as unplayable and falling back to the animations
|
|
/// (a silent decoder must never blank the alert box).</summary>
|
|
internal const double NoFrameFallbackSeconds = 1.0;
|
|
|
|
private static readonly TimeSpan TickInterval = TimeSpan.FromMilliseconds(33);
|
|
|
|
private readonly AlertRenderer _renderer;
|
|
private readonly Func<string, int, int, IAlertClipDecoder>? _clipDecoderFactory;
|
|
private readonly Func<Source, string?>? _clipPathResolver;
|
|
private readonly Action<AudioSample>? _audioSink;
|
|
private readonly Queue<ChatMessage> _queue = new();
|
|
private readonly DispatcherTimer? _ticker;
|
|
|
|
private ChatMessage? _current;
|
|
private Source? _currentBox;
|
|
private IAlertClipDecoder? _clip;
|
|
private VideoFrame? _latestClipFrame;
|
|
private bool _clipEof;
|
|
private double _clipEofElapsed;
|
|
private int _clipWidth;
|
|
private int _clipHeight;
|
|
private bool _clipSawFrame;
|
|
private bool _clipAudioHeard;
|
|
private string? _tickerText;
|
|
private double _elapsed;
|
|
private IEnumerable<Scene>? _scenes;
|
|
|
|
private int _contentVersion;
|
|
private int _frameVersion = -1;
|
|
private string? _frameKey;
|
|
private VideoFrame? _cachedFrame;
|
|
private DateTime _lastPreviewLog = DateTime.MinValue;
|
|
|
|
public AlertOverlayLayer(
|
|
AlertRenderer renderer,
|
|
Func<string, int, int, IAlertClipDecoder>? clipDecoderFactory = null,
|
|
Func<Source, string?>? clipPathResolver = null,
|
|
Action<AudioSample>? audioSink = null)
|
|
{
|
|
_renderer = renderer;
|
|
_clipDecoderFactory = clipDecoderFactory;
|
|
_clipPathResolver = clipPathResolver;
|
|
_audioSink = audioSink;
|
|
if (Application.Current != null)
|
|
{
|
|
_ticker = new DispatcherTimer { Interval = TickInterval };
|
|
_ticker.Tick += (_, _) =>
|
|
{
|
|
Advance(TickInterval.TotalSeconds);
|
|
RefreshAlertPreviews();
|
|
};
|
|
}
|
|
}
|
|
|
|
/// <summary>Production entry from the chat feed: enqueue one event. Messages
|
|
/// arrive on the chat poller thread, so this marshals to the UI thread (same
|
|
/// rule as <see cref="ChatOverlayLayer.OnMessageReceived"/>) — the preview
|
|
/// writes create WPF <see cref="WriteableBitmap"/> instances, and creating a
|
|
/// DependencySource off the UI thread crashes the WPF binding engine
|
|
/// (ArgumentException "Must create DependencySource on same Thread").</summary>
|
|
public void OnMessageReceived(ChatMessage message, IEnumerable<Scene> scenes)
|
|
{
|
|
var app = Application.Current;
|
|
if (app != null && !app.Dispatcher.CheckAccess())
|
|
{
|
|
app.Dispatcher.Invoke(() => Ingest(message, scenes));
|
|
return;
|
|
}
|
|
Ingest(message, scenes);
|
|
}
|
|
|
|
private void Ingest(ChatMessage message, IEnumerable<Scene> scenes)
|
|
{
|
|
_scenes = scenes;
|
|
Enqueue(message);
|
|
_ticker?.Start();
|
|
RefreshAlertPreviews();
|
|
}
|
|
|
|
/// <summary>Queue an event (pure — also the test seam). Chat rows carry
|
|
/// <see cref="ChatEventKind.None"/> and are ignored: no alert for a mention.</summary>
|
|
public void Enqueue(ChatMessage message)
|
|
{
|
|
if (message.Kind == ChatEventKind.None) return;
|
|
if (_queue.Count >= 10) return; // back-to-back floods drop the tail, never stall playback
|
|
_queue.Enqueue(message);
|
|
if (_current == null)
|
|
{
|
|
_current = _queue.Dequeue();
|
|
_elapsed = 0;
|
|
BeginClip();
|
|
}
|
|
_contentVersion++;
|
|
}
|
|
|
|
/// <summary>Advance the playback clock (the ticker's beat — and the test seam;
|
|
/// tests call this directly and never start the timer). Draining the queue to
|
|
/// nothing stops the ticker and leaves the box transparent.</summary>
|
|
public void Advance(double deltaSeconds)
|
|
{
|
|
// Video-backed alert: duration is clip-driven. The clip plays at media
|
|
// pace; once it EOFs the last frame sweeps through the fade-out before
|
|
// the alert moves on (a hard cut at EOF would pop, not fade).
|
|
if (_clip != null)
|
|
{
|
|
_elapsed += deltaSeconds;
|
|
// A decoder that starts but produces not a single frame (or slab of
|
|
// audio) is dead weight — the box would sit transparent for the whole
|
|
// alert. Give it a short grace, then fall back to the animations so an
|
|
// alert ALWAYS renders (the live decode failure at 2026-09-26 11:44 was
|
|
// this exact void, invisible in the log).
|
|
if (!_clipSawFrame && !_clipAudioHeard && _elapsed >= NoFrameFallbackSeconds)
|
|
{
|
|
System.Diagnostics.Debug.WriteLine("Alert clip yielded no frames in 1s — falling back to the animation");
|
|
StopClip();
|
|
_elapsed = 0;
|
|
_contentVersion++;
|
|
return;
|
|
}
|
|
if (_clipEof && _elapsed - _clipEofElapsed >= FadeDurationSeconds)
|
|
{
|
|
_clipEof = false;
|
|
StopClip();
|
|
_current = null;
|
|
_elapsed = 0;
|
|
AdvanceToNext();
|
|
}
|
|
_contentVersion++;
|
|
return;
|
|
}
|
|
|
|
if (_current == null)
|
|
{
|
|
if (_queue.Count == 0) return;
|
|
_current = _queue.Dequeue();
|
|
_elapsed = 0;
|
|
BeginClip();
|
|
_contentVersion++;
|
|
return;
|
|
}
|
|
|
|
_elapsed += deltaSeconds;
|
|
if (_elapsed >= DurationOf(_current))
|
|
{
|
|
_current = null;
|
|
_elapsed = 0;
|
|
AdvanceToNext();
|
|
}
|
|
_contentVersion++;
|
|
}
|
|
|
|
/// <summary>Whether an alert is currently playing or queued.</summary>
|
|
public bool IsPlaying => _current != null || _queue.Count > 0;
|
|
|
|
/// <summary>The live-output frame for one AlertBox element: the decoded clip
|
|
/// frame (faded) while a clip is active, else the animation frame, null
|
|
/// (transparent) when idle.</summary>
|
|
public VideoFrame? RenderFrame(Source alertBox)
|
|
{
|
|
if (_current == null)
|
|
{
|
|
_frameKey = null;
|
|
return null;
|
|
}
|
|
|
|
if (_clip != null)
|
|
return RenderClipFrame(alertBox);
|
|
|
|
var width = (int)Math.Max(1, alertBox.Width);
|
|
var height = (int)Math.Max(1, alertBox.Height);
|
|
var key = $"{width}x{height}|{_current.Id}|{Progress()}";
|
|
|
|
if (_frameKey == key && _frameVersion == _contentVersion)
|
|
return _cachedFrame;
|
|
|
|
var app = Application.Current;
|
|
if (app != null && !app.Dispatcher.CheckAccess())
|
|
{
|
|
var capturedKey = key;
|
|
var capturedVersion = _contentVersion;
|
|
return app.Dispatcher.Invoke(() =>
|
|
{
|
|
if (_frameKey == capturedKey && _frameVersion == capturedVersion)
|
|
return _cachedFrame;
|
|
return RenderCore(alertBox, width, height, capturedKey, capturedVersion);
|
|
});
|
|
}
|
|
|
|
return RenderCore(alertBox, width, height, key, _contentVersion);
|
|
}
|
|
|
|
/// <summary>The global ticker strip for the stream output (top edge, full
|
|
/// master width): a marquee of "viewer — event" while an alert is playing
|
|
/// and the config has the ticker on, else null. Pure byte-math per call so
|
|
/// the frame pump can read it on any thread (the text strip is rasterized
|
|
/// once on the UI thread and cached).</summary>
|
|
public VideoFrame? AlertTickerFrame
|
|
{
|
|
get
|
|
{
|
|
if (_current == null || string.IsNullOrWhiteSpace(_tickerText)) return null;
|
|
return AlertTickerRenderer.Render(_tickerText!, _elapsed);
|
|
}
|
|
}
|
|
|
|
private VideoFrame? RenderClipFrame(Source alertBox)
|
|
{
|
|
var frame = _latestClipFrame;
|
|
if (frame == null) return null;
|
|
|
|
var w = (int)Math.Max(1, alertBox.Width);
|
|
var h = (int)Math.Max(1, alertBox.Height);
|
|
if (frame.Width != w || frame.Height != h)
|
|
frame = StretchMath.BilinearScale(frame, w, h);
|
|
|
|
var fade = FadeAt();
|
|
if (fade >= 1.0) return frame;
|
|
|
|
// Straight-alpha source-over on the compositor side: scale the source
|
|
// alpha by the envelope so the clip blends out (a 680x200 copy per fade
|
|
// tick is trivial; outside the ramps fade==1 returns the raw frame).
|
|
var copy = new byte[frame.BgraPixels.Length];
|
|
Buffer.BlockCopy(frame.BgraPixels, 0, copy, 0, copy.Length);
|
|
var a = (byte)(fade * 255 + 0.5);
|
|
for (var i = 3; i < copy.Length; i += 4)
|
|
copy[i] = (byte)((copy[i] * a) >> 8);
|
|
return new VideoFrame(frame.Width, frame.Height, copy);
|
|
}
|
|
|
|
private double FadeAt()
|
|
{
|
|
if (_clip == null) return 1.0;
|
|
if (_clipEof)
|
|
return Math.Clamp(1.0 - (_elapsed - _clipEofElapsed) / FadeDurationSeconds, 0.0, 1.0);
|
|
return Math.Clamp(_elapsed / FadeDurationSeconds, 0.0, 1.0);
|
|
}
|
|
|
|
private void BeginClip()
|
|
{
|
|
StopClip();
|
|
_tickerText = null;
|
|
if (_current == null) return;
|
|
|
|
var box = _scenes?.SelectMany(s => s.Elements).OfType<Source>()
|
|
.FirstOrDefault(s => s.Type == SourceType.AlertBox);
|
|
_currentBox = box;
|
|
if (box != null && box.AlertShowTicker)
|
|
_tickerText = ComposeTicker(_current);
|
|
|
|
if (box == null || _clipDecoderFactory == null) return;
|
|
var path = _clipPathResolver?.Invoke(box);
|
|
if (string.IsNullOrWhiteSpace(path)) return;
|
|
|
|
_clipWidth = (int)Math.Max(1, box.Width);
|
|
_clipHeight = (int)Math.Max(1, box.Height);
|
|
try
|
|
{
|
|
_clipSawFrame = false;
|
|
_clipAudioHeard = false;
|
|
_clip = _clipDecoderFactory(path, _clipWidth, _clipHeight);
|
|
_clip.FrameAvailable += OnClipFrame;
|
|
_clip.AudioReady += OnClipAudio;
|
|
_clip.Completed += OnClipCompleted;
|
|
_clip.Start();
|
|
AppLog.Write($"Alert clip start: box {box.Id} path '{path}' {_clipWidth}x{_clipHeight}");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
AppLog.Write(ex, "Alert clip setup failed");
|
|
StopClip(); // decode setup failed (bad path) — fall back to animations
|
|
}
|
|
}
|
|
|
|
private void StopClip()
|
|
{
|
|
var clip = _clip;
|
|
_clip = null;
|
|
_latestClipFrame = null;
|
|
_clipEof = false;
|
|
_clipSawFrame = false;
|
|
_clipAudioHeard = false;
|
|
if (clip == null) return;
|
|
clip.FrameAvailable -= OnClipFrame;
|
|
clip.AudioReady -= OnClipAudio;
|
|
clip.Completed -= OnClipCompleted;
|
|
try { clip.Stop(); } catch { /* already stopped */ }
|
|
clip.Dispose();
|
|
}
|
|
|
|
private void OnClipFrame(VideoFrame frame)
|
|
{
|
|
_clipSawFrame = true;
|
|
_latestClipFrame = frame;
|
|
}
|
|
|
|
private void OnClipCompleted()
|
|
{
|
|
_clipEof = true;
|
|
_clipEofElapsed = _elapsed;
|
|
}
|
|
|
|
private void OnClipAudio(AudioSample sample)
|
|
{
|
|
if (_audioSink == null) return;
|
|
_clipAudioHeard = true;
|
|
var volume = (float)Math.Clamp(_currentBox?.AlertVideoVolume ?? 1.0, 0.0, 1.0);
|
|
if (volume <= 0f) return;
|
|
var fade = FadeAt();
|
|
if (fade <= 0.0) return;
|
|
|
|
var data = sample.Samples;
|
|
if (fade < 1.0 || Math.Abs(volume - 1f) > 0.001f)
|
|
{
|
|
data = new float[data.Length];
|
|
var gain = volume * (float)fade;
|
|
for (var i = 0; i < sample.Samples.Length; i++)
|
|
data[i] = sample.Samples[i] * gain;
|
|
}
|
|
_audioSink(new AudioSample(data, sample.SampleRate, sample.Channels));
|
|
}
|
|
|
|
private void AdvanceToNext()
|
|
{
|
|
if (_queue.Count > 0)
|
|
{
|
|
_current = _queue.Dequeue();
|
|
_elapsed = 0;
|
|
BeginClip();
|
|
}
|
|
else
|
|
{
|
|
_ticker?.Stop();
|
|
}
|
|
}
|
|
|
|
private VideoFrame? RenderCore(Source alertBox, int width, int height, string key, int version)
|
|
{
|
|
_frameKey = key;
|
|
_frameVersion = version;
|
|
if (_current == null) return _cachedFrame = null;
|
|
return _cachedFrame = _renderer.Render(_current, Progress(), width, height);
|
|
}
|
|
|
|
private double Progress()
|
|
{
|
|
if (_current == null) return 0;
|
|
return Math.Clamp(_elapsed / DurationOf(_current), 0.0, 1.0);
|
|
}
|
|
|
|
private static double DurationOf(ChatMessage m) => m.Kind switch
|
|
{
|
|
ChatEventKind.SuperChat => 4.6,
|
|
ChatEventKind.SuperSticker => 4.2,
|
|
ChatEventKind.NewMember => 4.0,
|
|
ChatEventKind.MemberGift => 5.0,
|
|
ChatEventKind.GiftReceived => 4.4,
|
|
ChatEventKind.MemberMilestone => 4.8,
|
|
_ => 4.0,
|
|
};
|
|
|
|
private static string ComposeTicker(ChatMessage m)
|
|
{
|
|
var label = m.Kind switch
|
|
{
|
|
ChatEventKind.SuperChat => "Super Chat",
|
|
ChatEventKind.SuperSticker => "Super Sticker",
|
|
ChatEventKind.NewMember => "New Member",
|
|
ChatEventKind.MemberGift => "Membership Gift",
|
|
ChatEventKind.GiftReceived => "Gifted Membership",
|
|
ChatEventKind.MemberMilestone => "Member Milestone",
|
|
_ => "Stream Alert",
|
|
};
|
|
var amount = m.Kind is ChatEventKind.SuperChat or ChatEventKind.SuperSticker
|
|
&& m.SuperChatDisplayString is { Length: > 0 } s
|
|
? $" · {s}"
|
|
: "";
|
|
return $"{m.AuthorName} — {label}{amount}";
|
|
}
|
|
|
|
/// <summary>Render the current state onto one AlertBox preview (mirrors chat):
|
|
/// the playing frame, or null when idle so the source shows nothing until an
|
|
/// event actually fires.</summary>
|
|
public void UpdatePreview(Source alertBox)
|
|
{
|
|
try
|
|
{
|
|
var frame = RenderFrame(alertBox);
|
|
// Preview eyes (TASK 47, 2026-09-26): a healthy decode painted nothing on
|
|
// the canvas. UpdatePreview swallows its failures to Debug, so say once a
|
|
// second whether the box is being handed a frame at all.
|
|
if (IsPlaying)
|
|
{
|
|
var now = DateTime.UtcNow;
|
|
if (now - _lastPreviewLog >= TimeSpan.FromSeconds(1))
|
|
{
|
|
_lastPreviewLog = now;
|
|
AppLog.Write($"Alert preview: box {alertBox.Id} frame="
|
|
+ (frame == null ? "null" : $"{frame.Width}x{frame.Height} a{frame.BgraPixels[3]}"));
|
|
}
|
|
}
|
|
if (frame == null)
|
|
{
|
|
alertBox.VideoImageSource = null;
|
|
return;
|
|
}
|
|
var width = (int)Math.Max(1, alertBox.Width);
|
|
var height = (int)Math.Max(1, alertBox.Height);
|
|
var bitmap = new WriteableBitmap(width, height, 96, 96, PixelFormats.Bgra32, null);
|
|
bitmap.WritePixels(new Int32Rect(0, 0, width, height), frame.BgraPixels, frame.Stride, 0);
|
|
alertBox.VideoImageSource = bitmap;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
System.Diagnostics.Debug.WriteLine($"Alert preview failed: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
/// <summary>Wire property-change subscriptions for all AlertBox sources across
|
|
/// every scene (geometry changes re-render), and clear/reset their previews.</summary>
|
|
public void UpdateConfiguredBoxes(IEnumerable<Scene> scenes)
|
|
{
|
|
_scenes = scenes;
|
|
foreach (var scene in scenes)
|
|
foreach (var alertBox in scene.Elements.OfType<Source>().Where(s => s.Type == SourceType.AlertBox))
|
|
{
|
|
alertBox.PropertyChanged += (_, e) =>
|
|
{
|
|
if (e.PropertyName is nameof(Source.Width) or nameof(Source.Height))
|
|
UpdatePreview(alertBox);
|
|
};
|
|
UpdatePreview(alertBox);
|
|
}
|
|
}
|
|
|
|
private void RefreshAlertPreviews()
|
|
{
|
|
if (_scenes == null) return;
|
|
foreach (var alertBox in _scenes.SelectMany(s => s.Elements).OfType<Source>()
|
|
.Where(s => s.Type == SourceType.AlertBox))
|
|
UpdatePreview(alertBox);
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
_ticker?.Stop();
|
|
_queue.Clear();
|
|
_current = null;
|
|
StopClip();
|
|
}
|
|
} |