The FFmpeg ProcessFailed event arrives off-thread. Start recovery on the UI dispatcher, publish retry status there, and route StopStream back to the dispatcher before changing bound state or alert collections. This prevents a recovery exception from becoming an unhandled WPF CollectionView crash. AttentionAlertStackTests covers worker-thread recovery reporting and stop. 616 tests green, clean build with 0 warnings.
304 lines
12 KiB
C#
304 lines
12 KiB
C#
using System;
|
|
using System.Diagnostics;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using ytLive.Helpers;
|
|
using ytLive.Services;
|
|
using ytLive.Services.Encoder;
|
|
|
|
namespace ytLive.ViewModels;
|
|
|
|
/// <summary>
|
|
/// TASK 32·1 — a dropped push is retried instead of ending the session. The loop itself lives in
|
|
/// <see cref="StreamRecovery"/> (pure, testable); this partial supplies the app-side steps and reports
|
|
/// each episode into the left panel's alert stack.
|
|
/// <para>Deliberately narrow: only the ENCODER dying is recoverable. A compositor fault still goes
|
|
/// through <see cref="OnFramePumpFailed"/> and ends the session, because retrying it would loop.</para>
|
|
/// </summary>
|
|
public partial class MainViewModel
|
|
{
|
|
private CancellationTokenSource? _recoveryCts;
|
|
|
|
/// <summary>True while a push is being relaunched — the top bar shows amber RECONNECTING.</summary>
|
|
public bool IsReconnecting
|
|
{
|
|
get => _isReconnecting;
|
|
private set
|
|
{
|
|
if (!SetProperty(ref _isReconnecting, value)) return;
|
|
RaiseTopBarModes();
|
|
OnPropertyChanged(nameof(ReconnectDetailText));
|
|
ApplyReconnectPollCadence();
|
|
}
|
|
}
|
|
|
|
private bool _isReconnecting;
|
|
|
|
/// <summary>True after a drop ended the session with the broadcast gone: the show is over and the
|
|
/// only honest action left is a new broadcast. Drives the red OFF AIR line and the
|
|
/// "Back on air" button (TASK 32·3).</summary>
|
|
public bool IsOffAirAfterDrop
|
|
{
|
|
get => _isOffAirAfterDrop;
|
|
private set
|
|
{
|
|
if (!SetProperty(ref _isOffAirAfterDrop, value)) return;
|
|
RaiseTopBarModes();
|
|
OnPropertyChanged(nameof(ShowPrimaryStartButton));
|
|
OnPropertyChanged(nameof(BackOnAirLabel));
|
|
}
|
|
}
|
|
|
|
private bool _isOffAirAfterDrop;
|
|
|
|
public string BackOnAirLabel => "● Back on air";
|
|
|
|
private RelayCommand? _backOnAirCommand;
|
|
|
|
/// <summary>One click: build a new broadcast from the saved details and push — no dialog, and the
|
|
/// title carries the resume marker so the channel shows what happened.</summary>
|
|
public RelayCommand BackOnAirCommand =>
|
|
_backOnAirCommand ??= new RelayCommand(_ => BackOnAir(), _ => IsOffAirAfterDrop && IsOffline);
|
|
|
|
/// <summary>How many times this session has been resumed — the title marker counts.</summary>
|
|
private int _resumeCount;
|
|
|
|
/// <summary>The title the dropped show was using, kept so the resumed broadcast is named after it
|
|
/// rather than after whatever the form happens to hold now.</summary>
|
|
private string? _lastLiveTitle;
|
|
|
|
private void BackOnAir()
|
|
{
|
|
if (!IsOffAirAfterDrop) return;
|
|
|
|
_resumeCount++;
|
|
StreamTitle = BroadcastTitle.ForResume(_lastLiveTitle ?? StreamTitle, _resumeCount);
|
|
WindowTitle = $"{StreamTitle} — LlamaCasty";
|
|
|
|
// Clear the state BEFORE starting: the go-live path reuses the primary-button gate, and a
|
|
// session that starts while this flag is set would show two ways to start at once.
|
|
IsOffAirAfterDrop = false;
|
|
|
|
AppLog.Write($"RECONNECT: back on air as a new broadcast — '{StreamTitle}'");
|
|
OnAirPillOn = true;
|
|
StartStreamingSession(alsoRecord: false); // the ordinary path: new broadcast, same cached stream
|
|
}
|
|
|
|
/// <summary>Test seam: puts the view-model in the post-drop state without a live session.</summary>
|
|
internal void MarkOffAirAfterDropForTest(string? title)
|
|
{
|
|
_lastLiveTitle = title;
|
|
IsOffAirAfterDrop = true;
|
|
}
|
|
|
|
/// <summary>The attempt currently in flight (0 when idle).</summary>
|
|
public int ReconnectAttempt
|
|
{
|
|
get => _reconnectAttempt;
|
|
private set
|
|
{
|
|
if (!SetProperty(ref _reconnectAttempt, value)) return;
|
|
OnPropertyChanged(nameof(ReconnectDetailText));
|
|
}
|
|
}
|
|
|
|
private int _reconnectAttempt;
|
|
|
|
/// <summary>Whole seconds since the drop — measured, never predicted.</summary>
|
|
public int ReconnectElapsedSeconds
|
|
{
|
|
get => _reconnectElapsedSeconds;
|
|
private set
|
|
{
|
|
if (!SetProperty(ref _reconnectElapsedSeconds, value)) return;
|
|
OnPropertyChanged(nameof(ReconnectDetailText));
|
|
}
|
|
}
|
|
|
|
private int _reconnectElapsedSeconds;
|
|
private System.Windows.Threading.DispatcherTimer? _reconnectTick;
|
|
|
|
/// <summary>While recovering, ask YouTube much more often — the status IS the sign, so it has to
|
|
/// keep up with a reconnect that can succeed or die inside a few seconds.</summary>
|
|
private void ApplyReconnectPollCadence()
|
|
{
|
|
if (IsReconnecting)
|
|
{
|
|
_healthPollTimer.Interval = TimeSpan.FromSeconds(4);
|
|
_reconnectTick ??= new System.Windows.Threading.DispatcherTimer
|
|
{
|
|
Interval = TimeSpan.FromSeconds(1),
|
|
};
|
|
_reconnectTick.Tick -= OnReconnectTick;
|
|
_reconnectTick.Tick += OnReconnectTick;
|
|
_reconnectTick.Start();
|
|
}
|
|
else
|
|
{
|
|
_healthPollTimer.Interval = TimeSpan.FromSeconds(30);
|
|
if (_reconnectTick != null)
|
|
{
|
|
_reconnectTick.Stop();
|
|
_reconnectTick.Tick -= OnReconnectTick;
|
|
}
|
|
}
|
|
}
|
|
|
|
private void OnReconnectTick(object? sender, EventArgs e)
|
|
=> ReconnectElapsedSeconds = (int)Math.Round(_reconnectStarted?.Elapsed.TotalSeconds ?? 0);
|
|
|
|
// Measured from the drop, so the displayed seconds are facts rather than a projection.
|
|
private System.Diagnostics.Stopwatch? _reconnectStarted;
|
|
|
|
private void OnEncoderFailed(object? sender, string message)
|
|
{
|
|
var dispatcher = System.Windows.Application.Current?.Dispatcher;
|
|
if (dispatcher != null && !dispatcher.CheckAccess())
|
|
{
|
|
dispatcher.BeginInvoke(new Action(() => OnEncoderFailed(sender, message)));
|
|
return;
|
|
}
|
|
|
|
if (!IsLive && !IsRecording) return;
|
|
if (_recoveryCts != null) return; // one recovery at a time
|
|
if (!IsLive) return; // a recording-only session has nothing to relive
|
|
|
|
AppLog.Write($"RECONNECT: encoder process failed: {message}");
|
|
_recoveryCts = new CancellationTokenSource();
|
|
_ = RecoverStreamAsync(message, _recoveryCts.Token);
|
|
}
|
|
|
|
private async Task RecoverStreamAsync(string reason, CancellationToken cancellationToken)
|
|
{
|
|
ReconnectElapsedSeconds = 0;
|
|
_reconnectStarted = System.Diagnostics.Stopwatch.StartNew();
|
|
IsReconnecting = true;
|
|
ReconnectAttempt = 0;
|
|
var dropped = _reconnectStarted;
|
|
|
|
var logFrom = AppLog.SnapshotCurrentLaunch().Count > 0
|
|
? AppLog.SnapshotCurrentLaunch()[^1].Sequence + 1
|
|
: 1;
|
|
|
|
var recovery = new StreamRecovery(
|
|
StreamRetryPolicy.Default,
|
|
new StreamRecoverySteps
|
|
{
|
|
RestartAudio = _ =>
|
|
{
|
|
// The audio pipe's server is single-instance and its client just died, so it has
|
|
// to be recreated before ffmpeg can attach — otherwise the restart fails on a
|
|
// missing input.
|
|
_audioMixer.StopLive();
|
|
_audioMixer.StartLive(InstanceProfile.AudioPipeName(EncoderOptions.DefaultAudioPipeName));
|
|
return Task.CompletedTask;
|
|
},
|
|
StartPush = async ct =>
|
|
{
|
|
await _framePump.StartAsync(ct);
|
|
return _framePump.IsRunning;
|
|
},
|
|
CheckBroadcast = async ct =>
|
|
{
|
|
if (_currentBroadcastId == null || _reusableStream?.Id == null)
|
|
return BroadcastLiveness.Unknown;
|
|
var status = await _youtubeStream.GetBroadcastStatusAsync(_currentBroadcastId);
|
|
return status switch
|
|
{
|
|
"complete" => BroadcastLiveness.Complete,
|
|
null => BroadcastLiveness.Unknown,
|
|
_ => BroadcastLiveness.Live,
|
|
};
|
|
},
|
|
Report = text => ReportRecoveryStatus(text, logFrom),
|
|
Log = line => AppLog.Write(line, AppLogMessageType.Error, AppLogErrorClass.Class2),
|
|
});
|
|
|
|
try
|
|
{
|
|
var result = await recovery.RunAsync(cancellationToken);
|
|
|
|
switch (result.Outcome)
|
|
{
|
|
case RecoveryOutcome.Recovered:
|
|
ReconnectAttempt = result.Attempts;
|
|
AppLog.WriteSuccess(
|
|
$"RECONNECT: recovered on attempt {result.Attempts} after {result.Elapsed.TotalSeconds:0.0} s");
|
|
break;
|
|
|
|
case RecoveryOutcome.Cancelled:
|
|
// The creator stopped the stream mid-recovery: nothing to report.
|
|
break;
|
|
|
|
default:
|
|
// Gave up, or the broadcast was already complete: this session is over. The alert
|
|
// stack keeps the reason; the normal stop path clears pills, audio and recording.
|
|
// Then the honest next step is offered as one click — a NEW broadcast (TASK 32·3),
|
|
// which is the only way back on air once YouTube has completed this one.
|
|
AppLog.Write("RECONNECT: session ended — the broadcast could not be recovered",
|
|
AppLogMessageType.Error, AppLogErrorClass.Class1);
|
|
var droppedTitle = StreamTitle;
|
|
StopStream();
|
|
MarkOffAirAfterDropForTest(null);
|
|
_lastLiveTitle = droppedTitle;
|
|
break;
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
AppLog.Write(ex, "RECONNECT: recovery loop faulted");
|
|
StopStream();
|
|
}
|
|
finally
|
|
{
|
|
_reconnectStarted = null;
|
|
IsReconnecting = false;
|
|
ReconnectAttempt = 0;
|
|
_recoveryCts?.Dispose();
|
|
_recoveryCts = null;
|
|
}
|
|
}
|
|
|
|
internal void ReportRecoveryStatus(string text, long logFrom)
|
|
{
|
|
var dispatcher = System.Windows.Application.Current?.Dispatcher;
|
|
if (dispatcher != null && !dispatcher.CheckAccess())
|
|
{
|
|
dispatcher.Invoke(() => ReportRecoveryStatus(text, logFrom));
|
|
return;
|
|
}
|
|
|
|
var progress = text.StartsWith("Stream dropped", StringComparison.Ordinal)
|
|
? ParseAttempt(text)
|
|
: 0;
|
|
if (progress > 0) ReconnectAttempt = progress;
|
|
|
|
var to = AppLog.SnapshotCurrentLaunch() is { Count: > 0 } entries
|
|
? entries[^1].Sequence
|
|
: logFrom;
|
|
PushAttentionAlert("stream-recovery", text, logFrom, Math.Max(logFrom, to));
|
|
}
|
|
|
|
private static int ParseAttempt(string text)
|
|
{
|
|
// "Stream dropped — reconnecting (attempt 2 of 4)"
|
|
var marker = text.IndexOf("attempt ", StringComparison.Ordinal);
|
|
if (marker < 0) return 0;
|
|
var rest = text[(marker + "attempt ".Length)..];
|
|
var space = rest.IndexOf(' ');
|
|
return int.TryParse(space > 0 ? rest[..space] : rest, out var attempt) ? attempt : 0;
|
|
}
|
|
|
|
/// <summary>The creator asked to stop: never race a stop against a relaunch.</summary>
|
|
private void CancelStreamRecovery() => _recoveryCts?.Cancel();
|
|
|
|
/// <summary>YouTube ended the chat: ask the broadcast status now rather than at the next tick.
|
|
/// The API decides — this signal only saves up to 30 s of lying to the creator.</summary>
|
|
private void OnChatEnded()
|
|
{
|
|
if (!IsLive) return;
|
|
AppLog.Write("Chat ended event received — checking the broadcast status now");
|
|
_ = PollHealthAsync();
|
|
}
|
|
}
|