feat(stream): retry a dropped push instead of ending the session (TASK 32·1)
A dead encoder used to run the whole stop path, so a one-second blip ended the broadcast. Now a dropped push is relaunched against the same ingest while the broadcast is still alive. - `Services/StreamRetryPolicy.cs` — immediate, +1 s, +2 s, +3 s. Minimum waits on purpose (creator ruling): YouTube's grace is short, so every second not pushing is grace spent. - `Services/StreamRecovery.cs` — the loop, with the audio pipe recreated before every attempt (its server is single-instance and unusable after the old client dies, so a naive relaunch fails on a missing input). After a push comes up it asks the API whether the broadcast survived: "complete" ends recovery as `BroadcastComplete` instead of showing a live sign over a dead stream; an API hiccup counts as recovered rather than ending a healthy session. - `FramePump.EncoderFailed` — a dead push is recoverable; a compositor fault still goes through `Failed` and ends the session, because retrying it would loop. - `MainViewModel.Recovery.cs` — wires the steps, reports each episode into the left panel alert stack (one row per episode, updated in place, log range painted), and cancels on Stop. - Alert stack polish: a rule and a gap now separate the stack from the scene layers (they read as one list before), and the licence indicator uses the creator's wording — `STEAM ID NOT PRESENT` / `Steam ID Limit: 30fps` / `Click for More Info`, becoming `STEAM ID PRESENT` / `60fps` with a close control once owned. Tests: `StreamRecoveryTests` (schedule, recover on attempt 2 with the pipe restarted each time, complete-broadcast honesty, unknown liveness, give-up, cancel, audio-restart fault), `FramePumpTests` now pin that process death raises `EncoderFailed` and not `Failed`. FAST gate: 538 passed (one documented pre-existing audio failure), 0 warnings.
This commit is contained in:
@@ -49,8 +49,11 @@
|
|||||||
ownership is verified (that close is permanent). Clicking a row runs its
|
ownership is verified (that close is permanent). Clicking a row runs its
|
||||||
action: the licence indicator opens About, an event opens the Logs tab with
|
action: the licence indicator opens About, an event opens the Logs tab with
|
||||||
that event's lines painted. -->
|
that event's lines painted. -->
|
||||||
<ItemsControl Grid.Row="0" ItemsSource="{Binding AttentionAlerts}" Margin="0,0,0,6"
|
<!-- Alerts and the panel content are different kinds of thing (state vs. the
|
||||||
Visibility="{Binding HasAttentionAlerts, Converter={StaticResource BoolToVis}}">
|
scene you are editing), so they are separated by space AND a rule: without it
|
||||||
|
the stack read as the first rows of the layer list (creator, 2026-11-05). -->
|
||||||
|
<StackPanel Grid.Row="0" Visibility="{Binding HasAttentionAlerts, Converter={StaticResource BoolToVis}}">
|
||||||
|
<ItemsControl ItemsSource="{Binding AttentionAlerts}" Margin="0,0,0,2">
|
||||||
<ItemsControl.ItemTemplate>
|
<ItemsControl.ItemTemplate>
|
||||||
<DataTemplate>
|
<DataTemplate>
|
||||||
<Border Margin="0,0,0,4" Padding="8,6" CornerRadius="4" Cursor="Hand"
|
<Border Margin="0,0,0,4" Padding="8,6" CornerRadius="4" Cursor="Hand"
|
||||||
@@ -89,6 +92,8 @@
|
|||||||
</DataTemplate>
|
</DataTemplate>
|
||||||
</ItemsControl.ItemTemplate>
|
</ItemsControl.ItemTemplate>
|
||||||
</ItemsControl>
|
</ItemsControl>
|
||||||
|
<Border Style="{StaticResource PropertyDivider}" Margin="0,4,0,10"/>
|
||||||
|
</StackPanel>
|
||||||
|
|
||||||
<!-- Edit mode (offline): layers + properties -->
|
<!-- Edit mode (offline): layers + properties -->
|
||||||
<Grid Grid.Row="1" Visibility="{Binding IsEditMode, Converter={StaticResource BoolToVis}}">
|
<Grid Grid.Row="1" Visibility="{Binding IsEditMode, Converter={StaticResource BoolToVis}}">
|
||||||
|
|||||||
+2
-1
@@ -13,7 +13,8 @@
|
|||||||
|
|
||||||
## Next action
|
## Next action
|
||||||
|
|
||||||
- **TASK 32·1 (blip retry)** — it pushes its episodes into the alert stack. Then 32·5 (ingest pre-flight),
|
- **TASK 32·1 shipped** (blip retry, immediate/+1/+2/+3 s, alert-stack episodes, complete-broadcast check).
|
||||||
|
Next: 32·5 (ingest pre-flight),
|
||||||
then 32·2/32·3 (countdown + Back on air) which need YOUR measurement take: pull the network on a
|
then 32·2/32·3 (countdown + Back on air) which need YOUR measurement take: pull the network on a
|
||||||
private stream and time how long YouTube holds the broadcast.
|
private stream and time how long YouTube holds the broadcast.
|
||||||
- Open loop: `32·4` is proven by construction only — kill ffmpeg mid-recording and play the file.
|
- Open loop: `32·4` is proven by construction only — kill ffmpeg mid-recording and play the file.
|
||||||
|
|||||||
@@ -129,6 +129,10 @@ public sealed class FramePump : IDisposable
|
|||||||
/// <summary>Raised when the encoder cannot start or dies mid-stream. The pump stops itself.</summary>
|
/// <summary>Raised when the encoder cannot start or dies mid-stream. The pump stops itself.</summary>
|
||||||
public event EventHandler<string>? Failed;
|
public event EventHandler<string>? Failed;
|
||||||
|
|
||||||
|
/// <summary>TASK 32·1: the ENCODER died (the push dropped) — recoverable, unlike a compositor fault.
|
||||||
|
/// Raised before the pump stops, so the session can relaunch the push instead of ending.</summary>
|
||||||
|
public event EventHandler<string>? EncoderFailed;
|
||||||
|
|
||||||
public FramePump(
|
public FramePump(
|
||||||
Func<Scene?> sceneProvider,
|
Func<Scene?> sceneProvider,
|
||||||
Func<SceneElement, VideoFrame?> frameResolver,
|
Func<SceneElement, VideoFrame?> frameResolver,
|
||||||
@@ -969,7 +973,9 @@ public sealed class FramePump : IDisposable
|
|||||||
private void OnProcessFailed(object? sender, string message)
|
private void OnProcessFailed(object? sender, string message)
|
||||||
{
|
{
|
||||||
_log?.Invoke($"FramePump: encoder process failed: {message}");
|
_log?.Invoke($"FramePump: encoder process failed: {message}");
|
||||||
Failed?.Invoke(this, message);
|
// A dead push is not a dead session: the broadcast may still be live, so this is the
|
||||||
|
// recoverable signal (TASK 32·1). Fatal pump faults still go through Failed.
|
||||||
|
EncoderFailed?.Invoke(this, message);
|
||||||
_ = StopAsync();
|
_ = StopAsync();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,150 @@
|
|||||||
|
using System;
|
||||||
|
using System.Threading;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
|
namespace ytLive.Services;
|
||||||
|
|
||||||
|
/// <summary>What YouTube says about the broadcast once a push is back.</summary>
|
||||||
|
public enum BroadcastLiveness
|
||||||
|
{
|
||||||
|
/// <summary>Could not tell (no token, network, API hiccup) — treat as recovered.</summary>
|
||||||
|
Unknown,
|
||||||
|
|
||||||
|
/// <summary>The broadcast is still live/ready: the push landed and the show continues.</summary>
|
||||||
|
Live,
|
||||||
|
|
||||||
|
/// <summary>The broadcast is complete. It cannot be resumed; the push is talking to a dead ingest.</summary>
|
||||||
|
Complete,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Everything the recovery loop needs from the app, as delegates so it can be tested
|
||||||
|
/// without ffmpeg, without a network, and without a window.</summary>
|
||||||
|
public sealed class StreamRecoverySteps
|
||||||
|
{
|
||||||
|
/// <summary>Recreates the audio pipe server. The old one is unusable after its client died
|
||||||
|
/// (single-instance server), and the encoder cannot start without it.</summary>
|
||||||
|
public required Func<CancellationToken, Task> RestartAudio { get; init; }
|
||||||
|
|
||||||
|
/// <summary>Starts the push. True when the encoder came up.</summary>
|
||||||
|
public required Func<CancellationToken, Task<bool>> StartPush { get; init; }
|
||||||
|
|
||||||
|
/// <summary>Asks YouTube whether the broadcast survived. Only called after a push starts.</summary>
|
||||||
|
public required Func<CancellationToken, Task<BroadcastLiveness>> CheckBroadcast { get; init; }
|
||||||
|
|
||||||
|
/// <summary>State for the alert stack: attempt numbers and the outcome.</summary>
|
||||||
|
public required Action<string> Report { get; init; }
|
||||||
|
|
||||||
|
/// <summary>The record: one line per attempt, into the app log.</summary>
|
||||||
|
public required Action<string> Log { get; init; }
|
||||||
|
|
||||||
|
public Func<TimeSpan, CancellationToken, Task> Delay { get; init; }
|
||||||
|
= (delay, ct) => delay <= TimeSpan.Zero ? Task.CompletedTask : Task.Delay(delay, ct);
|
||||||
|
}
|
||||||
|
|
||||||
|
public enum RecoveryOutcome
|
||||||
|
{
|
||||||
|
Recovered,
|
||||||
|
GaveUp,
|
||||||
|
Cancelled,
|
||||||
|
BroadcastComplete,
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed record RecoveryResult(RecoveryOutcome Outcome, int Attempts, TimeSpan Elapsed);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// TASK 32·1 — the blip retry. When the encoder dies while the session is live, relaunch the same push
|
||||||
|
/// against the same ingest, restarting the audio pipe each time (its server is single-instance and the
|
||||||
|
/// dead client leaves it unusable).
|
||||||
|
/// <para><b>The check that stops us lying:</b> a reconnect can succeed at the socket level after
|
||||||
|
/// YouTube has already ended the broadcast. So each successful start asks the API whether the broadcast
|
||||||
|
/// is still there; "complete" ends recovery instead of showing a live sign over a dead stream.</para>
|
||||||
|
/// </summary>
|
||||||
|
public sealed class StreamRecovery
|
||||||
|
{
|
||||||
|
private readonly StreamRetryPolicy _policy;
|
||||||
|
private readonly StreamRecoverySteps _steps;
|
||||||
|
|
||||||
|
public StreamRecovery(StreamRetryPolicy policy, StreamRecoverySteps steps)
|
||||||
|
{
|
||||||
|
_policy = policy;
|
||||||
|
_steps = steps;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<RecoveryResult> RunAsync(CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
var clock = System.Diagnostics.Stopwatch.StartNew();
|
||||||
|
var attempts = 0;
|
||||||
|
|
||||||
|
for (var attempt = 0; attempt < _policy.Attempts; attempt++)
|
||||||
|
{
|
||||||
|
if (cancellationToken.IsCancellationRequested) return Result(RecoveryOutcome.Cancelled);
|
||||||
|
attempts = attempt + 1;
|
||||||
|
|
||||||
|
var wait = _policy.WaitBefore(attempt);
|
||||||
|
if (wait > TimeSpan.Zero)
|
||||||
|
{
|
||||||
|
try { await _steps.Delay(wait, cancellationToken).ConfigureAwait(false); }
|
||||||
|
catch (OperationCanceledException) { return Result(RecoveryOutcome.Cancelled); }
|
||||||
|
}
|
||||||
|
|
||||||
|
if (cancellationToken.IsCancellationRequested) return Result(RecoveryOutcome.Cancelled);
|
||||||
|
|
||||||
|
_steps.Report($"Stream dropped — reconnecting (attempt {attempts} of {_policy.Attempts})");
|
||||||
|
_steps.Log($"RECONNECT: attempt {attempts} of {_policy.Attempts} — restarting the audio pipe and the push");
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await _steps.RestartAudio(cancellationToken).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_steps.Log($"RECONNECT: audio pipe restart failed on attempt {attempts}: {ex.Message}");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool started;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
started = await _steps.StartPush(cancellationToken).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_steps.Log($"RECONNECT: push restart failed on attempt {attempts}: {ex.Message}");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!started)
|
||||||
|
{
|
||||||
|
_steps.Log($"RECONNECT: attempt {attempts} of {_policy.Attempts} failed — the encoder did not come up");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var liveness = BroadcastLiveness.Unknown;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
liveness = await _steps.CheckBroadcast(cancellationToken).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
_steps.Log($"RECONNECT: could not check the broadcast on attempt {attempts}: {ex.Message}");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (liveness == BroadcastLiveness.Complete)
|
||||||
|
{
|
||||||
|
_steps.Report("Stream ended — YouTube reports this broadcast complete. It cannot be resumed.");
|
||||||
|
_steps.Log("RECONNECT: push came back, but the broadcast is complete — cannot resume");
|
||||||
|
return Result(RecoveryOutcome.BroadcastComplete);
|
||||||
|
}
|
||||||
|
|
||||||
|
_steps.Report($"Stream recovered after {clock.Elapsed.TotalSeconds:0.0} s");
|
||||||
|
_steps.Log($"RECONNECT: push re-established after {clock.Elapsed.TotalSeconds:0.0} s — still live");
|
||||||
|
return Result(RecoveryOutcome.Recovered);
|
||||||
|
}
|
||||||
|
|
||||||
|
_steps.Report($"Stream dropped — {_policy.Attempts} attempts failed. The stream is off air.");
|
||||||
|
_steps.Log($"RECONNECT: gave up after {_policy.Attempts} attempts — stream is off air");
|
||||||
|
return Result(RecoveryOutcome.GaveUp);
|
||||||
|
|
||||||
|
RecoveryResult Result(RecoveryOutcome outcome) => new(outcome, attempts, clock.Elapsed);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
namespace ytLive.Services;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// TASK 32·1: how many times to relaunch the push, and how long to wait between tries.
|
||||||
|
/// <para><b>Minimum waits, deliberately</b> (creator ruling 2026-11-05): YouTube is not generous about
|
||||||
|
/// holding a broadcast whose ingest has gone quiet, so every second spent not pushing is grace spent.
|
||||||
|
/// The first attempt is immediate — there is nothing to wait for when the socket has just died.</para>
|
||||||
|
/// <para>Pure so the schedule is asserted without waiting on it.</para>
|
||||||
|
/// </summary>
|
||||||
|
public sealed class StreamRetryPolicy
|
||||||
|
{
|
||||||
|
/// <summary>Immediate, then +1 s, +2 s, +3 s — four attempts inside about six seconds.</summary>
|
||||||
|
public static readonly StreamRetryPolicy Default = new(new[]
|
||||||
|
{
|
||||||
|
TimeSpan.Zero,
|
||||||
|
TimeSpan.FromSeconds(1),
|
||||||
|
TimeSpan.FromSeconds(2),
|
||||||
|
TimeSpan.FromSeconds(3),
|
||||||
|
});
|
||||||
|
|
||||||
|
public IReadOnlyList<TimeSpan> Waits { get; }
|
||||||
|
|
||||||
|
public StreamRetryPolicy(IReadOnlyList<TimeSpan> waits)
|
||||||
|
{
|
||||||
|
if (waits == null || waits.Count == 0)
|
||||||
|
throw new ArgumentException("A retry policy needs at least one attempt.", nameof(waits));
|
||||||
|
Waits = waits;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int Attempts => Waits.Count;
|
||||||
|
|
||||||
|
/// <summary>The wait BEFORE <paramref name="attemptIndex"/> (0-based).</summary>
|
||||||
|
public TimeSpan WaitBefore(int attemptIndex)
|
||||||
|
=> attemptIndex < 0 || attemptIndex >= Waits.Count ? TimeSpan.Zero : Waits[attemptIndex];
|
||||||
|
}
|
||||||
@@ -182,6 +182,25 @@ public class YouTubeStreamService
|
|||||||
/// "public"), read from YouTube rather than trusted from local state — the creator can
|
/// "public"), read from YouTube rather than trusted from local state — the creator can
|
||||||
/// change visibility mid-stream from the broadcast form. Null when signed out or the
|
/// change visibility mid-stream from the broadcast form. Null when signed out or the
|
||||||
/// read fails (the caller then does nothing destructive).</summary>
|
/// read fails (the caller then does nothing destructive).</summary>
|
||||||
|
/// <summary>TASK 32·1: the broadcast's lifecycle status (<c>created</c>/<c>ready</c>/<c>testing</c>/
|
||||||
|
/// <c>live</c>/<c>complete</c>), or null when it cannot be read. Used after a reconnect: a push can
|
||||||
|
/// succeed at the socket level while YouTube has already ended the broadcast.</summary>
|
||||||
|
public async Task<string?> GetBroadcastStatusAsync(string broadcastId)
|
||||||
|
{
|
||||||
|
if (!await EnsureToken()) return null;
|
||||||
|
|
||||||
|
_http.DefaultRequestHeaders.Authorization = new("Bearer", _auth.CurrentChannel!.AccessToken);
|
||||||
|
var response = await _http.GetAsync(
|
||||||
|
$"{ApiBase}/liveBroadcasts?part=status&id={Uri.EscapeDataString(broadcastId)}");
|
||||||
|
if (!response.IsSuccessStatusCode) return null;
|
||||||
|
|
||||||
|
var json = JsonSerializer.Deserialize<JsonElement>(await response.Content.ReadAsStringAsync());
|
||||||
|
if (!json.TryGetProperty("items", out var items) || items.GetArrayLength() == 0) return null;
|
||||||
|
if (!items[0].TryGetProperty("status", out var status)) return null;
|
||||||
|
if (!status.TryGetProperty("lifeCycleStatus", out var lifeCycle)) return null;
|
||||||
|
return lifeCycle.ValueKind == JsonValueKind.String ? lifeCycle.GetString() : null;
|
||||||
|
}
|
||||||
|
|
||||||
public async Task<string?> GetBroadcastPrivacyAsync(string broadcastId)
|
public async Task<string?> GetBroadcastPrivacyAsync(string broadcastId)
|
||||||
{
|
{
|
||||||
if (!await EnsureToken()) return null;
|
if (!await EnsureToken()) return null;
|
||||||
|
|||||||
@@ -48,7 +48,7 @@ blocker; the smallest slice is 32·4 (fragmented MP4 on the record block).
|
|||||||
| 55·2–3 | ☐ | Disk hygiene: delete `%TEMP%` alert clips on close + sweep at startup; low-disk guard before REC (slice 1 shipped) | [`task-55`](TASKS/task-55-disk-hygiene.md) |
|
| 55·2–3 | ☐ | Disk hygiene: delete `%TEMP%` alert clips on close + sweep at startup; low-disk guard before REC (slice 1 shipped) | [`task-55`](TASKS/task-55-disk-hygiene.md) |
|
||||||
| 40·A/C/D | ☐ | App Settings: active-camera picker; persist output resolution + surface transition defaults + confirm-before-End; accent consolidation | [`task-40`](TASKS/task-40-app-settings-round.md) |
|
| 40·A/C/D | ☐ | App Settings: active-camera picker; persist output resolution + surface transition defaults + confirm-before-End; accent consolidation | [`task-40`](TASKS/task-40-app-settings-round.md) |
|
||||||
| 3·20 | ☐ | RewardEvent persistence: canonical `RewardEvents` table + `superChatEvents.list` 30-day backfill (the session report and journey read it) | [`task-03`](TASKS/task-03-capture-pipeline.md) |
|
| 3·20 | ☐ | RewardEvent persistence: canonical `RewardEvents` table + `superChatEvents.list` 30-day backfill (the session report and journey read it) | [`task-03`](TASKS/task-03-capture-pipeline.md) |
|
||||||
| 32 | 🔶 | Stream resilience: **fragmented MP4 done 2026-11-05**; remaining — blip retry → measured grace + amber "RECONNECTING" → one-click Back on air → ingest pre-flight | [`task-32`](TASKS/task-32-stream-resilience.md) |
|
| 32 | 🔶 | Stream resilience: **fragmented MP4 + blip retry done 2026-11-05**; remaining — measured grace + RECONNECTING countdown → one-click Back on air → ingest pre-flight | [`task-32`](TASKS/task-32-stream-resilience.md) |
|
||||||
| 33 | ☐ | Bandwidth auto step-down: pure policy over dropped-frame rate, one-way within a broadcast | [`task-33`](TASKS/task-33-bandwidth-step-down.md) |
|
| 33 | ☐ | Bandwidth auto step-down: pure policy over dropped-frame rate, one-way within a broadcast | [`task-33`](TASKS/task-33-bandwidth-step-down.md) |
|
||||||
| 34 | ☐ | Scheduled streams (Text-drawer version) | [`task-34`](TASKS/task-34-scheduled-streams.md) |
|
| 34 | ☐ | Scheduled streams (Text-drawer version) | [`task-34`](TASKS/task-34-scheduled-streams.md) |
|
||||||
| 35 | ☐ | Scene-linked audio | [`task-35`](TASKS/task-35-scene-linked-audio.md) |
|
| 35 | ☐ | Scene-linked audio | [`task-35`](TASKS/task-35-scene-linked-audio.md) |
|
||||||
|
|||||||
@@ -11,12 +11,18 @@ the API — creator ruling 2026-09-01): no lying in the copy.
|
|||||||
|
|
||||||
**Slices (one integration test each, in order):**
|
**Slices (one integration test each, in order):**
|
||||||
|
|
||||||
1. ☐ **Blip retry** — app-side encoder restart: push dies while live (ProcessFailed / stdin
|
1. ✅ **Blip retry** (shipped 2026-11-05) — app-side encoder restart: push dies while live (ProcessFailed / stdin
|
||||||
backpressure death) → bounded-backoff relaunch (2s/5s/10s, N attempts) against the cached
|
backpressure death) → bounded-backoff relaunch (2s/5s/10s, N attempts) against the cached
|
||||||
constant reusable-stream ingest URL. Success = push running again while the broadcast is still
|
constant reusable-stream ingest URL. Success = push running again while the broadcast is still
|
||||||
live; viewers never know. Fixes the map's stale "ffmpeg does reconnect" claim (corrected
|
live; viewers never know. Fixes the map's stale "ffmpeg does reconnect" claim (corrected
|
||||||
2026-09-01: reconnect flags are input-side only). Test: fake process fails once then starts →
|
2026-09-01: reconnect flags are input-side only). Test: fake process fails once then starts →
|
||||||
pump recovers without surfacing to the VM.
|
pump recovers without surfacing to the VM.
|
||||||
|
**Shipped:** `Services/StreamRetryPolicy.cs` (immediate, +1 s, +2 s, +3 s — minimum waits, because
|
||||||
|
YouTube's grace is short), `Services/StreamRecovery.cs` (the loop, with the audio pipe recreated per
|
||||||
|
attempt because its single-instance server is unusable after the old client died), and
|
||||||
|
`FramePump.EncoderFailed` separating a dead push (recoverable) from a fatal pump fault. Each episode
|
||||||
|
reports into the left panel's alert stack; a reconnect that lands on a **complete** broadcast is
|
||||||
|
reported as such instead of showing a live sign over a dead stream. Tests: `StreamRecoveryTests`.
|
||||||
2. ☐ **Measured grace + the sign** — while the push is dead, escalate the existing health poll
|
2. ☐ **Measured grace + the sign** — while the push is dead, escalate the existing health poll
|
||||||
(30s → ~5s). ON-AIR dot goes **amber "RECONNECTING"** with a mm:ss countdown to the observed
|
(30s → ~5s). ON-AIR dot goes **amber "RECONNECTING"** with a mm:ss countdown to the observed
|
||||||
YouTube cutoff; the grace length is a **measured, pinned constant** — probe it on a real private
|
YouTube cutoff; the grace length is a **measured, pinned constant** — probe it on a real private
|
||||||
|
|||||||
@@ -29,22 +29,25 @@ public partial class MainViewModel
|
|||||||
|
|
||||||
public RelayCommand<AttentionAlert> ActivateAlertCommand => new(ActivateAlert);
|
public RelayCommand<AttentionAlert> ActivateAlertCommand => new(ActivateAlert);
|
||||||
|
|
||||||
/// <summary>The licence indicator's text: what this build is entitled to, and the maximum.
|
/// <summary>The licence indicator's text (creator wording, 2026-11-05): the limit this build runs
|
||||||
/// A status readout — deliberately not a description of what is missing (creator ruling: the
|
/// at, plus where to find out why. Not a warning — the resolution picker explains the detail, and
|
||||||
/// resolution picker explains that).</summary>
|
/// clicking the row opens About.</summary>
|
||||||
private string SteamLicenceText()
|
private string SteamLicenceText()
|
||||||
{
|
{
|
||||||
if (QualityOptions.Length == 0) return "Entitlement 30 fps · Maximum 60 fps";
|
var entitled = 30;
|
||||||
|
if (QualityOptions.Length > 0)
|
||||||
var max = QualityOptions.Max(option => option.Fps);
|
{
|
||||||
var entitled = QualityOptions
|
entitled = QualityOptions
|
||||||
.Where(option => HasProOutput || !option.RequiresPro)
|
.Where(option => HasProOutput || !option.RequiresPro)
|
||||||
.Select(option => option.Fps)
|
.Select(option => option.Fps)
|
||||||
.DefaultIfEmpty(max)
|
.DefaultIfEmpty(30)
|
||||||
.Max();
|
.Max();
|
||||||
return $"Entitlement {entitled} fps · Maximum {max} fps";
|
}
|
||||||
|
return $"Steam ID Limit: {entitled}fps\nClick for More Info";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private string SteamLicenceLabel() => IsAppOwned ? "STEAM ID PRESENT" : "STEAM ID NOT PRESENT";
|
||||||
|
|
||||||
/// <summary>Recomputes row 1 in place. Called at startup and whenever ownership changes.</summary>
|
/// <summary>Recomputes row 1 in place. Called at startup and whenever ownership changes.</summary>
|
||||||
private void RefreshSteamLicenceAlert()
|
private void RefreshSteamLicenceAlert()
|
||||||
{
|
{
|
||||||
@@ -60,7 +63,7 @@ public partial class MainViewModel
|
|||||||
|
|
||||||
var alert = new AttentionAlert(
|
var alert = new AttentionAlert(
|
||||||
SteamLicenceAlertId,
|
SteamLicenceAlertId,
|
||||||
"STEAM LICENCE",
|
SteamLicenceLabel(),
|
||||||
SteamLicenceText(),
|
SteamLicenceText(),
|
||||||
IsPinned: !IsAppOwned,
|
IsPinned: !IsAppOwned,
|
||||||
AttentionAlertAction.About);
|
AttentionAlertAction.About);
|
||||||
|
|||||||
@@ -0,0 +1,149 @@
|
|||||||
|
using System;
|
||||||
|
using System.Diagnostics;
|
||||||
|
using System.Threading;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using ytLive.Helpers;
|
||||||
|
using ytLive.Services;
|
||||||
|
using ytLive.Services.Encoder;
|
||||||
|
|
||||||
|
namespace ytLive.ViewModels;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// TASK 32·1 — a dropped push is retried instead of ending the session. The loop itself lives in
|
||||||
|
/// <see cref="StreamRecovery"/> (pure, testable); this partial supplies the app-side steps and reports
|
||||||
|
/// each episode into the left panel's alert stack.
|
||||||
|
/// <para>Deliberately narrow: only the ENCODER dying is recoverable. A compositor fault still goes
|
||||||
|
/// through <see cref="OnFramePumpFailed"/> and ends the session, because retrying it would loop.</para>
|
||||||
|
/// </summary>
|
||||||
|
public partial class MainViewModel
|
||||||
|
{
|
||||||
|
private CancellationTokenSource? _recoveryCts;
|
||||||
|
|
||||||
|
/// <summary>True while a push is being relaunched — slice 2 binds the amber sign to this.</summary>
|
||||||
|
public bool IsReconnecting { get; private set; }
|
||||||
|
|
||||||
|
/// <summary>The attempt currently in flight (0 when idle).</summary>
|
||||||
|
public int ReconnectAttempt { get; private set; }
|
||||||
|
|
||||||
|
private void OnEncoderFailed(object? sender, string message)
|
||||||
|
{
|
||||||
|
if (!IsLive && !IsRecording) return;
|
||||||
|
if (_recoveryCts != null) return; // one recovery at a time
|
||||||
|
if (!IsLive) return; // a recording-only session has nothing to relive
|
||||||
|
|
||||||
|
AppLog.Write($"RECONNECT: encoder process failed: {message}");
|
||||||
|
_recoveryCts = new CancellationTokenSource();
|
||||||
|
_ = RecoverStreamAsync(message, _recoveryCts.Token);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task RecoverStreamAsync(string reason, CancellationToken cancellationToken)
|
||||||
|
{
|
||||||
|
IsReconnecting = true;
|
||||||
|
ReconnectAttempt = 0;
|
||||||
|
|
||||||
|
var logFrom = AppLog.SnapshotCurrentLaunch().Count > 0
|
||||||
|
? AppLog.SnapshotCurrentLaunch()[^1].Sequence + 1
|
||||||
|
: 1;
|
||||||
|
|
||||||
|
var recovery = new StreamRecovery(
|
||||||
|
StreamRetryPolicy.Default,
|
||||||
|
new StreamRecoverySteps
|
||||||
|
{
|
||||||
|
RestartAudio = _ =>
|
||||||
|
{
|
||||||
|
// The audio pipe's server is single-instance and its client just died, so it has
|
||||||
|
// to be recreated before ffmpeg can attach — otherwise the restart fails on a
|
||||||
|
// missing input.
|
||||||
|
_audioMixer.StopLive();
|
||||||
|
_audioMixer.StartLive(EncoderOptions.DefaultAudioPipeName);
|
||||||
|
return Task.CompletedTask;
|
||||||
|
},
|
||||||
|
StartPush = async ct =>
|
||||||
|
{
|
||||||
|
await _framePump.StartAsync(ct);
|
||||||
|
return _framePump.IsRunning;
|
||||||
|
},
|
||||||
|
CheckBroadcast = async ct =>
|
||||||
|
{
|
||||||
|
if (_currentBroadcastId == null || _reusableStream?.Id == null)
|
||||||
|
return BroadcastLiveness.Unknown;
|
||||||
|
var status = await _youtubeStream.GetBroadcastStatusAsync(_currentBroadcastId);
|
||||||
|
return status switch
|
||||||
|
{
|
||||||
|
"complete" => BroadcastLiveness.Complete,
|
||||||
|
null => BroadcastLiveness.Unknown,
|
||||||
|
_ => BroadcastLiveness.Live,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
Report = text =>
|
||||||
|
{
|
||||||
|
var progress = text.StartsWith("Stream dropped", StringComparison.Ordinal)
|
||||||
|
? ParseAttempt(text)
|
||||||
|
: 0;
|
||||||
|
if (progress > 0) ReconnectAttempt = progress;
|
||||||
|
|
||||||
|
var to = AppLog.SnapshotCurrentLaunch() is { Count: > 0 } entries
|
||||||
|
? entries[^1].Sequence
|
||||||
|
: logFrom;
|
||||||
|
PushAttentionAlert(
|
||||||
|
"stream-recovery",
|
||||||
|
"STREAM",
|
||||||
|
text,
|
||||||
|
logFrom,
|
||||||
|
Math.Max(logFrom, to));
|
||||||
|
},
|
||||||
|
Log = line => AppLog.Write(line, AppLogMessageType.Error, AppLogErrorClass.Class2),
|
||||||
|
});
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var result = await recovery.RunAsync(cancellationToken);
|
||||||
|
|
||||||
|
switch (result.Outcome)
|
||||||
|
{
|
||||||
|
case RecoveryOutcome.Recovered:
|
||||||
|
ReconnectAttempt = result.Attempts;
|
||||||
|
AppLog.WriteSuccess(
|
||||||
|
$"RECONNECT: recovered on attempt {result.Attempts} after {result.Elapsed.TotalSeconds:0.0} s");
|
||||||
|
break;
|
||||||
|
|
||||||
|
case RecoveryOutcome.Cancelled:
|
||||||
|
// The creator stopped the stream mid-recovery: nothing to report.
|
||||||
|
break;
|
||||||
|
|
||||||
|
default:
|
||||||
|
// Gave up, or the broadcast was already complete: this session is over. The alert
|
||||||
|
// stack keeps the reason; the normal stop path clears pills, audio and recording.
|
||||||
|
AppLog.Write("RECONNECT: session ended — the broadcast could not be recovered",
|
||||||
|
AppLogMessageType.Error, AppLogErrorClass.Class1);
|
||||||
|
StopStream();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
AppLog.Write(ex, "RECONNECT: recovery loop faulted");
|
||||||
|
StopStream();
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
IsReconnecting = false;
|
||||||
|
ReconnectAttempt = 0;
|
||||||
|
_recoveryCts?.Dispose();
|
||||||
|
_recoveryCts = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int ParseAttempt(string text)
|
||||||
|
{
|
||||||
|
// "Stream dropped — reconnecting (attempt 2 of 4)"
|
||||||
|
var marker = text.IndexOf("attempt ", StringComparison.Ordinal);
|
||||||
|
if (marker < 0) return 0;
|
||||||
|
var rest = text[(marker + "attempt ".Length)..];
|
||||||
|
var space = rest.IndexOf(' ');
|
||||||
|
return int.TryParse(space > 0 ? rest[..space] : rest, out var attempt) ? attempt : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>The creator asked to stop: never race a stop against a relaunch.</summary>
|
||||||
|
private void CancelStreamRecovery() => _recoveryCts?.Cancel();
|
||||||
|
}
|
||||||
@@ -339,6 +339,8 @@ public partial class MainViewModel : ViewModelBase
|
|||||||
|
|
||||||
private async void StopStream()
|
private async void StopStream()
|
||||||
{
|
{
|
||||||
|
CancelStreamRecovery(); // a stop wins over a relaunch in flight
|
||||||
|
RemoveAttentionAlert("stream-recovery");
|
||||||
var wasLive = IsLive;
|
var wasLive = IsLive;
|
||||||
var wasRecording = IsRecording;
|
var wasRecording = IsRecording;
|
||||||
|
|
||||||
|
|||||||
@@ -457,6 +457,7 @@ public partial class MainViewModel : ViewModelBase
|
|||||||
transition: _transition,
|
transition: _transition,
|
||||||
sceneGraph: _sceneGraph);
|
sceneGraph: _sceneGraph);
|
||||||
_framePump.Failed += OnFramePumpFailed;
|
_framePump.Failed += OnFramePumpFailed;
|
||||||
|
_framePump.EncoderFailed += OnEncoderFailed;
|
||||||
_framePump.HealthUpdated += OnFramePumpHealthUpdated;
|
_framePump.HealthUpdated += OnFramePumpHealthUpdated;
|
||||||
|
|
||||||
LoadLayout();
|
LoadLayout();
|
||||||
|
|||||||
@@ -99,10 +99,10 @@ public sealed class AboutHubTests
|
|||||||
// The old red panel under the quality picker is gone (2026-11-05): the licence state
|
// The old red panel under the quality picker is gone (2026-11-05): the licence state
|
||||||
// lives in the left panel's alert stack now, as a pinned status indicator.
|
// lives in the left panel's alert stack now, as a pinned status indicator.
|
||||||
Assert.Null(window.FindName("EntitlementErrorPanel"));
|
Assert.Null(window.FindName("EntitlementErrorPanel"));
|
||||||
var licenceRow = vm.AttentionAlerts.Single(alert => alert.Label == "STEAM LICENCE");
|
var licenceRow = vm.AttentionAlerts.Single(alert => alert.Label == "STEAM ID NOT PRESENT");
|
||||||
Assert.True(licenceRow.IsPinned);
|
Assert.True(licenceRow.IsPinned);
|
||||||
Assert.Contains("Entitlement 30 fps", licenceRow.Text);
|
Assert.Contains("Steam ID Limit: 30fps", licenceRow.Text);
|
||||||
Assert.Contains("Maximum 60 fps", licenceRow.Text);
|
Assert.Contains("Click for More Info", licenceRow.Text);
|
||||||
|
|
||||||
var picker = (ComboBox)window.FindName("QualityPicker");
|
var picker = (ComboBox)window.FindName("QualityPicker");
|
||||||
Assert.True(vm.CanChangeQuality);
|
Assert.True(vm.CanChangeQuality);
|
||||||
|
|||||||
@@ -32,15 +32,15 @@ public sealed class AttentionAlertStackTests
|
|||||||
vm.EntitlementOverride = new FakeEntitlement(false, "Steam licensing is not configured in this build.");
|
vm.EntitlementOverride = new FakeEntitlement(false, "Steam licensing is not configured in this build.");
|
||||||
|
|
||||||
var row = Assert.Single(vm.AttentionAlerts);
|
var row = Assert.Single(vm.AttentionAlerts);
|
||||||
Assert.Equal("STEAM LICENCE", row.Label);
|
Assert.Equal("STEAM ID NOT PRESENT", row.Label);
|
||||||
Assert.True(row.IsPinned);
|
Assert.True(row.IsPinned);
|
||||||
Assert.True(row.ShowsPin);
|
Assert.True(row.ShowsPin);
|
||||||
Assert.False(row.CanDismiss);
|
Assert.False(row.CanDismiss);
|
||||||
Assert.Equal(AttentionAlertAction.About, row.Action);
|
Assert.Equal(AttentionAlertAction.About, row.Action);
|
||||||
|
|
||||||
// A status readout: what this build is entitled to, and the maximum.
|
// Creator wording: the limit this build runs at, and where to find out why.
|
||||||
Assert.Contains("Entitlement 30 fps", row.Text);
|
Assert.Contains("Steam ID Limit: 30fps", row.Text);
|
||||||
Assert.Contains("Maximum 60 fps", row.Text);
|
Assert.Contains("Click for More Info", row.Text);
|
||||||
Assert.DoesNotContain("unavailable", row.Text, StringComparison.OrdinalIgnoreCase);
|
Assert.DoesNotContain("unavailable", row.Text, StringComparison.OrdinalIgnoreCase);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -56,18 +56,18 @@ public sealed class AttentionAlertStackTests
|
|||||||
Assert.False(row.IsPinned);
|
Assert.False(row.IsPinned);
|
||||||
Assert.False(row.ShowsPin);
|
Assert.False(row.ShowsPin);
|
||||||
Assert.True(row.CanDismiss);
|
Assert.True(row.CanDismiss);
|
||||||
Assert.Contains("Entitlement 60 fps", row.Text);
|
Assert.Contains("Steam ID Limit: 60fps", row.Text);
|
||||||
|
|
||||||
vm.DismissAlertCommand.Execute(row);
|
vm.DismissAlertCommand.Execute(row);
|
||||||
Assert.Empty(vm.AttentionAlerts);
|
Assert.Empty(vm.AttentionAlerts);
|
||||||
|
|
||||||
// Permanent: a fresh view-model on the same profile must not bring it back…
|
// Permanent: a fresh view-model on the same profile must not bring it back…
|
||||||
var reloaded = new MainViewModel();
|
var reloaded = new MainViewModel();
|
||||||
Assert.DoesNotContain(reloaded.AttentionAlerts, alert => alert.Label == "STEAM LICENCE");
|
Assert.DoesNotContain(reloaded.AttentionAlerts, alert => alert.Label.StartsWith("STEAM ID"));
|
||||||
|
|
||||||
// …but losing ownership restores it as a pinned indicator: a change of state, not a preference.
|
// …but losing ownership restores it as a pinned indicator: a change of state, not a preference.
|
||||||
reloaded.EntitlementOverride = new FakeEntitlement(false, "Steam licensing is not configured in this build.");
|
reloaded.EntitlementOverride = new FakeEntitlement(false, "Steam licensing is not configured in this build.");
|
||||||
var restored = reloaded.AttentionAlerts.First(alert => alert.Label == "STEAM LICENCE");
|
var restored = reloaded.AttentionAlerts.First(alert => alert.Label == "STEAM ID NOT PRESENT");
|
||||||
Assert.True(restored.IsPinned);
|
Assert.True(restored.IsPinned);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -89,10 +89,11 @@ public sealed class AttentionAlertStackTests
|
|||||||
{
|
{
|
||||||
WithVm(vm =>
|
WithVm(vm =>
|
||||||
{
|
{
|
||||||
|
vm.EntitlementOverride = new FakeEntitlement(false, "Steam licensing is not configured in this build.");
|
||||||
vm.PushAttentionAlertForTest("drop-1", "STREAM", "Stream dropped — reconnecting (attempt 1 of 4)", 10, 12);
|
vm.PushAttentionAlertForTest("drop-1", "STREAM", "Stream dropped — reconnecting (attempt 1 of 4)", 10, 12);
|
||||||
vm.PushAttentionAlertForTest("drop-2", "STREAM", "Stream dropped — reconnecting (attempt 2 of 4)", 13, 15);
|
vm.PushAttentionAlertForTest("drop-2", "STREAM", "Stream dropped — reconnecting (attempt 2 of 4)", 13, 15);
|
||||||
|
|
||||||
Assert.Equal(new[] { "STEAM LICENCE", "STREAM", "STREAM" }, vm.AttentionAlerts.Select(a => a.Label));
|
Assert.Equal(new[] { "STEAM ID NOT PRESENT", "STREAM", "STREAM" }, vm.AttentionAlerts.Select(a => a.Label));
|
||||||
// Newest event sits directly under the indicator.
|
// Newest event sits directly under the indicator.
|
||||||
Assert.Contains("attempt 2", vm.AttentionAlerts[1].Text);
|
Assert.Contains("attempt 2", vm.AttentionAlerts[1].Text);
|
||||||
Assert.Contains("attempt 1", vm.AttentionAlerts[2].Text);
|
Assert.Contains("attempt 1", vm.AttentionAlerts[2].Text);
|
||||||
@@ -104,7 +105,7 @@ public sealed class AttentionAlertStackTests
|
|||||||
|
|
||||||
vm.DismissAlertCommand.Execute(vm.AttentionAlerts[1]);
|
vm.DismissAlertCommand.Execute(vm.AttentionAlerts[1]);
|
||||||
Assert.Equal(2, vm.AttentionAlerts.Count);
|
Assert.Equal(2, vm.AttentionAlerts.Count);
|
||||||
Assert.Equal("STEAM LICENCE", vm.AttentionAlerts[0].Label);
|
Assert.Equal("STEAM ID NOT PRESENT", vm.AttentionAlerts[0].Label);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -298,18 +298,23 @@ public class FramePumpTests
|
|||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task ProcessDeath_StopsPump_AndRaisesFailed()
|
public async Task ProcessDeath_StopsPump_AndRaisesEncoderFailed()
|
||||||
{
|
{
|
||||||
var encoder = new FakeEncoder();
|
var encoder = new FakeEncoder();
|
||||||
var failed = new TaskCompletionSource<string>(TaskCreationOptions.RunContinuationsAsynchronously);
|
var encoderFailed = new TaskCompletionSource<string>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
var fatal = false;
|
||||||
using var pump = NewPump(encoder);
|
using var pump = NewPump(encoder);
|
||||||
pump.Failed += (_, m) => failed.TrySetResult(m);
|
// TASK 32·1: a dead push is recoverable — the broadcast may still be live — so it is a
|
||||||
|
// separate signal from a fatal pump fault, and the session gets to relaunch it.
|
||||||
|
pump.EncoderFailed += (_, m) => encoderFailed.TrySetResult(m);
|
||||||
|
pump.Failed += (_, _) => fatal = true;
|
||||||
|
|
||||||
await pump.StartAsync();
|
await pump.StartAsync();
|
||||||
encoder.RaiseProcessFailed("FFmpeg exited with code 1");
|
encoder.RaiseProcessFailed("FFmpeg exited with code 1");
|
||||||
|
|
||||||
var message = await failed.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
var message = await encoderFailed.Task.WaitAsync(TimeSpan.FromSeconds(5));
|
||||||
Assert.Contains("1", message);
|
Assert.Contains("1", message);
|
||||||
|
Assert.False(fatal, "an encoder death must not be reported as a fatal pump failure");
|
||||||
|
|
||||||
// The pump self-stops (fire-and-forget); poll the definitive teardown
|
// The pump self-stops (fire-and-forget); poll the definitive teardown
|
||||||
// marker — the encoder's disposal — rather than the IsRunning flag, which
|
// marker — the encoder's disposal — rather than the IsRunning flag, which
|
||||||
|
|||||||
@@ -0,0 +1,162 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Threading;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using Xunit;
|
||||||
|
using ytLive.Services;
|
||||||
|
|
||||||
|
namespace ytLive.Tests;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// TASK 32·1 — blip retry, exercised through <see cref="StreamRecovery"/> with fake steps: no ffmpeg,
|
||||||
|
/// no network, no waiting on the real schedule. The rules that matter: the first attempt is immediate,
|
||||||
|
/// the audio pipe is recreated before every push, a reconnect that lands on a COMPLETE broadcast is
|
||||||
|
/// reported honestly instead of pretending to be live, and giving up is a definite outcome, not a hang.
|
||||||
|
/// </summary>
|
||||||
|
public class StreamRecoveryTests
|
||||||
|
{
|
||||||
|
private static StreamRetryPolicy NoWaits(int attempts = 4)
|
||||||
|
=> new(Enumerable.Repeat(TimeSpan.Zero, attempts).ToArray());
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void DefaultPolicy_StartsImmediately_ThenOneTwoThreeSeconds()
|
||||||
|
{
|
||||||
|
var policy = StreamRetryPolicy.Default;
|
||||||
|
|
||||||
|
Assert.Equal(4, policy.Attempts);
|
||||||
|
Assert.Equal(TimeSpan.Zero, policy.WaitBefore(0));
|
||||||
|
Assert.Equal(TimeSpan.FromSeconds(1), policy.WaitBefore(1));
|
||||||
|
Assert.Equal(TimeSpan.FromSeconds(2), policy.WaitBefore(2));
|
||||||
|
Assert.Equal(TimeSpan.FromSeconds(3), policy.WaitBefore(3));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task RecoversOnTheSecondAttempt_AndRestartsTheAudioPipeEachTime()
|
||||||
|
{
|
||||||
|
var pushes = 0;
|
||||||
|
var audioRestarts = 0;
|
||||||
|
var reported = new List<string>();
|
||||||
|
|
||||||
|
var recovery = new StreamRecovery(NoWaits(), new StreamRecoverySteps
|
||||||
|
{
|
||||||
|
RestartAudio = _ => { audioRestarts++; return Task.CompletedTask; },
|
||||||
|
StartPush = _ => Task.FromResult(++pushes > 1), // first start fails, second works
|
||||||
|
CheckBroadcast = _ => Task.FromResult(BroadcastLiveness.Live),
|
||||||
|
Report = reported.Add,
|
||||||
|
Log = _ => { },
|
||||||
|
});
|
||||||
|
|
||||||
|
var result = await recovery.RunAsync();
|
||||||
|
|
||||||
|
Assert.Equal(RecoveryOutcome.Recovered, result.Outcome);
|
||||||
|
Assert.Equal(2, result.Attempts);
|
||||||
|
Assert.Equal(2, pushes);
|
||||||
|
Assert.Equal(2, audioRestarts); // the pipe is unusable after the old client died
|
||||||
|
Assert.Contains(reported, text => text.Contains("reconnecting (attempt 1"));
|
||||||
|
Assert.Contains(reported, text => text.StartsWith("Stream recovered"));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ABroadcastThatAlreadyEnded_IsReportedAsComplete_NotAsRecovered()
|
||||||
|
{
|
||||||
|
var reported = new List<string>();
|
||||||
|
|
||||||
|
var recovery = new StreamRecovery(NoWaits(), new StreamRecoverySteps
|
||||||
|
{
|
||||||
|
RestartAudio = _ => Task.CompletedTask,
|
||||||
|
StartPush = _ => Task.FromResult(true),
|
||||||
|
CheckBroadcast = _ => Task.FromResult(BroadcastLiveness.Complete),
|
||||||
|
Report = reported.Add,
|
||||||
|
Log = _ => { },
|
||||||
|
});
|
||||||
|
|
||||||
|
var result = await recovery.RunAsync();
|
||||||
|
|
||||||
|
Assert.Equal(RecoveryOutcome.BroadcastComplete, result.Outcome);
|
||||||
|
Assert.Contains(reported, text => text.Contains("cannot be resumed"));
|
||||||
|
Assert.DoesNotContain(reported, text => text.StartsWith("Stream recovered"));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task UnknownLiveness_CountsAsRecovered()
|
||||||
|
{
|
||||||
|
// If the API hiccups we cannot prove the broadcast is gone; treating that as failure would
|
||||||
|
// end sessions that are actually fine.
|
||||||
|
var recovery = new StreamRecovery(NoWaits(), new StreamRecoverySteps
|
||||||
|
{
|
||||||
|
RestartAudio = _ => Task.CompletedTask,
|
||||||
|
StartPush = _ => Task.FromResult(true),
|
||||||
|
CheckBroadcast = _ => Task.FromResult(BroadcastLiveness.Unknown),
|
||||||
|
Report = _ => { },
|
||||||
|
Log = _ => { },
|
||||||
|
});
|
||||||
|
|
||||||
|
Assert.Equal(RecoveryOutcome.Recovered, (await recovery.RunAsync()).Outcome);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task EveryAttemptFailing_GivesUpWithTheCount()
|
||||||
|
{
|
||||||
|
var reported = new List<string>();
|
||||||
|
var recovery = new StreamRecovery(NoWaits(3), new StreamRecoverySteps
|
||||||
|
{
|
||||||
|
RestartAudio = _ => Task.CompletedTask,
|
||||||
|
StartPush = _ => Task.FromResult(false),
|
||||||
|
CheckBroadcast = _ => Task.FromResult(BroadcastLiveness.Live),
|
||||||
|
Report = reported.Add,
|
||||||
|
Log = _ => { },
|
||||||
|
});
|
||||||
|
|
||||||
|
var result = await recovery.RunAsync();
|
||||||
|
|
||||||
|
Assert.Equal(RecoveryOutcome.GaveUp, result.Outcome);
|
||||||
|
Assert.Equal(3, result.Attempts);
|
||||||
|
Assert.Contains(reported, text => text.Contains("off air"));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task CancellingMidRecovery_StopsWithoutTouchingThePush()
|
||||||
|
{
|
||||||
|
using var cts = new CancellationTokenSource();
|
||||||
|
var pushes = 0;
|
||||||
|
|
||||||
|
var recovery = new StreamRecovery(StreamRetryPolicy.Default, new StreamRecoverySteps
|
||||||
|
{
|
||||||
|
RestartAudio = _ => Task.CompletedTask,
|
||||||
|
StartPush = _ => { pushes++; return Task.FromResult(false); },
|
||||||
|
CheckBroadcast = _ => Task.FromResult(BroadcastLiveness.Live),
|
||||||
|
Report = _ => { },
|
||||||
|
Log = _ => { },
|
||||||
|
Delay = (_, ct) => { cts.Cancel(); ct.ThrowIfCancellationRequested(); return Task.CompletedTask; },
|
||||||
|
});
|
||||||
|
|
||||||
|
var result = await recovery.RunAsync(cts.Token);
|
||||||
|
|
||||||
|
Assert.Equal(RecoveryOutcome.Cancelled, result.Outcome);
|
||||||
|
Assert.True(pushes <= 1, $"cancelling must stop the loop, but the push was started {pushes} times");
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task AAudioRestartFault_IsAnAttemptFailure_NotACrash()
|
||||||
|
{
|
||||||
|
var attempts = 0;
|
||||||
|
var recovery = new StreamRecovery(NoWaits(2), new StreamRecoverySteps
|
||||||
|
{
|
||||||
|
RestartAudio = _ =>
|
||||||
|
{
|
||||||
|
attempts++;
|
||||||
|
throw new InvalidOperationException("pipe busy");
|
||||||
|
},
|
||||||
|
StartPush = _ => Task.FromResult(true),
|
||||||
|
CheckBroadcast = _ => Task.FromResult(BroadcastLiveness.Live),
|
||||||
|
Report = _ => { },
|
||||||
|
Log = _ => { },
|
||||||
|
});
|
||||||
|
|
||||||
|
var result = await recovery.RunAsync();
|
||||||
|
|
||||||
|
Assert.Equal(RecoveryOutcome.GaveUp, result.Outcome);
|
||||||
|
Assert.Equal(2, attempts);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user