Compare commits
9
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3239808a73 | ||
|
|
24aca7d04a | ||
|
|
b7ab169718 | ||
|
|
818d316477 | ||
|
|
a4dfca6b6c | ||
|
|
096bf6debe | ||
|
|
804b1d2f5f | ||
|
|
ef60f1f3b6 | ||
|
|
c768a8ee49 |
@@ -20,3 +20,4 @@ MARCOM.md
|
||||
CREDENTIALS.md
|
||||
.aider*
|
||||
ffmpeg/
|
||||
BuildCounter.txt
|
||||
|
||||
+61
-15
@@ -49,12 +49,19 @@
|
||||
<RowDefinition Height="Auto"/>
|
||||
<RowDefinition Height="Auto"/>
|
||||
<RowDefinition Height="Auto"/>
|
||||
<RowDefinition Height="*"/>
|
||||
<RowDefinition Height="Auto"/>
|
||||
</Grid.RowDefinitions>
|
||||
|
||||
<!-- LAYERS header + Add button -->
|
||||
<StackPanel Grid.Row="0" Orientation="Horizontal" Margin="0,8,0,0">
|
||||
<TextBlock Text="LAYERS" Style="{StaticResource SectionHeader}" VerticalAlignment="Center"/>
|
||||
<Path Data="M12,3 L21,8 L12,13 L3,8 Z M3,12 L12,17 L21,12 M3,16 L12,21 L21,16"
|
||||
Stroke="#38bdf8" StrokeThickness="1.8" StrokeLineJoin="Round"
|
||||
Width="16" Height="16" Stretch="Uniform" VerticalAlignment="Center" Margin="0,-4,6,0"/>
|
||||
<TextBlock Text="SCENE LAYERS" Style="{StaticResource SectionHeader}" VerticalAlignment="Center"/>
|
||||
<Border Width="20" Height="20" CornerRadius="10" Background="#334155" Margin="6,-4,0,0" VerticalAlignment="Center">
|
||||
<TextBlock Text="{Binding LayerCount}" Foreground="#38bdf8" FontSize="11" FontWeight="SemiBold"
|
||||
HorizontalAlignment="Center" VerticalAlignment="Center"/>
|
||||
</Border>
|
||||
<Button ToolTip="Add Layer" Style="{StaticResource IconButton}" Margin="4,-4,0,0"
|
||||
Click="AddSourceButton_Click">
|
||||
<Button.ContextMenu>
|
||||
@@ -82,7 +89,7 @@
|
||||
</Button>
|
||||
</StackPanel>
|
||||
|
||||
<Helpers:FocusPreservingListBox x:Name="SourceList" Grid.Row="1" Background="Transparent" BorderThickness="0"
|
||||
<Helpers:FocusPreservingListBox x:Name="SourceList" Grid.Row="1" MinHeight="84" MaxHeight="240" Background="Transparent" BorderThickness="0"
|
||||
ItemsSource="{Binding StagedScene.Elements}"
|
||||
SelectedItem="{Binding SelectedElement, Mode=TwoWay}"
|
||||
PreviewMouseLeftButtonDown="List_PreviewMouseLeftButtonDown"
|
||||
@@ -106,15 +113,17 @@
|
||||
</Helpers:FocusPreservingListBox.ItemContainerStyle>
|
||||
<ListBox.ItemTemplate>
|
||||
<DataTemplate>
|
||||
<Grid Margin="0,2"
|
||||
<Grid Margin="8,2,0,2"
|
||||
Tag="{Binding DataContext.CanChangeBackground, RelativeSource={RelativeSource AncestorType=Window}}">
|
||||
<Grid.ColumnDefinitions>
|
||||
<ColumnDefinition Width="Auto"/>
|
||||
<ColumnDefinition Width="*"/>
|
||||
<ColumnDefinition Width="Auto"/>
|
||||
<ColumnDefinition Width="Auto"/>
|
||||
<ColumnDefinition Width="Auto"/>
|
||||
</Grid.ColumnDefinitions>
|
||||
|
||||
|
||||
<Grid.ContextMenu>
|
||||
<ContextMenu DataContext="{Binding PlacementTarget.DataContext, RelativeSource={RelativeSource Self}}">
|
||||
<MenuItem Header="Change Capture…" Click="BackgroundMenu_ChangeCapture">
|
||||
@@ -171,7 +180,40 @@
|
||||
</ContextMenu>
|
||||
</Grid.ContextMenu>
|
||||
|
||||
<TextBlock Grid.Column="0" Text="{Binding Name}" VerticalAlignment="Center"
|
||||
<Grid Grid.Column="0" Width="16" Height="16" Margin="0,0,6,0" VerticalAlignment="Center">
|
||||
<Path Data="M7,8 H17 A4,4 0 0 1 21,12 V14 A3,3 0 0 1 17,16 L15,14 H9 L7,16 A3,3 0 0 1 3,14 V12 A4,4 0 0 1 7,8 Z M8,11 V13 M7,12 H9 M15,11 H15.01 M17,13 H17.01" Stroke="#38bdf8" StrokeThickness="1.6" StrokeLineJoin="Round" Stretch="Uniform">
|
||||
<Path.Style><Style TargetType="Path"><Setter Property="Visibility" Value="Collapsed"/><Style.Triggers><DataTrigger Binding="{Binding IsLiveCapture}" Value="True"><Setter Property="Visibility" Value="Visible"/></DataTrigger></Style.Triggers></Style></Path.Style>
|
||||
</Path>
|
||||
<Path Data="M3,5 H21 V19 H3 Z M3,16 L9,10 L13,14 L16,11 L21,16" Stroke="#38bdf8" StrokeThickness="1.6" StrokeLineJoin="Round" Stretch="Uniform">
|
||||
<Path.Style><Style TargetType="Path"><Setter Property="Visibility" Value="Collapsed"/><Style.Triggers><MultiDataTrigger><MultiDataTrigger.Conditions><Condition Binding="{Binding IsBackground}" Value="True"/><Condition Binding="{Binding IsLiveCapture}" Value="False"/></MultiDataTrigger.Conditions><Setter Property="Visibility" Value="Visible"/></MultiDataTrigger></Style.Triggers></Style></Path.Style>
|
||||
</Path>
|
||||
<Path Data="M3,7 H15 V17 H3 Z M15,11 L21,8 V16 L15,13" Stroke="#38bdf8" StrokeThickness="1.6" StrokeLineJoin="Round" Stretch="Uniform">
|
||||
<Path.Style><Style TargetType="Path"><Setter Property="Visibility" Value="Collapsed"/><Style.Triggers><DataTrigger Binding="{Binding IsWebcam}" Value="True"><Setter Property="Visibility" Value="Visible"/></DataTrigger></Style.Triggers></Style></Path.Style>
|
||||
</Path>
|
||||
<Path Data="M4,4 H20 V20 H4 Z M4,16 L9,11 L13,15 L16,12 L20,16 M15,8 H15.01" Stroke="#38bdf8" StrokeThickness="1.6" StrokeLineJoin="Round" Stretch="Uniform">
|
||||
<Path.Style><Style TargetType="Path"><Setter Property="Visibility" Value="Collapsed"/><Style.Triggers><DataTrigger Binding="{Binding IsImageSource}" Value="True"><Setter Property="Visibility" Value="Visible"/></DataTrigger></Style.Triggers></Style></Path.Style>
|
||||
</Path>
|
||||
<Path Data="M5,5 H19 M12,5 V19 M8,19 H16" Stroke="#38bdf8" StrokeThickness="1.6" StrokeLineJoin="Round" Stretch="Uniform">
|
||||
<Path.Style><Style TargetType="Path"><Setter Property="Visibility" Value="Collapsed"/><Style.Triggers><DataTrigger Binding="{Binding IsTextOverlay}" Value="True"><Setter Property="Visibility" Value="Visible"/></DataTrigger></Style.Triggers></Style></Path.Style>
|
||||
</Path>
|
||||
<Path Data="M12,3 A9,9 0 1 0 12.01,3 M12,7 V12 L15,14" Stroke="#38bdf8" StrokeThickness="1.6" StrokeLineJoin="Round" Stretch="Uniform">
|
||||
<Path.Style><Style TargetType="Path"><Setter Property="Visibility" Value="Collapsed"/><Style.Triggers><DataTrigger Binding="{Binding IsCountdown}" Value="True"><Setter Property="Visibility" Value="Visible"/></DataTrigger></Style.Triggers></Style></Path.Style>
|
||||
</Path>
|
||||
<Path Data="M12,3 A9,9 0 1 0 12.01,3 M3,12 H21 M12,3 C8,7 8,17 12,21 M12,3 C16,7 16,17 12,21" Stroke="#38bdf8" StrokeThickness="1.6" StrokeLineJoin="Round" Stretch="Uniform">
|
||||
<Path.Style><Style TargetType="Path"><Setter Property="Visibility" Value="Collapsed"/><Style.Triggers><DataTrigger Binding="{Binding IsWebSource}" Value="True"><Setter Property="Visibility" Value="Visible"/></DataTrigger></Style.Triggers></Style></Path.Style>
|
||||
</Path>
|
||||
<Path Data="M4,5 H20 V16 H10 L6,20 V16 H4 Z" Stroke="#38bdf8" StrokeThickness="1.6" StrokeLineJoin="Round" Stretch="Uniform">
|
||||
<Path.Style><Style TargetType="Path"><Setter Property="Visibility" Value="Collapsed"/><Style.Triggers><DataTrigger Binding="{Binding IsChatBox}" Value="True"><Setter Property="Visibility" Value="Visible"/></DataTrigger></Style.Triggers></Style></Path.Style>
|
||||
</Path>
|
||||
<Path Data="M4,5 H20 V19 H4 Z M10,9 L15,12 L10,15 Z" Stroke="#38bdf8" StrokeThickness="1.6" StrokeLineJoin="Round" Stretch="Uniform">
|
||||
<Path.Style><Style TargetType="Path"><Setter Property="Visibility" Value="Collapsed"/><Style.Triggers><DataTrigger Binding="{Binding IsMediaSource}" Value="True"><Setter Property="Visibility" Value="Visible"/></DataTrigger></Style.Triggers></Style></Path.Style>
|
||||
</Path>
|
||||
<Path Data="M6,16 V11 A6,6 0 0 1 18,11 V16 L20,18 H4 Z M10,20 H14" Stroke="#38bdf8" StrokeThickness="1.6" StrokeLineJoin="Round" Stretch="Uniform">
|
||||
<Path.Style><Style TargetType="Path"><Setter Property="Visibility" Value="Collapsed"/><Style.Triggers><DataTrigger Binding="{Binding IsAlertBox}" Value="True"><Setter Property="Visibility" Value="Visible"/></DataTrigger></Style.Triggers></Style></Path.Style>
|
||||
</Path>
|
||||
</Grid>
|
||||
|
||||
<TextBlock Grid.Column="1" Text="{Binding Name}" VerticalAlignment="Center"
|
||||
Padding="4,2" TextTrimming="CharacterEllipsis">
|
||||
<TextBlock.Style>
|
||||
<Style TargetType="TextBlock">
|
||||
@@ -185,7 +227,7 @@
|
||||
</TextBlock.Style>
|
||||
</TextBlock>
|
||||
|
||||
<TextBox Grid.Column="0" Text="{Binding Name, UpdateSourceTrigger=PropertyChanged}"
|
||||
<TextBox Grid.Column="1" Text="{Binding Name, UpdateSourceTrigger=PropertyChanged}"
|
||||
BorderThickness="0" FontSize="12"
|
||||
IsVisibleChanged="SourceNameBox_IsVisibleChanged"
|
||||
KeyDown="SourceNameBox_KeyDown" LostFocus="SourceNameBox_LostFocus">
|
||||
@@ -201,21 +243,21 @@
|
||||
</TextBox.Style>
|
||||
</TextBox>
|
||||
|
||||
<Button Grid.Column="1" ToolTip="Edit Layer Name"
|
||||
<Button Grid.Column="2" ToolTip="Edit Layer Name"
|
||||
Command="{Binding DataContext.EditElementCommand, RelativeSource={RelativeSource AncestorType=ListBox}}"
|
||||
CommandParameter="{Binding}" Style="{StaticResource IconButton}">
|
||||
<Path Data="M3,17.25V21h3.75L17.81,9.94l-3.75,-3.75L3,17.25zM20.71,7.04c0.39,-0.39 0.39,-1.02 0,-1.41l-2.34,-2.34c-0.39,-0.39 -1.02,-0.39 -1.41,0l-1.83,1.83 3.75,3.75 1.83,-1.83z"
|
||||
Fill="#d0d0d0" Width="13" Height="13" Stretch="Uniform"/>
|
||||
</Button>
|
||||
|
||||
<Button Grid.Column="2" Style="{StaticResource EyeButton}"
|
||||
<Button Grid.Column="3" Style="{StaticResource EyeButton}"
|
||||
Command="{Binding DataContext.ToggleElementVisibilityCommand, RelativeSource={RelativeSource AncestorType=ListBox}}"
|
||||
CommandParameter="{Binding}">
|
||||
<Path Style="{StaticResource EyeIconStyle}" Fill="#d0d0d0"
|
||||
Width="13" Height="13" Stretch="Uniform"/>
|
||||
</Button>
|
||||
|
||||
<Button Grid.Column="3" ToolTip="Remove Layer"
|
||||
<Button Grid.Column="4" ToolTip="Remove Layer"
|
||||
Command="{Binding DataContext.RemoveSourceCommand, RelativeSource={RelativeSource AncestorType=ListBox}}"
|
||||
CommandParameter="{Binding}" Style="{StaticResource IconButton}"
|
||||
Visibility="{Binding IsBackground, Converter={StaticResource HiddenBoolToVis}}">
|
||||
@@ -248,15 +290,19 @@
|
||||
|
||||
<!-- ELEMENTS panel: per-element config beneath Sources.
|
||||
Collapses entirely when no element is selected. -->
|
||||
<Rectangle Grid.Row="4" Height="1" Fill="#2c3a5c" Margin="0,10,0,2"
|
||||
Visibility="{Binding SelectedElement, Converter={StaticResource NotNullToVis}}"/>
|
||||
<StackPanel Grid.Row="5" Orientation="Horizontal" Margin="0,8,0,0"
|
||||
Visibility="{Binding SelectedElement, Converter={StaticResource NotNullToVis}}">
|
||||
<Rectangle Grid.Row="4" Height="1" Fill="#2c3a5c" Margin="0,22,0,10"/>
|
||||
<StackPanel Grid.Row="5" Orientation="Horizontal" Margin="0,8,0,0">
|
||||
<Path Data="M12,8 A4,4 0 1 0 12.01,8 M19.4,15 L21,16.2 L19.4,19 L17.2,18.3 A7,7 0 0 1 15,19.4 L14.5,21.5 H9.5 L9,19.4 A7,7 0 0 1 6.8,18.3 L4.6,19 L3,16.2 L4.6,15 A7,7 0 0 1 4.6,9 L3,7.8 L4.6,5 L6.8,5.7 A7,7 0 0 1 9,4.6 L9.5,2.5 H14.5 L15,4.6 A7,7 0 0 1 17.2,5.7 L19.4,5 L21,7.8 L19.4,9 A7,7 0 0 1 19.4,15 Z"
|
||||
Stroke="#38bdf8" StrokeThickness="1.6" StrokeLineJoin="Round"
|
||||
Width="16" Height="16" Stretch="Uniform" VerticalAlignment="Center" Margin="0,0,6,0"/>
|
||||
<TextBlock Text="PROPERTIES" Style="{StaticResource SectionHeader}" VerticalAlignment="Center"/>
|
||||
</StackPanel>
|
||||
<ScrollViewer x:Name="ElementsScrollViewer" Grid.Row="6" VerticalScrollBarVisibility="Auto" Background="Transparent"
|
||||
<TextBlock Grid.Row="6" Text="Select an item to edit its properties" Foreground="#94a3b8" FontSize="12"
|
||||
Margin="8,4,0,0" TextWrapping="Wrap"
|
||||
Visibility="{Binding SelectedElement, Converter={StaticResource NullToVis}}"/>
|
||||
<ScrollViewer x:Name="ElementsScrollViewer" Grid.Row="6" MaxHeight="420" VerticalScrollBarVisibility="Auto" Background="Transparent"
|
||||
Visibility="{Binding SelectedElement, Converter={StaticResource NotNullToVis}}">
|
||||
<StackPanel x:Name="ElementsPanel" Margin="0,4,0,0">
|
||||
<StackPanel x:Name="ElementsPanel" Margin="8,4,0,0">
|
||||
<!-- Opacity + mirror/shape (draggable elements: image, webcam) -->
|
||||
<StackPanel Visibility="{Binding SelectedElement.IsDraggable, Converter={StaticResource BoolToVis}}" Margin="0,0,0,8">
|
||||
<StackPanel Visibility="{Binding SelectedElement.IsChatBox, Converter={StaticResource InverseBoolToVis}}">
|
||||
|
||||
@@ -54,9 +54,12 @@ public partial class LeftPanel : UserControl
|
||||
{
|
||||
while (child != null && !ReferenceEquals(child, ancestor))
|
||||
{
|
||||
child = child is Visual or System.Windows.Media.Media3D.Visual3D
|
||||
var next = child is Visual or System.Windows.Media.Media3D.Visual3D
|
||||
? VisualTreeHelper.GetParent(child)
|
||||
: LogicalTreeHelper.GetParent(child);
|
||||
: null;
|
||||
child = next
|
||||
?? LogicalTreeHelper.GetParent(child)
|
||||
?? (child as FrameworkElement)?.TemplatedParent;
|
||||
}
|
||||
return child != null;
|
||||
}
|
||||
|
||||
@@ -180,49 +180,6 @@
|
||||
</Grid>
|
||||
</Viewbox>
|
||||
|
||||
<!-- OBS-style border: stroked rect (Traditional) / centered circle (Round).
|
||||
Animations are driven by code-behind in WebcamMenu_SetBorderAnimation
|
||||
because DataTrigger storyboards in DataTemplates cannot be stopped
|
||||
(no ExitActions namescope support). -->
|
||||
<Rectangle IsHitTestVisible="False"
|
||||
Stroke="{Binding BorderBrush}"
|
||||
StrokeThickness="{Binding BorderWidth}">
|
||||
<Rectangle.Style>
|
||||
<Style TargetType="Rectangle">
|
||||
<Setter Property="Visibility" Value="Collapsed"/>
|
||||
<Style.Triggers>
|
||||
<MultiDataTrigger>
|
||||
<MultiDataTrigger.Conditions>
|
||||
<Condition Binding="{Binding HasBorder}" Value="True"/>
|
||||
<Condition Binding="{Binding ClipShape}" Value="Traditional"/>
|
||||
</MultiDataTrigger.Conditions>
|
||||
<Setter Property="Visibility" Value="Visible"/>
|
||||
</MultiDataTrigger>
|
||||
</Style.Triggers>
|
||||
</Style>
|
||||
</Rectangle.Style>
|
||||
</Rectangle>
|
||||
<Ellipse IsHitTestVisible="False"
|
||||
Stroke="{Binding BorderBrush}"
|
||||
StrokeThickness="{Binding BorderWidth}"
|
||||
Width="{Binding RoundBorderSize}"
|
||||
Height="{Binding RoundBorderSize}"
|
||||
HorizontalAlignment="Center" VerticalAlignment="Center">
|
||||
<Ellipse.Style>
|
||||
<Style TargetType="Ellipse">
|
||||
<Setter Property="Visibility" Value="Collapsed"/>
|
||||
<Style.Triggers>
|
||||
<MultiDataTrigger>
|
||||
<MultiDataTrigger.Conditions>
|
||||
<Condition Binding="{Binding HasBorder}" Value="True"/>
|
||||
<Condition Binding="{Binding ClipShape}" Value="Round"/>
|
||||
</MultiDataTrigger.Conditions>
|
||||
<Setter Property="Visibility" Value="Visible"/>
|
||||
</MultiDataTrigger>
|
||||
</Style.Triggers>
|
||||
</Style>
|
||||
</Ellipse.Style>
|
||||
</Ellipse>
|
||||
|
||||
<Grid.ContextMenu>
|
||||
<ContextMenu DataContext="{Binding PlacementTarget.DataContext, RelativeSource={RelativeSource Self}}">
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
<UserControl x:Class="ytLive.Controls.SceneThumbnailStrip"
|
||||
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
|
||||
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml">
|
||||
<Border Background="#0d1117" CornerRadius="4" Margin="0,0,0,4" Padding="4">
|
||||
<Viewbox Stretch="Uniform">
|
||||
<Border Width="1000" Height="150" Background="#0d1117" CornerRadius="4" Margin="0,0,0,4" Padding="4">
|
||||
<ItemsControl ItemsSource="{Binding Scenes}">
|
||||
<ItemsControl.ItemsPanel>
|
||||
<ItemsPanelTemplate>
|
||||
@@ -12,7 +13,6 @@
|
||||
<DataTemplate>
|
||||
<Border CornerRadius="4" Margin="2" Padding="2"
|
||||
BorderThickness="2"
|
||||
BorderBrush="Transparent"
|
||||
Cursor="Hand"
|
||||
MouseLeftButtonDown="Thumbnail_MouseLeftButtonDown">
|
||||
<Border.Style>
|
||||
@@ -20,30 +20,62 @@
|
||||
<Setter Property="BorderBrush" Value="#333344"/>
|
||||
<Style.Triggers>
|
||||
<DataTrigger Binding="{Binding IsStaged}" Value="True">
|
||||
<Setter Property="BorderBrush" Value="#e94560"/>
|
||||
<Setter Property="BorderBrush" Value="#ff8c3a"/>
|
||||
</DataTrigger>
|
||||
<DataTrigger Binding="{Binding IsLive}" Value="True">
|
||||
<Setter Property="BorderBrush" Value="#22c55e"/>
|
||||
<Setter Property="BorderBrush" Value="#ff8c3a"/>
|
||||
</DataTrigger>
|
||||
</Style.Triggers>
|
||||
</Style>
|
||||
</Border.Style>
|
||||
<Grid>
|
||||
<Grid.RowDefinitions>
|
||||
<RowDefinition Height="*"/>
|
||||
<RowDefinition Height="Auto"/>
|
||||
</Grid.RowDefinitions>
|
||||
<Border Grid.Row="0" CornerRadius="3"
|
||||
Background="#0a0a1a" ClipToBounds="True">
|
||||
<Border CornerRadius="3" Background="#0a0a1a" ClipToBounds="True">
|
||||
<Image Source="{Binding Snapshot}"
|
||||
Stretch="UniformToFill"
|
||||
RenderOptions.BitmapScalingMode="LowQuality"/>
|
||||
</Border>
|
||||
<TextBlock Grid.Row="1" Foreground="#a0a0b0" FontSize="20"
|
||||
HorizontalAlignment="Center"
|
||||
Margin="0,2,0,0"
|
||||
|
||||
<Border HorizontalAlignment="Left" VerticalAlignment="Top" Margin="6" Padding="8,3"
|
||||
CornerRadius="3" Background="#ff8c3a">
|
||||
<Border.Style>
|
||||
<Style TargetType="Border">
|
||||
<Setter Property="Visibility" Value="Collapsed"/>
|
||||
<Style.Triggers>
|
||||
<DataTrigger Binding="{Binding IsStaged}" Value="True">
|
||||
<Setter Property="Visibility" Value="Visible"/>
|
||||
</DataTrigger>
|
||||
</Style.Triggers>
|
||||
</Style>
|
||||
</Border.Style>
|
||||
<StackPanel Orientation="Horizontal">
|
||||
<Ellipse Width="8" Height="8" Fill="#ff5451" VerticalAlignment="Center" Margin="0,0,5,0">
|
||||
<Ellipse.Triggers>
|
||||
<EventTrigger RoutedEvent="FrameworkElement.Loaded">
|
||||
<BeginStoryboard>
|
||||
<Storyboard RepeatBehavior="Forever" AutoReverse="True">
|
||||
<DoubleAnimation Storyboard.TargetProperty="Opacity" From="1" To="0.15" Duration="0:0:0.6"/>
|
||||
</Storyboard>
|
||||
</BeginStoryboard>
|
||||
</EventTrigger>
|
||||
</Ellipse.Triggers>
|
||||
</Ellipse>
|
||||
<TextBlock Text="PROGRAM" Foreground="White" FontSize="11" FontWeight="Bold" VerticalAlignment="Center"/>
|
||||
</StackPanel>
|
||||
</Border>
|
||||
|
||||
<TextBlock Text="{Binding HotkeyLabel, Mode=OneWay}" Foreground="White" FontSize="16" FontWeight="SemiBold"
|
||||
HorizontalAlignment="Right" VerticalAlignment="Top" Margin="6">
|
||||
<TextBlock.Effect>
|
||||
<DropShadowEffect BlurRadius="4" ShadowDepth="0" Opacity="0.9"/>
|
||||
</TextBlock.Effect>
|
||||
</TextBlock>
|
||||
|
||||
<TextBlock Text="{Binding Name, Mode=OneWay}" Foreground="White" FontSize="16" FontWeight="SemiBold"
|
||||
HorizontalAlignment="Left" VerticalAlignment="Bottom" Margin="6"
|
||||
TextTrimming="CharacterEllipsis">
|
||||
<Run Text="{Binding Name, Mode=OneWay}"/><Run Text=" "/><Run Text="{Binding HotkeyLabel, Mode=OneWay}" Foreground="#6a6a7a" FontWeight="SemiBold"/>
|
||||
<TextBlock.Effect>
|
||||
<DropShadowEffect BlurRadius="4" ShadowDepth="0" Opacity="0.9"/>
|
||||
</TextBlock.Effect>
|
||||
</TextBlock>
|
||||
</Grid>
|
||||
</Border>
|
||||
@@ -51,4 +83,5 @@
|
||||
</ItemsControl.ItemTemplate>
|
||||
</ItemsControl>
|
||||
</Border>
|
||||
</UserControl>
|
||||
</Viewbox>
|
||||
</UserControl>
|
||||
|
||||
@@ -8,9 +8,22 @@ namespace ytLive.Controls;
|
||||
|
||||
public partial class SceneThumbnailStrip : UserControl
|
||||
{
|
||||
private const double DesignAspect = 154.0 / 1000.0;
|
||||
|
||||
public SceneThumbnailStrip()
|
||||
{
|
||||
InitializeComponent();
|
||||
SizeChanged += (_, _) => Height = ActualWidth * DesignAspect;
|
||||
Loaded += (_, _) =>
|
||||
{
|
||||
if (DataContext is MainViewModel vm)
|
||||
foreach (var scene in vm.Scenes)
|
||||
{
|
||||
var staged = scene.IsStaged;
|
||||
scene.IsStaged = !staged;
|
||||
scene.IsStaged = staged;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private void Thumbnail_MouseLeftButtonDown(object sender, MouseButtonEventArgs e)
|
||||
|
||||
+1
-1
@@ -12,7 +12,7 @@ See [`schema.md`](../schema.md) for the memory-map conventions.
|
||||
| `TopBar.xaml(.cs)` | Row 0 session surface (2026-09-22 redesign): brand + gear menu (left), one segmented REC|ON-AIR mode switch + world actions (Record: Start Recording · Stream: Go Live + Test) + reality line while running (dot/word/elapsed) + End (center), account zone — Sign In or avatar w/ Change Account/Logout context menu (right). `RefreshAvatar()`, `GearButton_Click` |
|
||||
| `PreviewPane.xaml(.cs)` | Row 1 Col 1 center: the 1920×1080 `OverlayCanvas`/`SelectionOverlay`/`BrandFlashLayer`/`SocialBarElement`, text pull-out drawer, and the live-controls row (TRAX / game / mic meters + sliders + speakers + socials). **TASK 57:** dragging a webcam under the vertical tier routes through `MainViewModel.MoveWebcamInVerticalTier` (the six-spot snap inside the cropped band) instead of free-floating over the master, and a resize re-snaps the tile onto its spot (TASK 57 Unit 2: the resize routes through `MainViewModel.ResizeWebcamInVerticalTier` — the Chat camera's two sizes, then the snap) |
|
||||
| `SceneThumbnailStrip.xaml(.cs)` | Row 0 scene-picker strip (`ItemsControl` over `Scenes`), staged-red / live-green border highlight |
|
||||
| `LeftPanel.xaml(.cs)` | Row 1 Col 0 left panel: edit mode → layers+properties; live → YouTube chat. Source add/edit/remove, list drag-to-reorder, snapshot revert-accept. **TASK 51:** the webcam properties carry the Background section — `RemoveBackgroundPill` / `BlurBackgroundPill` (`PillToggle`, radio-exclusive via `WebcamSceneConfig.RemoveBackground`/`BlurBackground`) + `BackgroundStrengthSlider` (0..1, 5% snaps, shown only while an effect is on). **TASK 57:** the webcam properties carry a Geometry section (X/Y/W/H) — manual placement of the camera, bound to the **live** placement (the one the current tier shows), gated on `IsWebcamSelected` (not `SelectedElement.IsWebcam` — the TASK 51 test finds that panel by binding text and would match twice); the Shape button's `Visibility` is bound to `CanToggleClipShape`, so the circle option is NOT OFFERED in the vertical Chat layout. **TASK 58:** every drop-down, check box and field in the properties panel carries the theme's style (`YtComboBox`/`YtCheckBox`/`YtTextBoxCompact`) and paints no colour of its own — the guard is `ytLive.Tests/PropertiesStyleTests.cs`. **TASK 59:** the webcam's properties are a HIERARCHY — a `Shape` title owns the mirror switch and the shape choice (`Rect` | `Circle` as two `SegmentToggle` options, Rect selected by default, the circle option hidden where `CanToggleClipShape` is false), and every titled block (Geometry, Border Color, Border Thickness, Background) owns its content one indent deeper (Strength sits under Blur). **TASK 60:** the border-colour drop-down builds its menu from `Models/BorderColorPalette.cs` — each row is the hex plus the colour's NAME (`#e94560 Crimson`, tooltip "Crimson (#e94560)"); the swatch icon is unchanged. **TASK 61:** the four geometry fields (X/Y/W/H) share one fixed width (44px — the largest value, 1920, plus a sign) and their rows set `LastChildFill="False"`, so Y/H no longer stretch to fill the row. **TASK 62:** the border-colour field shows the palette's name beside the hex (`SelectedElement.BorderColorName`, empty for a custom hex) and keeps a fixed width so the name stays next to the hex. **TASK 63:** the border colour is picked with the **app's own drop-down** (`YtComboBox`) — the `...` button and its code-built `ContextMenu` are gone; the closed box shows the swatch + the colour's NAME, the rows show swatch + hex + name (the panel's own `DataTemplate`s + `Helpers/ClosedBoxTemplateSelector.cs`), and the hex field stays for a custom colour |
|
||||
| `LeftPanel.xaml(.cs)` | Row 1 Col 0 left panel: edit mode → layers+properties; live → YouTube chat. Source add/edit/remove, list drag-to-reorder, snapshot revert-accept. **TASK 51:** the webcam properties carry the Background section — `RemoveBackgroundPill` / `BlurBackgroundPill` (`PillToggle`, radio-exclusive via `WebcamSceneConfig.RemoveBackground`/`BlurBackground`) + `BackgroundStrengthSlider` (0..1, 5% snaps, shown only while an effect is on). **TASK 57:** the webcam properties carry a Geometry section (X/Y/W/H) — manual placement of the camera, bound to the **live** placement (the one the current tier shows), gated on `IsWebcamSelected` (not `SelectedElement.IsWebcam` — the TASK 51 test finds that panel by binding text and would match twice); the Shape button's `Visibility` is bound to `CanToggleClipShape`, so the circle option is NOT OFFERED in the vertical Chat layout. **TASK 58:** every drop-down, check box and field in the properties panel carries the theme's style (`YtComboBox`/`YtCheckBox`/`YtTextBoxCompact`) and paints no colour of its own — the guard is `ytLive.Tests/PropertiesStyleTests.cs`. **TASK 59:** the webcam's properties are a HIERARCHY — a `Shape` title owns the mirror switch and the shape choice (`Rect` | `Circle` as two `SegmentToggle` options, Rect selected by default, the circle option hidden where `CanToggleClipShape` is false), and every titled block (Geometry, Border Color, Border Thickness, Background) owns its content one indent deeper (Strength sits under Blur). **TASK 60:** the border-colour drop-down builds its menu from `Models/BorderColorPalette.cs` — each row is the hex plus the colour's NAME (`#e94560 Crimson`, tooltip "Crimson (#e94560)"); the swatch icon is unchanged. **TASK 61:** the four geometry fields (X/Y/W/H) share one fixed width (44px — the largest value, 1920, plus a sign) and their rows set `LastChildFill="False"`, so Y/H no longer stretch to fill the row. **TASK 62:** the border-colour field shows the palette's name beside the hex (`SelectedElement.BorderColorName`, empty for a custom hex) and keeps a fixed width so the name stays next to the hex. **TASK 63:** the border colour is picked with the **app's own drop-down** (`YtComboBox`) — the `...` button and its code-built `ContextMenu` are gone; the closed box shows the swatch + the colour's NAME, the rows show swatch + hex + name (the panel's own `DataTemplate`s + `Helpers/ClosedBoxTemplateSelector.cs`), and the hex field stays for a custom colour; `IsDescendantOf` falls back to `LogicalTreeHelper.GetParent` / `TemplatedParent` when `VisualTreeHelper.GetParent` returns `null` (at `PopupRoot`) so clicking a popup row does not trigger `MainWindow`'s deselection guard |
|
||||
| `OverlayHost.xaml(.cs)` | Root-grid overlay dialogs: Settings / Report Bug / Feature Request / About (+ licenses + license-key entry). `BackToAbout_Click` lives here |
|
||||
| `SceneThumbnailStrip.xaml.cs` | Code-behind for the strip above |
|
||||
| `LeftPanel.xaml.cs` | Code-behind for the panel above |
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
# DESIGN.md — LlamaCasty visual design tokens (redesign exploration)
|
||||
|
||||
> Source: Google Stitch export, 2026-10-26, from the proposed edit-screen mockup. Colors and type
|
||||
> only — no layout decisions are captured here. This is a design reference for the UX redesign
|
||||
> in progress; it is not yet wired into the app's theme (`Themes/Controls.xaml`).
|
||||
|
||||
## Color
|
||||
|
||||
### Surfaces (dark navy scale, darkest to lightest)
|
||||
|
||||
| Token | Hex | Use |
|
||||
|---|---|---|
|
||||
| `background` | `#0b1326` | App window background |
|
||||
| `surface-container-lowest` | `#060e20` | Deepest recess (scene strip background, canvas housing) |
|
||||
| `surface-container-low` | `#131b2e` | Panel background (sidebars, headers) |
|
||||
| `surface-container` | `#171f33` | Card / grouped-section background |
|
||||
| `surface-container-high` | `#222a3d` | Hover state, input fields |
|
||||
| `surface-container-highest` | `#2d3449` | Selected/active row background |
|
||||
| `surface-bright` | `#31394d` | Lightest surface, rare use (strong hover) |
|
||||
| `outline` | `#334155` | Borders, dividers |
|
||||
| `outline-variant` | `#1e293b` | Subtle dividers, inactive VU meter segments |
|
||||
|
||||
### Text
|
||||
|
||||
| Token | Hex | Use |
|
||||
|---|---|---|
|
||||
| `on-surface` | `#dae2fd` | Primary text |
|
||||
| `on-surface-variant` | `#94a3b8` | Secondary / label text |
|
||||
|
||||
### Accent — one role each, never mixed
|
||||
|
||||
| Token | Hex | Role |
|
||||
|---|---|---|
|
||||
| `tertiary` (electric blue) | `#38bdf8` | The one interactive accent: selection, focus, sliders, primary buttons, transform handles |
|
||||
| `primary` (red) | `#ff5451` | **Reserved for on-air/recording state only** — LIVE badge, program border, End/Go Live |
|
||||
| `primary-container` | `#b91a24` | Red pressed/hover state |
|
||||
| `secondary` (amber) | `#ffb95f` | Game audio channel, Super Chat highlight |
|
||||
| `secondary-container` | `#ee9800` | Amber pressed/hover state |
|
||||
| `error` | `#ef4444` | Error states only (distinct from the on-air red) |
|
||||
|
||||
**Rule carried from the critique sessions:** red must mean "live or recording" and nothing else.
|
||||
Don't reuse it for a Send button, a delete action, or any other control — that was the defect in
|
||||
the first Stitch draft of the live screen.
|
||||
|
||||
## Typography
|
||||
|
||||
| Token | Family | Use |
|
||||
|---|---|---|
|
||||
| `sans` | Inter (400/500/600/700) | All UI text — labels, buttons, panel titles |
|
||||
| `mono` | JetBrains Mono (500/600) | All numeric/telemetry values — bitrate, FPS, dB, coordinates, timers. Keeps digits from shifting width as values change. |
|
||||
|
||||
Icons: Material Symbols Outlined.
|
||||
|
||||
## Color roles (decided 2026-10-26)
|
||||
|
||||
- **Red `#ff5451`:** YouTube references, on-air/GO LIVE, and record only.
|
||||
- **Attention orange `#ff8c3a`:** dropped packets, the live screen indicator, and blinking warning lights.
|
||||
- **Amber `#ffb95f`:** game audio and Super Chat only.
|
||||
- **`error` `#ef4444`:** unchanged for now; it sits close to the on-air red and may need revisiting.
|
||||
|
||||
## Open questions for the next pass
|
||||
|
||||
- These tokens have not been reconciled against the app's current WPF theme
|
||||
(`Themes/Controls.xaml`). That comparison should happen once the redesign direction is locked,
|
||||
not before — do not retrofit the current theme to these values yet.
|
||||
- No elevation/shadow scale, corner-radius scale, or spacing scale has been captured yet. Add them
|
||||
here as they're decided, rather than letting them live only in Stitch exports.
|
||||
+49
-9
@@ -1,4 +1,26 @@
|
||||
# HANDOFF — 2026-10-22 (the creator's properties-panel asks shipped, incl. TASK 63, and pushed; the vv webcam Units 1+2 state below)
|
||||
# HANDOFF — 2026-10-26 (ai.md memory split, 3 units landed; see below — everything after this block is the prior 2026-10-22 state, still accurate)
|
||||
|
||||
## At a glance (2026-10-26 docs session)
|
||||
- **`ai.md` is being split into `docs/ai/*.md`** (user-approved: verbatim move, staged units, "ai.md
|
||||
only"). Three units landed as separate commits, **docs only, not pushed**:
|
||||
`804b1d2` (frame pipeline: cadence/timing/budget/compositor/pump),
|
||||
`096bf6d` (capture: backdrop/webcam/background effects/vertical tier),
|
||||
`a4dfca6` (output: ffmpeg/encoder/recording/audio). Each move is byte-identical in the new file;
|
||||
`ai.md` keeps a short stub with the load-bearing invariants + a link. `docs/ai/index.md` routes.
|
||||
**`ai.md` is now 1,289 lines (~32k tokens), down from 2,702 (~64k)** — more than half the
|
||||
mandatory-read cost is gone.
|
||||
- **Also landed (docs only, not pushed):** `ef60f1f` — distribution & packaging deferred until near
|
||||
v1 (creator ruling 2026-10-26); see "Next step" below for the still-open route contradiction.
|
||||
- **Remaining split candidates in `ai.md`** (not yet moved — smaller, more interleaved with law that
|
||||
should stay): Architecture's "Social bar" (46) + "Toast notifications" (24) + "Licensing — do not
|
||||
violate" (60); "YouTube Live API — design constraints" incl. "Native events & alerts" (349 total,
|
||||
99 of it alerts); "Monetization" (136); "Windows packaging & distribution" (135, frozen per the
|
||||
deferral above). None of these is urgent — pick back up only if the user asks to continue the split.
|
||||
- **Scope-check passed on all three commits**; no source touched this session.
|
||||
|
||||
---
|
||||
|
||||
## Prior state (2026-10-22) — the creator's properties-panel asks shipped, incl. TASK 63, and pushed; the vv webcam Units 1+2 state below
|
||||
|
||||
## At a glance
|
||||
- **Branch** main, tracking `origin/main`. **The six-commit properties-panel batch is PUSHED**
|
||||
@@ -52,7 +74,8 @@
|
||||
(`SceneElement.BorderColorName`, empty for a custom hex), and the hex field keeps a fixed
|
||||
width so the name stays next to the hex. The field stays editable (a custom hex can
|
||||
still be typed). Guard: `ytLive.Tests/BorderColorNameTests.cs`.
|
||||
- **TASK 63 - the border-colour picker IS the app's drop-down** (shipped 2026-10-21). The creator:
|
||||
- **TASK 63 - the border-colour picker IS the app's drop-down** (shipped 2026-10-21; **popup click
|
||||
deselection fixed 2026-10-22**). The creator:
|
||||
*"why does the drop down under the web cam properties not show the same style for the border
|
||||
color drop down list box?"* and *"we established look-and-feel standards. And yet you keep
|
||||
ignoring these and I'm wondering why."* **Root cause of the miss:** the TASK 58 sweep looked for
|
||||
@@ -66,8 +89,17 @@
|
||||
`WebcamBorderColorPicker_Click` are **gone**. The closed box != the rows only through
|
||||
`Helpers/ClosedBoxTemplateSelector.cs` (WPF copies `ItemTemplate` into the read-only
|
||||
`SelectionBoxItemTemplate`; the selector is told them apart by the `ContentPresenter`'s
|
||||
templated parent - `ComboBox` = closed box, `ComboBoxItem` = a row). Guard:
|
||||
`ytLive.Tests/BorderColorPickerTests.cs` (model + panel + the **real widget**). The TASK 60
|
||||
templated parent - `ComboBox` = closed box, `ComboBoxItem` = a row). **2026-10-22 creator bug fix:**
|
||||
clicking a colour in the open `ComboBox` popup registered as an exit from the properties listing
|
||||
because `LeftPanel.IsDescendantOf` used `child is Visual ? VisualTreeHelper.GetParent(child) :
|
||||
LogicalTreeHelper.GetParent(child)` — at `PopupRoot` (which IS a `Visual`),
|
||||
`VisualTreeHelper.GetParent` returns `null`, so `IsDescendantOf` stopped without crossing
|
||||
`PopupRoot`'s logical parent (`PART_Popup` → `ComboBox` → `ElementsScrollViewer`), and
|
||||
`MainWindow.Window_PreviewMouseLeftButtonDown` cleared `SelectedElement = null`. Fixed by falling
|
||||
back to `LogicalTreeHelper.GetParent` / `TemplatedParent` when `VisualTreeHelper.GetParent`
|
||||
returns `null`. Guard: `ytLive.Tests/BorderColorPickerTests.cs` (model + panel + the **real
|
||||
widget**, including `LeftPanel.IsClickInside` on an open popup row and clicking a row to change
|
||||
`BorderColor`). The TASK 60
|
||||
guard (`BorderColorPaletteTests`) was updated to guard the new wiring (the drop-down is bound to
|
||||
the palette; no second list of colours anywhere).
|
||||
- **TASK 55 slice 1 — the log is capped (shipped 2026-10-22).** `startup.log` was uncapped (23 MB
|
||||
@@ -233,11 +265,10 @@ Zombie entities deleted from the registry: `script.leak_strobe`, `script.doorbel
|
||||
the full top area and the circle option is not offered. **PASSES.**
|
||||
|
||||
## Next step
|
||||
- **Creator check of TASK 63 (needs the creator's eyes)** — open Live → Layers → webcam →
|
||||
Properties → Border Color: the drop-down must now LOOK like the Display/Font drop-downs (same
|
||||
dark themed chrome, same chevron), the CLOSED box must show the colour's name beside the hex
|
||||
field ("Crimson"), and the open list must show swatch + hex + name. The hex field still accepts a
|
||||
typed colour (the drop-down then shows the swatch with no name — "no name" is the honest answer).
|
||||
- **✅ TASK 63 + popup-click fix verified by creator (2026-10-22)** — clicking a colour in the
|
||||
Border Color `YtComboBox` dropdown now stays inside `LeftPanel` (`IsDescendantOf` crosses
|
||||
`PopupRoot` → `PART_Popup` via `LogicalTreeHelper.GetParent`), updates the webcam border colour,
|
||||
and does not exit the properties listing.
|
||||
- **Creator check (needs the creator's hands)** — the last piece of the vv feature: switch the
|
||||
resolution dropdown to "Vertical 1080p60", confirm the Live camera tile lands inside the band and
|
||||
the **Circle** option (TASK 59 turned the Shape button into a Rect | Circle choice) turns the tile
|
||||
@@ -245,6 +276,15 @@ Zombie entities deleted from the registry: `script.leak_strobe`, `script.doorbel
|
||||
and that dragging the tile up and enlarging it fills a whole 607×540 area, then
|
||||
switch back and confirm the desktop layout is untouched. The unit tests cover the geometry, not the
|
||||
eyeball.
|
||||
- **⏸ Distribution & packaging is DEFERRED (creator ruling 2026-10-26) — do not pick it up until
|
||||
we are close to v1.** The creator's reasons: the remaining programming tasks are far from
|
||||
release, and a **complete UX redesign is about to start** — so packaging/signing/updater/route
|
||||
decisions would chase scope that is about to move. Nothing distribution-related is urgent now.
|
||||
Frozen: TASK 48 items 6-8, the signing decision, the updater choice, and the **route contradiction**
|
||||
(`ai.md` says Store route DEAD → direct + Steam; `TASKS.md`/`TASKS/task-48-…`/this file still say
|
||||
Store MSIX + Store IAP). That contradiction is to be settled in ONE distribution sweep at the
|
||||
revisit, not patched piecemeal in the meantime. Detail: `TASKS.md` → Open items → the ⏸ TASK 48
|
||||
bullet.
|
||||
- Nothing else is queued for llamacasty — the creator queues work.
|
||||
|
||||
## Open (operational)
|
||||
|
||||
+2
-2
@@ -15,8 +15,8 @@ Cross-cutting utilities. See [`schema.md`](../schema.md) for the memory-map conv
|
||||
| `InverseBoolToVisibilityConverter.cs` / `NotNullToVisibilityConverter.cs` | XAML value converters for visibility bindings |
|
||||
| `AllTrueToVisibilityConverter.cs` | Multi-value AND-gate → Visible; gates the layer-row capture menu items on row-IsBackground + staged-scene-is-Live (TASK 25) |
|
||||
| `CameraConflictProbe.cs` | Best-effort diagnostic: when a camera won't start, enumerates running processes known to hold cameras (OBS, Zoom, Teams, NVIDIA Broadcast, browsers, etc.) — Windows doesn't expose "which process has this device" via any public API, so this is a suspect list, not a verdict |
|
||||
| `StripPool.cs` | Four DEDICATED background threads at `BelowNormal` priority that split one composite into horizontal row strips (the caller runs a strip too); started lazily, runs inline below 64 rows. UNIT 5 — do NOT demote the ThreadPool from `Parallel.For`, it leaks into unrelated work (see ai.md "Render budget") |
|
||||
| `FrameCadence.cs` | Pure arrival-cadence counter (`FrameCadence` + `FrameCadenceSummary.Describe`): frames, gaps, hitches, longest/avg gap for a window of a timestamped stream. Fed by `MediaCaptureFrameSource` (camera stamps vs our arrivals) and `FramePump` (distinct webcam Epochs that reached the output) — UNIT 0, see ai.md "Frame cadence" |
|
||||
| `StripPool.cs` | Four DEDICATED background threads at `BelowNormal` priority that split one composite into horizontal row strips (the caller runs a strip too); started lazily, runs inline below 64 rows. UNIT 5 — do NOT demote the ThreadPool from `Parallel.For`, it leaks into unrelated work (see [`docs/ai/frame-pipeline.md`](../docs/ai/frame-pipeline.md) → "Render budget") |
|
||||
| `FrameCadence.cs` | Pure arrival-cadence counter (`FrameCadence` + `FrameCadenceSummary.Describe`): frames, gaps, hitches, longest/avg gap for a window of a timestamped stream. Fed by `MediaCaptureFrameSource` (camera stamps vs our arrivals) and `FramePump` (distinct webcam Epochs that reached the output) — UNIT 0, see [`docs/ai/frame-pipeline.md`](../docs/ai/frame-pipeline.md) → "Frame cadence" |
|
||||
| `HexToBrushConverter.cs` | A "#RRGGBB" hex string → `SolidColorBrush` (transparent on bad input), so a template can paint a colour swatch straight from the palette. Registered in the theme as `HexToBrush` — the theme paints the colour, never the UI code (TASK 63) |
|
||||
| `ClosedBoxTemplateSelector.cs` | An `ItemTemplateSelector` that gives a ComboBox's CLOSED box a different template from its drop-down rows: WPF calls it with the `ContentPresenter`'s templated parent — the `ComboBox` for the closed box, the `ComboBoxItem` for a row. The only way to tell them apart (`SelectionBoxItemTemplate` is read-only and mirrors `ItemTemplate`) — TASK 63 |
|
||||
|
||||
|
||||
+2
-2
@@ -78,8 +78,8 @@
|
||||
<ColumnDefinition Width="3*"/>
|
||||
</Grid.ColumnDefinitions>
|
||||
|
||||
<!-- ═══ THUMBNAIL STRIP ═══ -->
|
||||
<controls:SceneThumbnailStrip Grid.Row="0" Grid.ColumnSpan="2"/>
|
||||
<!-- ═══ THUMBNAIL STRIP (over the preview column; left column stays empty) ═══ -->
|
||||
<controls:SceneThumbnailStrip Grid.Row="0" Grid.Column="1" Margin="8,0"/>
|
||||
|
||||
<!-- LEFT PANEL: one panel, two states.
|
||||
Offline (edit mode): LAYERS + PROPERTIES
|
||||
|
||||
@@ -84,6 +84,9 @@ public abstract class SceneElement : INotifyPropertyChanged
|
||||
|
||||
/// <summary>True only for AlertBox-type sources. Gates stream alerts properties panel.</summary>
|
||||
public virtual bool IsAlertBox => false;
|
||||
public virtual bool IsTextOverlay => false;
|
||||
public virtual bool IsMediaSource => false;
|
||||
public virtual bool IsLiveCapture => false;
|
||||
public virtual bool IsBackground { get; set; }
|
||||
|
||||
private bool _useDefaultBackground = true;
|
||||
|
||||
+4
-2
@@ -73,6 +73,7 @@ public class Source : SceneElement
|
||||
Raise(nameof(IsLiveCapture));
|
||||
Raise(nameof(DisplaySource));
|
||||
Raise(nameof(IsWebSource));
|
||||
Raise(nameof(IsTextOverlay));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -85,6 +86,7 @@ public class Source : SceneElement
|
||||
? ElementKind.Dynamic : ElementKind.Static;
|
||||
|
||||
public override bool IsWebSource => Type == SourceType.WebSource;
|
||||
public override bool IsTextOverlay => Type == SourceType.TextOverlay;
|
||||
|
||||
/// <summary>The permanent scene background (live desktop/game capture). Never removable/reorderable.</summary>
|
||||
private bool _isBackground;
|
||||
@@ -110,7 +112,7 @@ public class Source : SceneElement
|
||||
private string? _captureKey;
|
||||
public string? CaptureKey { get => _captureKey; set => Set(ref _captureKey, value); }
|
||||
|
||||
public bool IsLiveCapture => Type is SourceType.DisplayCapture or SourceType.WindowCapture;
|
||||
public override bool IsLiveCapture => Type is SourceType.DisplayCapture or SourceType.WindowCapture;
|
||||
|
||||
// Image (asset stored in the layout database)
|
||||
private string? _assetId;
|
||||
@@ -190,7 +192,7 @@ public class Source : SceneElement
|
||||
set => Set(ref _mediaPlaybackState, value);
|
||||
}
|
||||
|
||||
public bool IsMediaSource => Type == SourceType.MediaSource;
|
||||
public override bool IsMediaSource => Type == SourceType.MediaSource;
|
||||
|
||||
// Alert video config (TASK 47): the shipped default clip lives in the Asset
|
||||
// table (seeded like image assets); the creator's replacement is a FILE PATH
|
||||
|
||||
+31
-1
@@ -1227,7 +1227,7 @@ clothes, and the failure mode is invisible because the sentence still *sounds* a
|
||||
Corollary: **a stale external fact inside a shipped document is worse than a missing one** — it
|
||||
is trusted, acted on, and costed. When a doc makes a factual claim about the outside world
|
||||
(certificates, platform behaviour, policy text), re-verify it on touch, not on memory. This is
|
||||
the same class as the dead-Pin incident in `FfmpegLocator` (`ai.md` → FFmpeg locator): an
|
||||
the same class as the dead-Pin incident in `FfmpegLocator` ([`docs/ai/output.md`](docs/ai/output.md) → FFmpeg locator): an
|
||||
external thing changed, our record of it did not, and the mismatch was discovered by a user
|
||||
rather than by a test. **Records of the outside world rot silently and are believed until they
|
||||
cost money.**
|
||||
@@ -2275,6 +2275,36 @@ sizes to its **content**, not to the ComboBox — measured 142.7px of popup for
|
||||
whose rows read "swatch + hex + name". A long row is never clipped, and the popup may be wider
|
||||
than the closed box. Measure with `ActualWidth` in a throwaway test; do not guess widths.
|
||||
|
||||
**2026-10-22 follow-up — `PopupRoot` IS a `Visual`, yet its `VisualTreeHelper.GetParent` is `null`:**
|
||||
Switching from a code-built `ContextMenu` (which is not in `MainWindow`'s routed-event path) to a
|
||||
`ComboBox` (`PART_Popup`) exposed a latent bug in `LeftPanel.IsDescendantOf`:
|
||||
```csharp
|
||||
child = child is Visual or System.Windows.Media.Media3D.Visual3D
|
||||
? VisualTreeHelper.GetParent(child)
|
||||
: LogicalTreeHelper.GetParent(child);
|
||||
```
|
||||
When the user clicked a colour row inside the open `ComboBox` popup, `PreviewMouseLeftButtonDown`
|
||||
tunnelled through `MainWindow.Window_PreviewMouseLeftButtonDown` first. `IsDescendantOf` walked up
|
||||
`VisualTreeHelper.GetParent` to `PopupRoot` (the root `Visual` of the popup's separate Win32
|
||||
`HwndSource`). Because `PopupRoot` IS a `Visual`, the ternary took the `VisualTreeHelper.GetParent`
|
||||
branch and got `null` — never crossing `PopupRoot`'s **logical** parent
|
||||
(`LogicalTreeHelper.GetParent(popupRoot) == PART_Popup`). `LeftPanel.IsClickInside` returned
|
||||
`false`, and `MainWindow` set `SelectedElement = null` (exiting the properties listing before the
|
||||
`ComboBoxItem` click could register). Fix: always coalesce
|
||||
`VisualTreeHelper.GetParent(child) ?? LogicalTreeHelper.GetParent(child) ?? (child as FrameworkElement)?.TemplatedParent`.
|
||||
|
||||
## 2026-10-22 — WINDOWS GIT IDENTITY + PRE-TASK GIT STATUS (RECIPE)
|
||||
|
||||
- **Windows `git` vs WSL `~/.gitconfig`:** the creator's `user.name` / `user.email` lived only in
|
||||
WSL's `~/.gitconfig`, so running `git commit` from the Windows PowerShell host failed with
|
||||
`Author identity unknown`. Fixed permanently in `.git/config` (`[user] name = gramps`,
|
||||
`email = gramps@llamachile.shop`). If you ever need the repo's commit author without running a
|
||||
shell command, read `.git/logs/HEAD` directly with `read_file`.
|
||||
- **Check `git status` BEFORE starting a task** (and never run multiline/compound shell commands
|
||||
when a single simple command or `read_file`/`edit_file` works): `verify.sh` runs `scope-check.sh`
|
||||
over `git diff HEAD`, so any pre-existing uncommitted files in the worktree will fail step 3 at
|
||||
the end of a 2-minute build+test run unless spotted up front.
|
||||
|
||||
## 2026-10-22 — SCANNING A GIT HISTORY FOR SECRETS WITHOUT EVER PRINTING ONE (RECIPE)
|
||||
|
||||
A release gate ("the repo goes public — check the history first"). The rule from
|
||||
|
||||
+4
-4
@@ -21,7 +21,7 @@ External-facing logic: YouTube API, persistence. See
|
||||
| `BackgroundEffects/OnnxPersonSegmenter.cs` | **TASK 51 default segmenter:** MediaPipe Selfie Segmenter (landscape, Apache-2.0) as an embedded ONNX resource, run by onnxruntime on the CPU (2 intra-op threads); frame stretched to 256×144 RGB/255 NHWC; lazy session; never throws (fail → log once, null forever) |
|
||||
| `BackgroundEffects/WebcamEffectKey.cs` | **TASK 51:** `(Effect, Strength)` variant key, strength rounded to 0.01; `None` normalizes to strength 0 (the raw path) |
|
||||
| `BackgroundEffects/BackgroundEffectProcessor.cs` | **TASK 51 pure pixel stage:** `Apply(frame, BackgroundEffect)` — None = same instance, Remove = copy with alpha = feathered mask × source alpha (`IsOpaque=false`), Blur = background blurred at ¼ res (3 box passes) and mask-blended, person untouched; strength 0..1; blur radius is **quadratic** via `BlurRadius(strength)` — 1→6 small px (default 0.5 = 2, was a linear 1→12 ≈ 48 full px; the "blur heavy" fix 2026-10-09); `TrySegment` once + `Render` per effect; no mask/exception → passthrough + one log |
|
||||
| `BackgroundEffects/MaskSmoother.cs` | **TASK 51 jitter fix (2026-10-09):** temporally blends each raw mask with the previous smoothed one — `CurrentWeight` 0.75 × new + 0.25 × previous — so a per-frame segmentation verdict that wavers at an edge doesn't flutter the cut-out. Null input resets history; first frame/size change pass through; the returned mask never aliases the caller's array and the two buffers ping-pong (no per-frame LOH alloc: a 256×144 mask is 147 KB). State is **per camera session**, not in the segmenter (which is shared app-wide and contract-pinned to raw output). Weights from the OBS/Jitsi consensus — see `ai.md` "Webcam background effects" |
|
||||
| `BackgroundEffects/MaskSmoother.cs` | **TASK 51 jitter fix (2026-10-09):** temporally blends each raw mask with the previous smoothed one — `CurrentWeight` 0.75 × new + 0.25 × previous — so a per-frame segmentation verdict that wavers at an edge doesn't flutter the cut-out. Null input resets history; first frame/size change pass through; the returned mask never aliases the caller's array and the two buffers ping-pong (no per-frame LOH alloc: a 256×144 mask is 147 KB). State is **per camera session**, not in the segmenter (which is shared app-wide and contract-pinned to raw output). Weights from the OBS/Jitsi consensus — see [`docs/ai/capture.md`](../docs/ai/capture.md) → "Webcam background effects" |
|
||||
| `CameraDeviceInfo.cs` | `(Id, DisplayName)` for a physical capture device |
|
||||
| `ICameraEnumerator.cs` | `GetCamerasAsync()` — seam so the picker/`CameraManager` never touch WinRT (tests inject fakes) |
|
||||
| `ICameraFrameSource.cs` | `StartAsync`/`StopAsync`/`FrameAvailable(VideoFrame)` + **`SourceFailed(string)`** — seam for a running capture source (tests inject fakes) |
|
||||
@@ -30,7 +30,7 @@ External-facing logic: YouTube API, persistence. See
|
||||
| `IMicrophoneEnumerator.cs` | `GetMicrophonesAsync()` — seam so the mic picker never touches WinRT (tests inject fakes) |
|
||||
| `WinRtMicrophoneEnumerator.cs` | WinRT mic enumeration via `DeviceInformation.FindAllAsync(DeviceClass.AudioCapture)` (no NAudio needed — capture libs stay deferred to the audio pipeline milestone) |
|
||||
| `MediaCaptureFrameSource.cs` | CPU-first MediaCapture source (`MemoryPreference = Cpu`, BGRA8 via `CreateFrameReaderAsync`). **Validated, not trusted:** post-init checks (`VideoDeviceId` match, `FrameSources` non-empty, `VideoDeviceController.GetAvailableMediaStreamProperties` ≥ 1); `reader.StartAsync()` status read (throws on non-`Success`); `capture.Failed` + `CameraStreamStateChanged` subscribed → `SourceFailed` event. Fallback ladder: VideoPreview → VideoRecord retry. `SharingMode.Exclusive` + `DeviceLost` not in 19041 SDK projection; `CameraStreamState.Failed` compared by `(int)2` (projection omits member names) |
|
||||
| `CameraManager.cs` | Webcam capture ownership: refcounted by `DeviceId`, one shared `WriteableBitmap` per camera, dispatcher-coalesced ~render-rate UI updates (latest-frame drop); `PreviewBitmapChanged`/`CameraFailed` events. **First-frame proof** (4s timeout, configurable via ctor `TimeSpan?`): `AcquireAsync` returns true only after a real frame arrives — a reader that starts but never delivers (locked, suspended, dead) fails and surfaces `CameraFailed` with suspect-app names (`CameraConflictProbe`) instead of a silent empty box. `SourceFailed` forwarded from the frame source as async death signal. **TASK 51:** the one background-effects processing point — `SetRequestedEffects(device, keys)` + per-variant `GetLatestFrame`/`GetPreviewBitmap(device, key)` + `EffectPreviewBitmapChanged`; a background worker segments once per raw frame (mask temporally smoothed per camera — `MaskSmoother` — see `ai.md` "Webcam background effects") and renders every variant (latest-frame-drop); raw path untouched |
|
||||
| `CameraManager.cs` | Webcam capture ownership: refcounted by `DeviceId`, one shared `WriteableBitmap` per camera, dispatcher-coalesced ~render-rate UI updates (latest-frame drop); `PreviewBitmapChanged`/`CameraFailed` events. **First-frame proof** (4s timeout, configurable via ctor `TimeSpan?`): `AcquireAsync` returns true only after a real frame arrives — a reader that starts but never delivers (locked, suspended, dead) fails and surfaces `CameraFailed` with suspect-app names (`CameraConflictProbe`) instead of a silent empty box. `SourceFailed` forwarded from the frame source as async death signal. **TASK 51:** the one background-effects processing point — `SetRequestedEffects(device, keys)` + per-variant `GetLatestFrame`/`GetPreviewBitmap(device, key)` + `EffectPreviewBitmapChanged`; a background worker segments once per raw frame (mask temporally smoothed per camera — `MaskSmoother` — see [`docs/ai/capture.md`](../docs/ai/capture.md) → "Webcam background effects") and renders every variant (latest-frame-drop); raw path untouched |
|
||||
| `IFullScreenDetector.cs` | Seam for the win32 full-screen detector: `int? GetForegroundFullScreenMonitorIndex()`, `int PrimaryMonitorIndex()`, `IReadOnlyList<DisplayInfo> GetDisplays()` |
|
||||
| `Win32FullScreenDetector.cs` | `GetForegroundWindow` + `DwmGetWindowAttribute(DWMWA_EXTENDED_FRAME_BOUNDS)` + `MonitorFromWindow` + `GetMonitorInfo`; monitor covers all four edges → full-screen; own process excluded; monitor order = `EnumDisplayMonitors` order (static `GetMonitorHandle(int)` maps index→HMONITOR for the capture factory). `GetDisplays()` returns `DisplayInfo` (index/name/resolution/bounds/`IsPrimary`, friendly name via `EnumDisplayDevices`) for the in-app "Capture Display" picker; `PrimaryMonitorIndex()` is the auto-key fallback when no full-screen game is detected |
|
||||
| `IScreenCaptureSource.cs` | `Key` + `StartAsync`/`StopAsync`/`FrameAvailable(VideoFrame)` — seam so `ScreenCaptureManager` never touches WinRT (tests inject fakes) |
|
||||
@@ -42,7 +42,7 @@ External-facing logic: YouTube API, persistence. See
|
||||
| `WebView2Manager.cs` | **Web-source ownership (TASK 17, composition-capture since 2026-09-14):** per registered `Source { Type: WebSource }` it owns a `CoreWebView2CompositionController` + its visual tree (`RootVisualTarget` child under a 1920×1080 root `ContainerVisual` on one `Windows.UI.Composition.Compositor` created on the UI thread via the CoreMessaging `CreateDispatcherQueueController` P/Invoke), a wrapped `WebCaptureFrameSource`, and the shared crop-sized `WriteableBitmap` preview (dispatcher-coalesced copy at Render priority, `PreviewBitmapChanged` raised only on recreation). Nav + `TransparentBackgroundScript` injection unchanged. Public ctor `(Dispatcher)`; internal seam ctor `(Dispatcher, Func<string, IScreenCaptureSource>?)` for hermetic tests (no WinRT). Caller: `MainViewModel.InitWebView2()` (now invoked from `MainWindow_Loaded` — the parent HWND must exist). See `../ai.md` "Slice 14" |
|
||||
| `WebCaptureFrameSource.cs` | The `IScreenCaptureSource` for a web session: owns the `GraphicsCaptureItem` (`CreateFromVisual`) + a free-threaded `Direct3D11CaptureFramePool` (2 buffers) + session; readback via `SoftwareBitmap.CreateCopyFromSurfaceAsync` (BGRA, **`BitmapAlphaMode.Straight`** — the compositor blends straight alpha); per-frame `FindContentBounds` alpha-bbox (internal static, unit-tested) stamps `VideoFrame.CropBounds`; ring up to 2 fresh frames + Epoch reuse; `IsOpaque=false`. Mirror of `ScreenCaptureFrameSource` |
|
||||
| `Compositor/SceneCompositor.cs` | **The output compositor (TASK 4 ship step 1)**: renders a scene into the encoder's master `VideoFrame` (tightly-packed BGRA8), mirroring the XAML preview minus editing chrome — backdrop → background → elements (`UniformToFill` cover-crop, round clip, mirror, opacity, border) → branding flash → social bar (optional `socialBarFrame` + `socialBarTop` in master space, blitted last so the bar sits above the flash; the old `BlitFlash` generalized to `BlitOverlay` with source offsets). **TASK 47:** `Render` + `CompositeLayers` take a trailing `tickerFrame` blitted at (0,0) after the bar — the alert ticker is a message-scrolling dynamic overlay, never baked. Pure and WPF-free: frames injected via a `Func<SceneElement, VideoFrame?>` resolver (webcam → DeviceId, image → AssetId, backdrop → CaptureKey); output sized by `CompositorOptions` (16:9 = full master 1:1; vertical 9:16 = 607×1080 crop → 1080×1920 bilinear). Preview stays XAML (editing view); this is the output view — see `ai.md` "Scene compositor". **2026-10-16 (measurement only):** `Render`/`CompositeLayers`/`BakeStaticBase` take a trailing optional `CompositeTimings?` — every stage and every element timed (`#<i> <name>`, nested `raster`) and reported by the pump's parts line; zero cost for callers that pass nothing |
|
||||
| `Compositor/CompositeTimings.cs` | **Per-element composite timing (2026-10-16, measurement only)**: the collector the pump owns for a stats window — `Add(stage, ticks)` accumulates, `Describe(renders, minMs = 0.1)` prints the averaged breakdown slowest-first. Stage names: `backdrop`/`base`/`flash`/`social`/`ticker`/`zoom`/`scale`/`overlay`, `bake backdrop`/`bake overlays`/`bake #i name`, and **one entry per element `#<z-index> <name>`** with the nested `raster` sub-stage (paste-cache rasterize miss) — the line that names WHICH of the six elements eats the render budget. See `ai.md` "Per-element composite timing + scene dump" |
|
||||
| `Compositor/CompositeTimings.cs` | **Per-element composite timing (2026-10-16, measurement only)**: the collector the pump owns for a stats window — `Add(stage, ticks)` accumulates, `Describe(renders, minMs = 0.1)` prints the averaged breakdown slowest-first. Stage names: `backdrop`/`base`/`flash`/`social`/`ticker`/`zoom`/`scale`/`overlay`, `bake backdrop`/`bake overlays`/`bake #i name`, and **one entry per element `#<z-index> <name>`** with the nested `raster` sub-stage (paste-cache rasterize miss) — the line that names WHICH of the six elements eats the render budget. See [`docs/ai/frame-pipeline.md`](../docs/ai/frame-pipeline.md) → "Per-element composite timing + scene dump" |
|
||||
| `Compositor/CompositorOptions.cs` | The active tier's output rect (source space over the 1920×1080 master, integer-aligned — `MainViewModel.OutputRectX` can be 656.5) + target W×H |
|
||||
| `Compositor/StretchMath.cs` | Pure pixel math: the WPF `UniformToFill` cover-crop, clamped bilinear sample/scale (unit-tested half of the compositor) |
|
||||
| `Compositor/StaticPixelCache.cs` | Asset bytes → cached BGRA8 `VideoFrame`, decoded once per content hash — the output path's raw-pixel source (the preview uses ImageCache's `BitmapImage`) |
|
||||
@@ -58,7 +58,7 @@ External-facing logic: YouTube API, persistence. See
|
||||
| `Encoder/FfmpegArgs.cs` | Pure FFmpeg command-line builder: rawvideo `pipe:0` input, `anullsrc` silent audio (WASAPI replaces it), H.264+AAC, closed GOP (`-g fps×4 -keyint_min -sc_threshold 0 -bf 0`, `yuv420p`), FLV → RTMP |
|
||||
| `Encoder/FfmpegProgressParser.cs` | Pure parser for `frame=/fps=/size=/time=/bitrate=` stats lines → `FfmpegProgress` |
|
||||
| `Encoder/FfmpegEncoderPicker.cs` | Pure H.264 encoder picker from `-encoders` output: NVENC → QSV → AMF → OpenH264; **never returns libx264** (GPL) |
|
||||
| `Encoder/FramePump.cs` | **TASK 4 ship step 5**: the live frame producer — while live it snapshots the active scene each tick, resolves every element to its latest frame (`Func<SceneElement, VideoFrame?>` resolver), composites it into the tier's output frame, and paces frames into the encoder at the tier's FPS. All collaborators constructor-injected seams; free of WPF and the capture managers. `StartAsync` never throws (failures log + `Failed`); no RTMP URL = encoder skipped; `StopAsync` stops the encoder before awaiting the loop (backpressure deadlock); `ProcessFailed` self-stops. Since the bar bug-fix branch it takes an optional `socialBar: Func<(VideoFrame? Frame, SocialBarPosition Position)>?` seam — a pre-rendered bar strip composited last (above the flash) at the top edge or `SourceRectHeight − bar height`. **TASK 47:** adds an `alertTicker: Func<VideoFrame?>?` ctor seam (parallel to `socialBar`), read per tick and threaded through `RenderScene`/`RenderFull` + **mixed into `BuildFullRenderSignature`** so the ticker scroll changes the cache key. See `ai.md` "Live frame pipeline". **2026-10-16:** owns a `CompositeTimings` collector, passes it through `RenderScene`/`RenderFull`/`GetBakedBase`/`CompositeLayers`, logs `FramePump parts (N renders/5s): …` (per-stage breakdown, slowest first) beside the stats line each 5s window, and dumps the scene shape — `FramePump scene dump:` header + one line per element (z-index, name, kind, type, visibility, rect, opacity, clip) — at pump start and on every scene change |
|
||||
| `Encoder/FramePump.cs` | **TASK 4 ship step 5**: the live frame producer — while live it snapshots the active scene each tick, resolves every element to its latest frame (`Func<SceneElement, VideoFrame?>` resolver), composites it into the tier's output frame, and paces frames into the encoder at the tier's FPS. All collaborators constructor-injected seams; free of WPF and the capture managers. `StartAsync` never throws (failures log + `Failed`); no RTMP URL = encoder skipped; `StopAsync` stops the encoder before awaiting the loop (backpressure deadlock); `ProcessFailed` self-stops. Since the bar bug-fix branch it takes an optional `socialBar: Func<(VideoFrame? Frame, SocialBarPosition Position)>?` seam — a pre-rendered bar strip composited last (above the flash) at the top edge or `SourceRectHeight − bar height`. **TASK 47:** adds an `alertTicker: Func<VideoFrame?>?` ctor seam (parallel to `socialBar`), read per tick and threaded through `RenderScene`/`RenderFull` + **mixed into `BuildFullRenderSignature`** so the ticker scroll changes the cache key. See [`docs/ai/frame-pipeline.md`](../docs/ai/frame-pipeline.md) → "Live frame pipeline". **2026-10-16:** owns a `CompositeTimings` collector, passes it through `RenderScene`/`RenderFull`/`GetBakedBase`/`CompositeLayers`, logs `FramePump parts (N renders/5s): …` (per-stage breakdown, slowest first) beside the stats line each 5s window, and dumps the scene shape — `FramePump scene dump:` header + one line per element (z-index, name, kind, type, visibility, rect, opacity, clip) — at pump start and on every scene change |
|
||||
| `Encoder/FfmpegLocator.cs` | The default locator: PATH probe wins, then the cache, then pull+extract (`ffmpeg.exe` **+ the `libav*.dll` family** via a staging dir, failures via `AppLog`). Pinned to BtbN `autobuild-2026-08-09-13-03` **lgpl-shared** (dynamic linking = LGPL compliance without static-relink §6 material; no GPL libx264; NVENC/QSV/AMF + libopenh264 + native AAC — see `ai.md` → Licensing). Search dirs / tools dir / downloader constructor-injected for hermetic tests. Wired into the encoder since ship step 5 |
|
||||
| `INotificationService.cs` | **Toast seam (TASK 24)**: `Show(title, message, AppNotificationSeverity)` + `AppNotificationSeverity { Info, Success, Warning, Error }` + `Info/Success/Warning/Error` extension methods — non-blocking user-facing messages; VMs/services raise through this so failures surface without MessageBox blocking the UI thread |
|
||||
| `NotificationService.cs` | The default seam impl (Notification.Wpf 11.0.0, MIT): wraps a library `NotificationManager`; routes to the in-window `NotificationArea` named `ToastArea` (MainWindow bottom-right above the footer); lifetimes via the pure `BuildRequest` mapping + `Cards` tint table — Info/Success ~4s, Warning ~8s auto-dismiss, Error sticky (`NeverExpires()` = `TimeSpan.MaxValue`, never null); dark card tints matching the health-banner language; all Show calls marshal to the captured Dispatcher. See [`../ai.md`](../ai.md) "Toast notifications" |
|
||||
|
||||
@@ -63,7 +63,7 @@
|
||||
| 45 | TEST-tab chat fix #2: insert body must declare `snippet.type` (400 MISSING_REQUIRED_FIELD) | ✅ Done (2026-09-25) | [`TASKS/task-45-chat-insert-type.md`](TASKS/task-45-chat-insert-type.md) |
|
||||
| 46 | Drawers: click outside the rail closes whichever is open — TEST added to the existing Stream Settings + YPP dismiss behavior | ✅ Done (2026-09-25) | [`TASKS/task-46-drawer-click-outside-close.md`](TASKS/task-46-drawer-click-outside-close.md) |
|
||||
| 47 | Alert box video: built-in/custom alert clip (+ six-animation fallback) with read-time fade in/out + ticker (now in the preview too, 3 display methods) + alert volume in the live mix | ✅ Done (2026-09-26) | [`TASKS/task-47-alert-videos.md`](TASKS/task-47-alert-videos.md) |
|
||||
| 48 | Distribution & packaging: **MSIX + Store IAP** | 🔶 In progress — **✅ route decided 2026-09-27 (Store MSIX + Store IAP)**; **item 1 (bundle ffmpeg) SHIPPED 2026-09-28**; **item 2 (`Package.appxmanifest` + `scripts/package-msix.sh`) SHIPPED 2026-10-22 — a real 159 MB `LlamaCasty.msix` packs**; **item 3 (remove the Velopack updater) SHIPPED 2026-10-22** — the Store updates the app. Open: items 6-8 (recording invariant, disqualifiers, pre-submission gates); items 4/5 are the creator's (Certification Kit, Partner Center listing) | [`TASKS/task-48-distribution-msix.md`](TASKS/task-48-distribution-msix.md) |
|
||||
| 48 | Distribution & packaging: **MSIX + Store IAP** | ⏸ **DEFERRED 2026-10-26 — revisit near v1, not now** (creator: the remaining programming work is far from release, and a complete UX redesign comes first). Frozen until then: items 6-8, the signing decision, the updater choice, and the route question itself. History: **✅ route decided 2026-09-27 (Store MSIX + Store IAP)**; **item 1 (bundle ffmpeg) SHIPPED 2026-09-28**; **item 2 (`Package.appxmanifest` + `scripts/package-msix.sh`) SHIPPED 2026-10-22 — a real 159 MB `LlamaCasty.msix` packs**; **item 3 (remove the Velopack updater) SHIPPED 2026-10-22** — the Store updates the app. Open: items 6-8 (recording invariant, disqualifiers, pre-submission gates); items 4/5 are the creator's (Certification Kit, Partner Center listing) | [`TASKS/task-48-distribution-msix.md`](TASKS/task-48-distribution-msix.md) |
|
||||
| 49 | Chat profanity filter (local, opt-in, non-persistent, user word list) | ☐ Queued (2026-09-27) | [`TASKS/task-49-chat-profanity-filter.md`](TASKS/task-49-chat-profanity-filter.md) |
|
||||
| 50 | One-button "Publish this video" (resumable upload of the local recording) | ☐ Queued (2026-09-28) | [`TASKS/task-50-publish-recording.md`](TASKS/task-50-publish-recording.md) |
|
||||
| 51 | **Webcam background effects: Remove + Blur** | ✅ **Built 2026-10-09** (v1 requirement) — bundled MediaPipe Selfie Segmenter → ONNX (verified vs Google's runtime), Remove + Blur with strength, per-scene config persisted, one processing point in CameraManager, radio-pill UI in the webcam's Layers properties. Creator check 2026-10-09: **Remove jitter FIXED** (temporal mask smoothing 0.75/0.25, OBS/Jitsi-derived) and **Blur "heavy" FIXED** (quadratic radius 1→6 small px, default 0.5 = 2, StreamFX-consensus); recheck on the real camera pending | [`TASKS/task-51-webcam-bg-effects.md`](TASKS/task-51-webcam-bg-effects.md) |
|
||||
@@ -78,7 +78,7 @@
|
||||
| 60 | **The border-colour drop-down names every colour** | ✅ Done — 2026-10-21 (the creator's "the #hex values are good, but add a corresponding color name/label next to the value" — the 14 colours moved out of the click handler into `Models/BorderColorPalette.cs`, and each row now reads "#e94560 Crimson") | [`TASKS/task-60-border-color-names.md`](TASKS/task-60-border-color-names.md) |
|
||||
| 61 | **The geometry fields share one fixed width** | ✅ Done — 2026-10-21 (the creator's "the Y and H boxes should be the same width as the X and W boxes and those widths should be fixed to accomodate the max number of possible digits" — the Y/H fields used to stretch to fill the row; all four are now 44px, wide enough for 1920 plus a sign) | [`TASKS/task-61-geometry-field-widths.md`](TASKS/task-61-geometry-field-widths.md) |
|
||||
| 62 | **The border-colour field shows the colour's name next to the hex** | ✅ Done — 2026-10-21 (the creator: "the border color drop down box content should also include the name of the color next to the hex value as they in the drop-down list box" — the field keeps the hex editable and the palette's name sits right beside it, empty for a custom hex) | [`TASKS/task-62-border-colour-name-in-the-field.md`](TASKS/task-62-border-colour-name-in-the-field.md) |
|
||||
| 63 | **The border-colour picker is the app's own drop-down** | ✅ Done — 2026-10-21 (the creator's "why does the drop down under the web cam properties not show the same style for the border color drop down list box?" + "we established look-and-feel standards. And yet you keep ignoring these" — the hand-rolled `...` button and its code-built menu are gone; the border colour is now a `YtComboBox` like every other picker, the closed box shows the swatch + the colour's name, the rows show swatch + hex + name, and the hex field stays for a custom colour) | [`TASKS/task-63-border-colour-app-dropdown.md`](TASKS/task-63-border-colour-app-dropdown.md) |
|
||||
| 63 | **The border-colour picker is the app's own drop-down** | ✅ Done — 2026-10-21 (popup row click deselection fixed 2026-10-22: `LeftPanel.IsDescendantOf` now crosses `PopupRoot` → `PART_Popup` via `LogicalTreeHelper.GetParent` when `VisualTreeHelper.GetParent` returns `null`, so clicking a colour in the open `YtComboBox` updates `BorderColor` instead of exiting properties) | [`TASKS/task-63-border-colour-app-dropdown.md`](TASKS/task-63-border-colour-app-dropdown.md) |
|
||||
|
||||
---
|
||||
|
||||
@@ -182,7 +182,7 @@ second encoder path needing a "redirect". Record+simulcast is one ffmpeg with tw
|
||||
(user-queued): static (+) catalog rows (Background, YouTubeEvent, quoted labels) with reason-greying.
|
||||
- **🔶 Webcam cadence → resolution — IN PROGRESS, creator-queued 2026-10-05** (DHH/lipreading: the
|
||||
webcam judders, ~1 hitch in 5, invisible in the output file). **UNIT 0 shipped** (counters +
|
||||
`Helpers/FrameCadence.cs`; see `ai.md` → "Frame cadence"). **Needs a creator recording before the
|
||||
`Helpers/FrameCadence.cs`; see [`docs/ai/frame-pipeline.md`](docs/ai/frame-pipeline.md) → "Frame cadence"). **Needs a creator recording before the
|
||||
next unit.** Remaining units, all agreed 2026-10-05, one Good Dog test each:
|
||||
- **⛔ UNIT 1 IS NOT NEXT — re-scoped 2026-10-05.** The first real reading exonerated the camera:
|
||||
it delivers 30.0/s with zero hitches; the compositor manages **6.8 composites/s at ~141ms each**,
|
||||
@@ -191,7 +191,7 @@ second encoder path needing a "redirect". Record+simulcast is one ffmpeg with tw
|
||||
and schedule it AFTER the compositor work.
|
||||
- **✅ SHIPPED 2026-10-16 (measurement only): per-element composite timing + a scene dump.**
|
||||
`CompositeTimings` + the pump's `FramePump parts (N renders/5s): …` line and
|
||||
`FramePump scene dump:` block (see `ai.md` → "Per-element composite timing + scene dump").
|
||||
`FramePump scene dump:` block (see [`docs/ai/frame-pipeline.md`](docs/ai/frame-pipeline.md) → "Per-element composite timing + scene dump").
|
||||
One integration test: `ytLive.Tests/CompositeTimingTests.cs`. **NOW WAITING ON A CREATOR TAKE
|
||||
(Release):** the parts line must name which of the 6 elements eats the ~140ms and the dump must
|
||||
show the shape of the slow take vs the fast one (10:03 → 12:01, 29–34ms → 143–155ms, same shape).
|
||||
@@ -318,6 +318,27 @@ second encoder path needing a "redirect". Record+simulcast is one ffmpeg with tw
|
||||
problems:** the WebView2 Evergreen *runtime* is a machine prerequisite (item 2), and
|
||||
**Velopack's updater downloads and executes — its teardown is a compliance action.**
|
||||
Detail: `TASKS/task-48-distribution-msix.md` §1.
|
||||
- **⏸ TASK 48 / ALL distribution & packaging work — DEFERRED 2026-10-26 (creator ruling). Do not
|
||||
start, plan, or re-litigate any of it until we are close to v1.** Verbatim reason: *"we're not
|
||||
anywhere close to release just considering the remaining programming tasks"* and *"I am getting
|
||||
ready to do a complete UX redesign"* — packaging, signing, installer and updater decisions would
|
||||
be chasing UI and scope that are about to change, so the urgency is withdrawn, not the task.
|
||||
What is frozen:
|
||||
- **TASK 48 items 6-8** (recording invariant comment, disqualifier re-check, pre-submission gates)
|
||||
and the creator's items 4/5 (Certification Kit, Partner Center listing).
|
||||
- **The route question itself.** The map currently contradicts itself: `ai.md` →
|
||||
"Windows packaging & distribution" says the **Microsoft Store route is dead (2026-10-22)** —
|
||||
direct download from `llamacasty.com` + Steam, ship unsigned — and `MONETIZATION.md`/`MARCOM.md`
|
||||
agree, while *this file*, `TASKS/task-48-distribution-msix.md` and `HANDOFF.md` still read
|
||||
"Microsoft Store / Store IAP decided". **Resolve it in ONE distribution sweep at the revisit —
|
||||
do not patch it piecemeal now.**
|
||||
- **Code signing** (ship unsigned vs buy OV/EV) — current recorded posture: ship unsigned and
|
||||
accept the "unrecognized publisher" warning until download reputation accrues.
|
||||
- **The updater.** Deleting Velopack (TASK 48 slice 3) left the app with **no update path at all**,
|
||||
and `ytLive.Tests/StorePackagingTests.cs` currently *asserts that no updater exists*. That guard
|
||||
must be **rewritten with the chosen updater** at the revisit, never left to veto one.
|
||||
- The MSIX package + `scripts/package-msix.sh` still build (159 MB artifact) — a working artefact,
|
||||
submitted nowhere. Leave it alone.
|
||||
- **TASK 10 is now a teardown, not a build** (2026-09-27) — Store IAP deletes the Polar
|
||||
licensing path. The Polar fee tables, the perpetual-key model, and the customer-portal
|
||||
plumbing in `MONETIZATION.md` (gitignored, local) are obsolete; the **watermark-only
|
||||
|
||||
@@ -63,7 +63,7 @@ change to the file). This is the fix for the "committed syntax error" landmine.
|
||||
## Model / naming facts
|
||||
|
||||
- `Source.CaptureKey` is persisted (`window:<hwnd>`, `monitor:<n>`, `picker:<displayname>`);
|
||||
`Source.IsBackground` marks the one Background per scene. See ai.md §Screen backdrop capture.
|
||||
`Source.IsBackground` marks the one Background per scene. See [`docs/ai/capture.md`](../docs/ai/capture.md) → "Screen backdrop capture".
|
||||
- `WindowInfo` (Models) — `Title`, `ProcessName`, `Hwnd` (long), `ProcessId`;
|
||||
`Label => "{Title} — {ProcessName}"`; `CaptureKey => $"window:0x{Hwnd:X}"`.
|
||||
- `IWindowEnumerator` (Services) — `IReadOnlyList<WindowInfo> Enumerate()`; seam like
|
||||
|
||||
@@ -5,7 +5,13 @@
|
||||
> Carved out of TASK 36 item 6 (release engineering) on 2026-09-27 so distribution has one
|
||||
> owner instead of three scattered mentions.
|
||||
|
||||
**Status:** 🔶 In progress — **✅ ROUTE DECIDED 2026-09-27: Microsoft Store MSIX + Store IAP.**
|
||||
**Status:** ⏸ **DEFERRED 2026-10-26 (creator ruling) — revisit near v1, not now.** This file is the
|
||||
record of what was decided and built, not an active queue: the remaining programming work is far
|
||||
from release and a complete UX redesign comes first. See `TASKS.md` → Open items → the ⏸ TASK 48
|
||||
bullet for exactly what is frozen (items 6-8, signing, updater, and the route contradiction against
|
||||
`ai.md`). Do not start or re-litigate any of it before that revisit.
|
||||
|
||||
**Historical status:** 🔶 In progress — **✅ ROUTE DECIDED 2026-09-27: Microsoft Store MSIX + Store IAP.**
|
||||
Polar is deleted; every licensing subsystem goes with it.
|
||||
|
||||
**Goal:** get LlamaCasty in front of a user without a SmartScreen scarewall, ideally without
|
||||
|
||||
@@ -31,6 +31,17 @@ not *"is this control type themed?"*.
|
||||
matches `SelectedValue` ordinally (case-sensitively) and the palette mixes `#FFFFFF` and
|
||||
`#e94560`; the model matches by VALUE, so the hex the creator typed in any case still lands
|
||||
on the right row.
|
||||
- **Clicking a popup row stays inside `LeftPanel` (2026-10-22 fix):** a `ComboBox`'s open
|
||||
`PART_Popup` lives in its own `PopupRoot` visual tree (`VisualTreeHelper.GetParent(popupRoot)`
|
||||
is `null`), connected back to the `Popup` and `ComboBox` only via `PopupRoot`'s **logical**
|
||||
parent (`LogicalTreeHelper.GetParent(popupRoot)`). `LeftPanel.IsDescendantOf` previously used
|
||||
`child is Visual ? VisualTreeHelper.GetParent(child) : LogicalTreeHelper.GetParent(child)`,
|
||||
which returned `null` at `PopupRoot` because `PopupRoot` is a `Visual`. That caused
|
||||
`MainWindow.Window_PreviewMouseLeftButtonDown` to see `IsClickInside == false` on any click
|
||||
inside the open dropdown and clear `SelectedElement = null` — exiting the properties panel
|
||||
before the `ComboBoxItem` click could select the colour. `IsDescendantOf` now falls back to
|
||||
`LogicalTreeHelper.GetParent` and `TemplatedParent` whenever `VisualTreeHelper.GetParent`
|
||||
returns `null`.
|
||||
- **The closed box shows the swatch + the colour's NAME; the rows show swatch + hex + name**
|
||||
(the rows the creator approved in TASK 60). A ComboBox cannot do that with one
|
||||
`ItemTemplate` — WPF copies it into the read-only `SelectionBoxItemTemplate` — so
|
||||
|
||||
@@ -155,14 +155,10 @@ public partial class MainViewModel
|
||||
? OnAirImage
|
||||
: SceneThumbnailImage(scene);
|
||||
if (img == null)
|
||||
{
|
||||
var stagedLive = isStaged
|
||||
&& SceneCatalog.Is(scene.Name, SceneCatalog.Live);
|
||||
img = stagedLive ? null : LoadBackgroundImage(scene);
|
||||
}
|
||||
img = LoadBackgroundImage(scene);
|
||||
if (img != null)
|
||||
scene.Snapshot = img;
|
||||
else if (scene.Snapshot == null || (isStaged && OnAirImage == null))
|
||||
else if (scene.Snapshot == null)
|
||||
scene.Snapshot = CreatePlaceholderSnapshot(scene.Name);
|
||||
}
|
||||
}
|
||||
@@ -441,9 +437,9 @@ public partial class MainViewModel
|
||||
if (background != null)
|
||||
StampDefaultBackgroundAsset(background, scene.Name);
|
||||
}
|
||||
SeedSceneThumbnails();
|
||||
SeedOnAirImage();
|
||||
OnPropertyChanged(nameof(ShowSourcesEmptyHint));
|
||||
OnPropertyChanged(nameof(LayerCount));
|
||||
}
|
||||
|
||||
// ─── Scene strip tile artwork (creator ruling 2026-10-10) ───
|
||||
@@ -452,45 +448,14 @@ public partial class MainViewModel
|
||||
// renders (each scene's original background seen through a CRT screen). They
|
||||
// are seeded into the layout's Asset table once per scene; the preview pane
|
||||
// keeps rendering each scene's ORIGINAL background — never these tiles.
|
||||
internal static readonly Dictionary<string, string> SceneThumbnailSeedPaths = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
[SceneCatalog.Starting] = @"C:\Users\gramp\Downloads\Gemini_Generated_Image_uvq3i3uvq3i3uvq3.jpg",
|
||||
[SceneCatalog.Live] = @"C:\Users\gramp\Downloads\Gemini_Generated_Image_y27sgqy27sgqy27s.jpg",
|
||||
[SceneCatalog.Brb] = @"C:\Users\gramp\Downloads\Gemini_Generated_Image_lxcq0klxcq0klxcq.jpg",
|
||||
[SceneCatalog.Chat] = @"C:\Users\gramp\Downloads\Gemini_Generated_Image_nqosdenqosdenqos.jpg",
|
||||
[SceneCatalog.Ending] = @"C:\Users\gramp\Downloads\Gemini_Generated_Image_8je5n68je5n68je5.jpg",
|
||||
};
|
||||
|
||||
private void SeedSceneThumbnails()
|
||||
{
|
||||
var imported = false;
|
||||
foreach (var scene in Scenes)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(scene.ThumbnailAssetId)) continue;
|
||||
if (!SceneThumbnailSeedPaths.TryGetValue(scene.Name, out var path)) continue;
|
||||
if (!System.IO.File.Exists(path)) continue;
|
||||
try
|
||||
{
|
||||
// UpsertAsset is SHA256-keyed, so a re-seed never duplicates rows
|
||||
// and returns the same id. Only the Scene mapping is new state.
|
||||
var assetId = AddAsset(System.IO.File.ReadAllBytes(path));
|
||||
if (assetId == null) continue;
|
||||
scene.ThumbnailAssetId = assetId;
|
||||
imported = true;
|
||||
}
|
||||
catch { /* the strip keeps the scene's background-art fallback */ }
|
||||
}
|
||||
if (!imported) return;
|
||||
RefreshSnapshotsAsync();
|
||||
ScheduleSave();
|
||||
}
|
||||
internal static readonly Dictionary<string, string> SceneThumbnailSeedPaths = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// ─── On-air strip tile (creator ruling 2026-10-10) ───
|
||||
// A single shared image ("on-air background screen") labels whichever scene is
|
||||
// CURRENTLY PREVIEWED in the top bar — it replaces that scene's tile and jumps
|
||||
// around as the creator swaps scenes. Stored as an Asset row like any image,
|
||||
// tracked by the OnAirImageAssetId setting so the Save pruner keeps it alive.
|
||||
internal static string OnAirSeedPath = @"C:\Users\gramp\Downloads\Gemini_Generated_Image_4zajyl4zajyl4zaj.jpg";
|
||||
internal static string OnAirSeedPath = string.Empty;
|
||||
|
||||
private System.Windows.Media.Imaging.BitmapImage? OnAirImage;
|
||||
|
||||
|
||||
@@ -45,6 +45,7 @@ public partial class MainViewModel
|
||||
OnPropertyChanged(nameof(ShowEmptySceneHint));
|
||||
OnPropertyChanged(nameof(ShowPreviewPlaceholder));
|
||||
OnPropertyChanged(nameof(ShowSourcesEmptyHint));
|
||||
OnPropertyChanged(nameof(LayerCount));
|
||||
OnPropertyChanged(nameof(CanChangeBackground));
|
||||
OnPropertyChanged(nameof(CanAddWebcam));
|
||||
OnPropertyChanged(nameof(CanAddYouTubeChat));
|
||||
@@ -102,7 +103,10 @@ public partial class MainViewModel
|
||||
if (scene.IsHidden || scene == StagedScene) return;
|
||||
StagedScene = scene;
|
||||
if (IsLive)
|
||||
{
|
||||
LiveScene = scene;
|
||||
_transition.Start(SelectedTransitionType, TransitionDurationMs, null);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -186,6 +190,7 @@ public partial class MainViewModel
|
||||
}
|
||||
|
||||
public bool ShowEmptySceneHint => StagedScene != null && StagedScene.Elements.Count == 0;
|
||||
public int LayerCount => StagedScene?.Elements.Count(e => !e.IsZoomLayer) ?? 0;
|
||||
public bool ShowSourcesEmptyHint => StagedScene == null || StagedScene.Elements.Count == 0;
|
||||
|
||||
public bool IsCountdownAvailable => StagedScene != null
|
||||
|
||||
@@ -85,6 +85,7 @@ public partial class MainViewModel
|
||||
|
||||
OnPropertyChanged(nameof(ShowEmptySceneHint));
|
||||
OnPropertyChanged(nameof(ShowSourcesEmptyHint));
|
||||
OnPropertyChanged(nameof(LayerCount));
|
||||
UpdateActiveBackground();
|
||||
}
|
||||
|
||||
@@ -227,6 +228,7 @@ public partial class MainViewModel
|
||||
SelectedElement = source;
|
||||
OnPropertyChanged(nameof(ShowEmptySceneHint));
|
||||
OnPropertyChanged(nameof(ShowSourcesEmptyHint));
|
||||
OnPropertyChanged(nameof(LayerCount));
|
||||
UpdateActiveBackground();
|
||||
}
|
||||
|
||||
@@ -260,6 +262,7 @@ public partial class MainViewModel
|
||||
|
||||
OnPropertyChanged(nameof(ShowEmptySceneHint));
|
||||
OnPropertyChanged(nameof(ShowSourcesEmptyHint));
|
||||
OnPropertyChanged(nameof(LayerCount));
|
||||
UpdateActiveBackground();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,6 +65,7 @@ public partial class MainViewModel : ViewModelBase
|
||||
ResetHealth(StreamStatus.Offline);
|
||||
_audioMixer.StartLive(EncoderOptions.DefaultAudioPipeName);
|
||||
IsRecording = true;
|
||||
LiveScene = StagedScene;
|
||||
await _framePump.StartAsync();
|
||||
}
|
||||
|
||||
@@ -222,6 +223,7 @@ public partial class MainViewModel : ViewModelBase
|
||||
// only once frames flow. The liveChatId (snippet.liveChatId) does not exist
|
||||
// before that (2026-09-25 diagnosis), so chat resolution runs AFTER the pump
|
||||
// starts. Chat is non-fatal — if it fails, the stream continues without chat.
|
||||
LiveScene = StagedScene;
|
||||
await _framePump.StartAsync(); // never throws; failures log + surface via Failed
|
||||
|
||||
string? liveChatId = null;
|
||||
@@ -324,6 +326,7 @@ public partial class MainViewModel : ViewModelBase
|
||||
|
||||
StreamStatus = StreamStatus.Offline;
|
||||
IsRecording = false;
|
||||
LiveScene = null;
|
||||
// TASK 41: a stopped test broadcast forgets its drawer session (idempotent)
|
||||
// and drops the TEST face — a test never outlives its run.
|
||||
TestSession.EndTestSilently();
|
||||
|
||||
@@ -375,6 +375,7 @@ public partial class MainViewModel
|
||||
|
||||
OnPropertyChanged(nameof(ShowEmptySceneHint));
|
||||
OnPropertyChanged(nameof(ShowSourcesEmptyHint));
|
||||
OnPropertyChanged(nameof(LayerCount));
|
||||
UpdateActiveBackground();
|
||||
|
||||
// PreviewBitmapChanged fires only on the camera's first frame, so a config
|
||||
|
||||
@@ -438,7 +438,7 @@ public partial class MainViewModel : ViewModelBase
|
||||
{
|
||||
_isLoading = false;
|
||||
}
|
||||
StagedScene = Scenes.FirstOrDefault();
|
||||
StagedScene = Scenes.FirstOrDefault(s => SceneCatalog.Is(s.Name, SceneCatalog.Starting)) ?? Scenes.FirstOrDefault();
|
||||
UpdateActiveBackground();
|
||||
UpdateActiveChatBoxPreviews();
|
||||
UpdateActiveAlertBoxPreviews();
|
||||
|
||||
@@ -0,0 +1,484 @@
|
||||
# Capture — backdrop, webcam, background effects, and the vertical tier
|
||||
|
||||
> Split out of [`ai.md`](../../ai.md) (2026-10-26) so a session reads only what its task touches;
|
||||
> `ai.md` keeps a short invariant stub for this file. Directory map: [`index.md`](index.md).
|
||||
|
||||
Every frame source in the app lands here: the locked Background layer (screen/game/window capture),
|
||||
the webcam seam and its sessions, the MediaPipe background effects, and the per-scene camera placement
|
||||
the vertical 9:16 tier uses. Content is the architecture plus the measured history that produced it;
|
||||
superseded designs are marked inline.
|
||||
|
||||
---
|
||||
|
||||
## The locked Background layer — screen, game, window capture
|
||||
|
||||
### Screen backdrop capture (TASK 3 ship task #1) — model reworked by TASK 25
|
||||
|
||||
**The Background is a locked per-screen layer (TASK 25): exactly one per canonical
|
||||
scene, always named "Background", pinned at index 0, never addable/removable/reorderable,
|
||||
absent from the (+) menu. Flavor is decided by scene: Live = `DisplayCapture` (the real-time
|
||||
desktop/game surface with Show Desktop / monitor switch / fullscreen-game detect, falling back
|
||||
to `live-background.jpg`); Starting/BRB/Chat/Ending = static `Background` art layers.**
|
||||
`MainViewModel.NormalizeBackgrounds(scenes)` (internal static) heals any loaded layout to this
|
||||
model on every load: keeps the correctly-flavored row (or converts a survivor in place — note
|
||||
`Source.Type`'s setter derives `IsBackground`, so conversions must re-assert the flag), drops
|
||||
duplicates/stale rows, seeds missing ones via `CreateBackground(name)`, renames, moves to index 0;
|
||||
non-canonical scenes get `HasBackground=false` and lose any backgrounds. `HealBackgrounds()`
|
||||
(instance) runs Normalize + stamps default art (`StampDefaultBackgroundAsset` reads
|
||||
`Assets/{scene}-background.jpg` pack resources through `AddAsset`; custom Browse art wins).
|
||||
This replaces the old five-seeder cluster (`Seed{Starting,Brb,Ending,Chat}Background`,
|
||||
`SeedLiveBackgroundAsset`, `EnsureDefaultBackground`) and `EnforceBackgroundPolicy`.
|
||||
|
||||
- **Model (schema v6):** `Source.IsBackground` (persisted) marks the one Background per scene; `Source.CaptureKey`
|
||||
(persisted) names the target — `monitor:<n>`, `window:<hwnd>`, or `picker:<displayname>`. The Live row is
|
||||
a real `Source` of `Type DisplayCapture`, inserted **first** (`MainViewModel.EnsureBackground(scene)`,
|
||||
internal static — also used by the `StagedScene` setter so a scene switch can never expose a
|
||||
missing Background), fixed at X=0/Y=0/1920×1080, and excluded from drag/hit-test/remove/reorder (remove is guarded in `RemoveElement`;
|
||||
the element template sets `IsHitTestVisible=false` for
|
||||
backgrounds; the row's remove button and "Remove Source" menu item are hidden). **`Scene.HasBackground`
|
||||
(persisted) is true for all five canonical scenes** (see `SceneCatalog.HasBackground`). Capture controls —
|
||||
"Change Capture…"/"Refresh Capture"/"Capture Display"/"Show Desktop" — exist only while the **Live** scene is
|
||||
staged (`CanChangeBackground` = staged scene is Live; the preview CanvasGrid menu binds it directly, the layer-row
|
||||
menu MultiBindings it with the row's own `IsBackground` via `Helpers/AllTrueToVisibilityConverter`). Non-Live
|
||||
screens keep the properties panel pill ("Use default background") + Browse for custom art.
|
||||
- **Settings hygiene:** `LayoutStore.Save` purges orphaned `BackgroundUseDefault_{id}` /
|
||||
`BackgroundPath_{id}` Settings keys whose id is no longer a live Source row (TASK 25 heal drops duplicates; their keys must not accumulate).
|
||||
- **Layering invariant (regression lesson, 2026-08-23):** `ActiveBackgroundImage` renders *above*
|
||||
`BackgroundImage`, so it is fed by **static-art rows only** (`Type == Background`). The capture-flavored
|
||||
Live row must never populate it or it permanently covers the desktop/game capture; Live's canvas shows
|
||||
`DisplaySource` (capture, falling back to its AssetId art). Flavor-blind `IsBackground` lookups are for
|
||||
persistence/heal/minis — not this overlay.
|
||||
- **Detection (event-driven):** `Win32FullScreenDetector` hooks `SetWinEventHook(EVENT_SYSTEM_FOREGROUND)`
|
||||
— fires on the UI thread whenever the foreground window changes. The callback runs the same
|
||||
`GetForegroundWindow` + `DwmGetWindowAttribute(DWMWA_EXTENDED_FRAME_BOUNDS)` + `MonitorFromWindow` +
|
||||
`GetMonitorInfo` check; a window is full-screen when its frame covers all four monitor edges; own
|
||||
process is excluded; monitor **index = `EnumDisplayMonitors` enumeration order** (the same order
|
||||
`ScreenCaptureSourceFactory` uses to map index→HMONITOR via `Win32FullScreenDetector.GetMonitorHandle`).
|
||||
`IFullScreenDetector` also exposes `GetDisplays()` (`DisplayInfo`: index/name/resolution/bounds/`IsPrimary`,
|
||||
friendly name via `EnumDisplayDevices`) + `PrimaryMonitorIndex()` for the in-app "Capture Display"
|
||||
submenu + `FullscreenMonitorChanged` event + `StartWatching()`/`StopWatching()`. At launch
|
||||
`ReacquireScreenCaptures` heals backgrounds (`HealBackgrounds()`), then keys the Live background to the
|
||||
full-screen game's
|
||||
monitor; when **no game is detected**, `ResolveAutoCaptureKey()` returns `null` — stale `CaptureKey`
|
||||
values are cleared, no capture session is started, and `Source.DisplaySource` falls back to
|
||||
`_imageSource` (the static asset on the backdrop element). `StartWatching()` is called at the end of
|
||||
`ReacquireScreenCaptures`. `OnFullscreenMonitorChanged` auto-redesignates when a fullscreen game appears
|
||||
on a different monitor; null (no fullscreen / own window) is ignored — the existing capture keeps
|
||||
running so loading screens / transitions don't break the stream. `RefreshBackdropAutoCapture` (bound to
|
||||
"Refresh Capture" context menu + `Activated` handler) reads the current foreground directly as a manual
|
||||
fallback. Deduplication: `_lastReportedMonitor` prevents redundant redesignates from rapid foreground
|
||||
changes.
|
||||
- **Capture (WinRT GraphicsCapture):** `ScreenCaptureFrameSource` creates a free-threaded
|
||||
`Direct3D11CaptureFramePool` (2 buffers, `B8G8R8A8UIntNormalized`) + `GraphicsCaptureSession`; frames →
|
||||
`SoftwareBitmap.CreateCopyFromSurfaceAsync` (alpha ignored) → `VideoFrame` (BGRA8), bytes read via
|
||||
`WindowsRuntimeMarshal.TryGetDataUnsafe` (the same CsWinRT-safe read the webcam path uses) — the
|
||||
`IMemoryBufferByteAccess` ComImport cast threw `Invalid cast` on **every frame** under CsWinRT, which
|
||||
flooded `startup.log` (~5 MB in a session) and burned CPU, so it is gone. Surfaces larger than the
|
||||
1920×1080 master are downscaled to the master (`DownscaleBgra`, integer 8.8 fixed-point bilinear —
|
||||
slice 16: the same two-stage math as `SceneCompositor.Bilinear`; the previous double-per-pixel
|
||||
version was ~30-45ms quiet / ~150ms under 240Hz-HDR load and froze the desktop layer ~90% of a take).
|
||||
Conversions run as up to MaxConcurrentConversions (3) overlapping readbacks
|
||||
(slice 17: the OS readback, not the downscale, is the ~47ms wall — see Slice 17) with a
|
||||
monotonic LatestFrame publish gate (`MonotonicGate`: a slow OLDER completion can never
|
||||
overwrite a newer frame), and are spaced by a 10ms floor
|
||||
(`MinConvertInterval`): the monitor delivers at the **240Hz DWM cadence** (~4.2ms), far too fast for
|
||||
the ~60/s the pump can use, so the extra arrivals are dropped (`skip busy`/`skip cadence` telemetry).
|
||||
Hand-out buffers come from a **reuse-distance ring** (`FrameRingBuffer`, depth 8, redLine 4): a
|
||||
buffer is only rewritten ≥4 rents after its last hand-out, else a fresh one is allocated — a frame a
|
||||
consumer still holds (`session.LatestFrame` survives conversions; the dispatcher preview copy lags)
|
||||
can never be read-while-overwritten (the 1742 tear). Rolling telemetry (frames/s, conv avg/max ms,
|
||||
drops, ring allocs) is logged to `startup.log` every 2s while converting. Per-frame conversion
|
||||
failures are logged at most once per 5 s (`ErrorLogThrottle`). DRM-protected content delivers black
|
||||
frames (OS limitation, documented). Frame pool pauses while the app is minimized — capture keeps
|
||||
running, the pool just stops delivering.
|
||||
- **Ownership:** `ScreenCaptureManager` mirrors `CameraManager` — refcounted by target key, one shared
|
||||
`WriteableBitmap` per key, dispatcher-coalesced latest-frame copies, `PreviewBitmapChanged`/`CaptureFailed`
|
||||
events, `ReleaseAllAsync` on re-designation. `ScreenCaptureSourceFactory.Resolve(key)` parses the key into
|
||||
a source; `PickAsync()` shows the OS `GraphicsCapturePicker` ("Change Capture…", owner window set via the
|
||||
`IInitializeWithWindow` ComImport) and returns a **transient** `picker:` key — a reload falls back to
|
||||
auto-detection.
|
||||
- **"Capture Window…" — in-app window pin (TASK 38, 2026-09-21):** the Live backdrop context menus
|
||||
(PreviewPane canvas + layer-row) offer a submenu listing visible top-level windows of other processes
|
||||
(`Win32WindowEnumerator`, an `EnumWindows` pass filtered like `Win32FullScreenDetector`:
|
||||
visible, non-cloaked via `DWMWA_CLOAKED`, non-empty rect, other-process, titled). Picking one runs
|
||||
`RedesignateBackgroundAsync("window:0x{hwnd:X}")` — the same full-bleed layer-0 capture path as the
|
||||
game/desktop, keyed straight into `ScreenCaptureSourceFactory` (`window:` already resolved there).
|
||||
**Session-scoped, never persisted:** `window:` (and `picker:`) keys are `IsTransientCaptureKey` —
|
||||
`ReacquireScreenCaptures` strips them before auto-detection on every load, because an HWND can be
|
||||
recycled to an unrelated window after restart (resurrecting a dead pin could capture the wrong
|
||||
window). **Pin wins while alive:** `OnFullscreenMonitorChanged` skips re-designation while
|
||||
`IsAliveWindowPin(current.CaptureKey)` is true — the auto game/desktop detector never steals an
|
||||
explicit pin. **No dead ends:** if the pinned window's capture session dies mid-session,
|
||||
`OnScreenCaptureFailed` (transient key only) re-runs `ReacquireScreenCaptures` → auto-fallback
|
||||
(fullscreen game → desktop → static art). Seam: `IWindowEnumerator` + `WindowInfo`
|
||||
(`Label = "Title — Process"`), `WindowEnumeratorOverride` static test seam mirrors
|
||||
`CameraEnumeratorOverride`.
|
||||
- **CsWinRT projection gaps hand-rolled:** `Windows.Graphics.Direct3D11.Direct3D11Helper` is not projected,
|
||||
so `Direct3D11Helper` P/Invokes `d3d11.dll!D3D11CreateDevice` (hardware, BGRA_SUPPORT, explicit 11.1-first
|
||||
feature array) → QI `IDXGIDevice` → the WinRT interop export
|
||||
`CreateDirect3D11DeviceFromDXGIDevice` → `MarshalInterface<IDirect3DDevice>.FromAbi` (one shared device per
|
||||
process). Do **not** switch back to the QI-for-`IDirect3DDxgiInterfaceAccess` trick: the raw D3D11 device
|
||||
no longer exposes that interface on newer Windows (verified E_NOINTERFACE on build 26200, hardware and
|
||||
WARP alike) while `CreateDirect3D11DeviceFromDXGIDevice` keeps working.
|
||||
`IInitializeWithWindow` are ComImports in `CaptureInterop.cs`. All WinRT projections were verified by
|
||||
reflection against the built `Microsoft.Windows.SDK.NET.dll` before writing the interop.
|
||||
- **Known v1 limits:** full-desktop captures are CPU-copied at native resolution (GPU downscale = encoder
|
||||
task); multi-monitor live re-targeting beyond the auto-detected game is behind the picker; picker- and
|
||||
window-pin captures don't survive reload (session-scoped by design — see the "Capture Window…" bullet).
|
||||
- **Focus-loss capture lag (known OS limit, NOT an in-app throttle — deferred):** when the app window loses
|
||||
focus the preview visibly slows (mouse-movement lag). Nothing in the repo checks focus to slow capture —
|
||||
the only focus hooks (`FullscreenMonitorChanged` event + `RefreshBackdropAutoCapture`) re-target the backdrop, never
|
||||
pace it. The lag is inherited: `Windows.Graphics.Capture` is content-driven and DWM/OS-throttled while the
|
||||
app is in the background (worse under a full-screen game on 24H2/26100), GPU readback
|
||||
(`CreateCopyFromSurfaceAsync`) contends with the foreground game, the source's one-in-flight `_framePending`
|
||||
gate drops frames during stalls, and the manager's `DispatcherPriority.Render` copies only run as fast as
|
||||
WPF presents the window. Slice 16 shrank the per-conversion stall (fast downscale + 10ms cadence floor)
|
||||
but the OS-level delivery throttle under focus loss remains. Recorded 2026-08-13; no mitigation attempted
|
||||
yet (deferred by user decision).
|
||||
- **GPU posture:** same as webcam — CPU frames, WPF hardware-presents; D3DImage GPU compositing deferred
|
||||
to the encoder task.
|
||||
- **Preview watermark:** the "Preview" placeholder hides while a background capture renders —
|
||||
`ShowPreviewPlaceholder` also checks `BackgroundImage` (raised on change), so a scene with
|
||||
live capture shows the feed instead of the "nothing here" label.
|
||||
|
||||
---
|
||||
|
||||
## Webcam — capture seam, sessions, and background effects
|
||||
|
||||
### Webcam capture (TASK 3 milestone 1)
|
||||
|
||||
- **Seam-first:** everything above the WinRT layer speaks only `VideoFrame` (normalized tightly-packed
|
||||
BGRA8) + `CameraDeviceInfo`/`ICameraEnumerator`/`ICameraFrameSource` interfaces. Tests inject fakes;
|
||||
screen capture and background removal later feed the same seam.
|
||||
- **CPU-first:** `MediaCaptureInitializationSettings { MemoryPreference = Cpu, StreamingCaptureMode = Video,
|
||||
SharingMode = SharedReadOnly }`, frames pulled via
|
||||
`CreateFrameReaderAsync(colorSource, MediaEncodingSubtypes.Bgra8)` — the pipeline does any format
|
||||
conversion, so every `FrameArrived` yields a ready BGRA8 `SoftwareBitmap` (bytes read via
|
||||
`WindowsRuntimeMarshal.TryGetDataUnsafe`, not marshalled copies).
|
||||
- **Reader-output subtype ladder (2026-09-15 webcam-take fix):** the reader is created per-candidate
|
||||
by `ReaderSubtypeCandidates(activeSubtype)` — Bgra8 for uncompressed cameras (fast path), NV12 then
|
||||
source-default for cameras sitting in MJPG. A camera another app left in MJPG mode cannot be read as
|
||||
Bgra8 by a `MediaFrameReader` (`StartAsync` → `OutputFormatNotSupported`; the MJPG pipeline exposes
|
||||
NV12, not BGRA) — exactly what a browser grabbing the webcam produces. When the app can't re-negotiate
|
||||
(our `SetMediaStreamPropertiesAsync` refuses with "file in use / CaptureMode is SharedReadOnly" under
|
||||
msedge + NVIDIA Broadcast contention), an MJPG-active reader otherwise aborts the whole acquisition.
|
||||
Each candidate is allocated AND started under its OWN catch — WinRT answers an unsupported subtype
|
||||
with a THROW (`E_INVALIDARG`), not a status; one rejected format must never abort the ladder. Failures
|
||||
degrade to the next candidate; rejections are logged + collected into the final error; only the outer
|
||||
catch nets device-level errors (see `MyMistakes.md` → WINRT RESOURCE-ALLOCATION RECIPE). Anything not
|
||||
Bgra8 is converted in `OnFrameArrived`, which already handled non-BGRA software bitmaps.
|
||||
- **Source pick, not first hit:** the frame reader is bound to the first source that is `VideoPreview`
|
||||
(preferred) or `VideoRecord`, not blindly the first preview source. If a camera exposes neither, the
|
||||
failure names the device and the stream types it *does* expose. `SharedReadOnly` lets the capture
|
||||
coexist with other apps that share the camera.
|
||||
- **Known failure: NVIDIA Broadcast** — it opens the physical webcam exclusively, so `InitializeAsync`
|
||||
fails with "camera in use" (or, if init slips through, the device exposes no preview source). Fix:
|
||||
quit Broadcast while streaming, or pick its virtual "NVIDIA Broadcast" device from the picker and the
|
||||
app captures the processed feed. This is a real-device finding (Logitech `VID_046D&PID_082D`).
|
||||
Correction (2026-10-05): **Windows Studio Effects is NOT a second option of this kind** — it runs
|
||||
inside the physical camera's own driver, so it never appears as a selectable device. Only tools
|
||||
that install a virtual camera (NVIDIA Broadcast, XSplit VCam) do. "Pick the vcam" is a user action in
|
||||
the camera picker, never an automatic substitution: creator ruling 2026-10-05 — the app must not
|
||||
bypass the user's explicit camera choice, so there is deliberately **no fallback-camera setting**.
|
||||
- **TFM:** `net8.0-windows10.0.19041.0` (app + tests) pulls the WinRT projection from the SDK reference
|
||||
packs — no NuGet package, no capability manifest (unpackaged desktop app works; the Windows privacy
|
||||
camera toggle still applies). `EnableWindowsTargeting` keeps WSL builds working.
|
||||
- **One webcam, many scenes (schema v3) — app-level default (2026-09-17):** a singleton `Webcam`
|
||||
row holds the identity (`Id`/`DeviceId`/`Name`); each scene gets its own `WebcamSceneConfig`
|
||||
(position/size/clip/mirror/border/`IsVisible`). `Scene.Elements` holds images (`Source`) and, at
|
||||
most once, the webcam (`WebcamSceneConfig`); `Scene.WebcamConfig` is the accessor. `CameraManager`
|
||||
refcounts capture sessions by `DeviceId` (a session starts at `RefCount = 1`; repeat acquire bumps
|
||||
it; the last release stops + disposes). **The webcam is an app-level resource — ONE selection (the
|
||||
app default), usable in every scene that can host it.** "Add Webcam" places the existing default
|
||||
DIRECTLY (no picker); the Windows camera picker runs only for the initial selection
|
||||
(`_webcam == null`). Choosing a different camera in the picker swaps it app-wide via
|
||||
`SwapWebcamIdentityAsync` (same path "Change Webcam…" uses). **Per-scene gate, not app-wide**
|
||||
(TASK 26 superseded by creator directive 2026-09-17): `CanAddWebcam` = `StagedScene is { WebcamConfig: null } && IsWebcamAttainable`
|
||||
— a scene holding its own placement greys the row (one webcam per stream), but ANOTHER scene
|
||||
holding one does not, so Chat's camera never greys Live. **Attainable means a live lock, not just a
|
||||
saved identity:** `IsWebcamAttainable` = `_webcam != null && CameraManager.IsRunning(_webcam.DeviceId)`
|
||||
— an identity whose device was unplugged, or whose session can't start, leaves the row greyed with
|
||||
`WebcamAddToolTip` = "No webcam is currently available…" (identity alone no longer suffices). The
|
||||
max-1 reason gets the gentle "a second camera means you've graduated to OBS" line.
|
||||
Removing a scene's placement **keeps the identity** (it's the app default; `_webcam` is never
|
||||
nulled by removal) — Add stays offered for the same camera, because the app holds its own base lock
|
||||
(below).
|
||||
- **Startup resource lifecycle (2026-09-17):** the webcam is a resource the app *allocates and
|
||||
locks*, validated at startup — this answers "why is the WebCam row greyed" from inside the app
|
||||
instead of a DB spelunk. `MainViewModel.ValidateWebcamResourceStartupAsync` (fired fire-and-forget
|
||||
right after `LoadLayout`, stored as `WebcamStartupValidationTask` so tests await it) enumerates the
|
||||
OS once and tri-states on the count: **0** → app runs on, layer inactive, no alarm; **1** → attempt
|
||||
`CameraManager.AcquireAsync(device)` as an app-wide lock — on failure a **persistent red alert**
|
||||
(`WebcamLockAlert`, bottom of the Layers panel with a Retry button) re-polls every 5 s
|
||||
(`_webcamLockPollTimer`) until a lock succeeds, and any first real frame also clears it; **≥2** →
|
||||
deliberately no auto-lock — selection belongs to the App Settings dialog (gear) **queued — TASK 40
|
||||
Unit A (App Settings round, plan saved 2026-09-22)**.
|
||||
The single-camera branch **always acquires** (it no longer skips when configs already hold the
|
||||
session): that acquire is the app's own BASE ref, so removing every scene's placement leaves
|
||||
`RefCount = 1` and the session alive — the gate stays satisfied and the Web Cam row stays offered,
|
||||
i.e. the app default outlives the scenes that render it. Acquire on a running session just bumps
|
||||
the refcount. `CameraManager.IsRunning(deviceId)` = a session exists (started OR still starting —
|
||||
a mid-start session is not "locked yet", it's in flight), and a rolled-back (failed) session leaves
|
||||
the dictionary so the pass can retry. After a successful single-cam lock, the pass **adopts the
|
||||
camera as the app default** if no identity exists yet (fresh layout) — the Web Cam layer is then
|
||||
offered in Live/Chat immediately (`CanAddWebcam` true); an already-loaded identity is never
|
||||
overwritten. Test seams mirror `LayoutPathOverride`:
|
||||
`CameraEnumeratorOverride`/`CameraFrameSourceFactoryOverride` let `WebcamStartupResourceTests`
|
||||
drive the probe without real hardware (0-cam no-alarm, 1-cam locked → identity adopted, lock-fail
|
||||
→ alert → clears on retry). **Attribution correction:** the "NVIDIA Broadcast opens the webcam exclusively" failure
|
||||
below is a *suspect-list* claim — `CameraConflictProbe` reads only running process names, no device
|
||||
handles; contention symptoms fit shared-mode/bandwidth just as well (see `MyMistakes.md`).
|
||||
The **YouTube Chat layer follows the same one-per-layout rule** (TASK 27):
|
||||
`CanAddYouTubeChat` greys out the (+) item while any scene carries a ChatBox source (raised on
|
||||
staging + elements change; `AddSource` refuses a second as defense in depth). Chat layers created
|
||||
before the "YouTube Chat" default still carried the legacy name "Chat" (commit `65641d8` era) —
|
||||
LoadLayout heals exactly that un-renamed default to "YouTube Chat"; creator renames are untouched.
|
||||
- **Round→rect restores the aspect (persisted, schema v4):** `SceneElement.ToggleClipShape()`
|
||||
snapshots the rectangular Width/Height into public `RectWidth`/`RectHeight` before going Round and
|
||||
restores them when switching back — otherwise the Round resize lock (square) would leave a square
|
||||
behind. The rect dims are **persisted** (`WebcamSceneConfig.RectWidth`/`RectHeight`, nullable), so a
|
||||
reloaded Round webcam still restores its pre-Round aspect instead of staying square. A one-time
|
||||
`HealLegacySquareRect` (load only) widens a pre-v4 `Traditional` config that ended up square to 16:9
|
||||
(keeps height; Round and explicit rect dims are untouched).
|
||||
- **Device swap / layout reload:** `ChangeWebcamAsync` (picker) and `ReacquireWebcam` (after load)
|
||||
release the old device with `ReleaseAllAsync` — a forced full drop that zeroes the refcount and
|
||||
stops the source regardless of how many scenes held it (the per-config count isn't known once the
|
||||
scenes are replaced) — then `AcquireAsync` the new device once per config.
|
||||
- **Shared bitmap, coalesced updates:** one `WriteableBitmap` per active camera, created on the UI thread
|
||||
at the device's frame size (first frame), forwarded to every `WebcamSceneConfig.VideoImageSource` via
|
||||
`PreviewBitmapChanged`. Frames arrive on a worker thread; `CameraManager` coalesces onto the dispatcher
|
||||
(at most one pending copy per session, at `Render` priority, always copying the latest frame) so a 60fps
|
||||
device never drowns the render thread.
|
||||
- **Webcam added mid-session must get the live frames:** `PreviewBitmapChanged` fires **once** (the first
|
||||
frame creates the shared bitmap); later frames only mutate that bitmap in place, so a config that didn't
|
||||
exist at first-frame time would never receive it — the empty/transparent container you'd see adding a
|
||||
webcam to Chat while Live already had the camera. `CameraManager.GetPreviewBitmap(deviceId)` exposes the
|
||||
current shared bitmap; `AddWebcamToActiveSceneAsync` assigns it to the new config right before
|
||||
`AcquireAsync`, and `ReacquireWebcam` re-propagates it to every config after a reload.
|
||||
- **Clip/mirror/border:** per-element `ClipShape` (Traditional rectangle / Round ellipse) + `IsMirrored`
|
||||
(`ScaleX = -1`) + the OSB-standard static border (`BorderColor` `#RRGGBB` or `""`=none, `BorderOpacity`
|
||||
0–1, `BorderWidth` 0–20, `BorderAnimation` `None|Pulse|Chase|Rainbow|Shimmer|MarchingAnts|Glow|
|
||||
Electricity|Sparkles`). Rendered in the preview DataTemplate; toggled from the element's right-click
|
||||
context menu (webcam menu: Change Webcam…, Border Effect submenu — all 9 items enabled, values persist,
|
||||
rendering stays static until the animation tier ships — Border Color, Opacity/Thickness sliders, Hide in
|
||||
this scene, Remove); persisted in the layout DB. The Add menu shows when no webcam exists; the empty
|
||||
preview canvas has its own Show Webcam entry.
|
||||
- The Round webcam is an `Image Stretch="UniformToFill"` with an `EllipseGeometry` clip
|
||||
(`Center=0.5,0.5` `RadiusX/Y=0.5`), inside a `Viewbox Stretch="Uniform"` holding a `1x1` Grid, so it
|
||||
renders as a true circle (diameter = the shorter element dimension) instead of an oval stretched to
|
||||
the element rect — the traditional `Image` keeps `UniformToFill` over the full rect. The clip is
|
||||
geometry, **not an `ImageBrush`**: a brush re-rasterizes the frequently-updated `WriteableBitmap` per
|
||||
frame on the render thread, which is what made the live webcam crawl while round. The Round border is
|
||||
a centered `Ellipse` at `Width/Height = RoundBorderSize`.
|
||||
- Resizing locks to a square (`_resizeAspect = 1`) while `ClipShape == Round`.
|
||||
- **Webcam size clamp:** `ClampWebcamToBounds(config, sceneName)` (internal — test seam) enforces the
|
||||
max per dimension at resize + load and no less than 10% of the master (192×108). The cap is picked
|
||||
**by canonical scene name**: 50% per dimension (960×540) everywhere except the **Chat scene**, which
|
||||
may reach half the screen **area** (~1358×764 @16:9) so the viewer sees the creator better
|
||||
(`MaxWebcamWidthFor`/`MaxWebcamHeightFor`; a renamed Chat loses the bigger cap).
|
||||
`RoundBorderSize` follows the clamped height. `WebcamSafeguardTests` guards the clamp.
|
||||
- **Hit-testing:** a `Grid` without `Background` only hit-tests where its children draw, so clicks in
|
||||
the empty corners of a round clip fell through to `Window_PreviewMouseLeftButtonDown` and deselected
|
||||
the element — making the corner handle ungrabbable. The element Grid carries
|
||||
`Background="Transparent"` (whole rect draggable; the empty canvas Grid uses the same trick for
|
||||
right-click Show Webcam) and the `SelectionOverlay` (dashed border + corner dot) is
|
||||
`IsHitTestVisible="False"` so it never intercepts the click. Two things make the webcam menu work:
|
||||
(1) the `ContextMenu` pins its own `DataContext` to `PlacementTarget.DataContext` — a `ContextMenu`
|
||||
isn't in the visual tree, so without it the Click-handler `DataContext:` patterns (and the
|
||||
IsChecked/slider bindings) silently fail; (2) `Themes/Controls.xaml` ships a full dark `MenuItem`
|
||||
template — `PART_Popup` (submenu popups), a popup `ItemsPresenter` (the Border Opacity/Thickness
|
||||
sliders live in Items, so they render in a hover flyout), a `✓` checkmark column, and a `›` arrow
|
||||
driven by `HasItems`. An earlier bare `Border + Header` template dropped all three: submenus never
|
||||
opened, sliders never rendered, checkmarks never showed — the menu looked dead even though the
|
||||
Click handlers were fine.
|
||||
- **GPU posture:** webcam frames are CPU (GPU-agnostic; WPF hardware-presents the preview anyway). Hardware
|
||||
encoders (NVENC/AMF/QSV) matter for the encoder task, not capture. D3DImage GPU compositing is deferred
|
||||
to the encoder task.
|
||||
- **Background removal — REVERSED, now IN for v1 (creator ruling 2026-10-08, TASK 51).** The
|
||||
2026-10-05 "permanently out" ruling stood on "an in-app model means a runtime, a GPU/driver matrix,
|
||||
and per-scene tuning". The delivery shape changed: a ~430 KB Apache-2.0 model embedded in the exe,
|
||||
run on the CPU by onnxruntime — no download, no GPU requirement, no driver matrix. NVIDIA's Video
|
||||
Effects SDK stays **ruled out** (RTX-only + ~1 GB user-side download). See "Webcam background
|
||||
effects" below and `TASKS/task-51-webcam-bg-effects.md`.
|
||||
|
||||
### Webcam background effects (TASK 51 — built 2026-10-09; jitter fix 2026-10-09)
|
||||
|
||||
- **Model:** Google's **MediaPipe Selfie Segmenter, landscape** (Apache-2.0, model card 2021-05-06),
|
||||
official file `storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter_landscape/
|
||||
float16/latest/selfie_segmenter_landscape.tflite`. Converted to ONNX by our own
|
||||
`tools/selfie-seg/convert_tflite_to_onnx.py` (the graph ends in MediaPipe's custom op
|
||||
`Convolution2DTransposeBias`, which tf2onnx/tflite2onnx can't convert — 11 op types, so a direct
|
||||
translation was smaller than any workaround). **Verified against Google's own runtime:**
|
||||
`verify_against_mediapipe.py` feeds the same frame to MediaPipe's `ImageSegmenter` (original
|
||||
.tflite) and onnxruntime (our .onnx): max |diff| 5e-5, 100% binary agreement on two images.
|
||||
⛔ Do NOT swap in PINTO model-zoo #109 (it's ML Kit's model, extracted — not Apache provenance).
|
||||
obs-backgroundremoval is GPL-3.0 — license-compatible since LlamaCasty went GPL-3.0 (2026-10-09),
|
||||
but none of its code is used; if any is ever borrowed, it needs its copyright notice kept.
|
||||
- **Contract:** input `[1,144,256,3]` RGB /255 NHWC → output `[1,144,256,1]` person probability. The
|
||||
whole frame is stretched (not cropped) to 256×144, so the mask maps back by normalized coordinates.
|
||||
- **Shipping:** `Assets/Models/selfie_segmenter_landscape.onnx` is an **EmbeddedResource** (in git,
|
||||
inside the exe — nothing to lose next to it, nothing fetched at runtime). Runtime =
|
||||
`Microsoft.ML.OnnxRuntime` 1.31.0 (MIT), CPU execution provider, 2 intra-op threads. Note the native
|
||||
`onnxruntime.dll` is ~17 MB (win-x64); an RID-less build copies every RID's native folder — the MSIX
|
||||
publish (TASK 48) must pin the RID so only win-x64 ships.
|
||||
- **Code (`Services/BackgroundEffects/`):** `IPersonSegmenter` seam (`Segment(VideoFrame) →
|
||||
SegmentationMask?`, tests inject fakes); `OnnxPersonSegmenter` (lazy session from the embedded
|
||||
resource; **never throws** — a load/run failure logs once and returns null forever after);
|
||||
`BackgroundEffectProcessor.Apply(frame, BackgroundEffect)` — `None` = same instance (true
|
||||
passthrough), `Remove` = copy with alpha = bilinear-upscaled mask × source alpha, RGB untouched,
|
||||
`IsOpaque = false` (the compositor's straight-alpha blend shows the layers below), strength =
|
||||
edge feather (mask band 0.5±(0.05+0.45·s) → alpha); `Blur` (slice 2) = background downscaled 4×
|
||||
by box average, 3 separable box-blur passes and bilinear-upsampled, blended by the mask — person
|
||||
pixels (mask = 1) are copied
|
||||
untouched, opacity preserved. **Blur radius is quadratic** (`BlurRadius(strength)`: 1→6 small px,
|
||||
default 0.5 = 2 — the "blur heavy" fix 2026-10-09; the old LINEAR 1→12 small px ≈ 48 full px was
|
||||
far beyond what the established tools use: StreamFX defaults its Box blur Size to 5 and its author
|
||||
stops downscale-based blurs around 8 px). The mask is computed once per frame (`TrySegment`) and
|
||||
rendered per effect (`Render`), so several variants share one inference. No mask / segmenter
|
||||
exception → passthrough + one log line, never a dropped layer.
|
||||
- **Config (slice 3):** per scene, on `WebcamSceneConfig` — `BackgroundEffect` + `BackgroundStrength`
|
||||
(0..1, default 0.5), persisted as two additive guarded columns on `WebcamSceneConfig`
|
||||
(`BackgroundEffect TEXT DEFAULT 'None'`, `BackgroundStrength REAL DEFAULT 0.5`, no version bump —
|
||||
the TASK 47 convention). `RemoveBackground`/`BlurBackground` are the radio-pill bools the UI binds.
|
||||
- **Hookup (slice 4) — the ONE processing point is `CameraManager`.** Effects are per scene but the
|
||||
camera session is shared, so the manager renders **variants** keyed by `WebcamEffectKey(effect,
|
||||
strength rounded to 0.01)`. `MainViewModel.SyncWebcamEffects()` declares the union of every
|
||||
config's key (`SetRequestedEffects`; unused variants are dropped so their CPU stops) and points each
|
||||
config's `VideoImageSource` at its variant's bitmap; it re-runs on load, add, remove, device swap,
|
||||
and any `BackgroundEffect`/`BackgroundStrength` change (configs are watched via `PropertyChanged`,
|
||||
which also `ScheduleSave`s). Per raw frame a background worker (latest-frame-drop, one at a time)
|
||||
segments ONCE and renders every variant; each variant keeps its own latest frame (the resolver:
|
||||
`GetLatestFrame(device, key)`) and WriteableBitmap (`EffectPreviewBitmapChanged` on first
|
||||
creation). **The raw path is untouched** — a slow/failed model never stalls the plain webcam — and
|
||||
an effect variant shows **nothing** until its first processed frame (Remove never flashes the
|
||||
real room). The compositor needed no change: its sampler already honours source alpha.
|
||||
- **Temporal smoothing (2026-10-09, creator check: "removal is jittery"):** the segmenter classifies
|
||||
every frame independently, so a verdict that wavers at an edge makes the cut-out flutter. Fix:
|
||||
`MaskSmoother` blends each raw mask with the previous smoothed one — **0.75 × new + 0.25 × previous**
|
||||
(`CurrentWeight`, internal const). State is **per `CameraSession`** (one per-camera worker), NOT in
|
||||
the segmenter — the `OnnxPersonSegmenter` is shared app-wide and its bundled-model contract test
|
||||
pins the raw per-frame output. A null mask (model hiccup) resets history; the first frame and any
|
||||
size change pass through un-blended; the returned mask never aliases the caller's array and two
|
||||
buffers ping-pong (no per-frame allocation — a 256×144 mask is 147 KB, past the 85 KB LOH). Weights
|
||||
come from the established tools, not a guess: OBS's obs-backgroundremoval smoke-tests
|
||||
`temporal_smooth_factor` at its 0.85 default, Jitsi's webcam background filter (jitsi/jitsi-meet#17080)
|
||||
found 0.7/0.3 transformed best — a midpoint that still reacts to real movement; 0.5/0.5 leaves a
|
||||
smearing edge ghost. The blur "heavy" follow-up was fixed the same day: quadratic radius, 1→6
|
||||
small px (see the `Blur` bullet above).
|
||||
- **Perf (synthetic 1280×720, Debug build):** segment 1.6 ms, Remove 3.3 ms, Blur 17.5 ms per frame
|
||||
(Release is several × faster). Pixel loops are row-parallel, capped at min(4, cores/2); the
|
||||
bilinear upsampling is separable (horizontal rows precomputed per mask/small row, one vertical lerp
|
||||
per pixel) — the first naive per-pixel 2-D bilinear cost 46/80 ms. `CameraManager` logs the real
|
||||
average once per session after 120 processed frames (`startup.log`: "background effects avg …").
|
||||
`MainViewModel.PersonSegmenterOverride` is the hermetic-test seam.
|
||||
- **UI (slice 5, creator spec 2026-10-09):** the webcam's properties in the Layers panel
|
||||
(`Controls/LeftPanel.xaml`) gain a **Background** section — two `PillToggle` switches, Remove and
|
||||
Blur, bound to `RemoveBackground`/`BlurBackground` (radio-exclusive: one on turns the other off,
|
||||
turning the active one off = None) and a Strength slider (0..1, 5% snaps, visible only while an
|
||||
effect is on; Blur = amount, Remove = edge softness). Dragging the slider never blanks the webcam:
|
||||
a new strength variant inherits the dropped same-effect variant's last frame + bitmap
|
||||
(`CameraManager.SyncVariants`), and the VM keeps the shown bitmap until the new one renders.
|
||||
- **Tests:** `BackgroundEffectEngineTests` — the Remove integration test (fake segmenter), and a
|
||||
model contract test that runs the BUNDLED model through the C# preprocessing on a fixture and
|
||||
matches the MediaPipe-produced reference mask (≥99.5% agreement; fixture = MediaPipe's public test
|
||||
image `mozart_square.jpg`, public-domain painting, in `ytLive.Tests/Fixtures/`). `CameraManagerTests`
|
||||
— the ONE integration test for the jitter fix (a `SteppingMaskSegmenter` whose two consecutive
|
||||
masks disagree at an edge; asserts the second frame keeps a partial alpha where the raw mask would
|
||||
have cut to 0: x=6 → 58 upstairs of the raw 0, person core 255, background 0).
|
||||
`Blur_default_strength_is_gentle_and_full_strength_is_strong` pins the quadratic radius mapping
|
||||
(`BlurRadius(0.5)` = 2, `BlurRadius(1)` = 6) and the behaviour at a step edge: default smears
|
||||
light ~9px into the dark half vs ~26px at full strength.
|
||||
|
||||
---
|
||||
|
||||
## The vertical (9:16) tier — per-scene camera placement
|
||||
|
||||
### Vertical (9:16) tier camera placement (TASK 57 — "vv" Units 1+2, built 2026-10-21)
|
||||
|
||||
**The problem.** The vertical tier (1080×1920) is a centred **band crop** of the SAME 1920×1080
|
||||
master: `OutputRect*` = x 656.25, y 0, w 607.5, h 1080. A camera placed for 16:9 (the default
|
||||
webcam sits at x≈1408) lands **outside the cropped window entirely** — in the vertical stream the
|
||||
creator is simply not in the picture.
|
||||
|
||||
**The rule (creator's locked table, 2026-10-17).** Each scene keeps its **own** vertical
|
||||
placement — position, size and shape — and switching back to 16:9 restores the desktop layout
|
||||
exactly:
|
||||
|
||||
| mode | shape | size | position |
|
||||
|---|---|---|---|
|
||||
| 16:9 | as today | as today | as today |
|
||||
| **vv · Live** | circle **or** rect (the creator's choice, per scene) | small tile | one of **six** spots: top/bottom × left/centre/right |
|
||||
| vv · Chat | **rect only** (the circle option is not offered) | the small tile (normal), or enlarged to fill one 607×540 area | the same six spots; enlarged fills the top or the bottom area |
|
||||
| Starting / BRB / Ending | unchanged | unchanged | unchanged |
|
||||
|
||||
**Design — geometry only, no render-path change.** The preview and the compositor both read the
|
||||
element's OWN `X`/`Y`/`Width`/`Height` + `ClipShape`, so the tier switch **swaps the live
|
||||
geometry** instead of teaching the render path about tiers:
|
||||
|
||||
- `WebcamSceneConfig` carries the vertical placement (`VerticalX/Y/Width/Height/VerticalClipShape`
|
||||
+ `VerticalRectWidth/Height` — seven additive DB columns, NULL = "this scene has no vertical
|
||||
placement yet") and the **parked** 16:9 layout (`LandscapeX/…/LandscapeRectHeight`, transient).
|
||||
- `BeginVerticalLayout(...)` parks the live 16:9 layout and brings the vertical one in;
|
||||
`EndVerticalLayout()` commits the vertical placement back into the stored slot and restores the
|
||||
parked 16:9 layout. The live geometry IS the render input → **the compositor and
|
||||
`PreviewPane.xaml` are untouched**.
|
||||
- The save keeps the X/Y/W/H/ClipShape columns as the **16:9** layout (while the vertical tier is
|
||||
live, the parked values are written there and the live ones into the `Vertical*` columns).
|
||||
- `SceneElement.X/Y/Width/Height` became **virtual** so the webcam can CONSTRAIN every write
|
||||
(drag, resize handle, properties fields) into the band — one choke point instead of three
|
||||
writers re-implementing the rule. Outside the band = invisible in the vertical stream.
|
||||
|
||||
**The band and the six spots** (`Models/VerticalCameraLayout.cs`, pure math): the band is
|
||||
656.25…1263.75 × 0…1080; it splits into two 607×540 areas (top/bottom), each with three columns
|
||||
(left/centre/right) → six spots. A tile snaps to the spot its **centre** falls in, inset 32 master
|
||||
px from the band's outer edge (the app's own placement margin) and never outside the band. The
|
||||
default tile is **270×152 master px** (480×270 in the 1080×1920 output) in the **bottom-right**
|
||||
spot — the corner the 16:9 default webcam uses. The Live scene gets that tile automatically the
|
||||
first time the vertical tier is picked (otherwise the vertical stream has no camera at all); the
|
||||
other scenes keep their 16:9 placement until the creator drags them into the band.
|
||||
|
||||
**The Chat scene's own rules (Unit 2).** In the vertical tier the Chat camera is **rect only**:
|
||||
the circle is not *offered* (`MainViewModel.CanToggleClipShape` is false → the Shape button
|
||||
disappears) and the model refuses it too — `WebcamSceneConfig.ClipShape` is **overridden**, so a
|
||||
locked camera cannot become Round by ANY writer (the toggle, the button, the placement swap). The
|
||||
lock is `WebcamSceneConfig.VerticalShapeLocked`, set by the VM per scene and cleared **before** the
|
||||
16:9 layout comes back, so the desktop shape is restored. Its **size is discrete**: the small tile
|
||||
(normal) or one whole 607×540 area (enlarged) — the resize handle snaps to the nearer of the two
|
||||
(`VerticalCameraLayout.SizeFor`), and an enlarged camera **fills the area its centre is in** (top
|
||||
or bottom). The Chat scene also gets its tile automatically the first time the vertical tier is
|
||||
picked (like Live) — otherwise the vertical chat stream has no camera at all.
|
||||
|
||||
**Manual entry** (creator rule): the webcam's Layers properties gained a **Geometry** section
|
||||
(X/Y/W/H) — it edits the LIVE placement, i.e. the one the current tier shows. The webcam's
|
||||
geometry setters clamp it into the band, so a typed value can never push the camera out of the
|
||||
cropped window.
|
||||
|
||||
**Derivative work.** OBS has no built-in way to store a scene's layout per aspect ratio — the
|
||||
maintainers' answer is "create a new empty scene and add the scene you want to send as source"
|
||||
(obsproject.com/forum/threads/…161307), and Streamyard's dual-aspect guide says the same ("no
|
||||
built-in way to store separate layouts per aspect ratio, so you rely on manual workarounds and
|
||||
plugins" — streamyard.com/blog/dual-aspect-ratio-streaming-tool). Our per-scene, per-tier stored
|
||||
placement, resolved on the tier change, is the step past that.
|
||||
|
||||
**Tests.** `VerticalWebcamLayoutTests` — the ONE integration test for Unit 1: the vertical tier
|
||||
is picked → the Live camera comes INTO the band as a small tile → dragged to the top-right it
|
||||
lands top-right inside the band → back to 16:9 the desktop layout is restored exactly → and both
|
||||
placements survive a save + reload. (The store writes the webcam rows only when the app has a
|
||||
webcam identity — the config rows hang off it — so the test supplies one.)
|
||||
`VerticalChatCameraLayoutTests` — the ONE integration test for Unit 2: the vertical tier is picked
|
||||
→ the Chat camera comes into the band as a small RECT tile → the circle option is not offered and
|
||||
cannot be set even on the model → dragged up and enlarged it fills the **top** area exactly → back
|
||||
to 16:9 the desktop layout is restored exactly and the circle option is offered again.
|
||||
|
||||
**Known trap (see `MyMistakes.md`):** a second properties panel bound to `SelectedElement.IsWebcam`
|
||||
broke `WebcamBackgroundPillTests` — that test finds the webcam's panel by **binding text**, so a
|
||||
new panel must use a different binding (`IsWebcamSelected`) or the selector matches twice.
|
||||
|
||||
---
|
||||
|
||||
## Related in [`ai.md`](../../ai.md)
|
||||
|
||||
- **Resolution tiers** — `OutputRect*` and the tier list (the vertical tier's 607×1080 band).
|
||||
- **Key patterns** — MVVM/binding conventions the capture UI follows.
|
||||
- **Tests** — hermetic-real-window rules that camera tests must obey.
|
||||
- **Run** — launch/take procedure and the Release-only rule for measurements.
|
||||
@@ -0,0 +1,803 @@
|
||||
# Frame pipeline — compositor, render budget, measurement, and the live pump
|
||||
|
||||
> Split out of [`ai.md`](../../ai.md) (2026-10-26) so a session reads only what its task touches;
|
||||
> `ai.md` keeps a short invariant stub for this file. Directory map: [`index.md`](index.md).
|
||||
|
||||
This is the subsystem that turns scenes into the encoder's frames: the CPU compositor that renders
|
||||
the master, the budget it must fit inside, the instrumentation that measures it, and the
|
||||
`FramePump` that paces frames into ffmpeg. Everything here is current architecture plus the
|
||||
measured history that produced it; superseded designs are marked inline.
|
||||
|
||||
---
|
||||
|
||||
## Budget, measurement & the compositor
|
||||
|
||||
### Frame cadence — how a dropped frame became measurable (UNIT 0, 2026-10-05)
|
||||
|
||||
**The problem this exists to solve.** The recorded file cannot distinguish a dropped camera frame
|
||||
from a padded repeat. `-f rawvideo` carries no per-frame timestamps — ffmpeg assigns pts from frame
|
||||
order at the DECLARED fps — so a webcam delivering 25 of its 30 frames still muxes a full-length,
|
||||
correctly-timed 60fps file. Measured on take `ty-20261005-1001-821-0001.mp4` (85.6s, 1080p60): the
|
||||
lower-right webcam region held 25.6 distinct images/s against the camera's 30fps, with 1,153 normal
|
||||
33ms intervals, 281 gaps of 50–100ms, and 96 bunched 17ms gaps — roughly one hitch in five, which is
|
||||
what makes a face judder while the file looks flawless. (The "webcam renders at ~18fps" figure that
|
||||
used to sit in this map was WRONG: it came from a different session and counted every newly rendered
|
||||
layer, not webcam arrivals. Corrected 2026-10-05.)
|
||||
|
||||
**`Helpers/FrameCadence.cs`** — one pure counter, three feeds. `Add(timestamp)` on any shared
|
||||
monotonic clock; `Summarize()` → frames, rate (gaps ÷ span, NOT frames ÷ span — N frames span N-1
|
||||
periods), gaps, hitches (default threshold 50ms ≈ 1.5× a 30fps period), longest and average gap;
|
||||
`Describe(label)` gives every producer the same one-line shape so two lines can be read against each
|
||||
other. An out-of-order stamp counts as a frame but produces no gap AND does not become the reference
|
||||
for the next interval — otherwise one bad stamp invents a multi-second "hitch" twice.
|
||||
|
||||
**Three measurement points, and what each separates:**
|
||||
|
||||
| Line | Fed by | Answers |
|
||||
|---|---|---|
|
||||
| `Camera cadence WxH: arrivals … \| camera …` | `MediaCaptureFrameSource` | device/USB/driver cadence (`MediaFrame.SystemRelativeTime`, QPC domain, nullable) vs what the reader handed us. `TryAcquireLatestFrame` keeps only the LATEST frame, so falling behind is silent by design — this is where that shows |
|
||||
| `webcam …` on the pump's stats line | `FramePump.TimedResolver` | distinct webcam `VideoFrame.Epoch` values that reached a composite — what actually got into the output |
|
||||
| `frames per 5s from R renders` | `FramePump` | submits vs REAL composites — the duplicate-on-lag ratio, the only honest reading of how much of the stream is a repeat |
|
||||
|
||||
**Why the stats-line denominators changed (the actual bug).** Every average was divided by
|
||||
`statFrames`, the number of frames SUBMITTED — which deadline padding always fills to the full 60fps
|
||||
target. A pump fed one webcam frame per 200ms logged "300/300 frames per 5s, avg render 12.4ms"
|
||||
while producing 60 distinct composites: the render average was diluted ~5× and the loss was
|
||||
invisible. Render/resolve/wait are now divided by `statRenders`; submit stays per-frame (it really
|
||||
happens per submit). Regression test:
|
||||
`FramePumpTests.Stats_ReportRealRendersAndDistinctWebcamFrames_NotJustPaddedSubmits`.
|
||||
|
||||
**Reading the three together:** camera 30/s steady + arrivals 25/s = WE dropped it (reader or
|
||||
compositor). Ragged in both = the device, the USB link, or the driver. Pump `webcam` below
|
||||
`arrivals` = the compositor missed frames the reader had already delivered.
|
||||
|
||||
**✅ WHAT THE FIRST REAL READING SAID (2026-10-05) — the camera was never the problem.**
|
||||
`ty-20261005-1201-556-0001.mp4` + its log: the C920 delivers **30.0/s with ZERO gaps ≥50ms**, and the
|
||||
device's own stamps are identical, so the camera, USB link, and driver are all clean. The pump did
|
||||
**34 composites per 5s (6.8/s) at ~141ms each** (worst 294ms), and the file holds **6.8 distinct
|
||||
webcam images/s at 88.7% duplicates with no arrival hitches at all**. The judder is entirely us
|
||||
holding one face for 140ms and padding the rest of a 60fps container. `resolve` is 0.1ms, so it is
|
||||
pure pixel work in the composite, not frame lookup. Two consequences, both load-bearing:
|
||||
|
||||
- **720p is a sharpness win, NOT a smoothness win** — it triples the bytes stretched per render on a
|
||||
path already 4.4× over budget. Schedule the exclusive lock AFTER the compositor work or it makes
|
||||
the judder measurably worse.
|
||||
- **The 10:01 take hit 25.6 images/s and the 12:01 take 6.8, same scene shape
|
||||
(`full-render split=0 elements=6 dynamic=5`), same 640×480 camera.** Composites went 29–34ms →
|
||||
143–155ms between sessions with nothing logged explaining it. That comparison was made by EYE on
|
||||
two takes because the log carried no per-element breakdown and no scene dump — both now exist
|
||||
(see "Per-element composite timing + scene dump" below); naming the culprit is a reading, not a
|
||||
guess, and guessing is explicitly out of bounds. **Superseded 2026-10-16:** "same scene shape" was
|
||||
an eye comparison of takes whose geometry was never logged, and the next bad take still on disk is
|
||||
the *vertical* tier (below). A Release 1920×1080 take now measures 1.0ms with no element over
|
||||
0.4ms — see "Per-element composite timing + scene dump" → what the first instrumented take said.
|
||||
|
||||
**Counter subtlety, so it is not re-derived:** the pump's `webcam` line read 13/s while the file held
|
||||
6.8/s. Not a bug — the counter stamps a frame when the resolver FIRST sees it, and one composite takes
|
||||
141ms, so ~4 camera frames arrive during it (signature pass stamps one epoch, the actual draw uses a
|
||||
later one). The file is the truth; the pump counter is a pipeline-health signal and overcounts by
|
||||
design.
|
||||
|
||||
**Still unmeasured:** the camera is stuck at the C920 default 640×480 YUY2 @30 because all 185
|
||||
`SetMediaStreamPropertiesAsync` attempts are refused under `SharedReadOnly`, so 720p MJPG is available
|
||||
— but see above, it is a quality win only.
|
||||
|
||||
### Per-element composite timing + scene dump (UNIT 0 follow-up, 2026-10-16 — MEASUREMENT ONLY)
|
||||
|
||||
The gap the cadence work left open is now closed by instrumentation, and closed by nothing else: the
|
||||
141ms take was never compared to the 30ms take because **no line in the log said which element cost
|
||||
what, and no line said what the scene WAS.** Both now exist. No fix has been attempted, and none is
|
||||
allowed until a real reading names the culprit.
|
||||
|
||||
- **`Services/Compositor/CompositeTimings.cs`** — a collector owned by the pump, threaded through
|
||||
`SceneCompositor.Render` / `CompositeLayers` / `BakeStaticBase` / `BlitBakedElement` /
|
||||
`BlitCachedLayer` (and `SceneGraph.GetBakedBase` → bake path) as an optional last parameter, so
|
||||
every other caller (thumbnails, tests, transition preview) is untouched and costs nothing.
|
||||
`Add(stage, ticks)` accumulates; `Describe(renders, minMs = 0.1)` renders the averaged breakdown
|
||||
slowest-first, dropping sub-0.1ms noise.
|
||||
- **Stage names** (stable, greppable): `backdrop` (prefill + live capture + background art + their
|
||||
resolves), `base` (the cached-bake `BlockCopy`), `flash`, `social`, `ticker`, `zoom`, `scale`,
|
||||
`overlay` (the fully-static path's per-frame overlay copies), `bake backdrop`, `bake overlays`, and
|
||||
**one entry per element as `#<z-index> <name>`** — the index IS the z-order, so the log names the
|
||||
layer by both position and name. Bake-pass elements are prefixed `bake #…` so a one-off rebake
|
||||
cannot masquerade as a per-composite cost.
|
||||
- **`#<i> <name> raster` nests inside its element's total** and is the paste-cache rasterize miss
|
||||
(`BlitCachedLayer`'s `raster == null` branch) — the direct answer to "is one element re-sampling
|
||||
every tick, or is the paste itself slow". A stage's time includes that stage's frame resolve;
|
||||
`resolve` is separately totalled in the existing stats line.
|
||||
- **`FramePump parts (N renders/5s): …`** — logged once per 5s window (same window as
|
||||
`FramePump stats`), averaged per render so the numbers compare directly with `avg render`, then the
|
||||
collector is reset. **No parts line, or a short one, is a reading too:** the fully-static shortcut
|
||||
and C4 cache hits composite no elements (`cache 92R/130H` beside a missing parts line is expected).
|
||||
- **`FramePump scene dump: '<name>' elements=N split=S`** followed by one line per element
|
||||
(`[i] name kind=… type=… vis=… rect=x,y WxH op=… clip=…`), logged when the pump first runs and
|
||||
whenever the active scene changes. This is the record the 10:01/12:01 comparison never had.
|
||||
- Gate test: `ytLive.Tests/CompositeTimingTests.cs` (one `[Fact]`) — a REAL composite of a full-canvas
|
||||
element plus a 64×64 one must rank the full-canvas element first, carry the `raster` sub-stage and
|
||||
the fixed stages, and the dump must name both elements with z-index/type/rect and echo the split.
|
||||
- **`scripts/webcam-cadence.py`** — the other half of a reading: what the FILE holds (distinct webcam
|
||||
images/s + hold histogram), beside what the pump says it did. Recipe and its three traps (H.264
|
||||
re-encoding, the round-clip crop, scene switches): `MyMistakes.md` → "Measuring webcam CAMERA cadence
|
||||
in a recording".
|
||||
|
||||
**Read takes in Release.** The creator launches `dotnet run -c Release`; Debug pixel paths are
|
||||
several× slower, so a Debug number is not a reading of this pipeline (see
|
||||
[`ai.md` → Run](../../ai.md#run)).
|
||||
|
||||
**Derivative check (AGENTS.md):** this is not invented — OBS's own answer to "which source is slow" is
|
||||
the **Source Profiler**, which reports the *average execution time of each source's tick and render
|
||||
functions* ([docs.obsproject.com → Source Profiler](https://docs.obsproject.com/reference-libobs-util-source-profiler),
|
||||
per-source timers via `util/profiler.h`). Same shape here: one timer per element per composite, reported
|
||||
as an average, plus the stage split. The `#<i> <name> raster` sub-stage is the extra this case needs —
|
||||
it separates "re-sampled every tick" from "slow paste" without another session.
|
||||
|
||||
### ✅ What the first instrumented take said (2026-10-16, Release) — the judder DID NOT reproduce
|
||||
|
||||
Take `ty-20261016-1841-559-0002.mp4` (174.5s, 1920×1080@60, Release build `e95154d9` — a **recording**,
|
||||
not preview; scene `Live`, switched to `Chat` at 28s, back to `Live` at 44s) and its log window:
|
||||
|
||||
| Reading | This take |
|
||||
|---|---|
|
||||
| composites | **301 per 5s = 60/s**, every slot; 0 dropped, 0 stalls after warm-up |
|
||||
| avg render | **1.0–1.1ms**, worst 2.5ms — ~7% of the 16.67ms budget |
|
||||
| `parts` | `#5 HD Pro Webcam C920 0.4ms, #5 … raster 0.2ms, backdrop 0.2ms` — the other four elements are under the 0.1ms print floor |
|
||||
| C4 full-composite cache | **151R/150H** per window — half the ticks were a whole-frame `BlockCopy` |
|
||||
| webcam at the resolver | **30.0/s**, longest gap **50.0ms** (one extra 60Hz frame = sampling phase, not a drop) |
|
||||
| the file itself | **29.6 distinct webcam images/s, 82% of images held exactly 2 frames**, longest hold 267ms (`scripts/webcam-cadence.py`) |
|
||||
|
||||
Nothing in this take is over budget and **no element is expensive — there is no culprit here to fix.**
|
||||
The two stalls in the take are not per-composite cost: 124ms on the very first render (warm-up) and
|
||||
53ms on the switch into `Chat` (a rebake).
|
||||
|
||||
**Scale of the disagreement:** the historic slow readings are 141ms (Oct 5 12:01) and 45ms (Oct 9
|
||||
19:48) — 40–140× the ~1ms this take needed per composite. Three facts about those takes, all read off
|
||||
the same log:
|
||||
|
||||
- **Oct 9 19:48 (`ty-20261009-1948-121-0000-2.mp4`) was the VERTICAL tier**, not a 16:9 one:
|
||||
`FramePump started (1080×1920 @ 60 fps) scene='Live' comp=608x1080` — the 607×1080 centre crop
|
||||
bilinear-upscaled to 1080×1920 (2M px), 92 renders/5s, avg render 45.0ms, worst 278ms, 92 stalls,
|
||||
`cache 92R/0H`. The file on disk is 11.6s and every frame is identical (a static desktop), so it
|
||||
cannot be read for cadence — only its cost.
|
||||
- **Oct 5 12:01 (`…1201-556-0001.mp4`, no longer on disk) logged `FramePump started (1920×1080 @ 60
|
||||
fps)` with NO `scene=/comp=` suffix** — that suffix did not exist yet, so its source rect was never
|
||||
recorded and the map's "21:9 → 1920×803" explanation stays a hypothesis.
|
||||
- **A take can latch mid-take.** Oct 5 12:29 goes 1.5ms (avg render) → 3.0ms → **45.2ms within 15
|
||||
seconds and stays there** (`cache 10R/0H`, then `98R/0H`). So the slow state is not a fixed property
|
||||
of a scene geometry. The consistent signature of every slow window is `cache xR/**0H**`: the C4
|
||||
whole-frame cache never reused once, so every tick composited. The healthy take shows the other
|
||||
stable state (composite 1ms → every second tick is byte-identical → whole-frame reuse). **The latch
|
||||
is the thing to test on a reproducing take** — it is a hypothesis, not a reading; the `parts` line
|
||||
is what will decide it.
|
||||
|
||||
**What this does NOT say.** This take is the 1:1 master tier and its scene logs
|
||||
`full-render split=0 elements=6 dynamic=4` (the bad-period takes logged `dynamic=5`), so it never
|
||||
exercises the failing geometry. "The healthy path is healthy" is not "the judder is gone". Reproducing
|
||||
it needs a Release take in the failing configuration (vertical tier first, then the 21:9 display
|
||||
config) — that take's `parts` line names the stage, and only then is a fix unit allowed.
|
||||
|
||||
**Reading the `parts` line's denominators:** like `avg render`, it is divided by `statRenders`, which
|
||||
counts C4 REUSES as well as real composites (the 301 above = 151 real + 150 whole-frame copies). So in
|
||||
a healthy 30fps-camera take both averages are ~2× optimistic; the `cache NR/NH` pair on the same line
|
||||
says how much of the window was real composite work. `CompositeTimings.Describe` drops stages under
|
||||
0.1ms by default, and **a missing or short parts line is itself a reading** — a fully-static window
|
||||
(every tick a C4 hit) composites no elements at all.
|
||||
|
||||
### Render budget — getting a composite inside 16.67ms (UNITs 4-5, 2026-10-05)
|
||||
|
||||
The pump emits one encoder frame per 16.67ms slot and **pads with a duplicate when a composite
|
||||
overruns**, so the whole problem is "render + submit + untimed work ≤ 16.67ms, ideally ≤12ms to
|
||||
leave headroom". `FramePump` now reports per-stage timings (`social`/`ticker`/`flash`/`acquireScratch`)
|
||||
so a slow stage names itself, and logs the scene name + compositor source rect at pump start.
|
||||
|
||||
**Why a 21:9 monitor cost 141ms and a 16:9 one 33ms — the fast path is EXACTLY 1920×1080.** A
|
||||
21:9 display shrinks the source rect to 1920×803 ("fit", not "fill"), so the opaque-backdrop 1:1 row
|
||||
walk is skipped, the base is prefilled with opaque black (~2M byte writes), and the backdrop is
|
||||
resampled per-pixel (~1.54M pixels with an alpha branch). Every recorded A/B lines up:
|
||||
21:9 → 141ms, 16:9 → 30-33ms, 2560×1440 (fits to 1920×1080 exactly) → fast path.
|
||||
**Corrected 2026-10-16 — the "never Debug-vs-Release" half of that is withdrawn.** A Release
|
||||
1920×1080 take measures **1.0–1.1ms** where an Oct 9 take of the same tier measured 10.4ms, and
|
||||
`Build <hash>` carries no build configuration, so the config of every pre-2026-10-16 take is
|
||||
UNKNOWN: no historic render number may be compared against a Release take. The 21:9 explanation
|
||||
itself stays a hypothesis — the 141ms take's start line predates the `scene=/comp=` suffix, so its
|
||||
source rect was never recorded. A take can also latch mid-take (see the instrumentation section).
|
||||
|
||||
**Unit 4 — cheap wins, all pixel-identical:**
|
||||
- Opaque 1:1 fast paths expanded to the whole-canvas opaque case (a `BlockCopy`, not a per-pixel
|
||||
alpha branch).
|
||||
- **Prefill only the uncovered rows.** When the backdrop does not cover the canvas, the letterbox
|
||||
bands are filled instead of the whole buffer; with no backdrop at all it still fills everything.
|
||||
- **Per-column LUT** for the scaled blit (`BlitContentRaw`): the source column, its neighbour index
|
||||
and the 8.8 weights are computed once per column instead of a divide + clamp per pixel.
|
||||
The table comes from `ArrayPool<float>.Shared` and is returned in a `finally` — a single static
|
||||
buffer is a **data race** the moment strips run in parallel (see below).
|
||||
|
||||
**Unit 5 — strip-parallel composite: `Helpers/StripPool.cs`.** One composite splits into four
|
||||
horizontal row strips; the calling thread takes a strip too, so four workers never means five-way
|
||||
oversubscription. Below 64 rows the fan-out costs more than it saves and runs inline. Each strip must
|
||||
write only its own rows — the blits already partition by row, which is why the output stays
|
||||
byte-identical.
|
||||
|
||||
**The demotion must come from DEDICATED threads, not the shared pool.** The obvious
|
||||
`Thread.CurrentThread.Priority = BelowNormal` inside `Parallel.For` is wrong: `Parallel.For` runs on
|
||||
shared ThreadPool threads, so the demotion leaks into every unrelated work item that thread later
|
||||
picks up. Research (https://stackoverflow.com/questions/524483/net-backgroundworker-objects-thread-priority
|
||||
— "you have next to no control of which thread pool thread will actually run your task";
|
||||
https://stackoverflow.com/questions/1578599/c-sharp-lower-thread-priority-in-thread-pool — "You
|
||||
shouldn't mess with the priority of the regular ThreadPool, since you aren't the only consumer ...
|
||||
write your own producer/consumer queue, using your own Thread(s) as the consumer(s) - setting the
|
||||
thread priority yourself") says exactly that, and **a real failed attempt confirmed it**: a full test
|
||||
run after demoting the pool threads broke an unrelated audio-mix timing test. `StripPool` IS that
|
||||
producer/consumer queue — four `Thread`s with `Priority = BelowNormal`, `IsBackground = true`,
|
||||
started lazily on first use, each fed a `Queue<Strip>` + `SemaphoreSlim`. Four is the measured sweet
|
||||
spot: fewer leaves render over budget, more steals cycles from the game and the encoder and made
|
||||
render **worse**.
|
||||
|
||||
Gate test: `SceneCompositorTests.StripParallel_RepeatedRenders_AreByteIdentical` — the same scene
|
||||
(scaled round webcam + mirrored image + master-sized flash over a short backdrop, 240 rows so all
|
||||
three fanned-out paths run) must come out byte-for-byte identical across 8 renders, and must differ
|
||||
from a backdrop-only render so it cannot pass vacuously. It earned its keep immediately: it caught a
|
||||
loop-variable capture (`new Thread(() => Worker(i))` where `i` was `Workers` by the time the thread
|
||||
ran) that crashed the test host on first run.
|
||||
|
||||
### Scene compositor (TASK 4 ship step 1 — shipped 2026-08-10, plan in TASKS.md)
|
||||
|
||||
The encoder needs the master 1920×1080 frame **without** the preview's editing chrome (SelectionOverlay,
|
||||
DimRects, output-rect outline, badge, placeholder). WPF's `RenderTargetBitmap` is software-rendered and
|
||||
captures the visual tree *including* chrome, so the preview can't be captured — the output is a **second,
|
||||
parallel software compositor** over the `VideoFrame` (BGRA8) seam, and the XAML preview
|
||||
(`MainWindow.xaml` CanvasGrid + element DataTemplate) is the rendering contract it replicates. Two
|
||||
renderers must agree: geometry, `UniformToFill` cover-crop, round clip, mirror, border, z-order. Preview
|
||||
stays XAML (editing view); the compositor is the output view.
|
||||
|
||||
- **The `DimRects` shading is itself preview chrome, so it renders as a SIBLING of `CanvasGrid`
|
||||
(above it, below the `ZoomBox`), never a descendant** (2026-10-08, creator ruling): the zoom box's
|
||||
`VisualBrush` samples all of `CanvasGrid`, and if the dims were inside it the box would show the
|
||||
dim shading — especially after the box is dragged away and the fixed capture slides under the dims.
|
||||
|
||||
- **Render the active tier's output rect directly** (`CompositorOptions {SourceRectX/Y/W/H,
|
||||
OutputWidth, OutputHeight}`, fed from `MainViewModel.OutputRect*`): 16:9 = full 1920×1080 1:1; the
|
||||
vertical 9:16 tier = composite the centered 607×1080 crop then bilinear-upscale to 1080×1920.
|
||||
- **CPU posture:** with FFmpeg as a subprocess the master crosses a CPU readback to the pipe every frame
|
||||
anyway, so GPU compositing buys little at this layer count (2-3 live layers; static layers
|
||||
pre-composite once into a cached base). GPU effort belongs to **NVENC** (the encoder), not composition;
|
||||
if composition grows (wipes, filters, many layers), a D3D11 compositor can replace this one **behind
|
||||
the same seam** — the CPU master buffer stays the contract.
|
||||
- **Every global overlay must be composited on EVERY render path (bug found 2026-09-26).** The
|
||||
frame pump has three exits: `RenderFull` (no graph / no static base), `CompositeLayers` (static base
|
||||
+ dynamic tail), and the **fully-static shortcut**, which stretches the cached bake and returned it
|
||||
directly. That third path silently **discarded every per-frame overlay** — the social bar, the alert
|
||||
ticker and now the brand flash. A creator whose scene was one static background saw none of them,
|
||||
and the flash looked "preview-only forever" for a reason unrelated to the tier. `SceneCompositor.Overlay`
|
||||
is therefore `internal static` (it copies before blitting, so the cached bake is never mutated) and
|
||||
the shortcut now composites overlays onto a copy before stretching. **When adding an overlay, grep
|
||||
for all three paths** — a fourth exit that forgets it fails the same silent way.
|
||||
- **`VideoFrame` buffers are recycled by some producers, and `Epoch` is not optional.** A producer that
|
||||
hands out the same array across frames MUST bump `Epoch`, or `FramePump`'s buffer-identity paste cache
|
||||
false-hits and freezes stale content (the take-11 class of bug). The `C4` cache signature mixes
|
||||
`Epoch`, so a per-frame overlay naturally invalidates it. `BrandFlashPresenter` reuses one 8MB
|
||||
master buffer and stamps `Epoch` per read for exactly this reason — allocating a fresh 8MB frame per
|
||||
tick is not an option at 1080p30.
|
||||
- **Dev-only: two instances side by side (`Helpers/InstanceProfile.cs`).** Pre-1.0 the creator runs one
|
||||
instance to stream and another to screen-capture it. Nothing prevented a second instance (no mutex, no
|
||||
port); what broke it was shared state — the whole-scene read/write layout DB (clobber), Chromium's
|
||||
**exclusive** lock on the WebView2 user data folder (second instance won't start), the auth token store
|
||||
(a test instance overwrites the real YouTube sign-in), and `startup.log`. Set
|
||||
`YTLIVE_INSTANCE=<id>` and that process gets a private root at `%APPDATA%\ytLlive\instances\<id>\`
|
||||
for the DB, the auth file, the log and the WebView2 folder. Recording folder and the ffmpeg `tools\`
|
||||
cache stay shared on purpose; global hotkeys stay un-namespaced (Windows refusing the second
|
||||
`RegisterHotKey` is the correct answer). The **entire implementation is inside `#if DEBUG`** — a
|
||||
Release build compiles to `DataRoot => DefaultRoot` and `WebViewDataFolder => null`, and the call
|
||||
sites are unconditional so Release cannot drift. Proof + harness: `ytLive.Tests/InstanceIsolationTests.cs`.
|
||||
⚠️ Grepping the binary for `YTLIVE_INSTANCE` proves nothing — a `const` is inlined and appears in
|
||||
neither build; check the `InstanceVariable` **field** in metadata instead.
|
||||
- **Branding flash is composited by the output path too** (it's on the live output, per Monetization),
|
||||
passed in as a pre-rendered `VideoFrame?` — the compositor core stays pure byte-math, no WPF. Likely a
|
||||
bundled asset rather than runtime text rendering (deterministic, no font/layout risk).
|
||||
- **Frame sources are injected** via a `Func<SceneElement, VideoFrame?>` resolver
|
||||
(`SceneCompositor.Render(scene, frameFor, flashFrame, options)`) — the caller maps each element to
|
||||
its frame (webcam → `DeviceId`, image → `AssetId` via `StaticPixelCache`, backdrop → `CaptureKey`),
|
||||
so the compositor is pure, WPF-free, and hermetic to test. The capture managers wire into that
|
||||
resolver in the encoder step, not the compositor step. The master buffer (the compositor's return
|
||||
value) is the seam a future D3D11 compositor would honor identically.
|
||||
|
||||
---
|
||||
|
||||
## The live pump — frame producer to encoder
|
||||
|
||||
### Live frame pipeline (TASK 4 ship step 5 — shipped 2026-08-12, health stats 2026-08-13, plan in TASKS.md)
|
||||
|
||||
The **`FramePump`** (`Services/Encoder/`) is the live frame producer: while live it snapshots the active
|
||||
scene each tick, resolves every element to its latest frame, composites it into the tier's output frame,
|
||||
and paces frames into the encoder at the tier's FPS. **rawvideo is stamped by ARRIVAL:** ffmpeg assigns
|
||||
pts from frame order at the declared fps — supply rate = output speed. A starved producer (pump < fps)
|
||||
ships a time-lapse, truncated file with NO error (take-2 lesson, 2026-09-01); the pump therefore logs
|
||||
`FramePump stats: n/target frames per 5s from R renders, avg render Xms (resolve R2), avg submit Yms` so a slow stage names itself (the `frames … from R renders` pair and the render-count denominators arrived with UNIT 0 — see "Frame cadence" above; before that every average was divided by the SUBMITTED count, which deadline padding always fills to the full target, so a pump fed one webcam frame per 200ms still logged "300/300 frames" and its render average was diluted ~an order of magnitude) — and since 2026-09-04 so does the BUILD: every build gets a GUID stamped
|
||||
into `Helpers/BuildStamp` (csproj `GenerateBuildStamp` target, fresh per compile — no lying incremental build), logged at startup
|
||||
("Build xxxxxxxx (compiled ...)"). **Take 6 (stamp-less, ambiguous): render 35-41ms with slice 3 built or not unknown — attribution is why the stamp exists; takes 7+ are
|
||||
readable.** If the breakdown says resolve dominates, the suspects are pre-cached chat/config thrash or the web frame; if blit dominates, the general-path element sizes. **Pattern — everything is a constructor-injected
|
||||
seam:** `Func<Scene?>`, `Func<SceneElement, VideoFrame?>` resolver, `Func<CompositorOptions>`,
|
||||
`Func<EncoderOptions?>`, `Func<IFfmpegEncoder>`, `Action<string>` log, and an injectable pacing delay
|
||||
(default `Task.Delay`; tests inject `Task.Yield`). The pump is free of WPF and of the capture managers.
|
||||
|
||||
- **`StartAsync` never throws** — the VM fires-and-forgets it from the sync command handler; failures log
|
||||
+ surface via the `Failed` event. `BuildEncoderOptions` now always returns options (TASK 18) with
|
||||
`StreamEnabled`/`RecordEnabled`/`RecordPath` baked from the pills + `_activeRecordPath`; the pump skips the
|
||||
encoder only when `options == null`, i.e. "no output configured". `MainViewModel._rtmpUrlProvider` is that
|
||||
seam — a `Func<string?>` that now yields the reusable stream's ingest URL (TASK 9, shipped 2026-08-16):
|
||||
loaded from the `LayoutStore` cache at startup and set fresh by `PrepareAndStartLiveAsync` before the pump
|
||||
starts. **The pump reads the URL once at startup**, which is why go-live ensures the stream BEFORE
|
||||
`StartAsync`.
|
||||
- **Deadline pacing + row-blit render (take-3 starvation fix, 2026-09-03):** take 3 recorded 30s into a
|
||||
2.1s file and the 5s stats line named the cause in one number — `17/300 frames per 5s, avg render
|
||||
258.1ms, avg submit 1.5ms`. Two defects, both fixed: (1) the pump slept the FULL interval after each
|
||||
render, so period = render + interval — OBS's `video_thread` (libobs/media-io/video-io.c) pattern
|
||||
replaces it: absolute `nextTick += intervalTicks` deadline, sleep only the remainder, and on overrun
|
||||
skip the wait. (NOTE, slice 9: the original "skip the missed ticks (rebase)" half of this fix was
|
||||
wrong — see the slice 9 correction below.) (2) the compositor did
|
||||
per-pixel float sampling + `Math.Round` blending over all 2.07M master pixels (backdrop) and scanned
|
||||
the whole destination per overlay (a 64px social-bar strip cost 2M iterations). `SceneCompositor` now
|
||||
follows the libyuv pattern (BSD-3, chromium.googlesource.com/libyuv/libyuv — cited per the
|
||||
derivative-work rule): 1:1 aligned blits take a row-walk fast path (bilinear at scale 1 + integer
|
||||
offset is the identity) with per-pixel alpha branch and integer fixed-point blend; `BlitOverlay`
|
||||
clips to the intersection rect; a full-cover live backdrop skips the opaque-black pre-fill. The
|
||||
general per-pixel path handled scaled/round/mirrored elements. Test:
|
||||
`Pump_Paces_To_The_Deadline_Compensating_Render_Cost` records the requested wait
|
||||
via the pacing seam (a seam fake must genuinely await — a synchronous completed task runs the whole
|
||||
pump loop on `StartAsync`'s continuation and hangs the run; MyMistakes).
|
||||
- **Slice 2 of the render fix (2026-09-04, take 4):** the pacing held (file no longer truncated at
|
||||
loop level) but render stayed at 58.9ms — the per-pixel row walk was still 2M managed iterations and
|
||||
every tick allocated a fresh 8.3MB master buffer. Three changes: (1) **`VideoFrame.IsOpaque`** — a
|
||||
producer-contract flag (only the screen-capture and webcam paths set it; DWM/MediaCapture fill alpha
|
||||
255 by contract); a full-canvas, aligned, opacity-1 blit of an opaque frame is now ONE
|
||||
`Buffer.BlockCopy` (~1.5ms) instead of the loop, and the black pre-fill is skipped when the backdrop
|
||||
covers. (2) **integer fixed-point bilinear** in the general `BlitContent` path (row-hoisted invariants,
|
||||
no divisions, no `Math.Round`) — within ±1 of the float reference, inside the ±2 test tolerance.
|
||||
(3) **scratch pool in the pump** — `AcquireScratch`/`ReleaseScratch` recycle the master buffer
|
||||
(max 4, keyed by length, owned-by-reference so bake-cache/social-bar/static-art arrays can never be
|
||||
captured); release happens strictly AFTER `SubmitFrameAsync` returns (the write to stdin copies),
|
||||
and the free-list Contains guard makes the transition Cut path safe (`BlendFrame` returns the
|
||||
to-frame itself, aliasing the scratch). The per-tick `fromScene` render in the transition branch was
|
||||
**dead weight** (BlendFrame uses `TransitionService.FromFrame` captured at `Start`, never the pump's) —
|
||||
removed, and the pump's `fromSceneProvider` seam + `MainViewModel` call site went with it. Tests:
|
||||
`Pump_Pools_ScratchBuffers_Across_Frames_Without_Stale_Pixels` (the ONE: alternating backdrop colors
|
||||
pin every frame's content, repeated backing-array identity proves the pool recycles),
|
||||
`Composite_OpaqueFullCover_Backdrop_CopiesEveryPixel_Into_Scratch` (memcpy branch + scratch
|
||||
sentinel). `FakeEncoder.SubmitFrameAsync` now snapshots bytes like the real stdin write — holding the
|
||||
reference would race legitimate recycling. Take 5 must show `avg render ≤ ~10ms, ≈300/300`; the
|
||||
vertical tier's final 1080×1920 `BilinearScale` still allocates fresh per frame (same GC lesson when
|
||||
someone streams vertical — recorded as a follow-up, not silently "done").
|
||||
- **Slice 3 — chat raster-on-change (2026-09-04, take 5):** take 5 measured `138/300, avg render
|
||||
25.5ms` — the per-tick blits were cheap now, but `ResolveOutputFrame` → `RenderChatBox` ran a FULL
|
||||
WPF raster (`ChatBoxRenderer`: FormattedText + `RenderTargetBitmap` + CopyPixels + channel swap)
|
||||
EVERY tick whenever the message buffer was non-empty — and the buffer survives between sessions,
|
||||
so even a signed-out record-only take paid it. Established answer, same as OBS text sources:
|
||||
**raster on change, blit the cache every tick.** `ChatOverlayLayer` now keeps a content version
|
||||
(`Messages.CollectionChanged` → `_contentVersion++`) plus a config key (box size + all Chat*
|
||||
props); `RenderFrame` returns the cached `VideoFrame` by identity until either changes (the
|
||||
compositor only ever reads a cached frame). Test: `ChatOverlayLayerCacheTests` (RealApp, real
|
||||
renderer — `Same()` for unchanged inputs, `NotSame()` on message/config change, null on empty).
|
||||
Accepted cost pending take 6: one slow tick (~15-25ms) per arriving message; if chat-burst frame
|
||||
loss shows up, the next slice moves the re-render off-tick (debounced, dispatcher-side).
|
||||
- **Slice 5 — paste cache for non-opaque layers (2026-09-04, take 7/8 data):** the render/resolve
|
||||
split (added in the stamp commit) finally named the last thing honestly: `resolve ≈ 0` but
|
||||
`render 26-27ms` (124-135/300, ≈2.2x) — the chat fix HAD worked; the remaining cost was the
|
||||
compositor re-rasterizing EVERY layer per tick even when its pixels never changed (this creator's
|
||||
Live scene: chat 159k + web widget 271k + image 95k + webcam 156k px ≈ 680k samples @ ~38ns each).
|
||||
OBS's actual shape: sources cache their surface, the compositor pastes. `SceneCompositor` now
|
||||
routes non-opaque layers through `BlitCachedLayer`: a layer rasterizes ONCE into an element-space,
|
||||
TRANSPARENT-based frame keyed by (source-array identity, source W×H, ceil'd dst rect, round,
|
||||
mirror), then every later tick PASTES it (integer position, row alpha-blend, opacity applied at
|
||||
paste). **Correction (2026-09-12, the web-widget "black box"):** the raster builds on a
|
||||
TRANSPARENT base, but the sampler's PARTIAL-alpha branch used the opaque-dst blend — it
|
||||
premultiplied the color into RGB and forced `alpha=255`. A translucent widget pixel then pasted
|
||||
as opaque darkened ink (the box in recordings) while the raw-bitmap preview stayed correct.
|
||||
`BlitContentRaw` now takes `transparentDst` (the raster call passes `true` and writes straight
|
||||
color + straight alpha; the paste rows do the source-over). Master paths are bit-identical.
|
||||
Guard: `PasteCache_SemiTransparentLayer_RevealsBackdrop_NotOpaqueInk`. Producers hand out fresh immutable arrays, so array-identity keys can never serve stale
|
||||
content; dict bounded at 48, cleared wholesale on overflow. Only changing content (webcam device
|
||||
frames, web capture ticks, chat messages) resamples; the opaque backdrop keeps its memcpy path.
|
||||
Position/opacity drags are now near-free (no resample — paste params, not cache keys).
|
||||
ONE integration test: `PasteCache_RepeatRender_IsByteIdentical_And_ContentChangePropagates`
|
||||
(raster-vs-paste byte equality incl. round-clip margins + new-array propagation); the whole
|
||||
existing pixel suite guards sampler semantics. Take 9 must show `≈300/300, avg render ≤ ~8ms`;
|
||||
if it lands there, the recording saga closes. Follow-ups unchanged: vertical-tier scale alloc,
|
||||
debounced chat re-render under message bursts.
|
||||
- **Slice 6 — the sleep quantum WAS the ceiling (take 9, 2026-09-04):** period measured ~37ms while
|
||||
work (render+submit) was ~25 — the missing ~12ms is `Task.Delay` rounding EVERY request up to the
|
||||
Windows clock tick (documented ~15.6ms default; learn.microsoft.com/en-us/dotnet/api/system.threading.tasks.task.delay).
|
||||
A frame finishing 3ms early requests a 3ms wait and sleeps a full 15.6 — capping the producer at
|
||||
~27fps NO MATTER how fast the compositor ran. This is why takes 7→9 showed zero playback change
|
||||
despite real render wins: the sleep floor dominated everything above it. Established fix (game-loop
|
||||
canon — stackoverflow.com/questions/5441464; timeBeginPeriod — learn.microsoft.com/en-us/windows/win32/api/timeapi/nf-timeapi-timebeginperiod):
|
||||
`timeBeginPeriod(1)` for the pump's lifetime (paired `timeEndPeriod` in the finally), sleep only the
|
||||
BULK of the remainder (request minus 2ms), `Thread.SpinWait` the last ~2ms across the deadline;
|
||||
blown deadlines still rebase. (SUPERSEDED in part: slice 9 deleted the rebase — deadlines only
|
||||
advance; slice 10 replaced the burst re-write with one fresh composite per iteration.) Stats gained
|
||||
`avg wait Xms` so render+submit+wait must ≈ period —
|
||||
the accounting is closed, no stage can hide again. Same slice: the webcam's `IsOpaque` paste-cache
|
||||
bypass was removed (it re-sampled ~156k px every tick even between identical device frames; the
|
||||
cached paste beats the sampler on hits and costs the same on misses). Take 10 verdict: `≈300/300`
|
||||
honest fps — if short, the wait/render split names the remaining term with no ambiguity left.
|
||||
- **Slice 7 — the loop was ON the UI thread the whole time (take-8 numbers, 2026-09-04):** with
|
||||
the wait finally measured, take 8 (59a02a5b) showed the impossible pair — `render 22ms, wait 10ms`
|
||||
against a 16.7ms deadline: a blown deadline rebases with NO wait. The wait was the producer
|
||||
**queued behind the live preview on the dispatcher**: `PumpAsync` starts from a UI command handler
|
||||
and its `await` continuations inherit the UI `SynchronizationContext`, so the "WPF-free" compositor
|
||||
actually rendered on the UI thread and ran only when WPF let it. OBS runs its media loops on
|
||||
dedicated threads for exactly this reason. Fix: `_pumpTask = Task.Run(() => PumpAsync(...))`
|
||||
(no sync context inside → continuations stay on the pool). Side effects handled, not ignored:
|
||||
`StaticPixelCache.Get` now locks (pool miss-decodes race UI callers); `ChatOverlayLayer.RenderFrame`
|
||||
checks its cache off-thread but MARSHALS the rare raster miss to the dispatcher (DrawingVisual/
|
||||
RenderTargetBitmap are UI-thread objects) and re-validates there; pump events already marshalled.
|
||||
Also: `GCLatencyMode.SustainedLowLatency` for the pump's life, stats gained `worst render Xms`
|
||||
(spike visibility — bimodal averages hid them), webcam now routes through the paste cache (bypass
|
||||
re-sampled 156k px even between identical device frames). Tests: `Pump_Produces_OffTheStartingContext`
|
||||
(the ONE — inline-pumping SyncContext proves continuations never return to the starting thread) +
|
||||
70/70. Take 9 verdict: wait ≈ true remainder (period → 16.7, n → ~300); `worst render` names any
|
||||
remaining raster-miss spikes; render+submit+wait still == period — the accounting holds.
|
||||
- **Slice 8 — capture buffer ring + paste-cache Epoch (take 11, 2026-09-04):** take 11 validated
|
||||
the architecture — typical frames now land `work ~10ms + wait ~6.8ms = 16.7`, exactly the deadline
|
||||
(212/300). The whole remaining gap is PERIODIC 35-65ms worst-render spikes that worsened across
|
||||
the take (189→147 frames/5s) — the signature of gen2 GC pauses, now provable via `gen2 +N` in
|
||||
every stats window. Biggest churn was structural: `ScreenCaptureFrameSource` minted a fresh
|
||||
~8.3MB `byte[]` per DWM frame (~500MB/s LOH). It now rotates a shared-frame ring (4-deep at slice 8,
|
||||
deepened to 8-deep after the "flash" finding — consumer holds must never outlive depth × source
|
||||
period), size-matched per slot, with reused downscale row scratch. Recycled
|
||||
arrays would poison the paste cache (it keys on array IDENTITY), so `VideoFrame.Epoch` —
|
||||
monotonic per producer frame, 0 for fresh-array producers — joins the key. Regression test
|
||||
`PasteCache_RecycledArrayWithNewEpoch_ReRasterizes_NotStaleHits` fails on the old key by
|
||||
construction. The camera producer carried the same fresh-array churn (110-220MB/s at 30-60fps):
|
||||
it now rotates its own 8-deep ring + Epoch (same identity rule), and the WebView2 capture
|
||||
reuses a canvas scratch + 8-deep output ring + a reused WriteableBitmap instead of minting two
|
||||
fresh arrays + a new bitmap per 10Hz tick. Next suspect if gen2 stays hot: the WPF preview load
|
||||
itself driving gen2 — recorded as follow-up, untouched this slice.
|
||||
- **Slice 9 — the "rebase" WAS the recording time-lapse (2026-09-10, the recording playback-timing
|
||||
take):** slice 6's "blown deadlines still rebase" was the recording-timing bug all along. In a real
|
||||
take the pump emitted `215-219/300 per 5s` (~43fps) while `render+wait == period` still looked closed —
|
||||
the rebase ERASED every missed slot (deadline = wall-now again), so the missed ticks never showed in
|
||||
the stats. rawvideo has no per-frame timestamps: ffmpeg muxes by frame count at `-framerate 60`, so a
|
||||
43fps reality was authored into a 60fps container and **every recording played ~1.4x fast** (takes
|
||||
confirmed with a WSL ticker visible in the recording: file duration 11.44s vs ~15.5s wall). Two
|
||||
defects, both fixed the OBS way (`libobs/media-io/video-io.c` — the video thread NEVER resets its
|
||||
deadline; every interval tick outputs ONE frame, and a late render means repeated content — judder —
|
||||
never a skipped timestamp):
|
||||
(1) **count-based CFR emission:** `while (GetTimestamp() >= nextTick) { SubmitFrame(frame); nextTick += intervalTicks; statFrames++; }`
|
||||
submits exactly one frame per crossed slot, re-writing the current composite when the render overruns
|
||||
(duplicated footage = correct duration, not a time-lapse), and `nextTick` is NEVER reset to wall-now;
|
||||
(2) **`-re` removed from `FfmpegArgs`** — it was a second, fighting pacer on the rawvideo demux
|
||||
(its "Resumed reading … after a lag" grew 0.79s→4.82s across the take, leaving the pump behind its own
|
||||
honest-stats count). One fix covers the game background AND the webcam — both flow through this one
|
||||
pump. Muxed duration is now frame-count ÷ fps == wall time by construction. Test: the pump suite stays
|
||||
green unchanged — the pacing test's `<interval` wait assertion still holds because the frame emits at
|
||||
the next slot boundary, not immediately. (Unrelated pre-existing failure found the same day:
|
||||
`Composite_FullScene_MasterPixels` pixel (1380,700) cyan-vs-magenta — reproduces with this fix stashed,
|
||||
untouched by it, recorded as a follow-up.)
|
||||
- **Slice 10 — bounded encoder queue + drop policy + burned-in frame counter (2026-09-10, the
|
||||
playback-pacing take):** slice 9 made the DURATION right but the CREATOR still read the recorded
|
||||
ticker as "1...23...4...56..." — variable pacing. The aggregates (301/300, uniform PTS, 15.6s wall vs
|
||||
15.74s file) could not see it, and the cause was finally measured in `FfmpegEncoder.SubmitFrameAsync`:
|
||||
it BLOCKED on `WriteAsync(8.3MB) + FlushAsync` whenever ffmpeg lagged the pipe, and slice 9's
|
||||
burst `while` loop then re-wrote that SAME composite for every slot that ticked past — frozen
|
||||
content runs (the "smeared ticker"). Fixed the OBS way (the encoder queue in `libobs/obs-encoder.c`:
|
||||
the encoder's thread NEVER couples back into the video thread; overload = dropped content, never a
|
||||
frozen video thread):
|
||||
(1) **`FfmpegEncoder.SubmitFrameAsync` is now an ENQUEUE** into a bounded `Channel<byte[]>` (cap 120 ≈
|
||||
2s at 60fps) owned by a dedicated drain task with the stdin write + flush; the caller NEVER blocks on
|
||||
the pipe. Pixels are copied into an `ArrayPool` buffer before enqueue (the write now happens later on
|
||||
the drain thread, so the pump's scratch reusable the moment submit returns — same contract as before).
|
||||
(2) **drop-on-overflow (creator's choice: freshness over coverage):** when the queue is full the NEWEST
|
||||
frame is dropped and counted (`IFfmpegEncoder.DroppedFrames`, `Interlocked`); the session never freezes
|
||||
or smears — it drops. Stop flushes the whole queue then EOF (`Channel.TryComplete` → drain writes
|
||||
leftovers → closes stdin → ffmpeg finalizes+exits), so no accepted frame is ever lost at stop.
|
||||
(3) **pump loops ONE submit per iteration** (replacing slice 9's burst): every due slot gets a FRESH
|
||||
composite — no catch-up slot ever repeats frozen content; missed slots vanish as a count-based gap.
|
||||
The deadline counter is still never reset to wall-now (slice 9 ruling).
|
||||
(4) **burned-in frame counter (the new judge):** `_outputIndex` is burned into a 6-digit dot-matrix
|
||||
strip (white box + black 5×7 glyphs, bottom-right corner) of EVERY composite before submit. The WSL
|
||||
ticker is demoted because its own timers smear under Windows host load; decoding the recording and
|
||||
reading the strip is clock-independent: the number advances +1 per frame and jumps by exactly the
|
||||
counted drops (queue overflow OR skipped catch-up slots). Stats gained `worst submit Xms`, `dropped N`,
|
||||
`stalls K`; a stall logger names any iteration > 2× interval with its render/submit split — with the
|
||||
queue, submit is ~1ms, so a stall means render/resolve. The strip sits above the social bar (bar is
|
||||
composited, then burned over) — a debug judge, tiny at 1080p. Test:
|
||||
`Backpressure_QueueOverflow_DropsFrames_AndNeverBlocks` (the ONE: a 40ms-per-frame sink fake makes
|
||||
every enqueue fill the queue; submit must return instantly, drops are counted, stop flushes exactly
|
||||
submitted − dropped bytes). Full suite 289/290 (the pre-existing compositor pixel failure unchanged).
|
||||
Audio untouched (follow-up). WSL ticker-under-load reliability check still to run (informational — the
|
||||
burned counter is the judge).
|
||||
- **Slice 11 — web-layer animation ran at ~1/6 speed; capture cadence 10Hz→~30Hz + de-throttle
|
||||
(2026-09-10, the "widget animates too slow" take):** the recording is 60fps but the WebView2
|
||||
capture loop was a blind 100ms `DispatcherTimer` = 10Hz — a 60fps-designed widget was sampled 6×
|
||||
under its native rate (repeated-footage slow-mo). Two stacked throttles:
|
||||
> Superseded by **Slice 14** (below): `CaptureScheduler` and the cadence hooks were the polling
|
||||
> design; frame-driven composition capture replaced them — nothing here is current architecture.
|
||||
(1) **the 10Hz cap** (unconditional, code-proven) and
|
||||
(2) **Chromium hidden-page throttling** (`requestAnimationFrame` parked, JS timers clamped to ~1s
|
||||
per WebView2Feedback#1172/#3070 + Chrome-88 timer throttling) whenever the app window is unfocused
|
||||
or covered — the page is off-screen at (-5000,-5000), so it is a hidden page the moment the host
|
||||
window loses occlusion.
|
||||
Fixed:
|
||||
(1) **`CaptureScheduler` (new, `Services/CaptureScheduler.cs`)** replaces the per-session
|
||||
`DispatcherTimer`: a dispatcher timer that DROPS ticks while a capture is in flight (latest-wins,
|
||||
never queues) — the in-flight drop is what made raising the cadence safe (concurrent full-HD PNG
|
||||
`CapturePreviewAsync` calls would stack ~10-30ms encodes and publish stale-after-fresh). Interval is
|
||||
owner-configurable: recording 33ms (~30Hz), idle 200ms. Unit-tested without a WebView2 runtime
|
||||
(the ONE test: `CaptureScheduler_Drops_Ticks_While_Capture_InFlight_And_Resumes` — overlapping
|
||||
ticks dropped, capture resumes when idle, driven by a TCS so it is deterministic).
|
||||
(2) **de-throttle browser args**: the shared `CoreWebView2Environment` is created with
|
||||
`--disable-backgrounding-occluded-windows --disable-renderer-backgrounding
|
||||
--disable-features=CalculateNativeWinOcclusion` (the Electron/Streamlabs-class embedder answer for
|
||||
occluded-window animation throttling) BEFORE `EnsureCoreWebView2Async`.
|
||||
(3) **cadence hook**: `MainViewModel` sets `SetCaptureInterval(33)` on record/stream start and
|
||||
`(200)` on stop (`MainViewModel.Streaming.Operations.cs`).
|
||||
(4) **capture-cost telemetry**: first 30 captures per session log elapsed ms to startup.log —
|
||||
PNG-encode+decode cost decides whether ~30Hz stays or drops to ~20Hz; the FramePump drops frames
|
||||
(never time-lapses, slice 10) if UI-thread GC churn starves it, so the cost is observable.
|
||||
Full suite 290/291 passing, the sole failure the pre-existing compositor pixel test. The web-overlay
|
||||
transparency verification (the first-capture diagnostic bound to about:blank) and the audio-silence
|
||||
item remain open push-gate items — both untouched by this slice.
|
||||
- **Slice 12 (audio diagnostics, 2026-09-10):** the recorded audio is full-length silent AAC
|
||||
(−91dB, 1124 frames / 23.95s in the latest take) — the named-pipe delivered ~9.1MB of zeros to
|
||||
ffmpeg for the entire recording, so the loop ran and connected, but both WASAPI capture sources
|
||||
delivered nothing. Sources started fine (`Audio: using system default mic...` logged), no failure
|
||||
callbacks fired, and the mixer's existing integration test (`Mix_WithFiltersDuckAndGain_Lands_On_AudioPipe`)
|
||||
proves the loop→pipe math is sound. The fault is capture-side: either the default render endpoint
|
||||
carried nothing (audio played on a non-default device — common), or both endpoints were held
|
||||
exclusive, or genuinely nothing played. Added permanent per-5s live-loop telemetry to startup.log
|
||||
(`Audio live: pipe connected=, dropped writes=, micLevel=, loopLevel=, drained, peakMix`) and
|
||||
`NamedPipeAudioWriter.DroppedWrites` — names the exact stage on the next take without new
|
||||
code. `AudioMixer.FillAndMix` now returns `(MicRms, MicDrained, LoopDrained)` for the telemetry
|
||||
accumulation. Full suite 290/291, same pre-existing sole failure.
|
||||
- **Slice 13 (web transparency diagnostic re-point, 2026-09-10):** the "?" in MyMistakes'
|
||||
"RESOLVED (? VERIFY)" is being closed. The one-shot widget dump + alpha stats bound to the
|
||||
FIRST capture ever = the initial about:blank document — a blind instrument. NavigationCompleted
|
||||
for the REAL widget URL now arms `WidgetDumpRemaining = 5`; those 5 post-paint captures dump
|
||||
`%TEMP%\ytLive-web-<id>-w1..5.png` + the shared `AlphaStats(...)` line (alpha[min/max/mean/zero%])
|
||||
+ FindContentBounds result to startup.log. That line alone decides the fix branch: zero% ⇒ the
|
||||
pre-parse injection did not hold for this widget (opaque capture → chroma-key / stronger DOM
|
||||
injection); large zero% + tight contentBounds ⇒ the capture IS transparent and the recording's
|
||||
black block lives downstream (compositor blend/underlay). `AlphaStats` extracted as the shared
|
||||
sampler. No new tests (WebView2 runtime not instantiable in tests; the re-point is log-only).
|
||||
- **Slice 14 — web frames are now COMPOSITION-CAPTURED; the PNG poll + `CaptureScheduler` are gone
|
||||
(2026-09-14, the "still not 60fps" take):** slice 11's ~30Hz was still a `CapturePreviewAsync` PNG
|
||||
poll — every 1920×1080 full-HD encode+decode cost 35–165ms, so the session budget capped real
|
||||
cadence at ~20Hz for a 60fps-designed widget. Root fix = frame-driven capture, not faster polling:
|
||||
the WebView2 renderer now feeds `Windows.Graphics.Capture` directly through a
|
||||
`CoreWebView2CompositionController` (the mechanism `WebView2CompositionControl` and Flutter's
|
||||
`webview_windows` use — see `graphics_context.cc` `CreateGraphicsCaptureItemFromVisual`, which
|
||||
captures the root `surface_` visual).
|
||||
- **Pipeline (per session):** `CreateCoreWebView2CompositionControllerAsync(WindowHandle)` — the
|
||||
parent HWND must exist, so `MainWindow.InitWebView2()` moved from the ctor to **Loaded** (was
|
||||
`InitWebView2(WebViewHostPanel)`; the hidden XAML `WebViewHostPanel` overlay is deleted).
|
||||
`controller.RootVisualTarget` = a child `ContainerVisual` (`RelativeSizeAdjustment = 1,1`) under a
|
||||
root `ContainerVisual` (1920×1080, `IsVisible=true`) built on ONE `Windows.UI.Composition.Compositor`
|
||||
(created once on the UI thread); `Bounds = 0,0,1920,1080`, `BoundsMode=UseRawPixels`,
|
||||
`ShouldDetectMonitorScaleChanges=false`, `RasterizationScale=1.0`, `IsVisible=true`,
|
||||
`DefaultBackgroundColor=Transparent`; then `GraphicsCaptureItem.CreateFromVisual(root)`.
|
||||
Frames pull from a free-threaded `Direct3D11CaptureFramePool` (2 buffers) →
|
||||
`SoftwareBitmap.CreateCopyFromSurfaceAsync` (BGRA, `BitmapAlphaMode.Straight` — the compositor
|
||||
blends STRAIGHT alpha, so premultiplied readback would wreck corner anti-aliasing) → per-frame
|
||||
`FindContentBounds` alpha-bbox on the capture worker → `VideoFrame.CropBounds` stamped → ring up to
|
||||
2 fresh + Epoch (reuse; GC lesson slice 8) → dispatcher-coalesced copy (Render priority) into ONE
|
||||
crop-sized shared `WriteableBitmap`; `PreviewBitmapChanged` raised only on bitmap (re)creation.
|
||||
Also unpumped: the OBS-side de-throttle flags (slice 11) stay — they fix the hidden-page JS
|
||||
throttling; the capture path itself no longer polls.
|
||||
- **Implemented in:** new `Services/WebCaptureFrameSource.cs` (owns item+pool+session — mirrors
|
||||
`ScreenCaptureFrameSource`, which servers as the copy template) + reworked `Services/WebView2Manager.cs`
|
||||
(owns controllers, the visual tree, nav/script wiring, preview; internal seam ctor
|
||||
`(Dispatcher, Func<string, IScreenCaptureSource>?)` so the tests instantiate zero WinRT; public
|
||||
ctor `(Dispatcher)`).
|
||||
- **CoreMessaging DQ recipe (record-once):** the 19041 CsWinRT projection has NO
|
||||
`DispatcherQueueController.CreateOnCurrentThread()` (CS0117 — only `CreateOnDedicatedThread` +
|
||||
`FromAbi(IntPtr)`). P/Invoke `coreMessaging.dll!CreateDispatcherQueueController` with struct
|
||||
`DispatcherQueueOptions { DwSize, ThreadType = 2 (DQTYPE_THREAD_CURRENT), ApartmentType = 2 (DQTAT_COM_STA) }`,
|
||||
wrap via `DispatcherQueueController.FromAbi(ptr)` (mirror of `CaptureInterop`), only then
|
||||
`new Compositor()` — all once on the WPF UI thread (the app dispatches Render there).
|
||||
- **Compile notes (each cost a build cycle):** `Compositor` collides with the repo's OWN
|
||||
`ytLive.Services.Compositor` namespace → fully-qualify `Windows.UI.Composition.Compositor`;
|
||||
`CoreWebView2CompositionController` exposes **`Close()`**, not `Dispose()`; `Color` is ambiguous
|
||||
(`System.Drawing` vs `System.Windows.Media`) → `System.Drawing.Color.Transparent`.
|
||||
- **Dropped:** `Services/CaptureScheduler.cs` (DELETED — no cadence to schedule), all three
|
||||
`SetCaptureInterval` hooks in `MainViewModel.Streaming.Operations.cs`, the XAML `WebViewHostPanel`.
|
||||
`TransparentBackgroundScript` const is unchanged (a test pins it).
|
||||
- **Test model (Good Dog):** reworked `ytLive.Tests/WebView2ManagerTests.cs` — dropped the 4
|
||||
control-size tests + the `CaptureScheduler_Drops…` test; `FindContentBounds` tests moved to
|
||||
`WebCaptureFrameSource.FindContentBounds`; the ONE integration test
|
||||
`Frames_PublishCroppedPreview_And_CoalesceToLatest_CarryingCropBounds` drives the internal seam with
|
||||
`FakeWebSource : IScreenCaptureSource` + a real background-STA `DispatcherPump` (copied from
|
||||
`ScreenCaptureManagerTests`): register → one crop-sized preview bitmap published → `CropBounds` +
|
||||
`IsOpaque=false` on `GetLatestFrame` → back-to-back pumps coalesce to latest. Byte assertion compares
|
||||
the DENSE 2×2 crop (`CropBytes` helper — slices source rows with stride gaps, a contiguous range
|
||||
spans rows wrongly).
|
||||
- Full suite **293/293 green, 0 warnings**. Audio untouched. **NOT YET VERIFIED ON DEVICE** — the
|
||||
composition path needs a real 60fps-widget run (open item; see HANDOFF). Web work commits stay
|
||||
LOCAL (no push) until the user greenlights.
|
||||
- **Slice 15 — one frame per deadline SLOT: duplicate-on-lag, never skip (2026-09-14, device takes
|
||||
ty-1723/1726):** slice 10's freshness choice — a render overrun VANISHES the missed slots from the
|
||||
file — authored ACCELERATED playback: ~35ms render cost vs the 16.6ms slot, one fresh frame per
|
||||
35ms (log wall-1726: `FramePump stall: iteration 33ms (> 2× the 17ms interval): worst render
|
||||
33-46ms`), and a 60fps container muxed on arrival → 1723: 697 frames = 11.62s video vs 11.84s
|
||||
audio; 1726: 163 frames = 2.72s vs 2.93s. The video also ended 0.21–0.24s before the audio ("audio
|
||||
speeds up, cuts off at the end"). OBS's answer is duplicate-on-lag: `libobs/media-io/video-io.c`
|
||||
emits one frame per tick and a lagger DUPLICATES the newest frame, counted as "lagged frames due
|
||||
to rendering lag/stalls" (obs-output.c) — a wall-time hole never exists (docs.obsproject.com/
|
||||
backend-design: "If the video frame queue is full, it will duplicate the last frame"). The pump's
|
||||
submit is now a bounded catch-up: `while (now >= nextTick) { submit; nextTick += intervalTicks; }`
|
||||
over a `deadlineNow` captured once per iteration — the LATEST composite, fresh on the first missed
|
||||
slot, repeated (OBS's duplication) for the rest. Duration == wall under any render load, at the
|
||||
cost of a short judder during a stall. The burst is safe because `Channel.TryWrite`
|
||||
never blocks (slice 10's queue — the take-9 smear was the BLOCKING pipe-write re-copying a stale buffer
|
||||
during a long freeze; here each emit is nanoseconds). The burned `_outputIndex` (slice 10 judge)
|
||||
moved INSIDE the submit loop so every emitted slot carries its own +1 (and the old unconditional
|
||||
pre-gate bump no longer gaps the sequence on non-submitting fast-render iterations). **Good Dog
|
||||
test** `Pump_Overrun_Renders_EmitsEverySlot_NotSkipped`: 60fps, resolver sleeps 35ms, asserts ≥0.65
|
||||
of the wall slots are emitted (the skip-pump wrote ~1/35ms ≈ 200 in 7s; the slot pump ~415). Full
|
||||
suite **294/294 green, 0 warnings**. The +0.6s audio-late clap reading on 1726 was confounded by
|
||||
the 1.1x acceleration — re-measure on device; if a real residual remains it is the audio pipeline.
|
||||
No push (web/A/V work stays local).
|
||||
- **Slice 16 — the desktop capture conversion was the bottleneck: fast downscale +
|
||||
cadence throttle + reuse-distance ring (2026-09-14, device take ty-1742):** slice 15
|
||||
fixed pacing but the 1742 desktop layer was still jerky/frozen with a horizontal
|
||||
tear. Decoded to raw frames and audited: the **desktop band was frozen 21s of 23.35s
|
||||
(90%)** — ~6.1 content updates/s, freeze runs up to 2.28-2.78s, dup-run max 90
|
||||
frames (1.5s). The render stat (`worst render 33-36ms`) was real but MOOT: the
|
||||
capture CONVERSION was the wall. The monitor delivers at the **240Hz DWM cadence**
|
||||
(~4.2ms); with one-in-flight conversions and each 2560×1440→1920×1080
|
||||
`DownscaleBgra` at ~150ms under load, `LatestFrame` updated a handful of times/s —
|
||||
the desktop feed inside a 60fps file read ~90% frozen. (Webcam + audio were their
|
||||
own paths — fine, matching the report.) Three changes, all in
|
||||
`Services/ScreenCaptureFrameSource.cs` (+ `Services/FrameRingBuffer.cs`):
|
||||
(1) `DownscaleBgra` is now **integer 8.8 fixed-point, "shift only at the end"** —
|
||||
the exact two-stage math of `SceneCompositor.Bilinear` (MyMistakes take-4 rule),
|
||||
dropping double-per-pixel to a row-walk of integer ops (the capture downscale had
|
||||
stayed the naive float twin of the 258ms disaster); (2) a **10ms conversion floor**
|
||||
(`MinConvertInterval`) so the 240Hz tail stops queuing ~150ms of serialized
|
||||
conversion per second — the open edge sits just above the ~60/s the pump can use;
|
||||
(3) the ring is a **reuse-distance pool** (`FrameRingBuffer`, depth 8, redLine 4):
|
||||
a buffer is only rewritten ≥4 rents after its last hand-out else a fresh allocation,
|
||||
replacing the blind round-robin — since `ScreenCaptureManager` keeps
|
||||
`session.LatestFrame` across conversions and the dispatcher preview copy lags,
|
||||
"who released the buffer" needs a consumer API that doesn't exist; a reuse
|
||||
DISTANCE needs no cooperation (the 1742 new-top/old-bottom tear is that
|
||||
read-under-write, structural now). Telemetry added: a startup.log line every 2s
|
||||
(`frames/s, conv avg/max ms, skip busy/cadence, ring allocs`) so the device take
|
||||
can be judged numerically. **Good Dog test**
|
||||
`Ring_NoLap_ReusesOnlyAfterRedLineRents` (depth 3/redLine 4 exercises the red-line
|
||||
skip; the 8/4 config settles at 8 buffers and never grows). Full suite **295/295
|
||||
green, 0 warnings**. C4 (compositor Epoch-cached downscale / blit-on-change) was
|
||||
DEFERRED: the slice-15 approved plan assumed a native-res relocation; the
|
||||
measurement recast it — relocating a ~30ms float downscale to the render thread
|
||||
just moves the same cost into the slot budget. Re-measure on device; if render
|
||||
still >16.6ms slots after capture feeds ≤60 real updates/s, add C4. No push.
|
||||
- **Slice 17 — the OS readback was the real wall: overlapping conversions + monotonic
|
||||
publish + deeper pool (2026-09-14, device take ty-1824):** slice 16's downscale fix
|
||||
landed but the desktop was still ~90% frozen on the 1824 take (band 6.8/s updates, max
|
||||
freeze 4.85s). The new 2s telemetry was decisive: `conv avg 46-50ms max ~61ms` with
|
||||
`skip busy 37-83` — the **GPU→CPU readback (`CreateCopyFromSurfaceAsync`), not
|
||||
`DownscaleBgra`**, is the ~47ms wall (240Hz HDR compositing + encoder + 2 capture
|
||||
devices share the GPU); at one-in-flight that caps the desktop feed at ~17-20
|
||||
updates/s, which is the file's whole-frame ~7 content-moments/s. Two facts reshaped
|
||||
the fix: **(a)** shrinking the pool size does NOT scale the desktop (Microsoft docs:
|
||||
"If content is larger than the frame, the contents are **clipped**") — readback stays
|
||||
at native 2560×1440; **(b)** delivery is healthy (60-100 arrivals/s), so the lever is
|
||||
conversion throughput, not the pool size. Changes in `Services/ScreenCaptureFrameSource.cs`:
|
||||
conversions overlap up to **MaxConcurrentConversions = 3** (pool deepened to 5 buffers
|
||||
so in-flight frames fit), each completion publishes ONLY if its Epoch is strictly
|
||||
newer than the last published (`MonotonicGate` — a slow older completion must never
|
||||
overwrite a newer LatestFrame), Epoch increments via `Interlocked`, and the
|
||||
DownscaleBgra row-scratch became per-conversion locals (concurrent callers). The
|
||||
render side (33-42ms → ~30 unique composites/s) is the NEXT cap after capture speeds
|
||||
up — that's the C4 slice, queued right after this re-measure. **Good Dog test**
|
||||
`PublishGate_TryPublish_OnlyStrictlyNewerWins`. Full suite **296/296 green, 0
|
||||
warnings**. NOT YET DEVICE-VERIFIED; target: telemetry frames/s jumps ≥ ~30-40 and the
|
||||
band audit drops below ~50% frozen. No push.
|
||||
- **Slice 18 — C4: the FramePump blit-on-change composite cache (2026-09-15, take ty-1841
|
||||
verdict):** the 1841 take proved capture fixed (band ~20 fresh updates/s, no tears, pacing
|
||||
clean) but logged a FramePump **stall on EVERY iteration** (`totalMs 21-44`,
|
||||
`render=full-render split=0 elements=6 dynamic=4`, worst render 166ms startup spike) →
|
||||
~22-28 composites/s hard-caps the desktop inside the 60fps file. The SceneGraph split
|
||||
CANNOT fix this wired: `GetSplitPoint` returns **0** because the live-capture backdrop is
|
||||
element 0 and must stay dynamic — a baked capture goes stale — so every tick runs the full
|
||||
re-composite even when no input changed (the compositor's per-layer paste cache already makes
|
||||
identical frames identical per-element; the waste is re-COMPOSITING the whole 8.3MB frame).
|
||||
Same shape as OBS (sources cache their surface, the compositor renders on update only —
|
||||
the pattern this file has cited since the 2026-09-04 paste-cache slice) at the FRAME level.
|
||||
`FramePump.RenderFull` now wraps both full-render call sites (no-SceneGraph + split=0
|
||||
fallback): it computes `BuildFullRenderSignature` (a deterministic hash of the crop/output
|
||||
dims, the social bar identity + top, and — per element, MIRRORING the compositor's own
|
||||
resolution — the element ref, layout/visual bits, and the frame each element would resolve
|
||||
through the SAME resolver seam, using buffer-array identity + Epoch + CropBounds; a changed
|
||||
frame forces a re-composite, an unchanged one never serves stale bytes) and, when it matches
|
||||
the last composite, reuses the cache with ONE `Buffer.BlockCopy` (~3ms) instead. The cache
|
||||
buffer is a **separate long-lived array** — never the scratch pool — because the caller burns
|
||||
the frame counter and recycles the scratch AFTER `RenderScene` returns; the cache is written
|
||||
from the rendered scratch BEFORE returning (pre-burn, pre-recycle). Engagement is gated on
|
||||
the 1:1 config (`SourceRect == Output`, the only deployed tier); the vertical tier's cached
|
||||
fresh-scale-buffer follow-up stays exactly where it was. Telemetry: internal `CacheHits`/
|
||||
`CacheRenders` counters surface as `cache {renders}R/{hits}H` on the 5s stats line, and the
|
||||
ticks' key is an internal `OutputIndex` (burned counter, stable across one tick's resolver
|
||||
passes). **Good Dog test**
|
||||
`FullRenderCache_StaticInputs_RenderOnce_Then_Reuse_UntilInputChanges`: static scene →
|
||||
exactly ONE composite (renders==1, hits>0, byte-identical content above the burn strip),
|
||||
then a new frame (new array + monotonic Epoch) invalidates and propagates. Existing pool test
|
||||
now passes a STABLE scene (production hands `StagedScene`; a fresh scene per tick churns the
|
||||
element refs inside the signature and hides the cache — the tests that care about cache
|
||||
behavior must mirror production) and keys its red/blue alternation on `OutputIndex`/frame
|
||||
count so the tick's two resolver passes (signature + render) don't double-advance a call-count
|
||||
flip. Full suite **297/297 green, 0 warnings**; verify.sh scope-locked to FramePump.cs +
|
||||
FramePumpTests.cs + docs. Target next take: `cache` hits dominate on TV holds, `avg render`
|
||||
drops toward the BlockCopy, stalls vanish, then the band audit + clap re-measure decide push.
|
||||
No push.
|
||||
- **Stop ordering matters:** `StopAsync` stops the encoder — since slice 10 it FLUSHES the pending
|
||||
queue (`Channel.TryComplete` → drain writes the leftovers, closes stdin → EOF → ffmpeg finalizes+exits;
|
||||
an accepted frame is never lost) — **before** awaiting the loop. The old reverse-order deadlock was
|
||||
a submit stuck on pipe backpressure; the drain thread owns that wait now, and stopping the encoder
|
||||
first keeps the pump's own (non-blocking) submits from racing the completed queue as a spurious drop.
|
||||
`ProcessFailed` self-stops the pump. `Failed` while live flips
|
||||
`StreamStatus.Error` (minimal).
|
||||
- **Health stats (ship step 6, shipped 2026-08-13):** `HealthUpdated` is bound to the bottom bar —
|
||||
`MainViewModel.OnFramePumpHealthUpdated` marshals to the UI thread (the encoder's stderr loop raises on
|
||||
a background thread) and copies into `CurrentHealth` (the bottom bar's existing `CurrentHealth.*`
|
||||
bindings). `ResetHealth(status)` zeroes dropped/duration on go-live and on End so stats never linger
|
||||
from a previous session; bitrate/FPS stay on the tier's targets (`ApplyStreamQuality`). The bar shows
|
||||
real encoder values since TASK 9 (2026-08-16) wired the reusable stream URL into `_rtmpUrlProvider`.
|
||||
- **`MainViewModel` owns the resolver** (`ResolveOutputFrame`): `WebcamSceneConfig` →
|
||||
`CameraManager.GetLatestFrame(DeviceKeyForWebcam(WebcamId))` — **device-keyed** (configs carry the
|
||||
identity GUID; sessions key on the Windows device id — the old direct-GUID lookup returned null
|
||||
forever and the webcam was structurally absent from output; take-2 fix 2026-09-01,
|
||||
`WebcamOutputKeyTests`), `Source { IsLiveCapture, CaptureKey }` →
|
||||
`ScreenCaptureManager.GetLatestFrame(CaptureKey)` (the new accessor mirroring `CameraManager`), image/
|
||||
background → `StaticPixelCache.Get(AssetId)`. `BuildCompositorOptions` rounds the VM's `OutputRect*`
|
||||
doubles to ints — the vertical 607.5 half-pixel crop rounds to a perfectly-centered **608** (`Math.Round`,
|
||||
ToEven); `BuildEncoderOptions` fills W×H/FPS/bitrate from the tier once the URL provider yields one.
|
||||
- **Media source decoder (TASK 21, Increment B — 2026-08-31):** `MediaVideoSource` (`Services/MediaVideoSource.cs`)
|
||||
decodes a local file to `VideoFrame`s by spawning ffmpeg with
|
||||
`-f rawvideo -pix_fmt bgra -vf scale=WxH -an` and draining the raw BGRA stdout pipe through the pure
|
||||
`RawVideoFrameReader` (`Services/RawVideoFrameReader.cs`), raising `FrameAvailable` per frame. Implements
|
||||
`IMediaFrameSource` (`Services/IMediaFrameSource.cs`: `Key`, `FrameAvailable`, `Completed`, `StartAsync`/`StopAsync`).
|
||||
The subprocess sits behind `IDecodeProcess`/`FfmpegDecodeProcess` (binary-stdout mirror of the encoder's
|
||||
`IEncoderProcess`), and the path comes from `IFfmpegLocator` — codec-agnostic, uses the ffmpeg already shipped,
|
||||
testable with a fake process. Decode sessions are owned app-wide by `MediaVideoSourceManager`
|
||||
(`Services/MediaVideoSourceManager.cs`): refcounted by `MediaPath`, `Func<string, IMediaFrameSource?>` factory
|
||||
seam (mirror of `ScreenCaptureManager`), `AcquireAsync`/`ReleaseAsync`/`ReleaseAllAsync`/`GetLatestFrame`,
|
||||
coalescing each file's frames onto the UI dispatcher onto one shared `WriteableBitmap`. Frames emit as fast as
|
||||
the pipe produces them. Wired into the live pipeline (slice 1 step 4): `ResolveOutputFrame` reads
|
||||
`_mediaManager.GetLatestFrame(MediaPath)` for a `MediaSource`, and `Source.DisplaySource`
|
||||
(`Models/Source.cs`) routes `VideoImageSource` for `MediaSource` so the preview/canvas show decoded video.
|
||||
Slice 2a shipped the probe seam: `FfmpegFrameRateParser` (pure, `avg_frame_rate=` then
|
||||
`r_frame_rate=` rational N/N/M, unknown→null) + `IFrameRateProbe`/`FfmpegFrameRateProbe` (derives sibling `ffprobe.exe`
|
||||
from the located ffmpeg dir, reuses the `IDecodeProcess` seam for the ffprobe subprocess text; null if
|
||||
ffprobe absent) + `FfmpegLocator.ProbeFileName` now also extracts `ffprobe.exe` from the pinned archive
|
||||
(conditional — old caches without it just get no pacing). Slice 2b wired pacing: `MediaVideoSource`
|
||||
takes optional `IFrameRateProbe?` + `Func<TimeSpan,CancellationToken,Task>? delay` (default `Task.Delay`)
|
||||
seams, probes FPS once in `RunAsync`, and delays by 1/fps after each emitted frame; no probe/unknown → no
|
||||
pacing (ffmpeg's own pipe backpressure already throttles the decode). Slice 3 added loop control:
|
||||
`IMediaFrameSource.Looping` (bool); `MediaVideoSource` takes a `Func<IDecodeProcess>` process factory (a single
|
||||
`Process` can't be re-`Start()`ed, so each loop pass creates a fresh decoder) and wraps the decode in a
|
||||
`do…while (Looping)` — restart on natural EOF instead of raising `Completed`. Production wiring (MainViewModel
|
||||
media factory): passes `() => new FfmpegDecodeProcess()` as the factory, `FfmpegFrameRateProbe(new FfmpegLocator(),
|
||||
() => new FfmpegDecodeProcess())` as the probe.
|
||||
Still open: session acquisition on add/remove (UI picker) + wiring `Source.MediaIsLooping` into
|
||||
`IMediaFrameSource.Looping` (needs a manager-level per-path loop provider, comes with the picker slice).
|
||||
- **Social bar on the output (bar bug-fix branch):** the `FramePump` takes an optional
|
||||
`socialBar: Func<(VideoFrame? Frame, SocialBarPosition Position)>?` seam, re-read **every frame** (so a
|
||||
mid-stream position flip applies immediately). The strip is pre-rasterized by `Compositor/SocialBarRenderer.cs`
|
||||
(WPF glue — WPF glue here is fine because the strip is rendered once per config change on the UI thread,
|
||||
the resulting immutable frame is then composited pure-CPU by `SceneCompositor`), and `SceneCompositor.Render`
|
||||
blits it **last — above the branding flash** at `socialBarTop` (0 = top, `SourceRectHeight − barHeight` =
|
||||
bottom) in master space. `MainViewModel` owns the frame (`_socialBarFrame`, rebuilt by `RenderSocialBarFrame`
|
||||
on load/save/`NotifySocialsChanged`).
|
||||
- Known consideration: the pump reads the active scene on a background thread while the UI can still edit
|
||||
it; a concurrent-mutation exception is contained (logged + `Failed` + the pump stops) rather than
|
||||
crashing. The background thread + video pipeline is the new reality since this step.
|
||||
|
||||
---
|
||||
|
||||
## Related in [`ai.md`](../../ai.md)
|
||||
|
||||
This file owns the compositor and the pump. The neighbours stay in `ai.md`:
|
||||
|
||||
- **Live encoder + RTMP push** — `IFfmpegEncoder`/`FfmpegEncoder`, `FfmpegArgs`, locator and bundle.
|
||||
- **Media source** — `MediaVideoSource` (ffmpeg `rawvideo` decode) feeding the pump's resolver.
|
||||
- **Social bar** and **Native events & alerts** — the overlay strips the compositor blits last.
|
||||
- **Run** and **Tests** — how to launch in Release and how the suite is run.
|
||||
@@ -0,0 +1,20 @@
|
||||
# docs/ai — index
|
||||
|
||||
Subsystems split out of [`ai.md`](../../ai.md) (2026-10-26) so a session pays only for the
|
||||
subsystem its task touches. The split is a **verbatim move**: history, traps and measurements are
|
||||
unchanged, and `ai.md` keeps a short stub carrying the load-bearing invariants plus a link here, so
|
||||
no rule is reachable only from a file you forgot to open.
|
||||
|
||||
Read the file whose row matches the task. There is no required reading order.
|
||||
|
||||
| File | Covers | Read when |
|
||||
|------|--------|-----------|
|
||||
| [`frame-pipeline.md`](frame-pipeline.md) | Render budget & measurement (`FrameCadence`, per-element composite timing, scene dump), `SceneCompositor` design + `StripPool`, and `FramePump` from scene snapshot to encoder (pacing, pool/paste caches, backpressure, teardown, health stats) | Touching the compositor, the render loop, per-frame cost, `Services/Encoder/*`, or debugging dropped/juddering frames |
|
||||
| [`capture.md`](capture.md) | The locked Background layer (screen/game/window capture, WinRT GraphicsCapture interop), the webcam seam + `CameraManager` sessions, MediaPipe background effects, and the vertical 9:16 tier's per-scene camera placement | Touching any frame source, the Background row, camera acquisition/conflicts, background effects, or vertical-tier camera geometry |
|
||||
| [`output.md`](output.md) | The bundled `ffmpeg.exe` and the encoder subprocess (args, probe order, seams), local recording (REC/ON-AIR, filenames, private-broadcast close-out), and the WASAPI capture → filter → mix → named-pipe audio chain (TRAX, sync offset, ducking, limiter) | Touching ffmpeg args/bundling, encoder or recording behaviour, REC/ON-AIR, the audio mix and its meters, or loudness/sync problems |
|
||||
|
||||
## Split-out yet to come
|
||||
|
||||
The remaining `ai.md` blocks are being moved in staged units (capture, YouTube/alerts/auth, product,
|
||||
UI patterns/theming, audio/recording/encoder). Add the row when the file lands; do not create empty
|
||||
placeholders.
|
||||
@@ -0,0 +1,326 @@
|
||||
# Output — the ffmpeg bundle, the encoder, recording, and audio
|
||||
|
||||
> Split out of [`ai.md`](../../ai.md) (2026-10-26) so a session reads only what its task touches;
|
||||
> `ai.md` keeps a short invariant stub for this file. Directory map: [`index.md`](index.md).
|
||||
|
||||
Everything that leaves the app: locating and shipping `ffmpeg.exe`, how the encoder subprocess is
|
||||
driven (video stdin + audio named pipe), local recording as a first-class mode, and the WASAPI capture
|
||||
/ filter / mix chain that feeds it. Content is the decisions plus the failures that produced them;
|
||||
superseded designs are marked inline.
|
||||
|
||||
---
|
||||
|
||||
## The encoder
|
||||
|
||||
### FFmpeg locator (TASK 4 ship step 2 — shipped 2026-08-10; **rewritten to BUNDLED 2026-09-28**, plan in TASKS.md)
|
||||
|
||||
The encoder's one external dependency is `ffmpeg.exe`. **It is BUNDLED — never downloaded, never
|
||||
fetched at runtime.** `IFfmpegLocator` resolves an absolute path on demand: **the bundle first**
|
||||
(`<AppContext.BaseDirectory>\ffmpeg\ffmpeg.exe`), then **PATH** as the escape hatch for someone who
|
||||
deliberately installed their own. The order is the opposite of the original ("the user's own install
|
||||
wins") on purpose: a PATH `ffmpeg` may be a different major version or a **GPL** build, and the
|
||||
bundle is the one that was pinned, license-vetted and tested. If neither hits, it throws a loud
|
||||
`IOException` naming the missing folder and the fetch script — the old "found something on disk"
|
||||
cache path is gone deliberately, because silently satisfying a missing bundle with a stale copy is
|
||||
exactly the silent drift a certification-gated dependency must not have.
|
||||
|
||||
**The bundle is not in git** (135 MB of incompressible DLLs would tax every commit on this
|
||||
9p-bridged tree). `scripts/fetch-ffmpeg.sh` populates `./ffmpeg` from a pinned month-end BtbN
|
||||
LGPL-**shared** win64 artifact, **verifies SHA-256 before extracting**, and is idempotent; set
|
||||
`FFMPEG_ZIP=<path>` to reuse a pre-downloaded archive. `ytLive.csproj` copies `ffmpeg\**` to the
|
||||
output (`CopyToPublishDirectory="Always"` for the MSIX package) and the `GuardBundledFfmpeg` target
|
||||
**fails the build** if the folder is absent, so a fresh clone finds out before the creator presses
|
||||
Record rather than during a take. `ffmpeg.exe` + `ffprobe.exe` + the `libav*.dll` family must stay in
|
||||
one folder — Windows resolves a shared build's DLLs from the executable's own directory.
|
||||
`ffplay.exe` (18 MB) is deliberately **not** shipped; nothing calls it.
|
||||
|
||||
BtbN LGPL-shared (not gyan.dev, not static): it drops GPL-only libx264/x265 while keeping
|
||||
NVENC/QSV/AMF + libopenh264 + native AAC, and dynamic linking means LGPL compliance is "license text +
|
||||
source offer" with no static-relink (§6) material — see the Licensing guardrails below. libopenh264
|
||||
is **compiled into `avcodec-63.dll`** (verified 2026-09-28: the encoder is registered there, and there
|
||||
is no separate OpenH264 binary to ship), so Cisco's BSD terms + patent grant ride along inside item 1
|
||||
of `THIRD-PARTY-NOTICES.txt`. Pin: `autobuild-2026-08-31-13-27` (month-end on purpose — BtbN keeps
|
||||
~14 dailies but month-end builds ~2 years; a daily pin is what 404'd, see below).
|
||||
|
||||
**⛔ Why this was rewritten (2026-09-27 → shipped 2026-09-28).** The pinned-download design was a
|
||||
defect, not a cold-start edge case. It **failed in production on 2026-09-01** — the previous daily
|
||||
pin aged out of BtbN's 14-day retention and the download 404'd on the creator's first real recording
|
||||
attempt. Two facts made "the pin is a const, just bump it" wrong:
|
||||
- **Microsoft Store policy 10.2.2 forbids dynamic code inclusion.** Downloading an executable
|
||||
from the internet and running it is the exact pattern the policy names. An MSIX submission
|
||||
is rejected over it.
|
||||
- Bumping the pin is whack-a-mole against a retention window we do not control.
|
||||
|
||||
Bundling deletes the startup network dependency entirely. The LGPL-**shared** build choice is
|
||||
**unaffected** — dynamic linking was chosen for LGPL §6 compliance ("license text + source offer"),
|
||||
not to enable downloading. Analysis: `TASKS/research-store-certification.md` §4.
|
||||
**Guardrail:** `FfmpegLocatorTests.TheLocatorCarriesNoNetworkCapability` asserts the locator's
|
||||
source contains no `HttpClient`/URL/downloader at all, so the policy cannot rot back.
|
||||
|
||||
**Every other third-party binary already ships via NuGet — ffmpeg was the only download-and-execute.**
|
||||
Verified 2026-09-28: `e_sqlite3.dll` and `WebView2Loader.dll` land in `runtimes\win-x64\native\`
|
||||
(SQLitePCLRaw + WebView2 packages) and NAudio/Notification.Wpf/Microsoft.Data.Sqlite/ProtectedData
|
||||
are managed. Two gaps remain and are **not** fixed by bundling: the **WebView2 Evergreen runtime** is
|
||||
a machine prerequisite (a manifest framework-dependency + graceful-failure path, TASK 48 item 2), and
|
||||
**Velopack's updater downloads and executes** — its teardown is a compliance action, not just cleanup.
|
||||
|
||||
### Live encoder + RTMP push (TASK 4 ship step 3 — shipped 2026-08-12, plan in TASKS.md)
|
||||
|
||||
The encoder is a **thin orchestrator over `ffmpeg.exe`** — no H.264/AAC code in the app. It spawns the
|
||||
subprocess (path from `IFfmpegLocator`), feeds raw BGRA master frames into stdin, and parses `-stats`
|
||||
stderr lines into `StreamHealth` (bitrate/FPS/duration, dropped-from-frame-count). `FfmpegEncoder`
|
||||
(`IFfmpegEncoder` seam) holds: `StartAsync` (locate → probe `-encoders` → spawn → stderr loop →
|
||||
drain loop), `SubmitFrameAsync` (slice 10: bounded-queue ENQUEUE — never a pipe write; the drain task
|
||||
owns stdin writes), `StopAsync` (flush queue → stdin EOF → ffmpeg
|
||||
finalizes + exits by itself; a 10s watchdog kills it), `Dispose` (force-kill + wait), and the
|
||||
`HealthUpdated`/`ProcessFailed` events. **Pattern:** the encoder never touches `Process` — it drives the
|
||||
`IEncoderProcess` seam (`FfmpegEncoderProcess` wraps the real `Process`, redirected stdin/stdout/stderr
|
||||
+ exit control); a `Func<IEncoderProcess>` factory + the locator are constructor-injected, so the
|
||||
integration test fakes the whole subprocess (probe + encoder) with a Channel-backed `TextReader` whose
|
||||
`Complete()` is EOF (`null`), never a `ChannelClosedException`.
|
||||
|
||||
**Decisions (locked):** args are pure (`FfmpegArgs.Build`, no string building in the encoder):
|
||||
`-f rawvideo -pix_fmt bgra -video_size WxH -framerate FPS -i pipe:0` (**NO `-re`** — the FramePump
|
||||
is the pacer since slice 9, 2026-09-10; `-re` added a second, fighting clock on the rawvideo demux)
|
||||
+ a **real audio input — the
|
||||
mixer writes IEEE-float stereo to a Windows named pipe** (`-f f32le -ar 48000 -ac 2
|
||||
-i \\.\pipe\ytllive_audio`, name via `EncoderOptions.AudioPipeName`; replaced the old `-f lavfi -i
|
||||
anullsrc` silence in the TASK 8 audio milestone) + explicit `-map 0:v -map 1:a` + `-c:v <enc> -b:v K
|
||||
-maxrate K -bufsize 2K` + **`-g fps×4 -keyint_min fps×4 -sc_threshold 0 -bf 0 -pix_fmt yuv420p`**
|
||||
(≤4s keyframes, closed GOP, H.264 compliance) + `-c:a aac -ar 48000 -ac 2 -f flv <rtmpUrl>`.
|
||||
**Record output (TASK 18):** `FfmpegArgs.Build` emits one self-contained block per output, each its own
|
||||
`-map 0:v -map 1:a` + codec tags (`AddVideoTags` helper). Stream block = `-f flv <rtmpUrl>`; record block =
|
||||
`-f mp4 <RecordPath>`. `EncoderOptions.StreamEnabled`/`RecordEnabled`/`RecordPath` gate each block, so
|
||||
the engine runs record-only (no RTMP) or stream-only — **stream+record simultaneously is OUT by creator
|
||||
ruling (2026-09-01): the VOD is already the copy, and dual-encoding drags mid-range chassis and degrades
|
||||
BOTH outputs ("we're not them"). The old "cheap on NVENC" claim was an unverified assumption; the
|
||||
UI constraint is landed (2026-09-20): the REC/ON-AIR pills are radio-exclusive — arming one clears the
|
||||
other — while the dual-block machinery stays generic.**
|
||||
`FfmpegEncoder.StartAsync` throws unless at least one output is enabled.
|
||||
**Encoder choice is probed from the binary's `-encoders` listing** (`FfmpegEncoderPicker`, pure):
|
||||
NVENC → QSV → AMF → OpenH264 fallback, **never libx264** (GPL; see Licensing). `EncoderOptions.VideoEncoder`
|
||||
forces one and skips the probe. `EncoderOptions` also carries W×H/FPS/bitrate from the quality tier and
|
||||
`GopSize` = FPS×4. Constructed by `MainViewModel` since ship step 5 (`new FfmpegEncoder(new FfmpegLocator())`),
|
||||
driven by the `FramePump` below.
|
||||
|
||||
---
|
||||
|
||||
## Recording
|
||||
|
||||
### Local recording (TASK 18 — shipped 2026-08-29, VM + top-bar UX; plan in TASKS.md)
|
||||
|
||||
Recording is **independent from streaming** and needs no YouTube sign-in. A single segmented
|
||||
**REC|ON-AIR switch** in the top bar picks the mode (redesigned 2026-09-22 from the old two pills):
|
||||
**REC** (local file, works signed-out) / **ON-AIR** (streaming; **armable signed-out** — intent may
|
||||
light while offline, 2026-09-20: a greyed pill was the dead end the guard shipped to forbid). The switch
|
||||
selection is intent; `IsRecording`/`IsLive` are reality — the reality line (dot + word + elapsed) only
|
||||
lights when a session is actually running.
|
||||
|
||||
- **State model (`MainViewModel`):** pills `RecordPillOn`/`OnAirPillOn` stay the **routing model**
|
||||
(radio-exclusive — arming one clears the other, record-OR-live 2026-09-01; the ON-AIR setter stays
|
||||
armable signed-out (2026-09-20: intent must be able to light while offline)). **Top bar redesign
|
||||
(2026-09-22):** the bar is ONE surface rendered by the FIRST decision — Record or Stream — and each
|
||||
world is the whole bar (`ShowIdleWorld`/`ShowRecordAction`/`ShowStreamActions`, wired by
|
||||
`RaiseTopBarModes()`). The old sign-lights + two pill toggles + status dot + PRIVATE/TEST chips are
|
||||
gone; the mode is one **segmented REC|ON-AIR switch** (`SegmentToggle`/`SegmentLabel` styles), and
|
||||
"going back" is one tap on the other segment. Idle worlds: Record = switch + "Start Recording"
|
||||
(`PrimaryStartButtonLabel` names the armed world); Stream = switch
|
||||
+ "Go Live" + Test (Test is a **child of Stream** — visible only stream-armed AND signed-in) plus the
|
||||
account zone on the right. **The account zone is WORLD-INDEPENDENT (2026-09-23):**
|
||||
`ShowSignInButton` = signed-out and `ShowAvatarButton` = the signed-in avatar, in Record and Stream
|
||||
worlds alike — it's app-level identity, and keeping it put pins the bar height (it no longer pops in
|
||||
and out between worlds). Right-click the avatar = Change Account/Logout. Running retires the switch
|
||||
and worlds; one reality line
|
||||
`● word elapsed` + End replace them (`RunningVisible`/`RunningText`/`RunningDotBrush` — TEST/LIVE/REC,
|
||||
gold/red/green). The account **status dot is gone** — the avatar/Sign In pair carries connectedness.
|
||||
`ShowPrimaryStartButton` remains the always-idle face. `StopStream` clears both pills (see "Stop ends
|
||||
everything; failures roll back" below).
|
||||
- **Filenames** (pure `Services/RecordingFile`): auto-name `ty-<yyyyMMdd>-<HHmm start>-<fff>-0000.mp4` at
|
||||
start; **rename-on-stop** to `ty-…-<hh2mm2 actual length>.mp4` (`FinalizeRecordingAsync` after the pump
|
||||
stops & the file closes), numeric `-2`/`-3` suffix on collision (`UniquePath`). Length comes from
|
||||
`_liveElapsed`, which the session timer walks while `IsLive || IsRecording`. The **`<fff>` millisecond
|
||||
segment is load-bearing, not decoration** (2026-09-27): both names derive from `_recordStartTime`, so it
|
||||
survives the rename, and without it two instances started in the same minute compute the *same* start
|
||||
path and both ffmpeg processes write one file — which is exactly the dev multi-instance case
|
||||
(`Helpers/InstanceProfile`, one instance recording the other).
|
||||
- **Private broadcasts end the same way (creator ruling 2026-10-09).** After `StopStream`'s
|
||||
transition(complete), `OfferToKeepPrivateBroadcastAsync` reads the broadcast's privacy **from YouTube**
|
||||
(`GetBroadcastPrivacyAsync` — visibility can change mid-stream, local state isn't trusted) and, only if
|
||||
it is `private`, shows `KeepBroadcastDialog` (repo root): **Enter = Keep** (`IsDefault`), **Escape /
|
||||
Delete / the X = delete** via `DeleteFinishedBroadcastAsync` — `videos.delete?id=<broadcastId>` (a live
|
||||
broadcast's id IS its video id; 204), falling back to `liveBroadcasts.delete` on 404. An unreadable
|
||||
status asks nothing and deletes nothing; a failed delete toasts that the video is still on YouTube.
|
||||
Decision logic = `MainViewModel.ClosePrivateBroadcastAsync` (internal static, tested in
|
||||
`PrivateBroadcastCloseOutTests`); `KeepBroadcastPromptOverride` is the dialog seam.
|
||||
- **Save dialog: Cancel DISCARDS (creator ruling 2026-09-26).** The `RenameRecordingDialog` result is a
|
||||
two-outcome decision, and it lives in the extracted `MainViewModel.CompleteRecordingSave(startPath, dir,
|
||||
autoStem, chosenStem, creatorSaved)` (internal, so `ytLive.Tests` can drive it against real files without
|
||||
a frame pump — see `ytLive.Tests/RecordingSaveDialogTests.cs`):
|
||||
- `creatorSaved == false` (**Cancel / Escape / the X**) → **DELETE the temp file.** Nothing is moved,
|
||||
nothing is left in the videos folder. A failed delete returns `DiscardFailed` and the toast NAMES the
|
||||
file + folder, because a "discarded" recording still on disk is worse than no report.
|
||||
- `creatorSaved == true` (Save, or **Enter on the pre-filled default**) → `File.Move` to the final name.
|
||||
A blank/whitespace box still means "keep the auto name" — but only on an explicit Save.
|
||||
- **The bug this fixed:** the caller used to `File.Move` UNCONDITIONALLY and only overwrite `stem` when
|
||||
the dialog returned true, so `ShowDialog() == false` fell straight through into the save — Cancel
|
||||
silently kept the recording under the default name. The modal itself was always correct; the
|
||||
side effect was in the wrong place. **Any modal whose result gates a side effect must have every
|
||||
branch handled explicitly — a falsy result must never fall through into the effect.**
|
||||
- **Folder:** default `%APPDATA%\ytLlive\recordings\`, user-overridable via `ChooseRecordFolderCommand`
|
||||
(`OpenFolderDialog`), persisted through `LayoutStore.Load/SaveRecordFolder` (`RecordFolder` key).
|
||||
- **Explicit sign-out only:** `StopStream` no longer clears the session/token. Sign out via **Logout
|
||||
only** — Change Account re-authenticates *in place* and deliberately does not sign out first
|
||||
(2026-09-27; signing out first deleted `ytLlive.auth` before the browser even opened, so a
|
||||
cancelled switch lost a working session). Stopping a recording leaves the creator signed in.
|
||||
- **Stop ends everything; failures roll back (2026-09-01):** `StopStream` also clears both intent
|
||||
pills (`RecordPillOn`/`OnAirPillOn`) — a lit pill with no session behind it is a lie. Every
|
||||
frame-pump death and every go-live prep failure now runs the full `StopStream` teardown instead of
|
||||
leaving a limbo (`StreamStatus.Error` with `IsRecording=true` over a dead encoder — the first-launch
|
||||
zombie recording). Toast copy distinguishes "Recording stopped" from "stream pipeline stopped";
|
||||
`AudioMixer.StopLive` is idempotent-safe for rollbacks that never reached StartLive. Seams:
|
||||
`OnFramePumpFailed` + `IsRecording` setter made internal (test-only, InternalsVisibleTo).
|
||||
Test: `SessionTeardownTests`.
|
||||
- `EncoderOptions.StreamEnabled/RecordEnabled/RecordPath` gate outputs; record+stream is one ffmpeg with two
|
||||
output blocks (the dual-block machinery is unreachable from the UI since the pills are radio-exclusive).
|
||||
Rest of the engine (FramePump/encoder) is unchanged — `BuildEncoderOptions` always returns an options
|
||||
(flags from the pills + `_activeRecordPath`), so record-only reaches the pump.
|
||||
|
||||
---
|
||||
|
||||
## Audio
|
||||
|
||||
### Live audio capture (TASK 4 ship step 4 — shipped 2026-08-12; game audio bar 2026-08-13; **TASK 8 audio milestone: real stream audio + filters + duck + TRAX, shipped 2026-08-14**, plan in TASKS.md)
|
||||
|
||||
**Capture runs for the app's lifetime and is KISS by rule**: desktop/game audio is automatic (WASAPI
|
||||
loopback), the mic is the creator's only audio control (meter/mute/volume already shipped). The whole
|
||||
layer sits behind an **`IAudioSource` seam** (`Services/Audio/`: `Start`/`Stop`/`Started`/`SampleReady`/
|
||||
`Failed`, IDisposable) so the app never touches NAudio directly and tests inject hermetic fakes (no
|
||||
devices, no timers).
|
||||
|
||||
- **Sources (NAudio 2.2.1 — `NAudio.Wasapi` + `NAudio.WinMM` (for `WaveOutEvent`), MIT — item 9 in
|
||||
`THIRD-PARTY-NOTICES.txt`):** `WasapiLoopbackAudioSource` = `WasapiLoopbackCapture` on the default
|
||||
render device; `WasapiMicAudioSource` = `WasapiCapture` with the device resolved by `FriendlyName`
|
||||
matching `MicSourceName` (the app only persists the **DisplayName**), falling back to the default
|
||||
capture endpoint. Both run at the **device's own mix format** — NAudio 2.2.1 exposes no overridable
|
||||
`GetDefaultMixFormat` on `WasapiCapture`, so there is no forced 48 kHz path; the mixer's
|
||||
`TinyResampler` normalizes any rate/channel layout to 48 kHz stereo (this replaced the earlier
|
||||
"force IEEE-float 48k" plan — the resampler IS the design). Mic device resolution re-reads the name
|
||||
provider `Func<string?>` at each `Start`, so a mic picked mid-session takes effect **immediately**
|
||||
(the mixer restarts the mic on pick). Both sources raise `Started` once their capture loop actually
|
||||
begins — the mixer turns that into `MicConnected`.
|
||||
- **`AudioMixer`** owns both sources; **`StartMicCaptureAsync` starts the mixer once at startup** and
|
||||
`Shutdown` disposes it — NOT go-live — so the meters preview live. Mic samples: downmix to mono →
|
||||
`TinyResampler` → **`VoiceFilterChain` per sample** (bass `LowShelfFilter` 120 Hz +4 dB → treble
|
||||
`HighShelfFilter` 8 kHz +3 dB → `NoiseGate` (threshold 0.005, hysteresis 0.5, attack 0.5, release
|
||||
0.0005) → **`SpeechLeveler`** (2026-10-09: automatic gain steering speech RMS to −20 dBFS over a
|
||||
0.3 s window, gain −6..+20 dB, rises ≤10 dB/s / falls ≤30 dB/s, learns only while the gate is open
|
||||
and above −54 dBFS so pauses/hum are never pumped) → `Compressor` (threshold 0.5, ratio 4:1); all
|
||||
pure DSP) → ring buffer → the
|
||||
**post-filter** `AudioLevelMeter` (RMS, 0.2 exponential smoothing) → `MicLevelChanged` →
|
||||
marshalled to the UI thread → `AudioLevel`. Loopback samples: resample → ring buffer → second meter
|
||||
→ `LoopbackLevelChanged` → the game bar's `GameAudioLevel`. The raw linear level maps to the meter's
|
||||
display scale by `AudioLevelMeter.ToDisplay` (−60..0 dBFS → 0..1, **honest** — the +10 dB display boost
|
||||
was removed 2026-10-09; the mic meter is `ToDisplay(AudioLevel × MicVolume)`, true dB math, no ×1.2).
|
||||
**Why (2026-10-09):** a creator's YouTube upload measured −28.5 LUFS / −8.9 dBFS peak against YouTube's
|
||||
−14 LUFS reference (YouTube only turns loud videos down, never quiet ones up) while the boosted meter
|
||||
looked "dialed in". Nothing in the chain added gain — the compressor folds only above −6 dBFS and the
|
||||
limiter only caps. Re-running that same audio through the new chain (mic volume 0.8): **−17.9 LUFS,
|
||||
−0.7 dBFS peak**. Recording and streaming share this one mix, so both were (and now are) the same.
|
||||
Measure with the bundled ffmpeg: `ffmpeg -i file -af ebur128=peak=true:framelog=quiet -f null -`.
|
||||
**Mic connection state surfaces as events:** `MicConnected` (source `Started`), `MicFailed` (source
|
||||
`Failed`), and `RestartMic()` re-resolves + restarts just the mic (loopback keeps running). Failures
|
||||
log via `AppLog`; a mic failure zeroes the meter, a loopback failure never kills the mic. Meter `Push`
|
||||
is unconditional (a `?.` on the event would skip the meter update when nothing is subscribed yet).
|
||||
- **Go-live audio (TASK 8):** `BeginGoLive` → `StartLive(pipeName)` (idempotent) creates the named pipe
|
||||
server (`NamedPipeAudioWriter`, behind the `IAudioPipeWriter` seam) and a live loop task; every
|
||||
`mixInterval` (default 10 ms) `FillAndMix` reads a chunk from each ring buffer (silence-fills
|
||||
underruns), computes the post-filter mic RMS → `AutoDucker` (threshold 0.02, duck ×0.25 / −12 dB,
|
||||
attack 0.05, release 0.005, always on), applies the **honest gains** via `Func<double>` seams
|
||||
(`micGain = MicMuted ? 0 : MicVolume`; `loopbackGain = GameMuted ? 0 : GameAudioVolume`, × duck),
|
||||
mixes mono mic + stereo loopback into interleaved stereo, runs the
|
||||
**`MasterLimiter`** (a **−1 dBFS ceiling** so the honest gains can never sum past 0 dBFS and clip the
|
||||
AAC encode — instant attack per frame, smoothed release toward unity), and writes the floats to the pipe. **The gains are wired through `AudioGainProvider`**
|
||||
(`Services/Audio/AudioGainProvider.cs`) — the VM hands its `MicGain`/`LoopbackGain` Funcs to the
|
||||
mixer at construction, read live each tick, so the volume sliders and mute buttons are stream-honest.
|
||||
(Before this wiring the seams defaulted to unity and the controls were decorative.) `StopStream` →
|
||||
`StopLive()` (closes the pipe → ffmpeg audio EOF) **BEFORE** stopping the frame pump (video EOF) —
|
||||
the reverse order stalls on pipe backpressure. `FfmpegEncoder` itself is untouched; the VM owns the
|
||||
pipe lifecycle.
|
||||
- **Audio sync offset (TASK 22):** after the limiter, `FillAndMix` routes the whole interleaved stereo
|
||||
mix through a **`AudioSyncDelay`** (`Services/Audio/AudioSyncDelay.cs`) — a pure delay line whose
|
||||
offset comes from a `Func<int> syncOffsetMs` seam (the VM's global `AudioSyncOffsetMs`, **−500..+500
|
||||
ms**, persisted as `Audio.SyncOffsetMs`). This is OBS's documented fix for lip-sync: **positive**
|
||||
delays audio when it runs ahead of video; **negative** advances audio when it runs behind by eating
|
||||
the first `|N|` ms of the live stream head (see `AudioMixer.StartLive`; the delay line stays
|
||||
clamped 0..500 internally — negative bypasses it entirely). The advance is armed once at `StartLive`
|
||||
(eating the head mid-stream is impossible); positive is live-reactive (re-read per tick). A slider
|
||||
on the mic bar (`"AUDIO SYNC"`, locked while live/recording via `IsEditMode`) surfaces it
|
||||
(a one-time status dot next to it was removed 2026-09-23 per creator feedback — the slider +
|
||||
numeric tooltip carry the state). Changing the delay flushes the line (a live change clicks
|
||||
rather than smears).
|
||||
- **TRAX — free background music (TASK 8):** `MusicPlayer` = NAudio `MediaFoundationReader`
|
||||
(mp3/wav/m4a) → `VolumeWaveProvider16` at the hardcoded **0.20** bed (no slider) → `WaveOutEvent` on
|
||||
the default device, **looping on any clean natural end** (`PlaybackStopped` with `e.Exception == null`
|
||||
→ rewind + replay; the earlier `Position >= Length` check was unreliable for `MediaFoundationReader`
|
||||
and let tracks play once then stop). The desktop/game bar's mute toggles `LocalGain` (0 or 1;
|
||||
`UpdateTraxLocalGain` runs on every `GameMuted` change and on TRAX load), so muting kills TRAX
|
||||
in the headphones — matching the stream, where it rides the loopback channel. The volume slider
|
||||
drives system volume (`PushSystemVolume`) — which scales the HEADPHONES; the loopback capture is
|
||||
pre-endpoint-volume (disproven-then-fixed 2026-09-01), so the STREAM follows the knob via
|
||||
`AudioGainProvider.LoopbackGain = GameMuted ? 0 : GameAudioVolume`, and the game meter multiplies
|
||||
its display by the same. One knob, three honest paths. No third mixer input. The footer
|
||||
**TRAX** button lives **inside the game audio bar** (in the preview overlay), **left of the
|
||||
"Desktop Audio" label** (the creator's pick — it rides the desktop channel, so the music control
|
||||
sits with the desktop audio controls): status dot (**red** no track / **yellow** loaded stopped /
|
||||
**green** playing) + "TRAX"; **left-click** toggles play/pause (opens the in-app picker via `OpenFileDialog` when no track is loaded); **right-click** always opens
|
||||
the picker (`TraxButton_PreviewMouseRightButtonUp`, `e.Handled = true`, code-behind pattern); tooltip
|
||||
shows the loaded/playing track name or "No track — right-click to choose background music". The picked
|
||||
track persists via **schema v9** single-row `Music` (`TrackPath`/`IsEnabled`) in `LayoutStore`. Known
|
||||
wrinkle (out of scope, future feature): YouTube mutes VODs carrying copyrighted music — "music on
|
||||
live, off VOD" is tabled.
|
||||
- **⛔ The load is behind a bounded probe, because a share that is gone must never
|
||||
freeze startup (2026-10-04).** `LoadLayout` → `LoadTrax` used to call `MusicPlayer.Load`
|
||||
synchronously on the UI thread, and `new MediaFoundationReader(path)` does **not** fail
|
||||
fast on a dead SMB session — it burns the whole ARP/TCP/SMB timeout (~40s, measured)
|
||||
and *then* throws `0x80070035`. Measured: `OnStartup` 08:42:59 → `MainWindow loaded`
|
||||
08:43:43, with 42 of those 44s inside `LoadLayout` on that one track. The rule now:
|
||||
**a probe may time out; only a bind is allowed to be on the creator's path.**
|
||||
`IsTrackReachableAsync(path, budget, File.Exists)` runs the probe on a thread-pool
|
||||
thread under a `Task.WhenAny` ceiling (`TraxProbeBudget` = **2s**) and reports
|
||||
offline/missing/slow as a normal state, not an error. Only a probe that passed gets
|
||||
the real bind, and **that bind stays on the UI thread** with every other
|
||||
`MusicPlayer` mutation, so no new cross-thread race is introduced. The orphan probe
|
||||
thread stays parked until SMB gives up — deliberate: it is not the thread anyone
|
||||
waits on. Same principle covers the webcam: a device held by another app fails fast
|
||||
(sharing violation) but a virtual-camera driver can stall enumeration, so device
|
||||
probes belong off the startup path too, not behind a long timeout.
|
||||
- **Mic status dot (`Models/MicStatus.cs`)** on the preview-bottom MIC button: green = `MicConnected`,
|
||||
yellow = `MicFailed` (in use/unplugged), red = no mic device at startup (the mixer is never started,
|
||||
so loopback and the game bar can't run either — no capture devices at all). **The picked mic persists:
|
||||
`LayoutStore`'s `Settings` key/value table stores `MicSourceName` (saved on pick via
|
||||
`SaveMicSourceName`), and the VM restores it at construction BEFORE the mixer's first `Start` — so a
|
||||
restart reconnects the same already-vetted device (green) or reports it missing (yellow), instead of
|
||||
silently falling back to the default endpoint.**
|
||||
- **Game audio bar** (desktop/game, **always visible** — TASK 4's `IGameAudioDetector` show/hide gating was **removed 2026-08-15**: the bar used to appear only while a full-screen game with sound was up, which kept hiding the creator's desktop meter; the detector stack is gone, desktop audio is just automatic WASAPI loopback): overlaid **at the bottom of the preview window** (bottom-center, dark translucent
|
||||
chip, a mirror of the mic bar: meter + mute + volume slider). It's monitoring UI **in the preview
|
||||
grid** — the meter is display-only (fill = `ToDisplay(GameAudioLevel) × GameAudioVolume` — the ×volume
|
||||
term was missing until 2026-09-01: loopback capture does NOT shrink with the endpoint, so an
|
||||
unscaled meter left it pegged at 20% volume; `GameMeterHonestyTests` guards it), but its **volume
|
||||
slider drives system volume** (`PushSystemVolume` — live headphones follow the knob; the capture
|
||||
side is scaled in the mix, see TRAX note above) **and mute kills TRAX** (`MusicPlayer.LocalGain` = 0). Relabelled
|
||||
**"Desktop Audio"** (TASK 8) since TRAX rides the same channel.
|
||||
- **`WaveToFloat`** (pure, shared): WASAPI mix formats → interleaved float — IEEE float 32-bit direct,
|
||||
PCM 16-bit normalized to -1..1, `WaveFormatExtensible` with the IEEE-float subformat GUID
|
||||
(`NAudio.Dmo.AudioMediaSubtypes.MEDIASUBTYPE_IEEE_FLOAT`), trailing partial samples ignored.
|
||||
- Build **0 warnings**; **197 passing** (DSP/ring-buffer/ducker/resampler/pipe + mixer/hysteresis/
|
||||
game-detector/meter-scale unit tests + the TASK 8 integration test
|
||||
`AudioPipelineTests.Mix_WithFiltersDuckAndGain_Lands_On_AudioPipe` + the polish-batch integration
|
||||
test `Mix_HonorsProviderGains_AndGameMute_KillsTheLoopback` proving the provider gains reach the
|
||||
pipe and mute silences the loopback).
|
||||
---
|
||||
|
||||
## Related in [`ai.md`](../../ai.md) and [`frame-pipeline.md`](frame-pipeline.md)
|
||||
|
||||
- [`frame-pipeline.md`](frame-pipeline.md) — the `FramePump` that feeds this encoder (pacing, backpressure, teardown order).
|
||||
- **ai.md** — resolution tiers (W×H/FPS/bitrate), go-live/prep ordering, toast notifications, licensing guardrails.
|
||||
- [`TASKS/research-store-certification.md`](../../TASKS/research-store-certification.md) — §4, the ffmpeg bundling analysis.
|
||||
@@ -0,0 +1,11 @@
|
||||
# docs — index
|
||||
|
||||
Reference material that is too large for [`ai.md`](../ai.md)'s front half. Nothing here is
|
||||
mandatory reading on session start; open the file whose subsystem your task touches.
|
||||
|
||||
| Path | Purpose |
|
||||
|------|---------|
|
||||
| [`ai/`](ai/index.md) | Split-out subsystems of the AI guide — each started life verbatim inside `ai.md`, which keeps a short invariant stub pointing here |
|
||||
| `ytLlive-preview.jpg` | Preview screenshot used by [`README.md`](../README.md) |
|
||||
|
||||
See [`schema.md`](../schema.md) for the memory-map conventions.
|
||||
@@ -24,6 +24,7 @@ that every other memory file follows. Read `ai.md` first; this file explains
|
||||
| `MyMistakes.md` | Per-task failure log **+ RECIPES REGISTRY** (derived-solution rule): one-off solutions / how-tos get recorded here the same session, then grepp'd before ever re-deriving them (see `AGENTS.md` 🔬) | grep first on any "did this before?" wall |
|
||||
| `schema.md` | This file: the conventions below | when in doubt |
|
||||
| `<dir>/index.md` | Per-directory map (progressive disclosure): what lives there + links | when diving into code |
|
||||
| `docs/ai/*.md` | **Split-out subsystems** of `ai.md` — verbatim moves (from 2026-10-26); `ai.md` keeps a short invariant stub + link, and [`docs/ai/index.md`](docs/ai/index.md) routes | when the task touches a moved subsystem |
|
||||
| `Views/` | Reserved for Views; currently empty | — |
|
||||
|
||||
## Conventions
|
||||
|
||||
@@ -132,9 +132,107 @@ public sealed class BorderColorPickerTests
|
||||
$"the drop-down rows do not show the colour's name: '{rows}'");
|
||||
|
||||
window.Close();
|
||||
|
||||
// ── 4. clicking a row inside the open popup stays inside LeftPanel ───────
|
||||
// MainWindow_PreviewMouseLeftButtonDown deselects SelectedElement (exiting
|
||||
// the properties listing) whenever LeftPanel.IsClickInside(e.OriginalSource)
|
||||
// returns false. A ComboBox's PART_Popup lives in its own PopupRoot visual
|
||||
// tree whose VisualTreeHelper.GetParent is null — the only link back to the
|
||||
// ComboBox and ElementsScrollViewer is PopupRoot's LOGICAL parent.
|
||||
panel.DataContext = new PanelContext { SelectedElement = element };
|
||||
var hostWindow = new Window { Width = 320, Height = 720, Content = panel };
|
||||
hostWindow.Show();
|
||||
panel.UpdateLayout();
|
||||
Dispatcher.CurrentDispatcher.Invoke(() => { }, DispatcherPriority.Loaded);
|
||||
|
||||
var realCombo = FindVisualChildren<ComboBox>(panel)
|
||||
.Single(c => c.ItemsSource is IEnumerable<NamedColor> opts && opts.Any(o => o.Hex == "#e94560"));
|
||||
realCombo.IsDropDownOpen = true;
|
||||
realCombo.UpdateLayout();
|
||||
Dispatcher.CurrentDispatcher.Invoke(() => { }, DispatcherPriority.Loaded);
|
||||
|
||||
var crimsonEntry = BorderColorPalette.EntryFor("#e94560");
|
||||
var crimsonItem = realCombo.ItemContainerGenerator.ContainerFromItem(crimsonEntry) as ComboBoxItem;
|
||||
Assert.NotNull(crimsonItem);
|
||||
|
||||
// The click target inside the popup row (e.g. a TextBlock inside the item)
|
||||
// must be recognised as inside LeftPanel, or MainWindow clears SelectedElement.
|
||||
var clickTarget = FindVisualChildren<TextBlock>(crimsonItem!).FirstOrDefault()
|
||||
?? (DependencyObject)crimsonItem!;
|
||||
Assert.True(panel.IsClickInside(clickTarget),
|
||||
"clicking a colour row in the open ComboBox popup is treated as a click outside LeftPanel (exits properties)");
|
||||
|
||||
// Clicking the row in the open popup updates the webcam's BorderColor and closes the popup.
|
||||
crimsonItem!.RaiseEvent(new System.Windows.Input.MouseButtonEventArgs(
|
||||
System.Windows.Input.Mouse.PrimaryDevice, 0, System.Windows.Input.MouseButton.Left)
|
||||
{
|
||||
RoutedEvent = UIElement.MouseLeftButtonDownEvent,
|
||||
Source = clickTarget,
|
||||
});
|
||||
crimsonItem.RaiseEvent(new System.Windows.Input.MouseButtonEventArgs(
|
||||
System.Windows.Input.Mouse.PrimaryDevice, 0, System.Windows.Input.MouseButton.Left)
|
||||
{
|
||||
RoutedEvent = UIElement.MouseLeftButtonUpEvent,
|
||||
Source = clickTarget,
|
||||
});
|
||||
realCombo.UpdateLayout();
|
||||
Assert.Equal("#e94560", element.BorderColor);
|
||||
Assert.Equal(crimsonEntry, realCombo.SelectedItem);
|
||||
Assert.Contains("Crimson", Texts(realCombo));
|
||||
Assert.False(realCombo.IsDropDownOpen);
|
||||
|
||||
// And when starting from a typed custom hex (#123456), picking a palette
|
||||
// row in the open popup still lands cleanly on that palette colour.
|
||||
element.BorderColor = "#123456";
|
||||
realCombo.IsDropDownOpen = true;
|
||||
realCombo.UpdateLayout();
|
||||
Dispatcher.CurrentDispatcher.Invoke(() => { }, DispatcherPriority.Loaded);
|
||||
|
||||
var emeraldEntry = BorderColorPalette.EntryFor("#22c55e");
|
||||
var emeraldItem = realCombo.ItemContainerGenerator.ContainerFromItem(emeraldEntry) as ComboBoxItem;
|
||||
Assert.NotNull(emeraldItem);
|
||||
var emeraldTarget = FindVisualChildren<TextBlock>(emeraldItem!).FirstOrDefault()
|
||||
?? (DependencyObject)emeraldItem!;
|
||||
Assert.True(panel.IsClickInside(emeraldTarget));
|
||||
emeraldItem!.RaiseEvent(new System.Windows.Input.MouseButtonEventArgs(
|
||||
System.Windows.Input.Mouse.PrimaryDevice, 0, System.Windows.Input.MouseButton.Left)
|
||||
{
|
||||
RoutedEvent = UIElement.MouseLeftButtonDownEvent,
|
||||
Source = emeraldTarget,
|
||||
});
|
||||
emeraldItem.RaiseEvent(new System.Windows.Input.MouseButtonEventArgs(
|
||||
System.Windows.Input.Mouse.PrimaryDevice, 0, System.Windows.Input.MouseButton.Left)
|
||||
{
|
||||
RoutedEvent = UIElement.MouseLeftButtonUpEvent,
|
||||
Source = emeraldTarget,
|
||||
});
|
||||
realCombo.UpdateLayout();
|
||||
Assert.Equal("#22c55e", element.BorderColor);
|
||||
Assert.Equal(emeraldEntry, realCombo.SelectedItem);
|
||||
Assert.Contains("Emerald", Texts(realCombo));
|
||||
|
||||
hostWindow.Close();
|
||||
});
|
||||
}
|
||||
|
||||
private sealed class PanelContext
|
||||
{
|
||||
public SceneElement? SelectedElement { get; set; }
|
||||
public bool IsWebcamSelected => true;
|
||||
public bool CanToggleClipShape => true;
|
||||
}
|
||||
|
||||
private static IEnumerable<T> FindVisualChildren<T>(DependencyObject root) where T : DependencyObject
|
||||
{
|
||||
for (var i = 0; i < VisualTreeHelper.GetChildrenCount(root); i++)
|
||||
{
|
||||
var child = VisualTreeHelper.GetChild(root, i);
|
||||
if (child is T match) yield return match;
|
||||
foreach (var nested in FindVisualChildren<T>(child))
|
||||
yield return nested;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Every text in a visual tree, so a template's content can be read back.</summary>
|
||||
private static string Texts(DependencyObject root)
|
||||
{
|
||||
|
||||
+8
-2
@@ -40,7 +40,7 @@
|
||||
Baseline 241 = count at introduction, first display 13. Missing git or unparsed output
|
||||
falls back to the start (build never breaks). GUID + compile time stay in startup.log,
|
||||
where build-identity forensics actually live. -->
|
||||
<BuildNumberBaseline>241</BuildNumberBaseline>
|
||||
<BuildNumberBaseline>17</BuildNumberBaseline>
|
||||
<BuildNumberStart>13</BuildNumberStart>
|
||||
</PropertyGroup>
|
||||
<Exec Command=""C:\Program Files\Git\cmd\git.exe" rev-list --count HEAD"
|
||||
@@ -49,7 +49,13 @@
|
||||
</Exec>
|
||||
<PropertyGroup>
|
||||
<CommitCount>$(CommitCountRaw.Trim())</CommitCount>
|
||||
<BuildNumber>$([MSBuild]::Add($(BuildNumberStart), $([MSBuild]::Subtract($(CommitCount), $(BuildNumberBaseline)))))</BuildNumber>
|
||||
<BuildCounterFile>$(ProjectDir)BuildCounter.txt</BuildCounterFile>
|
||||
<PriorBuildCount Condition="Exists('$(BuildCounterFile)')">$([System.IO.File]::ReadAllText('$(BuildCounterFile)').Trim())</PriorBuildCount>
|
||||
<PriorBuildCount Condition="'$(PriorBuildCount)' == ''">423</PriorBuildCount>
|
||||
<BuildNumber>$([MSBuild]::Add($(PriorBuildCount), 1))</BuildNumber>
|
||||
</PropertyGroup>
|
||||
<WriteLinesToFile File="$(BuildCounterFile)" Lines="$(BuildNumber)" Overwrite="true" WriteOnlyWhenDifferent="false" />
|
||||
<PropertyGroup>
|
||||
<BuildNumber Condition="! $([System.Text.RegularExpressions.Regex]::IsMatch('$(BuildNumber)', '^[0-9]+$'))">$(BuildNumberStart)</BuildNumber>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
|
||||
Reference in New Issue
Block a user