From a814bcd50208621050bcebd9ad8bb2bcb63691b2 Mon Sep 17 00:00:00 2001 From: gramps Date: Thu, 5 Nov 2026 08:39:46 -0800 Subject: [PATCH] fix(chat): stop painting what YouTube removed, and retire the local word filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TASK 49 is retired (creator ruling): YouTube owns chat moderation. Its blocked-words filter and hold-for-review run upstream of `liveChatMessages.list` — held messages are never returned (https://stackoverflow.com/questions/74511524/) — so a second local list would duplicate a switch the creator already owns, with a second set of false positives and a second support surface. What we actually owed the platform, and had wrong: - The parser added every item to the overlay. A deleted message arrives as a `tombstone` carrying no text at all ("it has since been deleted … only liveChatId, type and publishedAt are present" — https://developers.google.com/youtube/data/live/reference/rest/v3/liveChatMessages), so every deletion rendered as a BLANK chat row. `hasDisplayContent:false` messages and chat-mode events did the same. - Deletion events (`messageDeletedEvent`/`messageRetractedEvent`/`userBannedEvent`) are documented but have not been delivered to clients for years (https://stackoverflow.com/questions/44346893/), so nothing is built on them; they are skipped if they ever arrive. - Found while testing: Super Chat read `displayString`, but YouTube's field is `amountDisplayString` — every amount silently fell back to a recomposed string, which mangles non-dollar locales. The model's own comment said this was the reason the field exists. Test: `ChatModerationParseTests` — tombstone, no-display-content, chat-mode events all produce no row; a real Super Chat still parses its pre-rendered amount (constraint 4: reward fields are never reinterpreted). Docs: TASK 49 marked retired with the citations, the rule recorded in decisions.md, HANDOFF updated. verify.sh: 550 passed, 0 failed, 0 warnings. --- HANDOFF.md | 5 ++ Services/YouTubeChatService.cs | 38 +++++++++- TASKS.md | 2 +- TASKS/task-49-chat-profanity-filter.md | 25 +++++- docs/ai/decisions.md | 8 ++ ytLive.Tests/ChatModerationParseTests.cs | 97 ++++++++++++++++++++++++ 6 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 ytLive.Tests/ChatModerationParseTests.cs diff --git a/HANDOFF.md b/HANDOFF.md index b26fe7e..792ee0a 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -14,6 +14,11 @@ the same settings block (one alert box per layout). - Tests: `ChromaKeyDetectorTests` + `ChromaKeyApplyTests` (pipe asks ffmpeg to key; offer → accept → keyed session feeds the resolver; survives save; auto-apply; alert clip keyed). App build 0 warnings. +- **TASK 49 retired (creator ruling):** YouTube owns chat moderation — blocked words and hold-for-review + run before the API. What we owed it was not a wordlist but not painting what YouTube removed: the chat + parser now skips `tombstone` (a deleted message's text is never sent) and `hasDisplayContent:false`. + Same commit fixed a live bug: Super Chat read `displayString` instead of `amountDisplayString`, so every + amount fell back to a recomposed string. ## Next action diff --git a/Services/YouTubeChatService.cs b/Services/YouTubeChatService.cs index 844d597..a960370 100644 --- a/Services/YouTubeChatService.cs +++ b/Services/YouTubeChatService.cs @@ -106,6 +106,27 @@ public class YouTubeChatService : IDisposable /// One liveChat/messages page → typed chat messages + pagination + /// the server's poll cadence. Internal + separated from the HTTP so the six /// event parses are deterministically testable without any network. + /// True for the types that carry something a viewer should read. `snippet.hasDisplayContent` + /// is authoritative when present; tombstones and chat-mode events are never shown (they exist to + /// mark a deletion or a mode change, and carry no message). + internal static bool HasDisplayableContent(JsonElement snippet, string type) + { + switch (type) + { + case "tombstone": + case "chatEndedEvent": + case "sponsorOnlyModeStartedEvent": + case "sponsorOnlyModeEndedEvent": + case "messageDeletedEvent": + case "messageRetractedEvent": + case "userBannedEvent": + return false; + } + + return !snippet.TryGetProperty("hasDisplayContent", out var has) + || has.ValueKind != JsonValueKind.False; + } + internal static ChatPage ParsePage(string json) { var data = JsonSerializer.Deserialize(json); @@ -126,9 +147,16 @@ public class YouTubeChatService : IDisposable { var snippet = item.GetProperty("snippet"); var type = snippet.TryGetProperty("type", out var t) && t.ValueKind == JsonValueKind.String - ? t.GetString() + ? t.GetString() ?? "textMessageEvent" : "textMessageEvent"; + // YouTube owns moderation: blocked words, held-for-review messages and deleted + // messages are filtered before the feed ever reaches us (2026-11-05 ruling — + // TASK 49's local wordlist retired). What DOES arrive are event markers rather + // than chat: a `tombstone` (a deleted message's placeholder, carrying no text at + // all) and the silent chat-mode events. They get no row on the overlay. + if (!HasDisplayableContent(snippet, type)) continue; + var message = new ChatMessage { Id = item.TryGetProperty("id", out var idProp) ? idProp.GetString() ?? Guid.NewGuid().ToString() : Guid.NewGuid().ToString(), @@ -157,9 +185,15 @@ public class YouTubeChatService : IDisposable message.Kind = ChatEventKind.SuperChat; if (snippet.TryGetProperty("superChatDetails", out var scDetails)) { - if (scDetails.TryGetProperty("displayString", out var display) + // YouTube names the pre-rendered amount `amountDisplayString`; the old + // `displayString` lookup never matched, so every Super Chat fell back to a + // recomposed amount and mangled non-dollar locales (fixed 2026-11-05). + if (scDetails.TryGetProperty("amountDisplayString", out var display) && display.ValueKind == JsonValueKind.String) message.SuperChatDisplayString = display.GetString(); + else if (scDetails.TryGetProperty("displayString", out var legacyDisplay) + && legacyDisplay.ValueKind == JsonValueKind.String) + message.SuperChatDisplayString = legacyDisplay.GetString(); if (scDetails.TryGetProperty("amountMicros", out var amt) && amt.ValueKind == JsonValueKind.Number) message.SuperChatAmount = amt.GetDouble() / 1_000_000.0; diff --git a/TASKS.md b/TASKS.md index 8ccae71..864f809 100644 --- a/TASKS.md +++ b/TASKS.md @@ -55,7 +55,7 @@ blocker; the smallest slice is 32·4 (fragmented MP4 on the record block). | 37 | ☐ | Defaults vs current layout split (build-id); enables one-click revert to defaults | [`task-37`](TASKS/task-37-defaults-current-split.md) | | — | ✅ | **Preview undo (Ctrl+Z / Ctrl+Shift+Z)** — done 2026-11-01: `Models/PreviewHistory` + `MainViewModel.Preview`, one step per gesture, gated on `EditingActive`, cleared on layout load. Property edits not yet in the stack | [`ui-surface`](docs/ai/ui-surface.md) | | 39·2 | ☐ | YPP slice 2: Analytics scopes, watch-hours/Shorts velocity + ETA | [`task-39`](TASKS/task-39-ypp-journey-tracker.md) | -| 49 | ☐ | Chat profanity filter: local, opt-in, **non-persistent**, user-supplied word list | [`task-49`](TASKS/task-49-chat-profanity-filter.md) | +| 49 | ❌ | Chat profanity filter — **retired 2026-11-05**: YouTube owns moderation (blocked words + hold for review run before the API). Replaced by not rendering YouTube's moderation markers (tombstones / no-display events) | [`task-49`](TASKS/task-49-chat-profanity-filter.md) | | 50 | ☐ | One-button publish of the local recording (resumable upload) | [`task-50`](TASKS/task-50-publish-recording.md) | | 54 | 🔶 | Freemium: EULA + privacy shipped; free Steam Demo and paid app use one codebase; paid app ID and Steam Demo setup still pending | [`task-54`](TASKS/task-54-marcom-free-merch.md) | | — | 🔶 | **Steam ownership** — `BIsSubscribedApp(paidAppId)` + Debug fake + app-wide `IsAppOwned`; paid app ID awaits Steamworks | [`commercial`](docs/ai/commercial.md) | diff --git a/TASKS/task-49-chat-profanity-filter.md b/TASKS/task-49-chat-profanity-filter.md index ca0d888..410031a 100644 --- a/TASKS/task-49-chat-profanity-filter.md +++ b/TASKS/task-49-chat-profanity-filter.md @@ -4,7 +4,30 @@ > **Research: [`TASKS/research-store-certification.md`](research-store-certification.md) §12.** > Carved out 2026-09-27; shares a scope lock with nothing else. -**Status:** ☐ Queued — not blocked (does not depend on the route decision) +**Status:** ❌ **Retired 2026-11-05** — YouTube owns chat moderation (creator ruling). What replaced it: +`Services/YouTubeChatService.cs` no longer turns YouTube's own moderation markers into chat rows +(`tombstone`, `hasDisplayContent:false`, chat-mode events), with tests in +`ytLive.Tests/ChatModerationParseTests.cs`. The reasoning and citations are below and in +`docs/ai/decisions.md`; the body of this file is kept so the option is not re-derived from scratch. + +### Why retired + +- YouTube's **blocked words** filter ("block live chat messages that contain or closely match certain + words") and **hold for review** both run upstream of the API: blocked/held messages are not returned + by `liveChatMessages.list` (community evidence: held messages never appear — + ). Ours would be a second list, a second set of false + positives and a second support surface for a switch the creator already owns. +- Deleted messages arrive as a `tombstone` — "signifies that a message used to exist … it has since + been deleted … only liveChatId, type and publishedAt are present" + (). The parser had + no filter, so a deletion rendered as a **blank chat row**. That is the actual defect, and it is fixed. +- Deletion events (`messageDeletedEvent`/`messageRetractedEvent`/`userBannedEvent`) are documented in + the resource but have not been delivered to clients for years + (), so no client-side moderation machinery can be built + on them. The parser skips them if they ever do arrive. + +The constraints below are therefore **moot for us and binding on YouTube** — except constraint 4, which is +why the Super Chat parse path still asserts that reward fields are never reinterpreted. **Goal:** let the creator decide what reaches the projected chat box, without the app ever becoming a moderator that stores what people said. diff --git a/docs/ai/decisions.md b/docs/ai/decisions.md index c7e8a0a..128b6cc 100644 --- a/docs/ai/decisions.md +++ b/docs/ai/decisions.md @@ -209,6 +209,14 @@ Do not build these; each was explicitly ruled out. never forced. The original objection was webcam background removal (ML model, lighting, support weight); none of it applies to a clip the creator supplied. The webcam still gets the shipped ONNX path, and NVIDIA-only effects stay out. +- **Chat profanity filtering is YouTube's job (2026-11-05, creator ruling):** TASK 49's local wordlist is + retired. YouTube's blocked-words filter and hold-for-review both run upstream of `liveChatMessages.list` + (held messages are never returned: ), so a local list would + duplicate a switch the creator already owns. The rule that replaces it: **never render what YouTube has + removed** — a deleted message arrives as a `tombstone` with no text + () and is skipped, as are + `hasDisplayContent:false` messages and chat-mode events. Deletion events are documented but not delivered + (), so no client-side moderation machinery is built on them. - Virtual camera output — a different product and a device-driver swamp; our output is one YouTube stream. - Replay buffer, stream clipping, multi-destination restreaming, an in-app bug reporter, and an diff --git a/ytLive.Tests/ChatModerationParseTests.cs b/ytLive.Tests/ChatModerationParseTests.cs new file mode 100644 index 0000000..b96fe4a --- /dev/null +++ b/ytLive.Tests/ChatModerationParseTests.cs @@ -0,0 +1,97 @@ +using System.Linq; +using Xunit; +using ytLive.Services; + +namespace ytLive.Tests; + +/// +/// 2026-11-05 ruling (retires TASK 49's local wordlist): YouTube owns chat moderation, so the +/// behaviour we owe it is not a second list — it is not showing what YouTube has removed. +/// The feed sends deleted messages as a tombstone placeholder carrying no text, and marks +/// non-displayable events with hasDisplayContent: false; both used to become blank rows on +/// the overlay (the parser had no filter at all). +/// +public class ChatModerationParseTests +{ + [Fact] + public void Tombstone_NeverBecomesAChatRow() + { + const string json = """ + { + "items": [ + {"id":"d1","snippet":{"type":"tombstone","liveChatId":"c","publishedAt":"2026-11-05T10:00:00.000Z"}}, + {"id":"m1","snippet":{"type":"textMessageEvent","hasDisplayContent":true, + "textMessageDetails":{"messageText":"hello"},"publishedAt":"2026-11-05T10:00:01.000Z"}, + "authorDetails":{"displayName":"Viewer","channelId":"UC1"}} + ], + "nextPageToken":"n" + } + """; + + var page = YouTubeChatService.ParsePage(json); + + Assert.Single(page.Messages); + Assert.Equal("hello", page.Messages[0].Message); + } + + [Fact] + public void MessagesWithoutDisplayContent_AreSkipped() + { + const string json = """ + { + "items": [ + {"id":"m1","snippet":{"type":"textMessageEvent","hasDisplayContent":false, + "publishedAt":"2026-11-05T10:00:00.000Z"}, + "authorDetails":{"displayName":"Muted","channelId":"UC1"}}, + {"id":"m2","snippet":{"type":"textMessageEvent","hasDisplayContent":true, + "textMessageDetails":{"messageText":"visible"},"publishedAt":"2026-11-05T10:00:01.000Z"}, + "authorDetails":{"displayName":"Viewer","channelId":"UC2"}} + ] + } + """; + + var page = YouTubeChatService.ParsePage(json); + + Assert.Single(page.Messages); + Assert.Equal("visible", page.Messages[0].Message); + } + + [Fact] + public void ChatModeEvents_AreNotChat() + { + const string json = """ + { + "items": [ + {"id":"e1","snippet":{"type":"sponsorOnlyModeStartedEvent","hasDisplayContent":false, + "publishedAt":"2026-11-05T10:00:00.000Z"}}, + {"id":"e2","snippet":{"type":"chatEndedEvent","publishedAt":"2026-11-05T10:00:01.000Z"}} + ] + } + """; + + Assert.Empty(YouTubeChatService.ParsePage(json).Messages); + } + + [Fact] + public void RealChatStillParses_WithAuthorAndSuperChat() + { + const string json = """ + { + "items": [ + {"id":"m1","snippet":{"type":"superChatEvent","hasDisplayContent":true, + "superChatDetails":{"amountMicros":12340000,"currency":"USD","amountDisplayString":"$12.34", + "userComment":"nice stream"},"publishedAt":"2026-11-05T10:00:00.000Z"}, + "authorDetails":{"displayName":"Funder","channelId":"UC9","isChatSponsor":true}} + ] + } + """; + + var message = Assert.Single(YouTubeChatService.ParsePage(json).Messages); + + Assert.Equal("Funder", message.AuthorName); + Assert.True(message.IsSuperChat); + // Constraint that survives the TASK 49 retirement: reward fields are never reinterpreted. + Assert.Equal("$12.34", message.SuperChatDisplayString); + Assert.Equal("nice stream", message.Message); + } +}