From 1abc86358a647f6dd6964b2a4804c09800019a9d Mon Sep 17 00:00:00 2001
From: gramps <gramps@llamachile.shop>
Date: Thu, 12 Nov 2026 14:07:09 -0800
Subject: [PATCH] feat(tools): avsync pairs clap transients when the mouth
 correlation is weak
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Mouth-motion against speech is a statistical proxy and it fails on ordinary talking clips: the
per-window estimates swing by 700 ms and the honest answer is "unmeasurable". A clap is not a
proxy — it has one unmistakable audio spike and one unmistakable visual spike, a fixed distance
apart, with no periodicity to lock onto wrongly.

So when the correlation is weak, the tool now detects sharp audio transients (99th percentile,
one event per 0.6 s), finds the biggest visual change within ±400 ms of each, and reports the
median of the cluster within 60 ms of it — at least three pairs or it still says unmeasurable.
The cluster matters: an arm entering the frame moves more than the clap itself and lands early
(two pairs on the validation clip came out at +248 and +283 ms against -285).

Validated on ty-20261112-1405-621-0000.mp4: the correlation path said +98 ms and the clap path
+116 ms (three tight pairs at +117, +115, +107), so the applied 350 ms should come down to about
240. Two methods, same answer, no listening required.
---
 scripts/avsync.py | 58 ++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 57 insertions(+), 1 deletion(-)

diff --git a/scripts/avsync.py b/scripts/avsync.py
index a452969..32189da 100644
--- a/scripts/avsync.py
+++ b/scripts/avsync.py
@@ -142,6 +142,53 @@ def offset_ms(audio: np.ndarray, motion: np.ndarray, fps: float, max_ms: float =
     return overall_lag, overall_corr, windows
 
 
+def clap_offset(audio: np.ndarray, motion: np.ndarray, fps: float) -> tuple[float, int]:
+    """Pair sharp audio transients with the nearest big visual change and take the median.
+
+    A clap is the only event in an ordinary recording with an unmistakable audio spike and an
+    unmistakable visual spike a fixed distance apart, which is why it beats correlating a mouth
+    against speech: no periodicity to lock onto wrongly. Returns (median_ms, pair_count) where a
+    POSITIVE value means the sound arrived later than the picture.
+    """
+    if fps <= 0 or len(motion) < 4:
+        return float("nan"), 0
+
+    win = 240                      # 5 ms amplitude windows
+    n = len(audio) // win
+    if n < 8:
+        return float("nan"), 0
+    amp = np.abs(audio[: n * win].reshape(n, win)).max(axis=1)
+    times = np.arange(n) * win / 48000.0
+
+    threshold = np.percentile(amp, 99.0)
+    events: list[float] = []
+    last = -9.0
+    for t in times[amp >= threshold]:
+        if t - last > 0.6:         # one event per clap, not one per sample of its decay
+            events.append(float(t))
+        last = t
+
+    deltas = []
+    for t in events:
+        lo, hi = int((t - 0.4) * fps), int((t + 0.4) * fps)
+        if lo < 0 or hi >= len(motion):
+            continue
+        peak = (lo + int(np.argmax(motion[lo:hi]))) / fps
+        deltas.append((t - peak) * 1000.0)
+
+    if len(deltas) < 3:
+        return float("nan"), len(deltas)
+
+    # Keep the cluster: an arm entering the frame moves more than the clap itself and lands early.
+    deltas.sort()
+    mid = len(deltas) // 2
+    median = deltas[mid] if len(deltas) % 2 else (deltas[mid - 1] + deltas[mid]) / 2
+    tight = [d for d in deltas if abs(d - median) <= 60]
+    if len(tight) < 3:
+        return float("nan"), len(tight)
+    return float(np.median(tight)), len(tight)
+
+
 def main() -> int:
     ap = argparse.ArgumentParser(description=__doc__)
     ap.add_argument("recording")
@@ -179,8 +226,17 @@ def main() -> int:
         print(f"        {start:5.1f}s {wlag:+7.0f} ms  corr {wcorr:.2f} {mark}")
 
     if lag != lag or corr < 0.35:
+        clap, pairs = clap_offset(samples, motion, fps)
+        if clap == clap:
+            print(f"offset  {clap:+8.0f} ms  (clap pairs: {pairs})")
+            direction = "sound LATER than the lips" if clap > 0 else "sound EARLIER than the lips"
+            if abs(clap) >= 20:
+                print(f"        {direction} — adjust the mic delay by {-clap:+.0f} ms")
+            else:
+                print("        within a frame or two — leave it alone")
+            return 0
         print("offset  unmeasurable — too little agreement between mouth motion and speech")
-        print("        record 3 claps with your hands in the webcam frame, 2 s apart")
+        print(f"        ({pairs} usable clap pairs) record 3 claps, hands in the webcam frame, 2 s apart")
         return 0
 
     print(f"offset  {lag:+8.0f} ms  (correlation {corr:.2f})")