feat(tools): avsync pairs clap transients when the mouth correlation is weak

Mouth-motion against speech is a statistical proxy and it fails on ordinary talking clips: the
per-window estimates swing by 700 ms and the honest answer is "unmeasurable". A clap is not a
proxy — it has one unmistakable audio spike and one unmistakable visual spike, a fixed distance
apart, with no periodicity to lock onto wrongly.

So when the correlation is weak, the tool now detects sharp audio transients (99th percentile,
one event per 0.6 s), finds the biggest visual change within ±400 ms of each, and reports the
median of the cluster within 60 ms of it — at least three pairs or it still says unmeasurable.
The cluster matters: an arm entering the frame moves more than the clap itself and lands early
(two pairs on the validation clip came out at +248 and +283 ms against -285).

Validated on ty-20261112-1405-621-0000.mp4: the correlation path said +98 ms and the clap path
+116 ms (three tight pairs at +117, +115, +107), so the applied 350 ms should come down to about
240. Two methods, same answer, no listening required.
This commit is contained in:
2026-11-12 14:07:09 -08:00
parent 514fb8480d
commit 1abc86358a
+57 -1
View File
@@ -142,6 +142,53 @@ def offset_ms(audio: np.ndarray, motion: np.ndarray, fps: float, max_ms: float =
return overall_lag, overall_corr, windows
def clap_offset(audio: np.ndarray, motion: np.ndarray, fps: float) -> tuple[float, int]:
"""Pair sharp audio transients with the nearest big visual change and take the median.
A clap is the only event in an ordinary recording with an unmistakable audio spike and an
unmistakable visual spike a fixed distance apart, which is why it beats correlating a mouth
against speech: no periodicity to lock onto wrongly. Returns (median_ms, pair_count) where a
POSITIVE value means the sound arrived later than the picture.
"""
if fps <= 0 or len(motion) < 4:
return float("nan"), 0
win = 240 # 5 ms amplitude windows
n = len(audio) // win
if n < 8:
return float("nan"), 0
amp = np.abs(audio[: n * win].reshape(n, win)).max(axis=1)
times = np.arange(n) * win / 48000.0
threshold = np.percentile(amp, 99.0)
events: list[float] = []
last = -9.0
for t in times[amp >= threshold]:
if t - last > 0.6: # one event per clap, not one per sample of its decay
events.append(float(t))
last = t
deltas = []
for t in events:
lo, hi = int((t - 0.4) * fps), int((t + 0.4) * fps)
if lo < 0 or hi >= len(motion):
continue
peak = (lo + int(np.argmax(motion[lo:hi]))) / fps
deltas.append((t - peak) * 1000.0)
if len(deltas) < 3:
return float("nan"), len(deltas)
# Keep the cluster: an arm entering the frame moves more than the clap itself and lands early.
deltas.sort()
mid = len(deltas) // 2
median = deltas[mid] if len(deltas) % 2 else (deltas[mid - 1] + deltas[mid]) / 2
tight = [d for d in deltas if abs(d - median) <= 60]
if len(tight) < 3:
return float("nan"), len(tight)
return float(np.median(tight)), len(tight)
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("recording")
@@ -179,8 +226,17 @@ def main() -> int:
print(f" {start:5.1f}s {wlag:+7.0f} ms corr {wcorr:.2f} {mark}")
if lag != lag or corr < 0.35:
clap, pairs = clap_offset(samples, motion, fps)
if clap == clap:
print(f"offset {clap:+8.0f} ms (clap pairs: {pairs})")
direction = "sound LATER than the lips" if clap > 0 else "sound EARLIER than the lips"
if abs(clap) >= 20:
print(f" {direction} — adjust the mic delay by {-clap:+.0f} ms")
else:
print(" within a frame or two — leave it alone")
return 0
print("offset unmeasurable — too little agreement between mouth motion and speech")
print(" record 3 claps with your hands in the webcam frame, 2 s apart")
print(f" ({pairs} usable clap pairs) record 3 claps, hands in the webcam frame, 2 s apart")
return 0
print(f"offset {lag:+8.0f} ms (correlation {corr:.2f})")