feat(tools): avsync pairs clap transients when the mouth correlation is weak
Mouth-motion against speech is a statistical proxy and it fails on ordinary talking clips: the per-window estimates swing by 700 ms and the honest answer is "unmeasurable". A clap is not a proxy — it has one unmistakable audio spike and one unmistakable visual spike, a fixed distance apart, with no periodicity to lock onto wrongly. So when the correlation is weak, the tool now detects sharp audio transients (99th percentile, one event per 0.6 s), finds the biggest visual change within ±400 ms of each, and reports the median of the cluster within 60 ms of it — at least three pairs or it still says unmeasurable. The cluster matters: an arm entering the frame moves more than the clap itself and lands early (two pairs on the validation clip came out at +248 and +283 ms against -285). Validated on ty-20261112-1405-621-0000.mp4: the correlation path said +98 ms and the clap path +116 ms (three tight pairs at +117, +115, +107), so the applied 350 ms should come down to about 240. Two methods, same answer, no listening required.
This commit is contained in:
+57
-1
@@ -142,6 +142,53 @@ def offset_ms(audio: np.ndarray, motion: np.ndarray, fps: float, max_ms: float =
|
||||
return overall_lag, overall_corr, windows
|
||||
|
||||
|
||||
def clap_offset(audio: np.ndarray, motion: np.ndarray, fps: float) -> tuple[float, int]:
|
||||
"""Pair sharp audio transients with the nearest big visual change and take the median.
|
||||
|
||||
A clap is the only event in an ordinary recording with an unmistakable audio spike and an
|
||||
unmistakable visual spike a fixed distance apart, which is why it beats correlating a mouth
|
||||
against speech: no periodicity to lock onto wrongly. Returns (median_ms, pair_count) where a
|
||||
POSITIVE value means the sound arrived later than the picture.
|
||||
"""
|
||||
if fps <= 0 or len(motion) < 4:
|
||||
return float("nan"), 0
|
||||
|
||||
win = 240 # 5 ms amplitude windows
|
||||
n = len(audio) // win
|
||||
if n < 8:
|
||||
return float("nan"), 0
|
||||
amp = np.abs(audio[: n * win].reshape(n, win)).max(axis=1)
|
||||
times = np.arange(n) * win / 48000.0
|
||||
|
||||
threshold = np.percentile(amp, 99.0)
|
||||
events: list[float] = []
|
||||
last = -9.0
|
||||
for t in times[amp >= threshold]:
|
||||
if t - last > 0.6: # one event per clap, not one per sample of its decay
|
||||
events.append(float(t))
|
||||
last = t
|
||||
|
||||
deltas = []
|
||||
for t in events:
|
||||
lo, hi = int((t - 0.4) * fps), int((t + 0.4) * fps)
|
||||
if lo < 0 or hi >= len(motion):
|
||||
continue
|
||||
peak = (lo + int(np.argmax(motion[lo:hi]))) / fps
|
||||
deltas.append((t - peak) * 1000.0)
|
||||
|
||||
if len(deltas) < 3:
|
||||
return float("nan"), len(deltas)
|
||||
|
||||
# Keep the cluster: an arm entering the frame moves more than the clap itself and lands early.
|
||||
deltas.sort()
|
||||
mid = len(deltas) // 2
|
||||
median = deltas[mid] if len(deltas) % 2 else (deltas[mid - 1] + deltas[mid]) / 2
|
||||
tight = [d for d in deltas if abs(d - median) <= 60]
|
||||
if len(tight) < 3:
|
||||
return float("nan"), len(tight)
|
||||
return float(np.median(tight)), len(tight)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("recording")
|
||||
@@ -179,8 +226,17 @@ def main() -> int:
|
||||
print(f" {start:5.1f}s {wlag:+7.0f} ms corr {wcorr:.2f} {mark}")
|
||||
|
||||
if lag != lag or corr < 0.35:
|
||||
clap, pairs = clap_offset(samples, motion, fps)
|
||||
if clap == clap:
|
||||
print(f"offset {clap:+8.0f} ms (clap pairs: {pairs})")
|
||||
direction = "sound LATER than the lips" if clap > 0 else "sound EARLIER than the lips"
|
||||
if abs(clap) >= 20:
|
||||
print(f" {direction} — adjust the mic delay by {-clap:+.0f} ms")
|
||||
else:
|
||||
print(" within a frame or two — leave it alone")
|
||||
return 0
|
||||
print("offset unmeasurable — too little agreement between mouth motion and speech")
|
||||
print(" record 3 claps with your hands in the webcam frame, 2 s apart")
|
||||
print(f" ({pairs} usable clap pairs) record 3 claps, hands in the webcam frame, 2 s apart")
|
||||
return 0
|
||||
|
||||
print(f"offset {lag:+8.0f} ms (correlation {corr:.2f})")
|
||||
|
||||
Reference in New Issue
Block a user