diff --git a/scripts/avsync.py b/scripts/avsync.py index a452969..32189da 100644 --- a/scripts/avsync.py +++ b/scripts/avsync.py @@ -142,6 +142,53 @@ def offset_ms(audio: np.ndarray, motion: np.ndarray, fps: float, max_ms: float = return overall_lag, overall_corr, windows +def clap_offset(audio: np.ndarray, motion: np.ndarray, fps: float) -> tuple[float, int]: + """Pair sharp audio transients with the nearest big visual change and take the median. + + A clap is the only event in an ordinary recording with an unmistakable audio spike and an + unmistakable visual spike a fixed distance apart, which is why it beats correlating a mouth + against speech: no periodicity to lock onto wrongly. Returns (median_ms, pair_count) where a + POSITIVE value means the sound arrived later than the picture. + """ + if fps <= 0 or len(motion) < 4: + return float("nan"), 0 + + win = 240 # 5 ms amplitude windows + n = len(audio) // win + if n < 8: + return float("nan"), 0 + amp = np.abs(audio[: n * win].reshape(n, win)).max(axis=1) + times = np.arange(n) * win / 48000.0 + + threshold = np.percentile(amp, 99.0) + events: list[float] = [] + last = -9.0 + for t in times[amp >= threshold]: + if t - last > 0.6: # one event per clap, not one per sample of its decay + events.append(float(t)) + last = t + + deltas = [] + for t in events: + lo, hi = int((t - 0.4) * fps), int((t + 0.4) * fps) + if lo < 0 or hi >= len(motion): + continue + peak = (lo + int(np.argmax(motion[lo:hi]))) / fps + deltas.append((t - peak) * 1000.0) + + if len(deltas) < 3: + return float("nan"), len(deltas) + + # Keep the cluster: an arm entering the frame moves more than the clap itself and lands early. + deltas.sort() + mid = len(deltas) // 2 + median = deltas[mid] if len(deltas) % 2 else (deltas[mid - 1] + deltas[mid]) / 2 + tight = [d for d in deltas if abs(d - median) <= 60] + if len(tight) < 3: + return float("nan"), len(tight) + return float(np.median(tight)), len(tight) + + def main() -> int: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("recording") @@ -179,8 +226,17 @@ def main() -> int: print(f" {start:5.1f}s {wlag:+7.0f} ms corr {wcorr:.2f} {mark}") if lag != lag or corr < 0.35: + clap, pairs = clap_offset(samples, motion, fps) + if clap == clap: + print(f"offset {clap:+8.0f} ms (clap pairs: {pairs})") + direction = "sound LATER than the lips" if clap > 0 else "sound EARLIER than the lips" + if abs(clap) >= 20: + print(f" {direction} — adjust the mic delay by {-clap:+.0f} ms") + else: + print(" within a frame or two — leave it alone") + return 0 print("offset unmeasurable — too little agreement between mouth motion and speech") - print(" record 3 claps with your hands in the webcam frame, 2 s apart") + print(f" ({pairs} usable clap pairs) record 3 claps, hands in the webcam frame, 2 s apart") return 0 print(f"offset {lag:+8.0f} ms (correlation {corr:.2f})")