From fb538010bdc9d181823349dc28f097623dc4991e Mon Sep 17 00:00:00 2001 From: Abhishek Date: Tue, 1 Sep 2026 17:16:28 +0530 Subject: [PATCH] Fix crash on VTT timestamps at or past 24:00:00 datetime(2000, 1, 1, hour, ...) raises ValueError when hour >= 24, which happens for long-form YouTube auto-captions (multi-hour streams/lectures) that use timestamps like 24:xx:xx or higher. Replaced the datetime-based +10ms calculation with plain integer arithmetic on total milliseconds, which has no upper bound on hours. --- english/test_vtt_auto_to_conll_u.py | 29 +++++++++++++++++++++++++++++ english/vtt_auto_to_conll-u.py | 21 +++++++++++++-------- 2 files changed, 42 insertions(+), 8 deletions(-) create mode 100644 english/test_vtt_auto_to_conll_u.py diff --git a/english/test_vtt_auto_to_conll_u.py b/english/test_vtt_auto_to_conll_u.py new file mode 100644 index 0000000..5c9f6b9 --- /dev/null +++ b/english/test_vtt_auto_to_conll_u.py @@ -0,0 +1,29 @@ +# Self-check for add_10ms() in vtt_auto_to_conll-u.py. +# Run directly: python test_vtt_auto_to_conll_u.py +# +# vtt_auto_to_conll-u.py can't be imported here (its module-level code +# depends on `somajo`, which isn't installed outside the pipeline's HPC +# environment -- see README Prerequisites). add_10ms() has no such +# dependency, so it's copied verbatim rather than importing the whole file. + +def add_10ms(timestamp): + h, m, s_ms = timestamp.split(":") + s, ms = s_ms.split(".") + total_ms = ((int(h) * 3600 + int(m) * 60 + int(s)) * 1000 + int(ms)) + 10 + h, rem_ms = divmod(total_ms, 3600000) + m, rem_ms = divmod(rem_ms, 60000) + s, ms = divmod(rem_ms, 1000) + return f"{h:02d}:{m:02d}:{s:02d}.{ms:03d}" + + +def test_add_10ms(): + assert add_10ms("00:00:01.500") == "00:00:01.510" + assert add_10ms("00:00:59.995") == "00:01:00.005" + # regression: hour>=24 crashed the old datetime(2000,1,1,hour,...) implementation + assert add_10ms("23:59:59.995") == "24:00:00.005" + assert add_10ms("25:10:00.000") == "25:10:00.010" + print("all tests passed") + + +if __name__ == "__main__": + test_add_10ms() diff --git a/english/vtt_auto_to_conll-u.py b/english/vtt_auto_to_conll-u.py index a100a12..80180f4 100644 --- a/english/vtt_auto_to_conll-u.py +++ b/english/vtt_auto_to_conll-u.py @@ -1,7 +1,18 @@ import sys, re -from datetime import datetime, timedelta from somajo import SoMaJo +def add_10ms(timestamp): + # Plain seconds arithmetic instead of datetime() -- datetime's hour field + # is capped at 23, but long VTT files (multi-hour streams/lectures) can + # have timestamps at or past 24:00:00. + h, m, s_ms = timestamp.split(":") + s, ms = s_ms.split(".") + total_ms = ((int(h) * 3600 + int(m) * 60 + int(s)) * 1000 + int(ms)) + 10 + h, rem_ms = divmod(total_ms, 3600000) + m, rem_ms = divmod(rem_ms, 60000) + s, ms = divmod(rem_ms, 1000) + return f"{h:02d}:{m:02d}:{s:02d}.{ms:03d}" + # We have 3-line blocks, and the new stuff can always be found in the last line, so we can probably ignore the secoond one. # ToDo: There are 10 msec pauses because of the line feeds. We should try to find out which way they are more likely. # ToDo: We may be able to use statistics for this (i.e. compare the average length of certain words at the beginning of sentence vs. somewhere else) @@ -22,13 +33,7 @@ def tokenize_word (intext): if match1: # This is the first line of the triplet currenttime = match1.group(1) - # For proper time calculations, we need to do some stuff: - # We have to do this manually, since fromisoformat becomes available only in Python 3.7, and we do not have this on the servers yet... - endtime_iso = datetime(2000, 1, 1, int(match1.group(2)), int(match1.group(3)), int(match1.group(4)), int(match1.group(5))*1000) - delta = timedelta(milliseconds=10) - endtime_iso += delta - endtime = endtime_iso.strftime("%H:%M:%S.%f")[:-3] - #endtime = match1.group(2) + endtime = add_10ms(f"{match1.group(2)}:{match1.group(3)}:{match1.group(4)}.{match1.group(5)}") next(infile) # This is the second line. We are never interested in the second line. thirdline = next(infile) thirdline = thirdline.strip()