support py3.11

Pipfile

@@ -55,13 +55,14 @@ mccabe = "==0.6.1"
 numba = ">=0.50.0"
 numpy = "<1.24.0"
 oauthlib = "==3.1.0"
-openai-whisper = "==20230314"
+openai-whisper = "==20231106"
 pbr = "==4.0.2"
 pkgconfig = "~=1.5.5"
 pluggy = "==0.13.1"
 py = "==1.10.0"
 pyasn1 = "==0.4.8"
 pyasn1-modules = "==0.2.7"
+pybind11 = "~=2.11.1"
 pycountry = "~=20.7.3"
 pydot = "==1.2.4"
 pydot-ng = "==1.0.0"
@@ -76,7 +77,7 @@ python-dateutil = "==2.7.2"
 pytz = "==2018.4"
 PyYAML = ">=4.2b1"
 rsa = "==4.7"
-scipy = "<=1.8.1"
+scipy = "<1.11.0"
 scikit-learn = ">=0.19.1"
 sentencepiece = "~=0.1.95"
 six = "~=1.15.0"

README.md

@@ -3,7 +3,7 @@
 </div>
 
 [![Build Status](https://github.com/baxtree/subaligner/actions/workflows/ci-pipeline.yml/badge.svg?branch=master)](https://github.com/baxtree/subaligner/actions/workflows/ci-pipeline.yml?query=branch%3Amaster) ![Codecov](https://img.shields.io/codecov/c/github/baxtree/subaligner)
-[![Python 3.10](https://img.shields.io/badge/python-3.10-blue.svg)](https://www.python.org/downloads/release/python-3100/) [![Python 3.9](https://img.shields.io/badge/python-3.9-blue.svg)](https://www.python.org/downloads/release/python-390/) [![Python 3.8](https://img.shields.io/badge/python-3.8-blue.svg)](https://www.python.org/downloads/release/python-380/)
+[![Python 3.11](https://img.shields.io/badge/python-3.11-blue.svg)](https://www.python.org/downloads/release/python-3110/) [![Python 3.10](https://img.shields.io/badge/python-3.10-blue.svg)](https://www.python.org/downloads/release/python-3100/) [![Python 3.9](https://img.shields.io/badge/python-3.9-blue.svg)](https://www.python.org/downloads/release/python-390/) [![Python 3.8](https://img.shields.io/badge/python-3.8-blue.svg)](https://www.python.org/downloads/release/python-380/)
 [![Documentation Status](https://readthedocs.org/projects/subaligner/badge/?version=latest)](https://subaligner.readthedocs.io/en/latest/?badge=latest)
 [![GitHub license](https://img.shields.io/github/license/baxtree/subaligner)](https://github.com/baxtree/subaligner/blob/master/LICENSE)
 [![PyPI](https://badge.fury.io/py/subaligner.svg)](https://badge.fury.io/py/subaligner)

requirements-arm64.txt

@@ -38,18 +38,19 @@ psutil==5.6.7
 py==1.10.0
 pyasn1==0.4.8
 pyasn1-modules==0.2.7
+pybind11~=2.11.1
 pycountry~=20.7.3
 pydot==1.2.4
 pydot-ng==1.0.0
 pydotplus==2.0.2
 pyprof2calltree==1.4.3
-pysrt==1.1.1
-pysubs2<=1.4.2
+pysrt==1.1.2
+pysubs2~=1.6.1
 pystack-debugger==0.8.0
 pytz==2018.4
 PyYAML>=4.2b1
 rsa==4.7
-scipy<=1.8.1
+scipy<1.11.0
 scikit-learn<1.2.0
 setuptools>=41.0.0
 six~=1.15.0

requirements-dev.txt

@@ -4,7 +4,7 @@ tox~=3.23.0
 pycodestyle==2.5.0
 twine<4.0.0
 snakeviz==2.1.0
-line-profiler==3.1.0
+line-profiler~=4.1.2
 scikit-build==0.11.1
 radish-bdd~=0.13.3
 pex<=2.1.80

requirements-llm.txt

@@ -1,4 +1,4 @@
 sentencepiece~=0.1.95
-torch<1.13.0
+torch<2.2.0
 transformers<4.27.0
-openai-whisper==20230314
+openai-whisper==20231106

requirements.txt

@@ -37,18 +37,19 @@ pluggy==0.13.1
 py==1.10.0
 pyasn1==0.4.8
 pyasn1-modules==0.2.7
+pybind11~=2.11.1
 pycountry~=20.7.3
 pydot==1.2.4
 pydot-ng==1.0.0
 pydotplus==2.0.2
 pyprof2calltree==1.4.3
-pysrt==1.1.1
-pysubs2<=1.4.2
+pysrt==1.1.2
+pysubs2~=1.6.1
 pystack-debugger==0.8.0
 pytz==2018.4
 PyYAML>=4.2b1
 rsa==4.7
-scipy<=1.8.1
+scipy<1.11.0
 scikit-learn<1.2.0
 six~=1.15.0
 tblib==1.3.2

setup.py

@@ -64,6 +64,7 @@ def get_tag(self):
           "Programming Language :: Python :: 3.8",
           "Programming Language :: Python :: 3.9",
           "Programming Language :: Python :: 3.10",
+          "Programming Language :: Python :: 3.11",
           "Intended Audience :: Developers",
       ],
       license="MIT",

subaligner/__main__.py

@@ -4,7 +4,7 @@
                   [-sil {afr,amh,ara,arg,asm,aze,ben,bos,bul,cat,ces,cmn,cym,dan,deu,ell,eng,epo,est,eus,fas,fin,fra,gla,gle,glg,grc,grn,guj,heb,hin,hrv,hun,hye,ina,ind,isl,ita,jbo,jpn,kal,kan,kat,kir,kor,kur,lat,lav,lfn,lit,mal,mar,mkd,mlt,msa,mya,nah,nep,nld,nor,ori,orm,pan,pap,pol,por,ron,rus,sin,slk,slv,spa,sqi,srp,swa,swe,tam,tat,tel,tha,tsn,tur,ukr,urd,vie,yue,zho}]
                   [-fos] [-tod TRAINING_OUTPUT_DIRECTORY] [-o OUTPUT] [-t TRANSLATE] [-os OFFSET_SECONDS]
                   [-ml {afr,amh,ara,arg,asm,aze,ben,bos,bul,cat,ces,cmn,cym,dan,deu,ell,eng,epo,est,eus,fas,fin,fra,gla,gle,glg,grc,grn,guj,heb,hin,hrv,hun,hye,ina,ind,isl,ita,jbo,jpn,kal,kan,kat,kir,kor,kur,lat,lav,lfn,lit,mal,mar,mkd,mlt,msa,mya,nah,nep,nld,nor,ori,orm,pan,pap,pol,por,ron,rus,sin,slk,slv,spa,sqi,srp,swa,swe,tam,tat,tel,tha,tsn,tur,ukr,urd,vie,yue,zho}]
-                  [-mr {whisper}] [-mf {tiny,tiny.en,small,medium,medium.en,base,base.en,large-v1,large-v2,large}] [-tr {helsinki-nlp,whisper,facebook-mbart}]
+                  [-mr {whisper}] [-mf {tiny,tiny.en,small,medium,medium.en,base,base.en,large-v1,large-v2,large-v3,large}] [-tr {helsinki-nlp,whisper,facebook-mbart}]
                   [-tf TRANSLATION_FLAVOUR] [-lgs] [-d] [-q] [-ver]
 
 Subaligner command line interface
@@ -32,7 +32,7 @@
                         Target video's main language as an ISO 639-3 language code [https://en.wikipedia.org/wiki/List_of_ISO_639-3_codes]
   -mr {whisper}, --transcription_recipe {whisper}
                         LLM recipe used for transcribing video files
-  -mf {tiny,tiny.en,small,medium,medium.en,base,base.en,large-v1,large-v2,large}, --transcription_flavour {tiny,tiny.en,small,medium,medium.en,base,base.en,large-v1,large-v2,large}
+  -mf {tiny,tiny.en,small,medium,medium.en,base,base.en,large-v1,large-v2,large-v3,large}, --transcription_flavour {tiny,tiny.en,small,medium,medium.en,base,base.en,large-v1,large-v2,large-v3,large}
                         Flavour variation for a specific LLM recipe supporting transcription
   -tr {helsinki-nlp,whisper,facebook-mbart}, --translation_recipe {helsinki-nlp,whisper,facebook-mbart}
                         LLM recipe used for translating subtitles
@@ -55,7 +55,6 @@
 import traceback
 import os
 import tempfile
-import pkg_resources
 
 
 def main():
@@ -245,11 +244,15 @@ def main():
                     parser.print_usage()
                     sys.exit(21)
             if FLAGS.translate is not None or FLAGS.mode == "transcribe":
-                if "transformers" not in {pkg.key for pkg in pkg_resources.working_set}:
+                try:
+                    import transformers
+                except ModuleNotFoundError:
                     print('ERROR: Alignment has been configured to use language models. Please install "subaligner[llm]" and run your command again.')
                     sys.exit(21)
             if FLAGS.stretch_on or FLAGS.mode == "script":
-                if "aeneas" not in {pkg.key for pkg in pkg_resources.working_set}:
+                try:
+                    import aeneas
+                except ModuleNotFoundError:
                     print('ERROR: Alignment has been configured to use extra features. Please install "subaligner[stretch]" and run your command again.')
                     sys.exit(21)
 

subaligner/llm.py

@@ -21,6 +21,7 @@ class WhisperFlavour(Enum):
     BASE_EN = "base.en"
     LARGE_V1 = "large-v1"
     LARGE_V2 = "large-v2"
+    LARGE_V3 = "large-v3"
     LARGE = "large"
 
 

subaligner/subaligner_1pass/__main__.py

@@ -31,7 +31,6 @@
 import traceback
 import os
 import tempfile
-import pkg_resources
 
 
 def main():
@@ -119,7 +118,9 @@ def main():
         parser.print_usage()
         sys.exit(21)
     if FLAGS.translate is not None:
-        if "transformers" not in {pkg.key for pkg in pkg_resources.working_set}:
+        try:
+            import transformers
+        except ModuleNotFoundError:
             print('ERROR: Alignment has been configured to perform translation. Please install "subaligner[llm]" and run your command again.')
             sys.exit(21)
 

subaligner/subaligner_2pass/__main__.py

@@ -38,7 +38,6 @@
 import traceback
 import os
 import tempfile
-import pkg_resources
 
 
 def main():
@@ -146,11 +145,15 @@ def main():
         parser.print_usage()
         sys.exit(21)
     if FLAGS.translate is not None:
-        if "transformers" not in {pkg.key for pkg in pkg_resources.working_set}:
+        try:
+            import transformers
+        except ModuleNotFoundError:
             print('ERROR: Alignment has been configured to perform translation. Please install "subaligner[llm]" and run your command again.')
             sys.exit(21)
     if FLAGS.stretch_on:
-        if "aeneas" not in {pkg.key for pkg in pkg_resources.working_set}:
+        try:
+            import aeneas
+        except ModuleNotFoundError:
             print('ERROR: Alignment has been configured to use extra features. Please install "subaligner[stretch]" and run your command again.')
             sys.exit(21)
 

subaligner/subaligner_batch/__main__.py

@@ -4,7 +4,7 @@
                         [-sil {afr,amh,ara,arg,asm,aze,ben,bos,bul,cat,ces,cmn,cym,dan,deu,ell,eng,epo,est,eus,fas,fin,fra,gla,gle,glg,grc,grn,guj,heb,hin,hrv,hun,hye,ina,ind,isl,ita,jbo,jpn,kal,kan,kat,kir,kor,kur,lat,lav,lfn,lit,mal,mar,mkd,mlt,msa,mya,nah,nep,nld,nor,ori,orm,pan,pap,pol,por,ron,rus,sin,slk,slv,spa,sqi,srp,swa,swe,tam,tat,tel,tha,tsn,tur,ukr,urd,vie,yue,zho}]
                         [-fos] [-tod TRAINING_OUTPUT_DIRECTORY] [-od OUTPUT_DIRECTORY] [-of {srt,ytt,ttml,txt,smi,xml,ssa,ass,dfxp,sub,scc,tmp,sami,vtt,stl,sbv}] [-t TRANSLATE]
                         [-ml {afr,amh,ara,arg,asm,aze,ben,bos,bul,cat,ces,cmn,cym,dan,deu,ell,eng,epo,est,eus,fas,fin,fra,gla,gle,glg,grc,grn,guj,heb,hin,hrv,hun,hye,ina,ind,isl,ita,jbo,jpn,kal,kan,kat,kir,kor,kur,lat,lav,lfn,lit,mal,mar,mkd,mlt,msa,mya,nah,nep,nld,nor,ori,orm,pan,pap,pol,por,ron,rus,sin,slk,slv,spa,sqi,srp,swa,swe,tam,tat,tel,tha,tsn,tur,ukr,urd,vie,yue,zho}]
-                        [-mr {whisper}] [-mf {tiny,tiny.en,small,medium,medium.en,base,base.en,large-v1,large-v2,large}] [-lgs] [-d] [-q] [-ver]
+                        [-mr {whisper}] [-mf {tiny,tiny.en,small,medium,medium.en,base,base.en,large-v1,large-v2,large-v3,large}] [-lgs] [-d] [-q] [-ver]
 
 Batch align multiple subtitle files and audiovisual files
 
@@ -36,7 +36,7 @@
                         Target video's main language as an ISO 639-3 language code [https://en.wikipedia.org/wiki/List_of_ISO_639-3_codes]
   -mr {whisper}, --transcription_recipe {whisper}
                         LLM recipe used for transcribing video files
-  -mf {tiny,tiny.en,small,medium,medium.en,base,base.en,large-v1,large-v2,large}, --transcription_flavour {tiny,tiny.en,small,medium,medium.en,base,base.en,large-v1,large-v2,large}
+  -mf {tiny,tiny.en,small,medium,medium.en,base,base.en,large-v1,large-v2,large-v3,large}, --transcription_flavour {tiny,tiny.en,small,medium,medium.en,base,base.en,large-v1,large-v2,large-v3,large}
                         Flavour variation for a specific LLM recipe supporting transcription
   -lgs, --languages     Print out language codes used for stretch and translation
   -d, --debug           Print out debugging information
@@ -52,7 +52,6 @@
 import sys
 import traceback
 import os
-import pkg_resources
 import tempfile
 
 
@@ -208,11 +207,15 @@ def main():
         parser.print_usage()
         sys.exit(21)
     if FLAGS.translate is not None or FLAGS.mode == "transcribe":
-        if "transformers" not in {pkg.key for pkg in pkg_resources.working_set}:
+        try:
+            import transformers
+        except ModuleNotFoundError:
             print('ERROR: Alignment has been configured to use language models. Please install "subaligner[llm]" and run your command again.')
             sys.exit(21)
     if FLAGS.stretch_on or FLAGS.mode == "script":
-        if "aeneas" not in {pkg.key for pkg in pkg_resources.working_set}:
+        try:
+            import aeneas
+        except ModuleNotFoundError:
             print('ERROR: Alignment has been configured to use extra features. Please install "subaligner[stretch]" and run your command again.')
             sys.exit(21)
     if FLAGS.mode == "transcribe":

subaligner/subaligner_convert/__main__.py

@@ -27,7 +27,6 @@
 import sys
 import tempfile
 import traceback
-import pkg_resources
 
 
 def main():
@@ -98,7 +97,9 @@ def main():
         parser.print_usage()
         sys.exit(21)
     if FLAGS.translate is not None:
-        if "transformers" not in {pkg.key for pkg in pkg_resources.working_set}:
+        try:
+            import transformers
+        except ModuleNotFoundError:
             print('ERROR: Alignment has been configured to perform translation. Please install "subaligner[llm]" and run your command again.')
             sys.exit(21)
 

tests/subaligner/resource/test.ass

@@ -12,20 +12,20 @@ Style: Default,Arial,20.0,&H00FFFFFF,&H000000FF,&H00000000,&H00000000,0,0,0,0,10
 
 [Events]
 Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
-Dialogue: 0,0:00:12.44,0:00:14.95,Default,,0,0,0,,I've spent nearly two decades
-Dialogue: 0,0:00:14.97,0:00:18.53,Default,,0,0,0,,observing what makes people luckier than others
-Dialogue: 0,0:00:18.55,0:00:22.11,Default,,0,0,0,,and trying to help people increase their luck.
-Dialogue: 0,0:00:22.67,0:00:25.56,Default,,0,0,0,,You see, I teach entrepreneurship,
+Dialogue: 0,0:00:12.44,0:00:14.96,Default,,0,0,0,,I've spent nearly two decades
+Dialogue: 0,0:00:14.98,0:00:18.53,Default,,0,0,0,,observing what makes people luckier than others
+Dialogue: 0,0:00:18.56,0:00:22.12,Default,,0,0,0,,and trying to help people increase their luck.
+Dialogue: 0,0:00:22.68,0:00:25.57,Default,,0,0,0,,You see, I teach entrepreneurship,
 Dialogue: 0,0:00:25.59,0:00:29.14,Default,,0,0,0,,and we all know that most new ventures fail,
-Dialogue: 0,0:00:29.16,0:00:32.79,Default,,0,0,0,,and innovators and entrepreneurs need all the luck they can get.
-Dialogue: 0,0:00:33.84,0:00:35.13,Default,,0,0,0,,So what is luck?
+Dialogue: 0,0:00:29.16,0:00:32.80,Default,,0,0,0,,and innovators and entrepreneurs need all the luck they can get.
+Dialogue: 0,0:00:33.85,0:00:35.14,Default,,0,0,0,,So what is luck?
 Dialogue: 0,0:00:35.16,0:00:41.20,Default,,0,0,0,,Luck is defined as success or failure apparently caused by chance.
-Dialogue: 0,0:00:42.29,0:00:43.66,Default,,0,0,0,,Apparently.
-Dialogue: 0,0:00:43.68,0:00:45.32,Default,,0,0,0,,That's the operative word.
+Dialogue: 0,0:00:42.30,0:00:43.66,Default,,0,0,0,,Apparently.
+Dialogue: 0,0:00:43.69,0:00:45.32,Default,,0,0,0,,That's the operative word.
 Dialogue: 0,0:00:46.35,0:00:48.36,Default,,0,0,0,,It looks like it's chance
-Dialogue: 0,0:00:48.38,0:00:53.54,Default,,0,0,0,,because we rarely see all the levers that come into play to make people lucky.
-Dialogue: 0,0:00:53.56,0:00:56.56,Default,,0,0,0,,But I've realized, by watching so long,
-Dialogue: 0,0:00:56.58,0:00:59.91,Default,,0,0,0,,that luck is rarely a lightning strike,
-Dialogue: 0,0:00:59.94,0:01:01.69,Default,,0,0,0,,isolated and dramatic.
-Dialogue: 0,0:01:01.69,0:01:01.69,Default,,0,0,0,,(SOUND EFFECT)
-Dialogue: 0,0:01:01.69,0:01:01.69,Default,,0,0,0,,(SOUND EFFECT)
+Dialogue: 0,0:00:48.39,0:00:53.54,Default,,0,0,0,,because we rarely see all the levers that come into play to make people lucky.
+Dialogue: 0,0:00:53.57,0:00:56.56,Default,,0,0,0,,But I've realized, by watching so long,
+Dialogue: 0,0:00:56.59,0:00:59.92,Default,,0,0,0,,that luck is rarely a lightning strike,
+Dialogue: 0,0:00:59.94,0:01:01.70,Default,,0,0,0,,isolated and dramatic.
+Dialogue: 0,0:01:01.70,0:01:01.70,Default,,0,0,0,,(SOUND EFFECT)
+Dialogue: 0,0:01:01.70,0:01:01.70,Default,,0,0,0,,(SOUND EFFECT)

tests/subaligner/resource/test.ssa

@@ -12,20 +12,20 @@ Style: Default,Arial,20.0,16777215,255,0,0,0,0,1,2.0,2.0,2,10,10,10,0,1
 
 [Events]
 Format: Marked, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
-Dialogue: Marked=0,0:00:12.44,0:00:14.95,Default,,0,0,0,,I've spent nearly two decades
-Dialogue: Marked=0,0:00:14.97,0:00:18.53,Default,,0,0,0,,observing what makes people luckier than others
-Dialogue: Marked=0,0:00:18.55,0:00:22.11,Default,,0,0,0,,and trying to help people increase their luck.
-Dialogue: Marked=0,0:00:22.67,0:00:25.56,Default,,0,0,0,,You see, I teach entrepreneurship,
+Dialogue: Marked=0,0:00:12.44,0:00:14.96,Default,,0,0,0,,I've spent nearly two decades
+Dialogue: Marked=0,0:00:14.98,0:00:18.53,Default,,0,0,0,,observing what makes people luckier than others
+Dialogue: Marked=0,0:00:18.56,0:00:22.12,Default,,0,0,0,,and trying to help people increase their luck.
+Dialogue: Marked=0,0:00:22.68,0:00:25.57,Default,,0,0,0,,You see, I teach entrepreneurship,
 Dialogue: Marked=0,0:00:25.59,0:00:29.14,Default,,0,0,0,,and we all know that most new ventures fail,
-Dialogue: Marked=0,0:00:29.16,0:00:32.79,Default,,0,0,0,,and innovators and entrepreneurs need all the luck they can get.
-Dialogue: Marked=0,0:00:33.84,0:00:35.13,Default,,0,0,0,,So what is luck?
+Dialogue: Marked=0,0:00:29.16,0:00:32.80,Default,,0,0,0,,and innovators and entrepreneurs need all the luck they can get.
+Dialogue: Marked=0,0:00:33.85,0:00:35.14,Default,,0,0,0,,So what is luck?
 Dialogue: Marked=0,0:00:35.16,0:00:41.20,Default,,0,0,0,,Luck is defined as success or failure apparently caused by chance.
-Dialogue: Marked=0,0:00:42.29,0:00:43.66,Default,,0,0,0,,Apparently.
-Dialogue: Marked=0,0:00:43.68,0:00:45.32,Default,,0,0,0,,That's the operative word.
+Dialogue: Marked=0,0:00:42.30,0:00:43.66,Default,,0,0,0,,Apparently.
+Dialogue: Marked=0,0:00:43.69,0:00:45.32,Default,,0,0,0,,That's the operative word.
 Dialogue: Marked=0,0:00:46.35,0:00:48.36,Default,,0,0,0,,It looks like it's chance
-Dialogue: Marked=0,0:00:48.38,0:00:53.54,Default,,0,0,0,,because we rarely see all the levers that come into play to make people lucky.
-Dialogue: Marked=0,0:00:53.56,0:00:56.56,Default,,0,0,0,,But I've realized, by watching so long,
-Dialogue: Marked=0,0:00:56.58,0:00:59.91,Default,,0,0,0,,that luck is rarely a lightning strike,
-Dialogue: Marked=0,0:00:59.94,0:01:01.69,Default,,0,0,0,,isolated and dramatic.
-Dialogue: Marked=0,0:01:01.69,0:01:01.69,Default,,0,0,0,,(SOUND EFFECT)
-Dialogue: Marked=0,0:01:01.69,0:01:01.69,Default,,0,0,0,,(SOUND EFFECT)
+Dialogue: Marked=0,0:00:48.39,0:00:53.54,Default,,0,0,0,,because we rarely see all the levers that come into play to make people lucky.
+Dialogue: Marked=0,0:00:53.57,0:00:56.56,Default,,0,0,0,,But I've realized, by watching so long,
+Dialogue: Marked=0,0:00:56.59,0:00:59.92,Default,,0,0,0,,that luck is rarely a lightning strike,
+Dialogue: Marked=0,0:00:59.94,0:01:01.70,Default,,0,0,0,,isolated and dramatic.
+Dialogue: Marked=0,0:01:01.70,0:01:01.70,Default,,0,0,0,,(SOUND EFFECT)
+Dialogue: Marked=0,0:01:01.70,0:01:01.70,Default,,0,0,0,,(SOUND EFFECT)