now handles stereo input

TunisianASR/results/14epoch_tunisian/1234/app.py

@@ -744,18 +744,9 @@ mixer.checkpointer.recover_if_possible(device="cpu")
 mixer.modules.eval()
 
 
-label_encoder = sb.dataio.encoder.CTCTextEncoder()
 
 
-# We dynamicaly add the tokenizer to our brain class.
-# NB: This tokenizer corresponds to the one used for the LM!!
 
-decoder = build_ctcdecoder(
-    labels,
-    kenlm_model_path= "arpas/everything.arpa",  # either .arpa or .bin file
-    alpha=0.5,  # tuned on a val set
-    beta=1,  # tuned on a val set
-)
 
 
 
@@ -785,7 +776,12 @@ def treat_wav_file(file_mic,file_upload ,asr=mixer, device="cpu") :
         wav = file_mic
     else:
         wav = file_upload
-    sig, sr = torchaudio.load(wav)
+    info = torchaudio.info(wav)
+    sr = info.sample_rate
+    sig = sb.dataio.dataio.read_audio(wav)
+    if len(sig.shape)>1 :
+        sig = torch.mean(sig, dim=1)
+    sig = torch.unsqueeze(sig, 0)
     tensor_wav = sig.to(device)
     resampled = torchaudio.functional.resample( tensor_wav, sr, 16000)
     sentence = asr.treat_wav(resampled)

TunisianASR/results/14epoch_tunisian/1234/env.log

@@ -473,7 +473,7 @@ youtube-dl==2021.6.6
 zipp==3.6.0
 ==============================
 Git revision:
-0fdcdc4
+bf7e6b5
 ==============================
 CUDA version:
 11.7

TunisianASR/results/14epoch_tunisian/1234/log.txt

@@ -1830,3 +1830,494 @@ zipp==3.6.0
 2023-09-25 12:30:16,221 - speechbrain.core - INFO - 314.4M trainable parameters in ASR
 2023-09-25 12:30:16,224 - speechbrain.utils.checkpoints - INFO - Loading a checkpoint from TunisianASR/results/14epoch_tunisian/1234/save/CKPT+2023-08-03+01-38-38+00
 2023-09-25 12:30:16,534 - speechbrain.utils.distributed - INFO - distributed_launch flag is disabled, this experiment will be executed without DDP.
+2023-09-25 15:14:42,745 - speechbrain.core - INFO - Beginning experiment!
+2023-09-25 15:14:42,745 - speechbrain.core - INFO - Experiment folder: TunisianASR/results/14epoch_tunisian/1234/
+2023-09-25 15:14:43,231 - speechbrain.utils.superpowers - DEBUG - abkhazia==1.0
+absl-py==0.11.0
+aiofiles==23.2.1
+aiohttp==3.8.0
+aiosignal==1.2.0
+alabaster==0.7.12
+alembic==1.7.4
+altair==4.2.0
+altgraph==0.17
+antlr4-python3-runtime==4.9.3
+anyio==3.6.2
+appdirs==1.4.4
+argcomplete==1.12.2
+argon2-cffi==20.1.0
+arrow==1.2.3
+asgiref==3.6.0
+asteroid-filterbanks==0.4.0
+astunparse==1.6.3
+async-generator==1.10
+async-timeout==4.0.0
+attrdict==2.0.1
+attrs==20.3.0
+audeer==1.16.0
+audformat==0.11.5
+audinterface==0.7.0
+audiofile==1.0.0
+audiomentations==0.25.0
+audioread==2.1.9
+audobject==0.4.14
+audresample==0.1.6
+-e git+https://github.com/facebookresearch/WavAugment.git@54afcdb00ccc852c2f030f239f8532c9562b550e#egg=augment
+autopage==0.4.0
+Babel==2.9.0
+backcall==0.2.0
+backports.cached-property==1.0.2
+beautifulsoup4==4.10.0
+black==19.10b0
+bleach==3.3.0
+blessed==1.20.0
+boto3==1.20.2
+botocore==1.23.2
+bpemb==0.3.4
+braceexpand==0.1.7
+cachetools==4.2.0
+certifi @ file:///croot/certifi_1671487769961/work/certifi
+cffi==1.14.3
+cfgv==3.2.0
+chardet==3.0.4
+charset-normalizer==2.0.7
+click==7.1.2
+cliff==3.9.0
+clldutils==3.5.4
+cloudpickle==2.2.1
+cmaes==0.8.2
+cmake==3.18.4.post1
+cmd2==2.2.0
+colorama==0.4.4
+colorlog==4.6.2
+configparser==5.1.0
+conllu==4.5.3
+croniter==1.3.15
+cryptography==38.0.4
+csrgraph==0.1.28
+csvw==1.8.1
+cycler==0.10.0
+Cython==0.29.21
+dataclasses==0.6
+dateutils==0.6.12
+decorator==4.4.2
+deepdiff==6.3.0
+deepspeech==0.9.1
+defusedxml==0.7.1
+Deprecated==1.2.14
+dill==0.3.3
+Distance==0.1.3
+distlib==0.3.1
+Django==3.2.16
+django-auditlog==2.2.1
+django-filter==22.1
+django-js-asset==1.2.2
+django-mptt==0.14.0
+djangorestframework==3.14.0
+docker-pycreds==0.4.0
+docopt==0.6.2
+docutils==0.16
+drf-excel==2.2.0
+drf-flex-fields==1.0.0
+drf-renderer-xlsx==0.4.1
+easyocr==1.2.1
+editdistance==0.6.0
+einops==0.3.2
+emoji==2.2.0
+entrypoints==0.3
+et-xmlfile==1.1.0
+exceptiongroup==1.1.0
+farasapy==0.0.14
+fastapi==0.98.0
+fastjsonschema==2.17.1
+fasttext==0.9.2
+ffmpeg-python==0.2.0
+ffmpy==0.3.0
+filelock==3.0.12
+flair==0.12.2
+flake8==3.7.9
+flatbuffers==1.12
+frozendict==2.0.7
+frozenlist==1.2.0
+fsspec==2021.11.0
+ftfy==6.1.1
+future==0.18.2
+g2p-en==2.1.0
+gast==0.3.3
+gdown==4.4.0
+gdrive==0.1.5
+gensim==4.0.1
+gitdb==4.0.9
+GitPython==3.1.24
+google-api-core==2.11.1
+google-api-python-client==2.43.0
+google-auth==1.24.0
+google-auth-httplib2==0.1.0
+google-auth-oauthlib==0.5.3
+google-pasta==0.2.0
+googleapis-common-protos==1.59.1
+gradio==3.44.4
+gradio-client==0.5.1
+greenlet==1.1.2
+grpcio==1.32.0
+h11==0.14.0
+h5features==1.3.2
+h5py==2.10.0
+hierarchy==0.4.0
+hmmlearn==0.2.8
+htk-io==0.5
+httpcore==0.16.3
+httplib2==0.22.0
+httpx==0.23.3
+huggingface-hub==0.15.1
+hydra-colorlog==0.1.4
+hydra-core==1.3.2
+hyperopt==0.2.7
+HyperPyYAML==1.1.0
+hypothesis==6.61.2
+identify==1.5.10
+idna==2.10
+imageio==2.9.0
+imagesize==1.2.0
+importlib-metadata==4.8.1
+importlib-resources==5.2.2
+inflect==5.3.0
+inquirer==3.1.3
+ipadic==1.0.0
+ipyevents==2.0.1
+ipykernel==5.3.4
+ipython==7.19.0
+ipython-genutils==0.2.0
+ipywebrtc==0.6.0
+ipywidgets==7.6.3
+iso-639==0.4.5
+isodate==0.6.0
+isort==4.3.21
+itsdangerous==2.1.2
+Janome==0.5.0
+jedi==0.17.2
+jeepney==0.8.0
+jieba==0.42.1
+Jinja2==3.0.3
+jiwer==2.2.0
+jmespath==0.10.0
+joblib==0.17.0
+jsonschema==3.2.0
+julius==0.2.7
+jupyter-client==6.1.7
+jupyter-core==4.7.0
+jupyterlab-pygments==0.1.2
+jupyterlab-widgets==1.0.0
+kaitaistruct==0.9
+kaldi-io==0.9.4
+kaldi-python-io==1.2.2
+kaldiio==2.17.2
+kenlm @ https://github.com/kpu/kenlm/archive/master.zip
+Keras-Preprocessing==1.1.2
+kiwisolver==1.3.1
+lang-trans==0.6.0
+langdetect==1.0.9
+latexcodec==2.0.1
+ldap3==2.9.1
+librosa==0.9.0
+lightning-cloud==0.5.37
+lightning-utilities==0.8.0
+linkify-it-py==1.0.3
+lit==16.0.6
+llvmlite==0.35.0
+lxml==4.9.0
+Mako==1.1.5
+Markdown==3.3.3
+markdown-it-py==3.0.0
+MarkupSafe==2.1.3
+marshmallow==3.14.0
+matplotlib==3.3.3
+mccabe==0.6.1
+mcd==0.4
+mdit-py-plugins==0.3.3
+mdurl==0.1.2
+mecab-python3==1.0.3
+megatron-lm==2.2.0
+metrics==0.3.3
+mido==1.2.10
+mistune==0.8.4
+more-itertools==8.6.0
+mpld3==0.3
+mpmath==1.2.1
+multidict==5.2.0
+multiprocess==0.70.11.1
+nbclient==0.5.3
+nbconvert==5.6.1
+nbformat==5.9.0
+NEMO==4.3.2
+nemo-toolkit==1.4.0
+nest-asyncio==1.5.1
+networkx==2.8.8
+nltk==3.2.4
+nodeenv==1.5.0
+normalize==2.0.2
+notebook==6.3.0
+numba==0.52.0
+numpy==1.19.4
+nvidia-cublas-cu11==11.10.3.66
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+nvidia-cuda-runtime-cu11==11.7.99
+nvidia-cudnn-cu11==8.5.0.96
+nvidia-cufft-cu11==10.9.0.58
+nvidia-curand-cu11==10.2.10.91
+nvidia-cusolver-cu11==11.4.0.1
+nvidia-cusparse-cu11==11.7.4.91
+nvidia-nccl-cu11==2.14.3
+nvidia-nvtx-cu11==11.7.91
+oauthlib==3.1.0
+omegaconf==2.3.0
+onnx==1.10.2
+OpenCC==1.1.2
+opencv-python==4.4.0.46
+openpyxl==3.0.9
+opensmile==2.2.0
+opt-einsum==3.3.0
+optuna==2.10.0
+ordered-set==4.1.0
+orjson==3.8.4
+oyaml==1.0
+packaging==22.0
+pandas==1.2.5
+pandocfilters==1.4.3
+pangu==4.0.6.1
+parameterized==0.8.1
+parso==0.7.1
+pathlib2==2.3.7.post1
+pathspec==0.5.5
+pathtools==0.1.2
+pbr==5.6.0
+pefile==2019.4.18
+pescador==2.1.0
+pesq==0.0.3
+pexpect==4.8.0
+phonemizer==2.2.1
+pickleshare==0.7.5
+Pillow==9.3.0
+pip-api==0.0.23
+pipreqs==0.4.11
+pluggy==0.13.1
+pooch==1.3.0
+portalocker==2.3.2
+pptree==3.1
+pre-commit==2.9.0
+preprocessing==0.1.13
+pretty-midi==0.2.9
+prettytable==2.2.1
+primePy==1.3
+progressbar2==3.53.1
+prometheus-client==0.10.1
+promise==2.3
+prompt-toolkit==3.0.8
+protobuf==3.20.3
+psutil==5.6.6
+ptyprocess==0.6.0
+py==1.9.0
+py-espeak-ng==0.1.8
+py4j==0.10.9.7
+pyannote.audio==2.1.1
+pyannote.core==4.5
+pyannote.database==4.1.3
+pyannote.metrics==3.2.1
+pyannote.pipeline==2.3
+pyannotebook==0.1.0.dev0
+PyArabic==0.6.15
+pyarrow==3.0.0
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pybind11==2.8.1
+pybtex==0.24.0
+pybtex-docutils==1.0.1
+pycodestyle==2.5.0
+pycparser==2.20
+pycryptodome==3.16.0
+pyctcdecode==0.4.0
+pydantic==1.10.4
+pyDeprecate==0.3.1
+pydub==0.25.1
+pyflakes==2.1.1
+Pygments==2.15.1
+pygtrie==2.5.0
+PyJWT==2.7.0
+pymodbus==2.5.3
+pyparsing==2.4.7
+pyperclip==1.8.2
+pypinyin==0.43.0
+pyrsistent==0.17.3
+pyserial==3.5
+PySocks==1.7.1
+pystoi==0.3.3
+pytest==5.4.1
+pytest-runner==5.3.1
+python-bidi==0.4.2
+python-crfsuite==0.9.7
+python-dateutil==2.8.2
+python-editor==1.0.4
+python-Levenshtein==0.12.2
+python-multipart==0.0.5
+python-utils==2.4.0
+pytorch-lightning==1.6.5
+pytorch-metric-learning==1.7.3
+pytorch-revgrad==0.2.0
+pytube==11.0.1
+pytz==2022.6
+PyWavelets==1.1.1
+PyYAML==6.0
+pyzmq==20.0.0
+rapidfuzz==1.8.2
+readchar==4.0.5
+regex==2020.11.13
+requests==2.28.1
+requests-oauthlib==1.3.0
+resampy==0.2.2
+rfc3986==1.4.0
+rich==13.4.2
+richenum==1.3.1
+rsa==4.7
+ruamel.yaml==0.17.21
+ruamel.yaml.clib==0.2.7
+s3m==1.1.0
+s3transfer==0.5.0
+sacrebleu==2.0.0
+sacremoses==0.0.44
+safetensors==0.3.1
+scikit-image==0.18.1
+scikit-learn==0.23.2
+scipy==1.5.4
+-e git+https://github.com/sanghack81/SDCIT@00d060dde733fde9345154a494f81e97fb395ca7#egg=SDCIT
+seaborn==0.11.1
+SecretStorage==3.3.3
+segments==2.1.3
+segtok==1.5.11
+semantic-version==2.10.0
+semver==2.13.0
+Send2Trash==1.5.0
+sentencepiece==0.1.99
+sentry-sdk==1.4.3
+shellingham==1.4.0
+shortuuid==1.0.7
+SIDEKIT==1.3.8.5.2
+simplejson==3.17.5
+singledispatchmethod==1.0
+six==1.15.0
+smart-open==5.0.0
+smmap==5.0.0
+sniffio==1.3.0
+snowballstemmer==2.0.0
+sortedcollections==2.1.0
+sortedcontainers==2.4.0
+sounddevice==0.4.5
+SoundFile==0.10.3.post1
+soupsieve==2.3
+sox==1.4.1
+sparsemax==0.1.9
+speechbrain==0.5.14
+sphfile==1.0.3
+Sphinx==3.3.1
+sphinx-rtd-theme==0.2.4
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-bibtex==2.4.1
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==1.0.3
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.4
+SQLAlchemy==1.4.25
+sqlitedict==2.1.0
+sqlparse==0.4.2
+stanza==1.4.2
+starlette==0.27.0
+starsessions==1.3.0
+stevedore==3.4.0
+subprocess32==3.5.4
+sympy==1.9
+tabulate==0.8.9
+tensorboard==2.4.0
+tensorboard-plugin-wit==1.7.0
+tensorboardX==2.6.1
+tensorflow==2.4.0
+tensorflow-estimator==2.4.0
+termcolor==1.1.0
+terminado==0.9.4
+testpath==0.4.4
+threadpoolctl==2.1.0
+tifffile==2020.12.8
+tikzplotlib==0.9.8
+tinycss2==1.2.1
+tkseem==0.0.3
+tokenizers==0.13.3
+toml==0.10.2
+toolz==0.12.0
+torch==1.13.1
+torch-audiomentations==0.11.0
+torch-pitch-shift==1.2.4
+torch-stft==0.1.4
+torchaudio==0.13.1
+torchmetrics==0.11.4
+torchvision==0.14.1
+tornado==6.1
+tqdm==4.61.1
+trackrip==1.2.1
+traitlets==5.9.0
+transformer-smaller-training-vocab==0.3.1
+transformers==4.30.2
+triton==2.0.0
+typed-ast==1.4.1
+typer==0.4.0
+typing-extensions==4.4.0
+uc-micro-py==1.0.1
+Unidecode==1.3.2
+uritemplate==3.0.1
+urllib3==1.26.2
+uvicorn==0.20.0
+versioneer==0.28
+virtualenv==20.2.1
+wandb==0.12.6
+wcwidth==0.2.5
+webdataset==0.1.62
+webencodings==0.5.1
+websocket-client==1.6.1
+websockets==10.4
+Werkzeug==1.0.1
+wget==3.2
+widgetsnbextension==3.5.1
+Wikipedia-API==0.6.0
+wordninja==2.0.0
+wrapt==1.12.1
+xmltodict==0.13.0
+xxhash==2.0.0
+yamllint==1.23.0
+yarg==0.1.9
+yarl==1.7.2
+yaspin==2.1.0
+youtokentome==1.0.6
+youtube-dl==2021.6.6
+zipp==3.6.0
+
+
+2023-09-25 15:14:43,263 - speechbrain.utils.superpowers - DEBUG - bf7e6b5
+
+
+2023-09-25 15:14:43,325 - speechbrain.pretrained.fetching - INFO - Fetch hyperparams.yaml: Using existing file/symlink in pretrained_models/asr-wav2vec2-commonvoice-fr/hyperparams.yaml.
+2023-09-25 15:14:43,325 - speechbrain.pretrained.fetching - INFO - Fetch custom.py: Linking to local file in /home/salah/Code-Switched-Tunisian-SpeechToText/asr-wav2vec2-commonvoice-fr/custom.py.
+2023-09-25 15:14:46,203 - speechbrain.lobes.models.huggingface_wav2vec - WARNING - speechbrain.lobes.models.huggingface_wav2vec - wav2vec 2.0 is frozen.
+2023-09-25 15:14:46,206 - speechbrain.utils.parameter_transfer - DEBUG - Collecting files (or symlinks) for pretraining in pretrained_models/asr-wav2vec2-commonvoice-fr.
+2023-09-25 15:14:46,207 - speechbrain.pretrained.fetching - INFO - Fetch wav2vec2.ckpt: Using existing file/symlink in pretrained_models/asr-wav2vec2-commonvoice-fr/wav2vec2.ckpt.
+2023-09-25 15:14:46,207 - speechbrain.pretrained.fetching - INFO - Fetch asr.ckpt: Using existing file/symlink in pretrained_models/asr-wav2vec2-commonvoice-fr/asr.ckpt.
+2023-09-25 15:14:46,207 - speechbrain.pretrained.fetching - INFO - Fetch tokenizer.ckpt: Using existing file/symlink in pretrained_models/asr-wav2vec2-commonvoice-fr/tokenizer.ckpt.
+2023-09-25 15:14:46,208 - speechbrain.utils.parameter_transfer - INFO - Loading pretrained files for: wav2vec2, asr, tokenizer
+2023-09-25 15:14:50,143 - speechbrain.lobes.models.huggingface_wav2vec - WARNING - speechbrain.lobes.models.huggingface_wav2vec - wav2vec 2.0 feature extractor is frozen.
+2023-09-25 15:14:50,144 - speechbrain.core - INFO - Info: auto_mix_prec arg from hparam file is used
+2023-09-25 15:14:50,145 - speechbrain.core - INFO - Info: ckpt_interval_minutes arg from hparam file is used
+2023-09-25 15:14:50,148 - speechbrain.core - INFO - 314.4M trainable parameters in ASRCV
+2023-09-25 15:14:50,157 - speechbrain.utils.checkpoints - INFO - Loading a checkpoint from EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00
+2023-09-25 15:14:51,202 - speechbrain.core - INFO - Info: auto_mix_prec arg from hparam file is used
+2023-09-25 15:14:51,202 - speechbrain.core - INFO - Info: ckpt_interval_minutes arg from hparam file is used
+2023-09-25 15:14:51,207 - speechbrain.core - INFO - 314.4M trainable parameters in ASR
+2023-09-25 15:14:51,211 - speechbrain.utils.checkpoints - INFO - Loading a checkpoint from TunisianASR/results/14epoch_tunisian/1234/save/CKPT+2023-08-03+01-38-38+00
+2023-09-25 15:14:52,244 - speechbrain.utils.distributed - INFO - distributed_launch flag is disabled, this experiment will be executed without DDP.

app.py

@@ -744,18 +744,9 @@ mixer.checkpointer.recover_if_possible(device="cpu")
 mixer.modules.eval()
 
 
-label_encoder = sb.dataio.encoder.CTCTextEncoder()
 
 
-# We dynamicaly add the tokenizer to our brain class.
-# NB: This tokenizer corresponds to the one used for the LM!!
 
-decoder = build_ctcdecoder(
-    labels,
-    kenlm_model_path= "arpas/everything.arpa",  # either .arpa or .bin file
-    alpha=0.5,  # tuned on a val set
-    beta=1,  # tuned on a val set
-)
 
 
 
@@ -785,7 +776,12 @@ def treat_wav_file(file_mic,file_upload ,asr=mixer, device="cpu") :
         wav = file_mic
     else:
         wav = file_upload
-    sig, sr = torchaudio.load(wav)
+    info = torchaudio.info(wav)
+    sr = info.sample_rate
+    sig = sb.dataio.dataio.read_audio(wav)
+    if len(sig.shape)>1 :
+        sig = torch.mean(sig, dim=1)
+    sig = torch.unsqueeze(sig, 0)
     tensor_wav = sig.to(device)
     resampled = torchaudio.functional.resample( tensor_wav, sr, 16000)
     sentence = asr.treat_wav(resampled)

results/non_semi_final_stac/app.py

@@ -744,18 +744,9 @@ mixer.checkpointer.recover_if_possible(device="cpu")
 mixer.modules.eval()
 
 
-label_encoder = sb.dataio.encoder.CTCTextEncoder()
 
 
-# We dynamicaly add the tokenizer to our brain class.
-# NB: This tokenizer corresponds to the one used for the LM!!
 
-decoder = build_ctcdecoder(
-    labels,
-    kenlm_model_path= "arpas/everything.arpa",  # either .arpa or .bin file
-    alpha=0.5,  # tuned on a val set
-    beta=1,  # tuned on a val set
-)
 
 
 
@@ -785,7 +776,12 @@ def treat_wav_file(file_mic,file_upload ,asr=mixer, device="cpu") :
         wav = file_mic
     else:
         wav = file_upload
-    sig, sr = torchaudio.load(wav)
+    info = torchaudio.info(wav)
+    sr = info.sample_rate
+    sig = sb.dataio.dataio.read_audio(wav)
+    if len(sig.shape)>1 :
+        sig = torch.mean(sig, dim=1)
+    sig = torch.unsqueeze(sig, 0)
     tensor_wav = sig.to(device)
     resampled = torchaudio.functional.resample( tensor_wav, sr, 16000)
     sentence = asr.treat_wav(resampled)

results/non_semi_final_stac/env.log

@@ -473,7 +473,7 @@ youtube-dl==2021.6.6
 zipp==3.6.0
 ==============================
 Git revision:
-0fdcdc4
+bf7e6b5
 ==============================
 CUDA version:
 11.7