Unable to generate vad audio

[RichardQin1]

environment：
python 3.8
conda install pytorch==1.10.0 torchvision==0.11.0 torchaudio==0.10.0 cudatoolkit=11.3 -c pytorch -c conda-forge
nemo-toolkit[all]==1.19.1

code：

from nemo.collections.asr.models.msdd_models import NeuralDiarizer
from nemo.collections.asr.models import EncDecRNNTBPEModel
from nemo.collections.asr.models import EncDecSpeakerLabelModel
import pandas as pd
import torch
import json
from omegaconf import OmegaConf
import uuid

device = "cuda" if torch.cuda.is_available() else "cpu"

#model = NeuralDiarizer.from_pretrained("diar_msdd_telephonic").to(device)
model =  NeuralDiarizer.save_to("/root/nas-private/nemotest/models/diar_msdd_telephonic.nemo").to(device)
speaker_model = EncDecSpeakerLabelModel.restore_from("/root/nas-private/nemotest/speakerverification_en_titanet_large/speakerverification_en_titanet_large.nemo").to(device)
model.eval()

def run_diarization(path1):
    annotation =model(path1, num_workers=0, batch_size=16)
    rttm = annotation.to_rttm()
    df = pd.DataFrame(columns=['start_time', 'end_time', 'speaker', 'text'])
    lines = rttm.splitlines()
    if len(lines) == 0:
        df.loc[0] = 0, 0, 'No speaker found'
        return df
    start_time, duration, prev_speaker = float(lines[0].split()[3]), float(lines[0].split()[4]), lines[0].split()[7]
    end_time = float(start_time) + float(duration)
    df.loc[0] = start_time, end_time, prev_speaker, ''

    for line in lines[1:]:
        split = line.split()
        start_time, duration, cur_speaker = float(split[3]), float(split[4]), split[7]
        end_time = float(start_time) + float(duration)
        if cur_speaker == prev_speaker:
            df.loc[df.index[-1], 'end_time'] = end_time
        else:
            df.loc[len(df)] = start_time, end_time, cur_speaker, ''
        prev_speaker = cur_speaker

    hyp = get_transcripts(df, path1)

    assert len(hyp) == len(df)

    for i in range(len(df)):
        df.loc[i, 'text'] = hyp[i]
    df.to
    return df


def create_manifest(df, audio_path):
    filename = '/tmp/' + str(uuid.uuid4()) + '.json'
    with open(filename, 'w') as f:
        for i in range(len(df)):
            start_time = df.iloc[i]['start_time']
            end_time = df.iloc[i]['end_time']
            speaker = df.iloc[i]['speaker']
            dic = {"audio_filepath": audio_path, "duration": end_time - start_time, "label": speaker,
                   "offset": start_time}
            json.dump(dic, f)
            f.write('\n')

    return filename


def get_transcripts(df, audio_path):
    filename = create_manifest(df, audio_path)
    model = EncDecRNNTBPEModel.from_pretrained(model_name="nvidia/stt_en_fastconformer_transducer_large").to(device)
    model.eval()
    config = OmegaConf.create({"manifest_filepath": filename, 'batch_size': 2})
    dataloader = model._setup_transcribe_dataloader(config)

    hypotheses = []
    all_hypotheses = []

    for test_batch in (dataloader):
        encoded, encoded_len = model.forward(
            input_signal=test_batch[0].to(device), input_signal_length=test_batch[1].to(device)
        )
        best_hyp, all_hyp = model.decoding.rnnt_decoder_predictions_tensor(
            encoded,
            encoded_len,
            return_hypotheses=False,
            partial_hypotheses=None, )

        hypotheses += best_hyp
        if all_hyp is not None:
            all_hypotheses += all_hyp
        else:
            all_hypotheses += best_hyp

        del encoded
        del test_batch

    return hypotheses

if __name__ == '__main__':
    print_hi(run_diarization('testfile'))

log ：

l[NeMo W 2023-09-14 02:25:40 experimental:27] Module <class 'nemo.collections.asr.modules.audio_modules.SpectrogramToMultichannelFeatures'> is experimental, not ready for production and is not fully supported. Use at your own risk.
[NeMo W 2023-09-14 02:25:41 modelPT:161] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
    Train config : 
    manifest_filepath: null
    emb_dir: null
    sample_rate: 16000
    num_spks: 2
    soft_label_thres: 0.5
    labels: null
    batch_size: 15
    emb_batch_size: 0
    shuffle: true
    
[NeMo W 2023-09-14 02:25:41 modelPT:168] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
    Validation config : 
    manifest_filepath: null
    emb_dir: null
    sample_rate: 16000
    num_spks: 2
    soft_label_thres: 0.5
    labels: null
    batch_size: 15
    emb_batch_size: 0
    shuffle: false
    
[NeMo W 2023-09-14 02:25:41 modelPT:174] Please call the ModelPT.setup_test_data() or ModelPT.setup_multiple_test_data() method and provide a valid configuration file to setup the test data loader(s).
    Test config : 
    manifest_filepath: null
    emb_dir: null
    sample_rate: 16000
    num_spks: 2
    soft_label_thres: 0.5
    labels: null
    batch_size: 15
    emb_batch_size: 0
    shuffle: false
    seq_eval_mode: false
    
[NeMo W 2023-09-14 02:25:45 modelPT:161] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
    Train config : 
    manifest_filepath: /manifests/ami_train_0.63.json,/manifests/freesound_background_train.json,/manifests/freesound_laughter_train.json,/manifests/fisher_2004_background.json,/manifests/fisher_2004_speech_sampled.json,/manifests/google_train_manifest.json,/manifests/icsi_all_0.63.json,/manifests/musan_freesound_train.json,/manifests/musan_music_train.json,/manifests/musan_soundbible_train.json,/manifests/mandarin_train_sample.json,/manifests/german_train_sample.json,/manifests/spanish_train_sample.json,/manifests/french_train_sample.json,/manifests/russian_train_sample.json
    sample_rate: 16000
    labels:
    - background
    - speech
    batch_size: 256
    shuffle: true
    is_tarred: false
    tarred_audio_filepaths: null
    tarred_shard_strategy: scatter
    augmentor:
      shift:
        prob: 0.5
        min_shift_ms: -10.0
        max_shift_ms: 10.0
      white_noise:
        prob: 0.5
        min_level: -90
        max_level: -46
        norm: true
      noise:
        prob: 0.5
        manifest_path: /manifests/noise_0_1_musan_fs.json
        min_snr_db: 0
        max_snr_db: 30
        max_gain_db: 300.0
        norm: true
      gain:
        prob: 0.5
        min_gain_dbfs: -10.0
        max_gain_dbfs: 10.0
        norm: true
    num_workers: 16
    pin_memory: true
    
[NeMo W 2023-09-14 02:25:45 modelPT:168] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
    Validation config : 
    manifest_filepath: /manifests/ami_dev_0.63.json,/manifests/freesound_background_dev.json,/manifests/freesound_laughter_dev.json,/manifests/ch120_moved_0.63.json,/manifests/fisher_2005_500_speech_sampled.json,/manifests/google_dev_manifest.json,/manifests/musan_music_dev.json,/manifests/mandarin_dev.json,/manifests/german_dev.json,/manifests/spanish_dev.json,/manifests/french_dev.json,/manifests/russian_dev.json
    sample_rate: 16000
    labels:
    - background
    - speech
    batch_size: 256
    shuffle: false
    val_loss_idx: 0
    num_workers: 16
    pin_memory: true
    
[NeMo W 2023-09-14 02:25:45 modelPT:174] Please call the ModelPT.setup_test_data() or ModelPT.setup_multiple_test_data() method and provide a valid configuration file to setup the test data loader(s).
    Test config : 
    manifest_filepath: null
    sample_rate: 16000
    labels:
    - background
    - speech
    batch_size: 128
    shuffle: false
    test_loss_idx: 0
    
[NeMo W 2023-09-14 02:25:46 modelPT:161] If you intend to do training or fine-tuning, please call the ModelPT.setup_training_data() method and provide a valid configuration file to setup the train data loader.
    Train config : 
    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/train.json
    sample_rate: 16000
    labels: null
    batch_size: 64
    shuffle: true
    is_tarred: false
    tarred_audio_filepaths: null
    tarred_shard_strategy: scatter
    augmentor:
      noise:
        manifest_path: /manifests/noise/rir_noise_manifest.json
        prob: 0.5
        min_snr_db: 0
        max_snr_db: 15
      speed:
        prob: 0.5
        sr: 16000
        resample_type: kaiser_fast
        min_speed_rate: 0.95
        max_speed_rate: 1.05
    num_workers: 15
    pin_memory: true
    
[NeMo W 2023-09-14 02:25:46 modelPT:168] If you intend to do validation, please call the ModelPT.setup_validation_data() or ModelPT.setup_multiple_validation_data() method and provide a valid configuration file to setup the validation data loader(s). 
    Validation config : 
    manifest_filepath: /manifests/combined_fisher_swbd_voxceleb12_librispeech/dev.json
    sample_rate: 16000
    labels: null
    batch_size: 128
    shuffle: false
    num_workers: 15
    pin_memory: true
    
splitting manifest:   0%|                                                                                                                                           | 0/1 [00:00<?, ?it/s][NeMo W 2023-09-14 02:25:47 nemo_logging:349] /root/nas-private/qinjiatest/lib/python3.8/site-packages/nemo/collections/asr/parts/utils/vad_utils.py:143: UserWarning: PySoundFile failed. Trying audioread instead.
      x, _sr = librosa.load(filepath, sr=sr, offset=in_offset, duration=in_duration)
    
[NeMo W 2023-09-14 02:25:47 nemo_logging:349] /root/nas-private/qinjiatest/lib/python3.8/site-packages/librosa/core/audio.py:183: FutureWarning: librosa.core.audio.__audioread_load
    	Deprecated as of librosa version 0.10.0.
    	It will be removed in librosa version 1.0.
      y, sr_native = __audioread_load(path, offset, duration, dtype)
    
splitting manifest: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.30s/it]
creating speech segments: 0it [00:00, ?it/s]
[NeMo W 2023-09-14 02:25:47 clustering_diarizer:279] no vad file found for asrfile due to zero or negative duration
[NeMo W 2023-09-14 02:25:47 speaker_utils:642] asrfile is ignored since the file does not contain any speech signal to be processed.
Traceback (most recent call last):
  File "test.py", line 100, in <module>
    print(run_diarization('/root/nas-private/nemotest/split.mp3'))
  File "test.py", line 22, in run_diarization
    annotation = model('/root/nas-private/nemotest/asrfile', num_workers=0, batch_size=16)
  File "/root/nas-private/qinjiatest/lib/python3.8/site-packages/nemo/collections/asr/models/msdd_models.py", line 1514, in __call__
    self.diarize()
  File "/root/nas-private/qinjiatest/lib/python3.8/site-packages/torch/autograd/grad_mode.py", line 28, in decorate_context
    return func(*args, **kwargs)
  File "/root/nas-private/qinjiatest/lib/python3.8/site-packages/nemo/collections/asr/models/msdd_models.py", line 1180, in diarize
    self.clustering_embedding.prepare_cluster_embs_infer()
  File "/root/nas-private/qinjiatest/lib/python3.8/site-packages/nemo/collections/asr/models/msdd_models.py", line 699, in prepare_cluster_embs_infer
    self.emb_sess_test_dict, self.emb_seq_test, self.clus_test_label_dict, _ = self.run_clustering_diarizer(
  File "/root/nas-private/qinjiatest/lib/python3.8/site-packages/nemo/collections/asr/models/msdd_models.py", line 866, in run_clustering_diarizer
    scores = self.clus_diar_model.diarize(batch_size=self.cfg_diar_infer.batch_size)
  File "/root/nas-private/qinjiatest/lib/python3.8/site-packages/nemo/collections/asr/models/clustering_diarizer.py", line 437, in diarize
    self._perform_speech_activity_detection()
  File "/root/nas-private/qinjiatest/lib/python3.8/site-packages/nemo/collections/asr/models/clustering_diarizer.py", line 336, in _perform_speech_activity_detection
    validate_vad_manifest(self.AUDIO_RTTM_MAP, vad_manifest=self._speaker_manifest_path)
  File "/root/nas-private/qinjiatest/lib/python3.8/site-packages/nemo/collections/asr/parts/utils/speaker_utils.py", line 645, in validate_vad_manifest
    raise ValueError("All files present in manifest contains silence, aborting next steps")
ValueError: All files present in manifest contains silence, aborting next steps

i dont know why vad is 0it. think you for your help

[titu1994]

@fayejf @stevehuang52

[stevehuang52]

looking at it~

[stevehuang52]

Seems like there's no RTTM files provided in manifest and no VAD model checkpoint is provided to the diarizer. Could you please try this script instead? https://github.com/NVIDIA/NeMo/blob/main/examples/speaker_tasks/diarization/neural_diarizer/multiscale_diar_decoder_infer.py