Train Tacotron2 model with Japanese

[easyautoml]

Hello everyone,

I’m currently training a Tacotron2 model using the Kokoro-speech-tiny dataset, which contains approximately 6,000 samples and 14 hours of audio.

I trained the model for 100 epochs, which took me two days, but the output from the trained model only produces noise.

I checked the tokenized data and the mel-spectrograms used as input for the model. The mel-spectrograms seem fine because they convert well into audio when using a vocoder.

However, from TensorBoard, I noticed that the average alignment error remains high and hasn’t improved throughout the epochs.

Here are my settings. Could you please provide some advice on how I can improve the model to generate intelligible speech?

{
    "return_wav": true,
    "output_path": "./output",
    "logger_uri": null,
    "run_name": "kokoro-ddc",
    "project_name": null,
    "run_description": "tacotron2 with DDC and differential spectral loss.",
    "print_step": 25,
    "plot_step": 10,
    "model_param_stats": false,
    "wandb_entity": null,
    "dashboard_logger": "tensorboard",
    "save_on_interrupt": true,
    "log_model_step": null,
    "save_step": 100,
    "save_n_checkpoints": 5,
    "save_checkpoints": true,
    "save_all_best": false,
    "save_best_after": 100,
    "target_loss": null,
    "print_eval": false,
    "test_delay_epochs": 10,
    "run_eval": true,
    "run_eval_steps": null,
    "distributed_backend": "nccl",
    "distributed_url": "tcp://localhost:54321",
    "mixed_precision": true,
    "precision": "fp16",
    "epochs": 100,
    "batch_size": 32,
    "eval_batch_size": 16,
    "grad_clip": 1.0,
    "scheduler_after_epoch": true,
    "lr": 0.0001,
    "optimizer": "RAdam",
    "optimizer_params": {
        "betas": [
            0.9,
            0.998
        ],
        "weight_decay": 1e-06
    },
    "lr_scheduler": "NoamLR",
    "lr_scheduler_params": {
        "warmup_steps": 4000
    },
    "use_grad_scaler": false,
    "allow_tf32": false,
    "cudnn_enable": true,
    "cudnn_deterministic": false,
    "cudnn_benchmark": true,
    "training_seed": 54321,
    "model": "Tacotron2",
    "num_loader_workers": 4,
    "num_eval_loader_workers": 0,
    "use_noise_augment": false,
    "audio": {
        "fft_size": 1024,
        "win_length": 1024,
        "hop_length": 256,
        "frame_shift_ms": null,
        "frame_length_ms": null,
        "stft_pad_mode": "reflect",
        "sample_rate": 22050,
        "resample": false,
        "preemphasis": 0.0,
        "ref_level_db": 20,
        "do_sound_norm": false,
        "log_func": "np.log10",
        "do_trim_silence": true,
        "trim_db": 60, // with value lower than 60 db, it will be trimmed
        "do_rms_norm": false,
        "db_level": null,
        "power": 1.5,
        "griffin_lim_iters": 60,
        "num_mels": 80, // original value = 80. Change to 560
        "mel_fmin": 50.0,
        "mel_fmax": 7600.0,
        "spec_gain": 1,
        "do_amp_to_db_linear": true,
        "do_amp_to_db_mel": true,
        "pitch_fmax": 640.0,
        "pitch_fmin": 0.0,
        "signal_norm": true,
        "min_level_db": -100,
        "symmetric_norm": true,
        "max_norm": 4.0,
        "clip_norm": true,
        "stats_path": "./data/scale_stats.npy"
    },
    "use_phonemes": true,
    "phonemizer": "ja_jp_phonemizer",
    "phoneme_language": "ja-jp",
    "compute_input_seq_cache": false,
    "text_cleaner": "basic_cleaners",
    "enable_eos_bos_chars": false,
    "test_sentences_file": null,
    "phoneme_cache_path": "./phoneme_cache/",
    "characters": {
        "characters_class": null,
        "vocab_dict": null,
        "pad": "_",
        "eos": "~",
        "bos": "^",
        "blank": null,
        "characters": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!'(),-.:;?* ",
        "punctuations": "!'(),-.:;?* ",
        "phonemes": "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz",
        "is_unique": true,
        "is_sorted": true
    },
    "add_blank": false,
    "batch_group_size": 4,
    "loss_masking": true,
    "min_audio_len": 1,
    "max_audio_len": Infinity,
    "min_text_len": 1,
    "max_text_len": Infinity,
    "compute_f0": false,
    "compute_energy": false,
    "compute_linear_spec": false,
    "precompute_num_workers": 0,
    "start_by_longest": false,
    "shuffle": false,
    "drop_last": false,
    "datasets": [
        {
            "formatter": "kokoro",
            "dataset_name": "",
            "path": "./data",
            "meta_file_train": "metadata.csv",
            "ignored_speakers": null,
            "language": "",
            "phonemizer": "",
            "meta_file_val": null,
            "meta_file_attn_mask": ""
        }
    ],
    "test_sentences": [
        "こんにちは、初めまして"
    ],
    "eval_split_max_size": null,
    "eval_split_size": 0.01,
    "use_speaker_weighted_sampler": false,
    "speaker_weighted_sampler_alpha": 1.0,
    "use_language_weighted_sampler": false,
    "language_weighted_sampler_alpha": 1.0,
    "use_length_weighted_sampler": false,
    "length_weighted_sampler_alpha": 1.0,
    "use_gst": false,
    "gst": {
        "gst_style_input_wav": null,
        "gst_style_input_weights": null,
        "gst_embedding_dim": 512,
        "gst_use_speaker_embedding": false,
        "gst_num_heads": 4,
        "gst_num_style_tokens": 10
    },
    "gst_style_input": null,
    "use_capacitron_vae": false,
    "capacitron_vae": null,
    "num_speakers": 1,
    "num_chars": 0,
    "r": 1,
    "gradual_training": [],
    "memory_size": -1,
    "prenet_type": "original",
    "prenet_dropout": true,
    "prenet_dropout_at_inference": false,
    "stopnet": true,
    "separate_stopnet": true,
    "stopnet_pos_weight": 15.0,
    "max_decoder_steps": 500,
    "encoder_in_features": 512,
    "decoder_in_features": 512,
    "decoder_output_dim": 80,
    "out_channels": 80,
    "attention_type": "original",
    "attention_heads": 4,
    "attention_norm": "sigmoid",
    "attention_win": false,
    "windowing": false,
    "use_forward_attn": false,
    "forward_attn_mask": false,
    "transition_agent": false,
    "location_attn": true,
    "bidirectional_decoder": false,
    "double_decoder_consistency": true,
    "ddc_r": 7,
    "speakers_file": null,
    "use_speaker_embedding": false,
    "speaker_embedding_dim": 512,
    "use_d_vector_file": false,
    "d_vector_file": false,
    "d_vector_dim": null,
    "seq_len_norm": false,
    "decoder_loss_alpha": 0.5,
    "postnet_loss_alpha": 0.25,
    "postnet_diff_spec_alpha": 0.25,
    "decoder_diff_spec_alpha": 0.25,
    "decoder_ssim_alpha": 0.5,
    "postnet_ssim_alpha": 0.25,
    "ga_alpha": 5.0
}