glow-tts training #1703

MaksimSerg · 2022-06-29T03:34:32Z

MaksimSerg
Jun 29, 2022

Hello! I train glow-tts network in Russian (dataset RUSLAN). At 10k steps I already begin to understand speech, but after 40k there is no improvement in quality. I think she's learning too fast... What settings can I change?
The latest training is on the charts. Before that, it reached 140k, also without a visible change in the range of 40k-140k

Original speech

007619_RUSLAN.mov

Synthesized speech

test.mov

MaksimSerg · 2022-06-29T03:35:52Z

MaksimSerg
Jun 29, 2022
Author

My config:

{
    "output_path": "/home/smaksimov/TTS/recipes/ljspeech/glow_tts",
    "logger_uri": null,
    "run_name": "run",
    "project_name": null,
    "run_description": "\ud83d\udc38Coqui trainer run.",
    "print_step": 50,
    "plot_step": 100,
    "model_param_stats": false,
    "wandb_entity": null,
    "dashboard_logger": "tensorboard",
    "log_model_step": null,
    "save_step": 5000,
    "save_n_checkpoints": 20,
    "save_checkpoints": true,
    "save_all_best": false,
    "save_best_after": 10000,
    "target_loss": null,
    "print_eval": false,
    "test_delay_epochs": 0,
    "run_eval": true,
    "distributed_backend": "nccl",
    "distributed_url": "tcp://localhost:54321",
    "mixed_precision": false,
    "epochs": 3000,
    "batch_size": 48,
    "eval_batch_size": 16,
    "grad_clip": 5.0,
    "scheduler_after_epoch": true,
    "lr": 0.001,
    "optimizer": "RAdam",
    "optimizer_params": {
        "betas": [
            0.9,
            0.998
        ],
        "weight_decay": 1e-06
    },
    "lr_scheduler": "NoamLR",
    "lr_scheduler_params": {
        "warmup_steps": 4000
    },
    "use_grad_scaler": false,
    "cudnn_enable": true,
    "cudnn_deterministic": false,
    "cudnn_benchmark": false,
    "training_seed": 54321,
    "model": "glow_tts",
    "num_loader_workers": 8,
    "num_eval_loader_workers": 8,
    "use_noise_augment": true,
    "audio": {
        "fft_size": 1024,
        "win_length": 1024,
        "hop_length": 256,
        "frame_shift_ms": null,
        "frame_length_ms": null,
        "stft_pad_mode": "reflect",
        "sample_rate": 22050,
        "resample": false,
        "preemphasis": 0.5,
        "ref_level_db": 0,
        "do_sound_norm": false,
        "log_func": "np.log10",
        "do_trim_silence": true,
        "trim_db": 20,
        "do_rms_norm": false,
        "db_level": null,
        "power": 1.5,
        "griffin_lim_iters": 60,
        "num_mels": 80,
        "mel_fmin": 75.0,
        "mel_fmax": 7600.0,
        "spec_gain": 10,
        "do_amp_to_db_linear": true,
        "do_amp_to_db_mel": true,
        "pitch_fmax": 640.0,
        "pitch_fmin": 0.0,
        "signal_norm": true,
        "min_level_db": -100,
        "symmetric_norm": true,
        "max_norm": 4.0,
        "clip_norm": true,
        "stats_path": null
    },
    "use_phonemes": false,
    "phonemizer": "gruut",
    "phoneme_language": "ru-ru",
    "compute_input_seq_cache": true,
    "text_cleaner": "basic_cleaners",
    "enable_eos_bos_chars": false,
    "test_sentences_file": "",
    "phoneme_cache_path": "/home/smaksimov/TTS/recipes/ljspeech/glow_tts/phoneme_cache",
    "characters": {
        "characters_class": "TTS.tts.utils.text.characters.IPAPhonemes",
        "vocab_dict": null,
        "pad": "<PAD>",
        "eos": "<EOS>",
        "bos": "<BOS>",
        "blank": "<BLNK>",
        "characters": "\u0430\u0431\u0432\u0433\u0434\u0435\u0436\u0437\u0438\u0439\u043a\u043b\u043c\u043d\u043e\u043f\u0440\u0441\u0442\u0443\u0444\u0445\u0446\u0447\u0448\u0449\u044a\u044b\u044c\u044d\u044e\u044f\u0451abcdefghijklmnopqrstuvwxyz",
        "punctuations": "!(),-.:;?-\u2013\u2014 \"'\u2019",
        "phonemes": null,
        "is_unique": false,
        "is_sorted": true
    },
    "add_blank": false,
    "batch_group_size": 0,
    "loss_masking": true,
    "sort_by_audio_len": false,
    "min_audio_len": 1,
    "max_audio_len": Infinity,
    "min_text_len": 20,
    "max_text_len": 100,
    "compute_f0": false,
    "compute_linear_spec": false,
    "precompute_num_workers": 0,
    "start_by_longest": false,
    "datasets": [
        {
            "name": "ljspeech",
            "path": "/home/smaksimov/TTS/recipes/ljspeech/glow_tts/../LJSpeech-1.2/",
            "meta_file_train": "metadata_train.csv",
            "ignored_speakers": null,
            "language": "ru",
            "meta_file_val": "metadata_val.csv",
            "meta_file_attn_mask": ""
        }
    ],
    "test_sentences": [
        "\u0428\u043b\u0430 \u0441\u0430\u0448\u0430 \u043f\u043e \u0448\u043e\u0441\u0441\u0435 \u0438 \u0441\u043e\u0441\u0430\u043b\u0430 \u0441\u0443\u0448\u043a\u0443.",
        "\u0422\u0435\u043a\u0441\u0442, \u043d\u0430\u043f\u0438\u0441\u0430\u043d\u043d\u044b\u0439 \u0440\u0443\u0441\u0441\u043a\u0438\u043c\u0438 \u0431\u0443\u043a\u0432\u0430\u043c\u0438.",
        "\u0414\u043e\u0431\u0440\u043e\u0435 \u0443\u0442\u0440\u043e. \u0414\u043e\u0431\u0440\u044b\u0439 \u0434\u0435\u043d\u044c. \u0414\u043e\u0431\u0440\u044b\u0439 \u0432\u0435\u0447\u0435\u0440.",
        "\u0412\u0430\u0441 \u043f\u0440\u0438\u0432\u0435\u0442\u0441\u0442\u0432\u0443\u0435\u0442 \u0441\u0438\u0441\u0442\u0435\u043c\u0430 \u041a\u041e\u041c\u041f\u0410\u041a\u0421.",
        "\u041e\u0431\u043d\u0430\u0440\u0443\u0436\u0435\u043d\u044b \u043d\u0435\u0438\u0441\u043f\u0440\u0430\u0432\u043d\u044b\u0435 \u043a\u0430\u043d\u0430\u043b\u044b!"
    ],
    "eval_split_max_size": null,
    "eval_split_size": 0.01,
    "use_speaker_weighted_sampler": false,
    "speaker_weighted_sampler_alpha": 1.0,
    "use_language_weighted_sampler": false,
    "language_weighted_sampler_alpha": 1.0,
    "num_chars": 79,
    "encoder_type": "rel_pos_transformer",
    "encoder_params": {
        "kernel_size": 3,
        "dropout_p": 0.1,
        "num_layers": 6,
        "num_heads": 2,
        "hidden_channels_ffn": 768,
        "input_length": null
    },
    "use_encoder_prenet": true,
    "hidden_channels_enc": 192,
    "hidden_channels_dec": 192,
    "hidden_channels_dp": 256,
    "dropout_p_dp": 0.1,
    "dropout_p_dec": 0.05,
    "mean_only": true,
    "out_channels": 80,
    "num_flow_blocks_dec": 12,
    "inference_noise_scale": 0.667,
    "kernel_size_dec": 5,
    "dilation_rate": 1,
    "num_block_layers": 4,
    "num_speakers": 0,
    "c_in_channels": 0,
    "num_splits": 4,
    "num_squeeze": 2,
    "sigmoid_scale": false,
    "d_vector_dim": 0,
    "data_dep_init_steps": 10,
    "style_wav_for_test": null,
    "length_scale": 1.0,
    "use_speaker_embedding": false,
    "speakers_file": null,
    "use_d_vector_file": false,
    "d_vector_file": false,
    "min_seq_len": 3,
    "max_seq_len": 500,
    "r": 3
}

1 reply

iprovalo Jun 30, 2022

@MaksimSerg I am training Russian ljspeech dataset on glow tts. I am facing similar issues. The speech is already recognizable but it won't improve. When I combine the model with mb melgan vocoder, it is hardly recognizable during the inference.

I am describing various configs I am using here: #1701