test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.chdir(\"/notebooks/bert-vits2/Bert-VITS2-Cantonese\")\n",
    "\n",
    "from models import (\n",
    "    SynthesizerTrn,\n",
    ")\n",
    "import math\n",
    "import torch\n",
    "from text.symbols import symbols, num_tones, num_languages, yue_symbols\n",
    "import utils\n",
    "from torch import nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "hps = utils.get_hparams_from_file(\"/notebooks/bert-vits2/dataset/commonvoice16_1_bertvits2/configs/config.json\")\n",
    "mas_noise_scale_initial = 0.0\n",
    "noise_scale_delta = 0.0\n",
    "\n",
    "def load_model(model_file: str):\n",
    "    net_g = SynthesizerTrn(\n",
    "        len(symbols),\n",
    "        hps.data.filter_length // 2 + 1,\n",
    "        hps.train.segment_size // hps.data.hop_length,\n",
    "        n_speakers=hps.data.n_speakers,\n",
    "        mas_noise_scale_initial=mas_noise_scale_initial,\n",
    "        noise_scale_delta=noise_scale_delta,\n",
    "        **hps.model,\n",
    "    )\n",
    "\n",
    "    optim_g = torch.optim.AdamW(\n",
    "        filter(lambda p: p.requires_grad, net_g.parameters()),\n",
    "        hps.train.learning_rate,\n",
    "        betas=hps.train.betas,\n",
    "        eps=hps.train.eps,\n",
    "    )\n",
    "\n",
    "    _, optim_g, g_resume_lr, epoch_str = utils.load_checkpoint(\n",
    "        model_file,\n",
    "        net_g,\n",
    "        optim_g,\n",
    "        skip_optimizer=(\n",
    "            hps.train.skip_optimizer if \"skip_optimizer\" in hps.train else True\n",
    "        ),\n",
    "    )\n",
    "\n",
    "    return net_g, optim_g"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/peft/lib/python3.8/site-packages/torch/nn/utils/weight_norm.py:28: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm.\n",
      "  warnings.warn(\"torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm.\")\n",
      "ERROR:utils:enc_p.emb.weight is not in the checkpoint\n",
      "ERROR:utils:enc_p.tone_emb.weight is not in the checkpoint\n",
      "ERROR:utils:enc_p.language_emb.weight is not in the checkpoint\n",
      "ERROR:utils:enc_p.yue_bert_proj.weight is not in the checkpoint\n",
      "ERROR:utils:enc_p.yue_bert_proj.bias is not in the checkpoint\n",
      "ERROR:utils:emb_g.weight is not in the checkpoint\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(SynthesizerTrn(\n",
       "   (enc_p): TextEncoder(\n",
       "     (emb): Embedding(155, 192)\n",
       "     (tone_emb): Embedding(19, 192)\n",
       "     (language_emb): Embedding(4, 192)\n",
       "     (bert_proj): Conv1d(1024, 192, kernel_size=(1,), stride=(1,))\n",
       "     (ja_bert_proj): Conv1d(1024, 192, kernel_size=(1,), stride=(1,))\n",
       "     (en_bert_proj): Conv1d(1024, 192, kernel_size=(1,), stride=(1,))\n",
       "     (yue_bert_proj): Conv1d(1024, 192, kernel_size=(1,), stride=(1,))\n",
       "     (encoder): Encoder(\n",
       "       (spk_emb_linear): Linear(in_features=512, out_features=192, bias=True)\n",
       "       (drop): Dropout(p=0.1, inplace=False)\n",
       "       (attn_layers): ModuleList(\n",
       "         (0-5): 6 x MultiHeadAttention(\n",
       "           (conv_q): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           (conv_k): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           (conv_v): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           (conv_o): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           (drop): Dropout(p=0.1, inplace=False)\n",
       "         )\n",
       "       )\n",
       "       (norm_layers_1): ModuleList(\n",
       "         (0-5): 6 x LayerNorm()\n",
       "       )\n",
       "       (ffn_layers): ModuleList(\n",
       "         (0-5): 6 x FFN(\n",
       "           (conv_1): Conv1d(192, 768, kernel_size=(3,), stride=(1,))\n",
       "           (conv_2): Conv1d(768, 192, kernel_size=(3,), stride=(1,))\n",
       "           (drop): Dropout(p=0.1, inplace=False)\n",
       "         )\n",
       "       )\n",
       "       (norm_layers_2): ModuleList(\n",
       "         (0-5): 6 x LayerNorm()\n",
       "       )\n",
       "     )\n",
       "     (proj): Conv1d(192, 384, kernel_size=(1,), stride=(1,))\n",
       "   )\n",
       "   (dec): Generator(\n",
       "     (conv_pre): Conv1d(192, 512, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "     (ups): ModuleList(\n",
       "       (0): ConvTranspose1d(512, 256, kernel_size=(16,), stride=(8,), padding=(4,))\n",
       "       (1): ConvTranspose1d(256, 128, kernel_size=(16,), stride=(8,), padding=(4,))\n",
       "       (2): ConvTranspose1d(128, 64, kernel_size=(8,), stride=(2,), padding=(3,))\n",
       "       (3): ConvTranspose1d(64, 32, kernel_size=(2,), stride=(2,))\n",
       "       (4): ConvTranspose1d(32, 16, kernel_size=(2,), stride=(2,))\n",
       "     )\n",
       "     (resblocks): ModuleList(\n",
       "       (0): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "           (1): Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,))\n",
       "           (2): Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(5,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "         )\n",
       "       )\n",
       "       (1): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(256, 256, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "           (1): Conv1d(256, 256, kernel_size=(7,), stride=(1,), padding=(9,), dilation=(3,))\n",
       "           (2): Conv1d(256, 256, kernel_size=(7,), stride=(1,), padding=(15,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(256, 256, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "         )\n",
       "       )\n",
       "       (2): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(256, 256, kernel_size=(11,), stride=(1,), padding=(5,))\n",
       "           (1): Conv1d(256, 256, kernel_size=(11,), stride=(1,), padding=(15,), dilation=(3,))\n",
       "           (2): Conv1d(256, 256, kernel_size=(11,), stride=(1,), padding=(25,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(256, 256, kernel_size=(11,), stride=(1,), padding=(5,))\n",
       "         )\n",
       "       )\n",
       "       (3): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "           (1): Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,))\n",
       "           (2): Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(5,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "         )\n",
       "       )\n",
       "       (4): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(128, 128, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "           (1): Conv1d(128, 128, kernel_size=(7,), stride=(1,), padding=(9,), dilation=(3,))\n",
       "           (2): Conv1d(128, 128, kernel_size=(7,), stride=(1,), padding=(15,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(128, 128, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "         )\n",
       "       )\n",
       "       (5): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(128, 128, kernel_size=(11,), stride=(1,), padding=(5,))\n",
       "           (1): Conv1d(128, 128, kernel_size=(11,), stride=(1,), padding=(15,), dilation=(3,))\n",
       "           (2): Conv1d(128, 128, kernel_size=(11,), stride=(1,), padding=(25,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(128, 128, kernel_size=(11,), stride=(1,), padding=(5,))\n",
       "         )\n",
       "       )\n",
       "       (6): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(64, 64, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "           (1): Conv1d(64, 64, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,))\n",
       "           (2): Conv1d(64, 64, kernel_size=(3,), stride=(1,), padding=(5,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(64, 64, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "         )\n",
       "       )\n",
       "       (7): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(64, 64, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "           (1): Conv1d(64, 64, kernel_size=(7,), stride=(1,), padding=(9,), dilation=(3,))\n",
       "           (2): Conv1d(64, 64, kernel_size=(7,), stride=(1,), padding=(15,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(64, 64, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "         )\n",
       "       )\n",
       "       (8): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(64, 64, kernel_size=(11,), stride=(1,), padding=(5,))\n",
       "           (1): Conv1d(64, 64, kernel_size=(11,), stride=(1,), padding=(15,), dilation=(3,))\n",
       "           (2): Conv1d(64, 64, kernel_size=(11,), stride=(1,), padding=(25,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(64, 64, kernel_size=(11,), stride=(1,), padding=(5,))\n",
       "         )\n",
       "       )\n",
       "       (9): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "           (1): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,))\n",
       "           (2): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(5,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "         )\n",
       "       )\n",
       "       (10): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(32, 32, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "           (1): Conv1d(32, 32, kernel_size=(7,), stride=(1,), padding=(9,), dilation=(3,))\n",
       "           (2): Conv1d(32, 32, kernel_size=(7,), stride=(1,), padding=(15,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(32, 32, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "         )\n",
       "       )\n",
       "       (11): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(32, 32, kernel_size=(11,), stride=(1,), padding=(5,))\n",
       "           (1): Conv1d(32, 32, kernel_size=(11,), stride=(1,), padding=(15,), dilation=(3,))\n",
       "           (2): Conv1d(32, 32, kernel_size=(11,), stride=(1,), padding=(25,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(32, 32, kernel_size=(11,), stride=(1,), padding=(5,))\n",
       "         )\n",
       "       )\n",
       "       (12): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "           (1): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,))\n",
       "           (2): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(5,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "         )\n",
       "       )\n",
       "       (13): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(16, 16, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "           (1): Conv1d(16, 16, kernel_size=(7,), stride=(1,), padding=(9,), dilation=(3,))\n",
       "           (2): Conv1d(16, 16, kernel_size=(7,), stride=(1,), padding=(15,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(16, 16, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "         )\n",
       "       )\n",
       "       (14): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(16, 16, kernel_size=(11,), stride=(1,), padding=(5,))\n",
       "           (1): Conv1d(16, 16, kernel_size=(11,), stride=(1,), padding=(15,), dilation=(3,))\n",
       "           (2): Conv1d(16, 16, kernel_size=(11,), stride=(1,), padding=(25,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(16, 16, kernel_size=(11,), stride=(1,), padding=(5,))\n",
       "         )\n",
       "       )\n",
       "     )\n",
       "     (conv_post): Conv1d(16, 1, kernel_size=(7,), stride=(1,), padding=(3,), bias=False)\n",
       "     (cond): Conv1d(512, 512, kernel_size=(1,), stride=(1,))\n",
       "   )\n",
       "   (enc_q): PosteriorEncoder(\n",
       "     (pre): Conv1d(1025, 192, kernel_size=(1,), stride=(1,))\n",
       "     (enc): WN(\n",
       "       (in_layers): ModuleList(\n",
       "         (0-15): 16 x Conv1d(192, 384, kernel_size=(5,), stride=(1,), padding=(2,))\n",
       "       )\n",
       "       (res_skip_layers): ModuleList(\n",
       "         (0-14): 15 x Conv1d(192, 384, kernel_size=(1,), stride=(1,))\n",
       "         (15): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (drop): Dropout(p=0, inplace=False)\n",
       "       (cond_layer): Conv1d(512, 6144, kernel_size=(1,), stride=(1,))\n",
       "     )\n",
       "     (proj): Conv1d(192, 384, kernel_size=(1,), stride=(1,))\n",
       "   )\n",
       "   (flow): TransformerCouplingBlock(\n",
       "     (flows): ModuleList(\n",
       "       (0): TransformerCouplingLayer(\n",
       "         (pre): Conv1d(96, 192, kernel_size=(1,), stride=(1,))\n",
       "         (enc): Encoder(\n",
       "           (spk_emb_linear): Linear(in_features=512, out_features=192, bias=True)\n",
       "           (drop): Dropout(p=0.1, inplace=False)\n",
       "           (attn_layers): ModuleList(\n",
       "             (0-3): 4 x MultiHeadAttention(\n",
       "               (conv_q): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_k): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_v): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_o): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (drop): Dropout(p=0.1, inplace=False)\n",
       "             )\n",
       "           )\n",
       "           (norm_layers_1): ModuleList(\n",
       "             (0-3): 4 x LayerNorm()\n",
       "           )\n",
       "           (ffn_layers): ModuleList(\n",
       "             (0-3): 4 x FFN(\n",
       "               (conv_1): Conv1d(192, 768, kernel_size=(5,), stride=(1,))\n",
       "               (conv_2): Conv1d(768, 192, kernel_size=(5,), stride=(1,))\n",
       "               (drop): Dropout(p=0.1, inplace=False)\n",
       "             )\n",
       "           )\n",
       "           (norm_layers_2): ModuleList(\n",
       "             (0-3): 4 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (post): Conv1d(192, 96, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (1): Flip()\n",
       "       (2): TransformerCouplingLayer(\n",
       "         (pre): Conv1d(96, 192, kernel_size=(1,), stride=(1,))\n",
       "         (enc): Encoder(\n",
       "           (spk_emb_linear): Linear(in_features=512, out_features=192, bias=True)\n",
       "           (drop): Dropout(p=0.1, inplace=False)\n",
       "           (attn_layers): ModuleList(\n",
       "             (0-3): 4 x MultiHeadAttention(\n",
       "               (conv_q): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_k): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_v): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_o): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (drop): Dropout(p=0.1, inplace=False)\n",
       "             )\n",
       "           )\n",
       "           (norm_layers_1): ModuleList(\n",
       "             (0-3): 4 x LayerNorm()\n",
       "           )\n",
       "           (ffn_layers): ModuleList(\n",
       "             (0-3): 4 x FFN(\n",
       "               (conv_1): Conv1d(192, 768, kernel_size=(5,), stride=(1,))\n",
       "               (conv_2): Conv1d(768, 192, kernel_size=(5,), stride=(1,))\n",
       "               (drop): Dropout(p=0.1, inplace=False)\n",
       "             )\n",
       "           )\n",
       "           (norm_layers_2): ModuleList(\n",
       "             (0-3): 4 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (post): Conv1d(192, 96, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (3): Flip()\n",
       "       (4): TransformerCouplingLayer(\n",
       "         (pre): Conv1d(96, 192, kernel_size=(1,), stride=(1,))\n",
       "         (enc): Encoder(\n",
       "           (spk_emb_linear): Linear(in_features=512, out_features=192, bias=True)\n",
       "           (drop): Dropout(p=0.1, inplace=False)\n",
       "           (attn_layers): ModuleList(\n",
       "             (0-3): 4 x MultiHeadAttention(\n",
       "               (conv_q): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_k): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_v): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_o): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (drop): Dropout(p=0.1, inplace=False)\n",
       "             )\n",
       "           )\n",
       "           (norm_layers_1): ModuleList(\n",
       "             (0-3): 4 x LayerNorm()\n",
       "           )\n",
       "           (ffn_layers): ModuleList(\n",
       "             (0-3): 4 x FFN(\n",
       "               (conv_1): Conv1d(192, 768, kernel_size=(5,), stride=(1,))\n",
       "               (conv_2): Conv1d(768, 192, kernel_size=(5,), stride=(1,))\n",
       "               (drop): Dropout(p=0.1, inplace=False)\n",
       "             )\n",
       "           )\n",
       "           (norm_layers_2): ModuleList(\n",
       "             (0-3): 4 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (post): Conv1d(192, 96, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (5): Flip()\n",
       "       (6): TransformerCouplingLayer(\n",
       "         (pre): Conv1d(96, 192, kernel_size=(1,), stride=(1,))\n",
       "         (enc): Encoder(\n",
       "           (spk_emb_linear): Linear(in_features=512, out_features=192, bias=True)\n",
       "           (drop): Dropout(p=0.1, inplace=False)\n",
       "           (attn_layers): ModuleList(\n",
       "             (0-3): 4 x MultiHeadAttention(\n",
       "               (conv_q): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_k): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_v): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_o): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (drop): Dropout(p=0.1, inplace=False)\n",
       "             )\n",
       "           )\n",
       "           (norm_layers_1): ModuleList(\n",
       "             (0-3): 4 x LayerNorm()\n",
       "           )\n",
       "           (ffn_layers): ModuleList(\n",
       "             (0-3): 4 x FFN(\n",
       "               (conv_1): Conv1d(192, 768, kernel_size=(5,), stride=(1,))\n",
       "               (conv_2): Conv1d(768, 192, kernel_size=(5,), stride=(1,))\n",
       "               (drop): Dropout(p=0.1, inplace=False)\n",
       "             )\n",
       "           )\n",
       "           (norm_layers_2): ModuleList(\n",
       "             (0-3): 4 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (post): Conv1d(192, 96, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (7): Flip()\n",
       "     )\n",
       "   )\n",
       "   (sdp): StochasticDurationPredictor(\n",
       "     (log_flow): Log()\n",
       "     (flows): ModuleList(\n",
       "       (0): ElementwiseAffine()\n",
       "       (1): ConvFlow(\n",
       "         (pre): Conv1d(1, 192, kernel_size=(1,), stride=(1,))\n",
       "         (convs): DDSConv(\n",
       "           (drop): Dropout(p=0.0, inplace=False)\n",
       "           (convs_sep): ModuleList(\n",
       "             (0): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(1,), groups=192)\n",
       "             (1): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,), groups=192)\n",
       "             (2): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(9,), dilation=(9,), groups=192)\n",
       "           )\n",
       "           (convs_1x1): ModuleList(\n",
       "             (0-2): 3 x Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           )\n",
       "           (norms_1): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "           (norms_2): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (proj): Conv1d(192, 29, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (2): Flip()\n",
       "       (3): ConvFlow(\n",
       "         (pre): Conv1d(1, 192, kernel_size=(1,), stride=(1,))\n",
       "         (convs): DDSConv(\n",
       "           (drop): Dropout(p=0.0, inplace=False)\n",
       "           (convs_sep): ModuleList(\n",
       "             (0): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(1,), groups=192)\n",
       "             (1): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,), groups=192)\n",
       "             (2): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(9,), dilation=(9,), groups=192)\n",
       "           )\n",
       "           (convs_1x1): ModuleList(\n",
       "             (0-2): 3 x Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           )\n",
       "           (norms_1): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "           (norms_2): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (proj): Conv1d(192, 29, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (4): Flip()\n",
       "       (5): ConvFlow(\n",
       "         (pre): Conv1d(1, 192, kernel_size=(1,), stride=(1,))\n",
       "         (convs): DDSConv(\n",
       "           (drop): Dropout(p=0.0, inplace=False)\n",
       "           (convs_sep): ModuleList(\n",
       "             (0): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(1,), groups=192)\n",
       "             (1): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,), groups=192)\n",
       "             (2): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(9,), dilation=(9,), groups=192)\n",
       "           )\n",
       "           (convs_1x1): ModuleList(\n",
       "             (0-2): 3 x Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           )\n",
       "           (norms_1): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "           (norms_2): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (proj): Conv1d(192, 29, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (6): Flip()\n",
       "       (7): ConvFlow(\n",
       "         (pre): Conv1d(1, 192, kernel_size=(1,), stride=(1,))\n",
       "         (convs): DDSConv(\n",
       "           (drop): Dropout(p=0.0, inplace=False)\n",
       "           (convs_sep): ModuleList(\n",
       "             (0): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(1,), groups=192)\n",
       "             (1): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,), groups=192)\n",
       "             (2): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(9,), dilation=(9,), groups=192)\n",
       "           )\n",
       "           (convs_1x1): ModuleList(\n",
       "             (0-2): 3 x Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           )\n",
       "           (norms_1): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "           (norms_2): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (proj): Conv1d(192, 29, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (8): Flip()\n",
       "     )\n",
       "     (post_pre): Conv1d(1, 192, kernel_size=(1,), stride=(1,))\n",
       "     (post_proj): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "     (post_convs): DDSConv(\n",
       "       (drop): Dropout(p=0.5, inplace=False)\n",
       "       (convs_sep): ModuleList(\n",
       "         (0): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(1,), groups=192)\n",
       "         (1): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,), groups=192)\n",
       "         (2): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(9,), dilation=(9,), groups=192)\n",
       "       )\n",
       "       (convs_1x1): ModuleList(\n",
       "         (0-2): 3 x Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (norms_1): ModuleList(\n",
       "         (0-2): 3 x LayerNorm()\n",
       "       )\n",
       "       (norms_2): ModuleList(\n",
       "         (0-2): 3 x LayerNorm()\n",
       "       )\n",
       "     )\n",
       "     (post_flows): ModuleList(\n",
       "       (0): ElementwiseAffine()\n",
       "       (1): ConvFlow(\n",
       "         (pre): Conv1d(1, 192, kernel_size=(1,), stride=(1,))\n",
       "         (convs): DDSConv(\n",
       "           (drop): Dropout(p=0.0, inplace=False)\n",
       "           (convs_sep): ModuleList(\n",
       "             (0): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(1,), groups=192)\n",
       "             (1): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,), groups=192)\n",
       "             (2): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(9,), dilation=(9,), groups=192)\n",
       "           )\n",
       "           (convs_1x1): ModuleList(\n",
       "             (0-2): 3 x Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           )\n",
       "           (norms_1): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "           (norms_2): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (proj): Conv1d(192, 29, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (2): Flip()\n",
       "       (3): ConvFlow(\n",
       "         (pre): Conv1d(1, 192, kernel_size=(1,), stride=(1,))\n",
       "         (convs): DDSConv(\n",
       "           (drop): Dropout(p=0.0, inplace=False)\n",
       "           (convs_sep): ModuleList(\n",
       "             (0): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(1,), groups=192)\n",
       "             (1): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,), groups=192)\n",
       "             (2): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(9,), dilation=(9,), groups=192)\n",
       "           )\n",
       "           (convs_1x1): ModuleList(\n",
       "             (0-2): 3 x Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           )\n",
       "           (norms_1): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "           (norms_2): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (proj): Conv1d(192, 29, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (4): Flip()\n",
       "       (5): ConvFlow(\n",
       "         (pre): Conv1d(1, 192, kernel_size=(1,), stride=(1,))\n",
       "         (convs): DDSConv(\n",
       "           (drop): Dropout(p=0.0, inplace=False)\n",
       "           (convs_sep): ModuleList(\n",
       "             (0): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(1,), groups=192)\n",
       "             (1): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,), groups=192)\n",
       "             (2): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(9,), dilation=(9,), groups=192)\n",
       "           )\n",
       "           (convs_1x1): ModuleList(\n",
       "             (0-2): 3 x Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           )\n",
       "           (norms_1): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "           (norms_2): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (proj): Conv1d(192, 29, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (6): Flip()\n",
       "       (7): ConvFlow(\n",
       "         (pre): Conv1d(1, 192, kernel_size=(1,), stride=(1,))\n",
       "         (convs): DDSConv(\n",
       "           (drop): Dropout(p=0.0, inplace=False)\n",
       "           (convs_sep): ModuleList(\n",
       "             (0): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(1,), groups=192)\n",
       "             (1): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,), groups=192)\n",
       "             (2): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(9,), dilation=(9,), groups=192)\n",
       "           )\n",
       "           (convs_1x1): ModuleList(\n",
       "             (0-2): 3 x Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           )\n",
       "           (norms_1): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "           (norms_2): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (proj): Conv1d(192, 29, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (8): Flip()\n",
       "     )\n",
       "     (pre): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "     (proj): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "     (convs): DDSConv(\n",
       "       (drop): Dropout(p=0.5, inplace=False)\n",
       "       (convs_sep): ModuleList(\n",
       "         (0): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(1,), groups=192)\n",
       "         (1): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,), groups=192)\n",
       "         (2): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(9,), dilation=(9,), groups=192)\n",
       "       )\n",
       "       (convs_1x1): ModuleList(\n",
       "         (0-2): 3 x Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (norms_1): ModuleList(\n",
       "         (0-2): 3 x LayerNorm()\n",
       "       )\n",
       "       (norms_2): ModuleList(\n",
       "         (0-2): 3 x LayerNorm()\n",
       "       )\n",
       "     )\n",
       "     (cond): Conv1d(512, 192, kernel_size=(1,), stride=(1,))\n",
       "   )\n",
       "   (dp): DurationPredictor(\n",
       "     (drop): Dropout(p=0.5, inplace=False)\n",
       "     (conv_1): Conv1d(192, 256, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "     (norm_1): LayerNorm()\n",
       "     (conv_2): Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "     (norm_2): LayerNorm()\n",
       "     (proj): Conv1d(256, 1, kernel_size=(1,), stride=(1,))\n",
       "     (cond): Conv1d(512, 192, kernel_size=(1,), stride=(1,))\n",
       "   )\n",
       "   (emb_g): Embedding(648, 512)\n",
       " ),\n",
       " AdamW (\n",
       " Parameter Group 0\n",
       "     amsgrad: False\n",
       "     betas: [0.8, 0.99]\n",
       "     capturable: False\n",
       "     differentiable: False\n",
       "     eps: 1e-09\n",
       "     foreach: None\n",
       "     fused: None\n",
       "     lr: 0.0002\n",
       "     maximize: False\n",
       "     weight_decay: 0.01\n",
       " ))"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "load_model(\"/notebooks/bert-vits2/dataset/commonvoice16_1_bertvits2/backup_models/G_0.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint_dict = torch.load(\"/notebooks/bert-vits2/dataset/commonvoice16_1_bertvits2/backup_models/G_0.pth\", map_location=\"cpu\")\n",
    "\n",
    "def resize_embedding_layer(weight, new_size):\n",
    "    old_vocab_size = weight.size(0)\n",
    "    if new_size < old_vocab_size:\n",
    "        return weight[:new_size, :]\n",
    "    elif new_size == old_vocab_size:\n",
    "        return weight\n",
    "    else:\n",
    "        new_weight = weight.new_zeros(new_size - old_vocab_size, weight.size(1))\n",
    "        embedding_dim = weight.size(1)\n",
    "        avg_weight = weight.mean(dim=0, keepdim=True)\n",
    "        noise_weight = torch.empty_like(new_weight)\n",
    "        noise_weight.normal_(mean=0, std=(1.0 / math.sqrt(embedding_dim)))\n",
    "        new_weight = avg_weight + noise_weight\n",
    "\n",
    "        return torch.cat([weight, new_weight], dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint_dict['model']['enc_p.emb.weight'] = resize_embedding_layer(checkpoint_dict['model']['enc_p.emb.weight'], len(symbols))\n",
    "checkpoint_dict['model']['enc_p.tone_emb.weight'] = resize_embedding_layer(checkpoint_dict['model']['enc_p.tone_emb.weight'], num_tones)\n",
    "checkpoint_dict['model']['enc_p.language_emb.weight'] = resize_embedding_layer(checkpoint_dict['model']['enc_p.language_emb.weight'], num_languages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(checkpoint_dict, \"/notebooks/bert-vits2/dataset/commonvoice16_1_bertvits2/resized_models/G_0.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:utils:enc_p.yue_bert_proj.weight is not in the checkpoint\n",
      "ERROR:utils:enc_p.yue_bert_proj.bias is not in the checkpoint\n",
      "ERROR:utils:emb_g.weight is not in the checkpoint\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(SynthesizerTrn(\n",
       "   (enc_p): TextEncoder(\n",
       "     (emb): Embedding(155, 192)\n",
       "     (tone_emb): Embedding(19, 192)\n",
       "     (language_emb): Embedding(4, 192)\n",
       "     (bert_proj): Conv1d(1024, 192, kernel_size=(1,), stride=(1,))\n",
       "     (ja_bert_proj): Conv1d(1024, 192, kernel_size=(1,), stride=(1,))\n",
       "     (en_bert_proj): Conv1d(1024, 192, kernel_size=(1,), stride=(1,))\n",
       "     (yue_bert_proj): Conv1d(1024, 192, kernel_size=(1,), stride=(1,))\n",
       "     (encoder): Encoder(\n",
       "       (spk_emb_linear): Linear(in_features=512, out_features=192, bias=True)\n",
       "       (drop): Dropout(p=0.1, inplace=False)\n",
       "       (attn_layers): ModuleList(\n",
       "         (0-5): 6 x MultiHeadAttention(\n",
       "           (conv_q): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           (conv_k): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           (conv_v): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           (conv_o): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           (drop): Dropout(p=0.1, inplace=False)\n",
       "         )\n",
       "       )\n",
       "       (norm_layers_1): ModuleList(\n",
       "         (0-5): 6 x LayerNorm()\n",
       "       )\n",
       "       (ffn_layers): ModuleList(\n",
       "         (0-5): 6 x FFN(\n",
       "           (conv_1): Conv1d(192, 768, kernel_size=(3,), stride=(1,))\n",
       "           (conv_2): Conv1d(768, 192, kernel_size=(3,), stride=(1,))\n",
       "           (drop): Dropout(p=0.1, inplace=False)\n",
       "         )\n",
       "       )\n",
       "       (norm_layers_2): ModuleList(\n",
       "         (0-5): 6 x LayerNorm()\n",
       "       )\n",
       "     )\n",
       "     (proj): Conv1d(192, 384, kernel_size=(1,), stride=(1,))\n",
       "   )\n",
       "   (dec): Generator(\n",
       "     (conv_pre): Conv1d(192, 512, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "     (ups): ModuleList(\n",
       "       (0): ConvTranspose1d(512, 256, kernel_size=(16,), stride=(8,), padding=(4,))\n",
       "       (1): ConvTranspose1d(256, 128, kernel_size=(16,), stride=(8,), padding=(4,))\n",
       "       (2): ConvTranspose1d(128, 64, kernel_size=(8,), stride=(2,), padding=(3,))\n",
       "       (3): ConvTranspose1d(64, 32, kernel_size=(2,), stride=(2,))\n",
       "       (4): ConvTranspose1d(32, 16, kernel_size=(2,), stride=(2,))\n",
       "     )\n",
       "     (resblocks): ModuleList(\n",
       "       (0): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "           (1): Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,))\n",
       "           (2): Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(5,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "         )\n",
       "       )\n",
       "       (1): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(256, 256, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "           (1): Conv1d(256, 256, kernel_size=(7,), stride=(1,), padding=(9,), dilation=(3,))\n",
       "           (2): Conv1d(256, 256, kernel_size=(7,), stride=(1,), padding=(15,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(256, 256, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "         )\n",
       "       )\n",
       "       (2): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(256, 256, kernel_size=(11,), stride=(1,), padding=(5,))\n",
       "           (1): Conv1d(256, 256, kernel_size=(11,), stride=(1,), padding=(15,), dilation=(3,))\n",
       "           (2): Conv1d(256, 256, kernel_size=(11,), stride=(1,), padding=(25,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(256, 256, kernel_size=(11,), stride=(1,), padding=(5,))\n",
       "         )\n",
       "       )\n",
       "       (3): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "           (1): Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,))\n",
       "           (2): Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(5,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "         )\n",
       "       )\n",
       "       (4): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(128, 128, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "           (1): Conv1d(128, 128, kernel_size=(7,), stride=(1,), padding=(9,), dilation=(3,))\n",
       "           (2): Conv1d(128, 128, kernel_size=(7,), stride=(1,), padding=(15,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(128, 128, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "         )\n",
       "       )\n",
       "       (5): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(128, 128, kernel_size=(11,), stride=(1,), padding=(5,))\n",
       "           (1): Conv1d(128, 128, kernel_size=(11,), stride=(1,), padding=(15,), dilation=(3,))\n",
       "           (2): Conv1d(128, 128, kernel_size=(11,), stride=(1,), padding=(25,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(128, 128, kernel_size=(11,), stride=(1,), padding=(5,))\n",
       "         )\n",
       "       )\n",
       "       (6): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(64, 64, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "           (1): Conv1d(64, 64, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,))\n",
       "           (2): Conv1d(64, 64, kernel_size=(3,), stride=(1,), padding=(5,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(64, 64, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "         )\n",
       "       )\n",
       "       (7): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(64, 64, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "           (1): Conv1d(64, 64, kernel_size=(7,), stride=(1,), padding=(9,), dilation=(3,))\n",
       "           (2): Conv1d(64, 64, kernel_size=(7,), stride=(1,), padding=(15,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(64, 64, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "         )\n",
       "       )\n",
       "       (8): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(64, 64, kernel_size=(11,), stride=(1,), padding=(5,))\n",
       "           (1): Conv1d(64, 64, kernel_size=(11,), stride=(1,), padding=(15,), dilation=(3,))\n",
       "           (2): Conv1d(64, 64, kernel_size=(11,), stride=(1,), padding=(25,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(64, 64, kernel_size=(11,), stride=(1,), padding=(5,))\n",
       "         )\n",
       "       )\n",
       "       (9): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "           (1): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,))\n",
       "           (2): Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(5,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(32, 32, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "         )\n",
       "       )\n",
       "       (10): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(32, 32, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "           (1): Conv1d(32, 32, kernel_size=(7,), stride=(1,), padding=(9,), dilation=(3,))\n",
       "           (2): Conv1d(32, 32, kernel_size=(7,), stride=(1,), padding=(15,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(32, 32, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "         )\n",
       "       )\n",
       "       (11): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(32, 32, kernel_size=(11,), stride=(1,), padding=(5,))\n",
       "           (1): Conv1d(32, 32, kernel_size=(11,), stride=(1,), padding=(15,), dilation=(3,))\n",
       "           (2): Conv1d(32, 32, kernel_size=(11,), stride=(1,), padding=(25,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(32, 32, kernel_size=(11,), stride=(1,), padding=(5,))\n",
       "         )\n",
       "       )\n",
       "       (12): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "           (1): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,))\n",
       "           (2): Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(5,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(16, 16, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "         )\n",
       "       )\n",
       "       (13): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(16, 16, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "           (1): Conv1d(16, 16, kernel_size=(7,), stride=(1,), padding=(9,), dilation=(3,))\n",
       "           (2): Conv1d(16, 16, kernel_size=(7,), stride=(1,), padding=(15,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(16, 16, kernel_size=(7,), stride=(1,), padding=(3,))\n",
       "         )\n",
       "       )\n",
       "       (14): ResBlock1(\n",
       "         (convs1): ModuleList(\n",
       "           (0): Conv1d(16, 16, kernel_size=(11,), stride=(1,), padding=(5,))\n",
       "           (1): Conv1d(16, 16, kernel_size=(11,), stride=(1,), padding=(15,), dilation=(3,))\n",
       "           (2): Conv1d(16, 16, kernel_size=(11,), stride=(1,), padding=(25,), dilation=(5,))\n",
       "         )\n",
       "         (convs2): ModuleList(\n",
       "           (0-2): 3 x Conv1d(16, 16, kernel_size=(11,), stride=(1,), padding=(5,))\n",
       "         )\n",
       "       )\n",
       "     )\n",
       "     (conv_post): Conv1d(16, 1, kernel_size=(7,), stride=(1,), padding=(3,), bias=False)\n",
       "     (cond): Conv1d(512, 512, kernel_size=(1,), stride=(1,))\n",
       "   )\n",
       "   (enc_q): PosteriorEncoder(\n",
       "     (pre): Conv1d(1025, 192, kernel_size=(1,), stride=(1,))\n",
       "     (enc): WN(\n",
       "       (in_layers): ModuleList(\n",
       "         (0-15): 16 x Conv1d(192, 384, kernel_size=(5,), stride=(1,), padding=(2,))\n",
       "       )\n",
       "       (res_skip_layers): ModuleList(\n",
       "         (0-14): 15 x Conv1d(192, 384, kernel_size=(1,), stride=(1,))\n",
       "         (15): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (drop): Dropout(p=0, inplace=False)\n",
       "       (cond_layer): Conv1d(512, 6144, kernel_size=(1,), stride=(1,))\n",
       "     )\n",
       "     (proj): Conv1d(192, 384, kernel_size=(1,), stride=(1,))\n",
       "   )\n",
       "   (flow): TransformerCouplingBlock(\n",
       "     (flows): ModuleList(\n",
       "       (0): TransformerCouplingLayer(\n",
       "         (pre): Conv1d(96, 192, kernel_size=(1,), stride=(1,))\n",
       "         (enc): Encoder(\n",
       "           (spk_emb_linear): Linear(in_features=512, out_features=192, bias=True)\n",
       "           (drop): Dropout(p=0.1, inplace=False)\n",
       "           (attn_layers): ModuleList(\n",
       "             (0-3): 4 x MultiHeadAttention(\n",
       "               (conv_q): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_k): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_v): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_o): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (drop): Dropout(p=0.1, inplace=False)\n",
       "             )\n",
       "           )\n",
       "           (norm_layers_1): ModuleList(\n",
       "             (0-3): 4 x LayerNorm()\n",
       "           )\n",
       "           (ffn_layers): ModuleList(\n",
       "             (0-3): 4 x FFN(\n",
       "               (conv_1): Conv1d(192, 768, kernel_size=(5,), stride=(1,))\n",
       "               (conv_2): Conv1d(768, 192, kernel_size=(5,), stride=(1,))\n",
       "               (drop): Dropout(p=0.1, inplace=False)\n",
       "             )\n",
       "           )\n",
       "           (norm_layers_2): ModuleList(\n",
       "             (0-3): 4 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (post): Conv1d(192, 96, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (1): Flip()\n",
       "       (2): TransformerCouplingLayer(\n",
       "         (pre): Conv1d(96, 192, kernel_size=(1,), stride=(1,))\n",
       "         (enc): Encoder(\n",
       "           (spk_emb_linear): Linear(in_features=512, out_features=192, bias=True)\n",
       "           (drop): Dropout(p=0.1, inplace=False)\n",
       "           (attn_layers): ModuleList(\n",
       "             (0-3): 4 x MultiHeadAttention(\n",
       "               (conv_q): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_k): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_v): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_o): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (drop): Dropout(p=0.1, inplace=False)\n",
       "             )\n",
       "           )\n",
       "           (norm_layers_1): ModuleList(\n",
       "             (0-3): 4 x LayerNorm()\n",
       "           )\n",
       "           (ffn_layers): ModuleList(\n",
       "             (0-3): 4 x FFN(\n",
       "               (conv_1): Conv1d(192, 768, kernel_size=(5,), stride=(1,))\n",
       "               (conv_2): Conv1d(768, 192, kernel_size=(5,), stride=(1,))\n",
       "               (drop): Dropout(p=0.1, inplace=False)\n",
       "             )\n",
       "           )\n",
       "           (norm_layers_2): ModuleList(\n",
       "             (0-3): 4 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (post): Conv1d(192, 96, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (3): Flip()\n",
       "       (4): TransformerCouplingLayer(\n",
       "         (pre): Conv1d(96, 192, kernel_size=(1,), stride=(1,))\n",
       "         (enc): Encoder(\n",
       "           (spk_emb_linear): Linear(in_features=512, out_features=192, bias=True)\n",
       "           (drop): Dropout(p=0.1, inplace=False)\n",
       "           (attn_layers): ModuleList(\n",
       "             (0-3): 4 x MultiHeadAttention(\n",
       "               (conv_q): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_k): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_v): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_o): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (drop): Dropout(p=0.1, inplace=False)\n",
       "             )\n",
       "           )\n",
       "           (norm_layers_1): ModuleList(\n",
       "             (0-3): 4 x LayerNorm()\n",
       "           )\n",
       "           (ffn_layers): ModuleList(\n",
       "             (0-3): 4 x FFN(\n",
       "               (conv_1): Conv1d(192, 768, kernel_size=(5,), stride=(1,))\n",
       "               (conv_2): Conv1d(768, 192, kernel_size=(5,), stride=(1,))\n",
       "               (drop): Dropout(p=0.1, inplace=False)\n",
       "             )\n",
       "           )\n",
       "           (norm_layers_2): ModuleList(\n",
       "             (0-3): 4 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (post): Conv1d(192, 96, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (5): Flip()\n",
       "       (6): TransformerCouplingLayer(\n",
       "         (pre): Conv1d(96, 192, kernel_size=(1,), stride=(1,))\n",
       "         (enc): Encoder(\n",
       "           (spk_emb_linear): Linear(in_features=512, out_features=192, bias=True)\n",
       "           (drop): Dropout(p=0.1, inplace=False)\n",
       "           (attn_layers): ModuleList(\n",
       "             (0-3): 4 x MultiHeadAttention(\n",
       "               (conv_q): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_k): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_v): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (conv_o): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "               (drop): Dropout(p=0.1, inplace=False)\n",
       "             )\n",
       "           )\n",
       "           (norm_layers_1): ModuleList(\n",
       "             (0-3): 4 x LayerNorm()\n",
       "           )\n",
       "           (ffn_layers): ModuleList(\n",
       "             (0-3): 4 x FFN(\n",
       "               (conv_1): Conv1d(192, 768, kernel_size=(5,), stride=(1,))\n",
       "               (conv_2): Conv1d(768, 192, kernel_size=(5,), stride=(1,))\n",
       "               (drop): Dropout(p=0.1, inplace=False)\n",
       "             )\n",
       "           )\n",
       "           (norm_layers_2): ModuleList(\n",
       "             (0-3): 4 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (post): Conv1d(192, 96, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (7): Flip()\n",
       "     )\n",
       "   )\n",
       "   (sdp): StochasticDurationPredictor(\n",
       "     (log_flow): Log()\n",
       "     (flows): ModuleList(\n",
       "       (0): ElementwiseAffine()\n",
       "       (1): ConvFlow(\n",
       "         (pre): Conv1d(1, 192, kernel_size=(1,), stride=(1,))\n",
       "         (convs): DDSConv(\n",
       "           (drop): Dropout(p=0.0, inplace=False)\n",
       "           (convs_sep): ModuleList(\n",
       "             (0): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(1,), groups=192)\n",
       "             (1): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,), groups=192)\n",
       "             (2): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(9,), dilation=(9,), groups=192)\n",
       "           )\n",
       "           (convs_1x1): ModuleList(\n",
       "             (0-2): 3 x Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           )\n",
       "           (norms_1): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "           (norms_2): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (proj): Conv1d(192, 29, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (2): Flip()\n",
       "       (3): ConvFlow(\n",
       "         (pre): Conv1d(1, 192, kernel_size=(1,), stride=(1,))\n",
       "         (convs): DDSConv(\n",
       "           (drop): Dropout(p=0.0, inplace=False)\n",
       "           (convs_sep): ModuleList(\n",
       "             (0): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(1,), groups=192)\n",
       "             (1): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,), groups=192)\n",
       "             (2): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(9,), dilation=(9,), groups=192)\n",
       "           )\n",
       "           (convs_1x1): ModuleList(\n",
       "             (0-2): 3 x Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           )\n",
       "           (norms_1): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "           (norms_2): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (proj): Conv1d(192, 29, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (4): Flip()\n",
       "       (5): ConvFlow(\n",
       "         (pre): Conv1d(1, 192, kernel_size=(1,), stride=(1,))\n",
       "         (convs): DDSConv(\n",
       "           (drop): Dropout(p=0.0, inplace=False)\n",
       "           (convs_sep): ModuleList(\n",
       "             (0): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(1,), groups=192)\n",
       "             (1): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,), groups=192)\n",
       "             (2): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(9,), dilation=(9,), groups=192)\n",
       "           )\n",
       "           (convs_1x1): ModuleList(\n",
       "             (0-2): 3 x Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           )\n",
       "           (norms_1): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "           (norms_2): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (proj): Conv1d(192, 29, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (6): Flip()\n",
       "       (7): ConvFlow(\n",
       "         (pre): Conv1d(1, 192, kernel_size=(1,), stride=(1,))\n",
       "         (convs): DDSConv(\n",
       "           (drop): Dropout(p=0.0, inplace=False)\n",
       "           (convs_sep): ModuleList(\n",
       "             (0): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(1,), groups=192)\n",
       "             (1): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,), groups=192)\n",
       "             (2): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(9,), dilation=(9,), groups=192)\n",
       "           )\n",
       "           (convs_1x1): ModuleList(\n",
       "             (0-2): 3 x Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           )\n",
       "           (norms_1): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "           (norms_2): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (proj): Conv1d(192, 29, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (8): Flip()\n",
       "     )\n",
       "     (post_pre): Conv1d(1, 192, kernel_size=(1,), stride=(1,))\n",
       "     (post_proj): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "     (post_convs): DDSConv(\n",
       "       (drop): Dropout(p=0.5, inplace=False)\n",
       "       (convs_sep): ModuleList(\n",
       "         (0): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(1,), groups=192)\n",
       "         (1): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,), groups=192)\n",
       "         (2): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(9,), dilation=(9,), groups=192)\n",
       "       )\n",
       "       (convs_1x1): ModuleList(\n",
       "         (0-2): 3 x Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (norms_1): ModuleList(\n",
       "         (0-2): 3 x LayerNorm()\n",
       "       )\n",
       "       (norms_2): ModuleList(\n",
       "         (0-2): 3 x LayerNorm()\n",
       "       )\n",
       "     )\n",
       "     (post_flows): ModuleList(\n",
       "       (0): ElementwiseAffine()\n",
       "       (1): ConvFlow(\n",
       "         (pre): Conv1d(1, 192, kernel_size=(1,), stride=(1,))\n",
       "         (convs): DDSConv(\n",
       "           (drop): Dropout(p=0.0, inplace=False)\n",
       "           (convs_sep): ModuleList(\n",
       "             (0): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(1,), groups=192)\n",
       "             (1): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,), groups=192)\n",
       "             (2): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(9,), dilation=(9,), groups=192)\n",
       "           )\n",
       "           (convs_1x1): ModuleList(\n",
       "             (0-2): 3 x Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           )\n",
       "           (norms_1): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "           (norms_2): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (proj): Conv1d(192, 29, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (2): Flip()\n",
       "       (3): ConvFlow(\n",
       "         (pre): Conv1d(1, 192, kernel_size=(1,), stride=(1,))\n",
       "         (convs): DDSConv(\n",
       "           (drop): Dropout(p=0.0, inplace=False)\n",
       "           (convs_sep): ModuleList(\n",
       "             (0): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(1,), groups=192)\n",
       "             (1): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,), groups=192)\n",
       "             (2): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(9,), dilation=(9,), groups=192)\n",
       "           )\n",
       "           (convs_1x1): ModuleList(\n",
       "             (0-2): 3 x Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           )\n",
       "           (norms_1): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "           (norms_2): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (proj): Conv1d(192, 29, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (4): Flip()\n",
       "       (5): ConvFlow(\n",
       "         (pre): Conv1d(1, 192, kernel_size=(1,), stride=(1,))\n",
       "         (convs): DDSConv(\n",
       "           (drop): Dropout(p=0.0, inplace=False)\n",
       "           (convs_sep): ModuleList(\n",
       "             (0): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(1,), groups=192)\n",
       "             (1): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,), groups=192)\n",
       "             (2): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(9,), dilation=(9,), groups=192)\n",
       "           )\n",
       "           (convs_1x1): ModuleList(\n",
       "             (0-2): 3 x Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           )\n",
       "           (norms_1): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "           (norms_2): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (proj): Conv1d(192, 29, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (6): Flip()\n",
       "       (7): ConvFlow(\n",
       "         (pre): Conv1d(1, 192, kernel_size=(1,), stride=(1,))\n",
       "         (convs): DDSConv(\n",
       "           (drop): Dropout(p=0.0, inplace=False)\n",
       "           (convs_sep): ModuleList(\n",
       "             (0): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(1,), groups=192)\n",
       "             (1): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,), groups=192)\n",
       "             (2): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(9,), dilation=(9,), groups=192)\n",
       "           )\n",
       "           (convs_1x1): ModuleList(\n",
       "             (0-2): 3 x Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "           )\n",
       "           (norms_1): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "           (norms_2): ModuleList(\n",
       "             (0-2): 3 x LayerNorm()\n",
       "           )\n",
       "         )\n",
       "         (proj): Conv1d(192, 29, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (8): Flip()\n",
       "     )\n",
       "     (pre): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "     (proj): Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "     (convs): DDSConv(\n",
       "       (drop): Dropout(p=0.5, inplace=False)\n",
       "       (convs_sep): ModuleList(\n",
       "         (0): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(1,), groups=192)\n",
       "         (1): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(3,), dilation=(3,), groups=192)\n",
       "         (2): Conv1d(192, 192, kernel_size=(3,), stride=(1,), padding=(9,), dilation=(9,), groups=192)\n",
       "       )\n",
       "       (convs_1x1): ModuleList(\n",
       "         (0-2): 3 x Conv1d(192, 192, kernel_size=(1,), stride=(1,))\n",
       "       )\n",
       "       (norms_1): ModuleList(\n",
       "         (0-2): 3 x LayerNorm()\n",
       "       )\n",
       "       (norms_2): ModuleList(\n",
       "         (0-2): 3 x LayerNorm()\n",
       "       )\n",
       "     )\n",
       "     (cond): Conv1d(512, 192, kernel_size=(1,), stride=(1,))\n",
       "   )\n",
       "   (dp): DurationPredictor(\n",
       "     (drop): Dropout(p=0.5, inplace=False)\n",
       "     (conv_1): Conv1d(192, 256, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "     (norm_1): LayerNorm()\n",
       "     (conv_2): Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(1,))\n",
       "     (norm_2): LayerNorm()\n",
       "     (proj): Conv1d(256, 1, kernel_size=(1,), stride=(1,))\n",
       "     (cond): Conv1d(512, 192, kernel_size=(1,), stride=(1,))\n",
       "   )\n",
       "   (emb_g): Embedding(648, 512)\n",
       " ),\n",
       " AdamW (\n",
       " Parameter Group 0\n",
       "     amsgrad: False\n",
       "     betas: [0.8, 0.99]\n",
       "     capturable: False\n",
       "     differentiable: False\n",
       "     eps: 1e-09\n",
       "     foreach: None\n",
       "     fused: None\n",
       "     lr: 0.0002\n",
       "     maximize: False\n",
       "     weight_decay: 0.01\n",
       " ))"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "load_model(\"/notebooks/bert-vits2/dataset/commonvoice16_1_bertvits2/resized_models/G_0.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'!', \"'\", ',', '-', '.', '?', '_', 'z', '…'}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the symbols\n",
    "\n",
    "from text import cleaned_text_to_sequence\n",
    "\n",
    "# test symbols\n",
    "lines = []\n",
    "my_phones = set()\n",
    "\n",
    "with open(\"/notebooks/bert-vits2/dataset/commonvoice16_1_bertvits2/esd.list.cleaned\", encoding=\"utf-8\") as f:\n",
    "    lines.extend(f.readlines())\n",
    "\n",
    "for line in lines:\n",
    "  wav_path, _, language_str, text, phones, tone, word2ph = line.strip().split(\"|\")\n",
    "  phone = phones.split(\" \")\n",
    "  my_phones.update(phone)\n",
    "  # tone = [int(i) for i in tone.split(\" \")]\n",
    "  # word2ph = [int(i) for i in word2ph.split(\" \")]\n",
    "  # word2ph = [i for i in word2ph]\n",
    "  # phone, tone, language = cleaned_text_to_sequence(phone, tone, language_str)\n",
    "\n",
    "my_phones - set(yue_symbols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}