AniVoiceChanger_colab.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/SociallyIneptWeeb/AniVoiceChanger/blob/main/AniVoiceChanger_colab.ipynb)"
      ],
      "metadata": {
        "id": "wEsZxTheMO8_"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jwu07JgqoFON",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Mount Drive\n",
        "\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ge_97mfpgqTm",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Clone repository\n",
        "!git init\n",
        "!git remote add origin https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI.git\n",
        "!git fetch origin 195a14e5c51ec02774c4d1961d0a4b67755e25c8 --depth=1\n",
        "!git reset --hard FETCH_HEAD"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Set Mode and Parameters\n",
        "#@markdown ## Mode\n",
        "#@markdown To run the WebUI for training a voice model, set the mode to Training.\n",
        "\n",
        "#@markdown To run the voice changer server for main_colab.py to connect to, set the mode to Inference.\n",
        "MODE = 'Training' #@param ['Training', 'Inference']\n",
        "\n",
        "#@markdown If MODE: Training, specify the path to a zip file containing the voice clips in your google drive to be used for training. If MODE: Inference, ignore.\n",
        "DATASET = 'char_voice_lines.zip' #@param {type:\"string\"}\n",
        "DATASET = '/content/drive/MyDrive/' + DATASET\n",
        "\n",
        "if MODE == 'Training':\n",
        "  !mkdir -p dataset\n",
        "  !unzip -d dataset -B {DATASET}\n",
        "  # rename duplicate filenames in dataset\n",
        "  !ls -a /content/dataset/\n",
        "  !rename 's/(\\w+)\\.(\\w+)~(\\d*)/$1_$3.$2/' /content/dataset/*.*~*\n",
        "\n",
        "#@markdown ## Upload a trained model\n",
        "#@markdown Only fill the below fields if you would like to continue training a previously trained model or use it for inference. Specify the name and epoch number of the model to be used or trained. The folder containing the trained files in your google drive will be used.\n",
        "\n",
        "MODELNAME = \"\"  #@param {type:\"string\"}\n",
        "MODELEPOCH = 2333333  #@param {type:\"integer\"}\n",
        "if MODELNAME:\n",
        "  !mkdir -p /content/logs/{MODELNAME}\n",
        "  !cp /content/drive/MyDrive/{MODELNAME}_files/*.index /content/logs/{MODELNAME}/\n",
        "  !cp /content/drive/MyDrive/{MODELNAME}_files/{MODELNAME}.pth /content/weights/\n",
        "  if MODE == 'Training':\n",
        "    !cp /content/drive/MyDrive/{MODELNAME}_files/D_{MODELEPOCH}.pth /content/logs/{MODELNAME}/\n",
        "    !cp /content/drive/MyDrive/{MODELNAME}_files/G_{MODELEPOCH}.pth /content/logs/{MODELNAME}/\n",
        "    !cp /content/drive/MyDrive/{MODELNAME}_files/*.npy /content/logs/{MODELNAME}/"
      ],
      "metadata": {
        "id": "S8wlJabmgeBK",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pqE0PrnuRqI2",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Install requirements\n",
        "!pip install -r requirements.txt\n",
        "!apt -y install -qq aria2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UG3XpUwEomUz",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Download pretrained models\n",
        "\n",
        "# v1\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/pretrained/D32k.pth -d /content/pretrained -o D32k.pth\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/pretrained/D40k.pth -d /content/pretrained -o D40k.pth\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/pretrained/D48k.pth -d /content/pretrained -o D48k.pth\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/pretrained/G32k.pth -d /content/pretrained -o G32k.pth\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/pretrained/G40k.pth -d /content/pretrained -o G40k.pth\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/pretrained/G48k.pth -d /content/pretrained -o G48k.pth\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/pretrained/f0D32k.pth -d /content/pretrained -o f0D32k.pth\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/pretrained/f0D40k.pth -d /content/pretrained -o f0D40k.pth\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/pretrained/f0D48k.pth -d /content/pretrained -o f0D48k.pth\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/pretrained/f0G32k.pth -d /content/pretrained -o f0G32k.pth\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/pretrained/f0G40k.pth -d /content/pretrained -o f0G40k.pth\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/pretrained/f0G48k.pth -d /content/pretrained -o f0G48k.pth\n",
        "\n",
        "# v2\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/pretrained_v2/D40k.pth -d /content/pretrained_v2 -o D40k.pth\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/pretrained_v2/G40k.pth -d /content/pretrained_v2 -o G40k.pth\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/pretrained_v2/f0D40k.pth -d /content/pretrained_v2 -o f0D40k.pth\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/pretrained_v2/f0G40k.pth -d /content/pretrained_v2 -o f0G40k.pth"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HugjmZqZRuiF",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Download Vocal Separation model\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/uvr5_weights/HP2-人声vocals+非人声instrumentals.pth -d /content/uvr5_weights -o HP2-人声vocals+非人声instrumentals.pth\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/uvr5_weights/HP5-主旋律人声vocals+其他instrumentals.pth -d /content/uvr5_weights -o HP5-主旋律人声vocals+其他instrumentals.pth"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2RCaT9FTR0ej",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Download hubert_base model\n",
        "!aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/hubert_base.pt -d /content -o hubert_base.pt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7vh6vphDwO0b",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Run WebUI for Training, skip if Inference\n",
        "if MODE == 'Training':\n",
        "  # %load_ext tensorboard\n",
        "  # %tensorboard --logdir /content/Retrieval-based-Voice-Conversion-WebUI/logs\n",
        "  !python3 infer-web.py --colab --pycmd python3"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FgJuNeAwx5Y_",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title Manually back up the trained model files to Google Drive for Mode: \"Training\"\n",
        "#@markdown MODELNAME should be the EXPERIMENT_NAME that you typed.\n",
        "\n",
        "#@markdown If the name of the model is john, and in the logs/john folder is a file called D_2333333.pth, set the MODELNAME: john and MODELEPOCH: 2333333\n",
        "\n",
        "if MODE == 'Training':\n",
        "\n",
        "#@markdown Model name\n",
        "  MODELNAME = \"\"  #@param {type:\"string\"}\n",
        "  if MODELNAME:\n",
        "#@markdown Epoch number\n",
        "    MODELEPOCH = 2333333  #@param {type:\"integer\"}\n",
        "#@markdown Save intermediate models if you would like to continue training the model later\n",
        "    SAVE_INTERMEDIATE = True  #@param {type:\"boolean\"}\n",
        "\n",
        "    !mkdir -p /content/drive/MyDrive/{MODELNAME}_files\n",
        "\n",
        "    if SAVE_INTERMEDIATE:\n",
        "      !cp /content/logs/{MODELNAME}/G_{MODELEPOCH}.pth /content/drive/MyDrive/{MODELNAME}_files/\n",
        "      !cp /content/logs/{MODELNAME}/D_{MODELEPOCH}.pth /content/drive/MyDrive/{MODELNAME}_files/\n",
        "      !cp /content/logs/{MODELNAME}/total_*.npy /content/drive/MyDrive/{MODELNAME}_files/\n",
        "\n",
        "    !cp /content/logs/{MODELNAME}/added_*.index /content/drive/MyDrive/{MODELNAME}_files/\n",
        "    !cp /content/weights/{MODELNAME}.pth /content/drive/MyDrive/{MODELNAME}_files/"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Inference\n",
        "\n",
        "The code below is to be run for Mode: Inference\n",
        "\n",
        "When prompted `Proceed (Y/n)?`, click beside it, type `Y` and press `Enter`.\n",
        "\n",
        "If `WARNING: The following packages were previously imported in this runtime: [numpy] You must restart the runtime in order to use newly installed versions.` is seen in the output, click `Restart Runtime` and then continue running the next cell."
      ],
      "metadata": {
        "id": "0JTjkTndoHb6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Install specific numpy version. If needed, click Restart Runtime before running the bottom two cells.\n",
        "if MODE == 'Inference':\n",
        "  !pip uninstall numpy\n",
        "  !pip install numpy==1.23.5"
      ],
      "metadata": {
        "id": "cfJV7kbKqtnw",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "try:\n",
        "  MODE\n",
        "except NameError:\n",
        "  MODE = 'Inference'\n",
        "\n",
        "#@title Set your NGROK_AUTH_TOKEN.\n",
        "if MODE == 'Inference':\n",
        "  !pip install flask-ngrok3 -q\n",
        "\n",
        "#@markdown Obtain your Ngrok auth token from [here](https://dashboard.ngrok.com/get-started/your-authtoken)\n",
        "  NGROK_AUTH_TOKEN = '' #@param {type:\"string\"}"
      ],
      "metadata": {
        "id": "OL-1YvcbOiWd",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "if MODE != 'Inference':\n",
        "  raise Exception('Mode is not set to Inference.')\n",
        "\n",
        "\n",
        "#@title Run RVC Inference server\n",
        "import json\n",
        "import sys\n",
        "import wave\n",
        "from pathlib import Path\n",
        "\n",
        "BASE_DIR = Path('/content')\n",
        "sys.path.append(str(BASE_DIR))\n",
        "\n",
        "import torch\n",
        "from multiprocessing import cpu_count\n",
        "from flask_ngrok3 import run_with_ngrok\n",
        "from flask import Flask, request, send_file\n",
        "\n",
        "from vc_infer_pipeline import VC\n",
        "from infer_pack.models import (\n",
        "    SynthesizerTrnMs256NSFsid,\n",
        "    SynthesizerTrnMs256NSFsid_nono,\n",
        "    SynthesizerTrnMs768NSFsid,\n",
        "    SynthesizerTrnMs768NSFsid_nono,\n",
        ")\n",
        "from my_utils import load_audio\n",
        "from fairseq import checkpoint_utils\n",
        "from scipy.io import wavfile\n",
        "\n",
        "\n",
        "INPUT_VOICE_PATH = 'input.mp3'\n",
        "OUTPUT_VOICE_PATH = 'output.wav'\n",
        "MODEL_NAME = ''\n",
        "DEVICE = 'cuda:0'\n",
        "cpt = None\n",
        "\n",
        "\n",
        "class Config:\n",
        "  def __init__(self, device, is_half):\n",
        "    self.device = device\n",
        "    self.is_half = is_half\n",
        "    self.n_cpu = 0\n",
        "    self.gpu_name = None\n",
        "    self.gpu_mem = None\n",
        "    self.x_pad, self.x_query, self.x_center, self.x_max = self.device_config()\n",
        "\n",
        "  def device_config(self) -> tuple:\n",
        "    if torch.cuda.is_available():\n",
        "      i_device = int(self.device.split(\":\")[-1])\n",
        "      self.gpu_name = torch.cuda.get_device_name(i_device)\n",
        "\n",
        "      if (\n",
        "        (\"16\" in self.gpu_name and \"V100\" not in self.gpu_name.upper())\n",
        "        or \"P40\" in self.gpu_name.upper()\n",
        "        or \"1060\" in self.gpu_name\n",
        "        or \"1070\" in self.gpu_name\n",
        "        or \"1080\" in self.gpu_name\n",
        "      ):\n",
        "        print(\"16 series/10 series P40 forced single precision\")\n",
        "        self.is_half = False\n",
        "        for config_file in [\"32k.json\", \"40k.json\", \"48k.json\"]:\n",
        "          with open(f\"configs/{config_file}\", \"r\") as f:\n",
        "            strr = f.read().replace(\"true\", \"false\")\n",
        "          with open(f\"configs/{config_file}\", \"w\") as f:\n",
        "            f.write(strr)\n",
        "        with open(\"trainset_preprocess_pipeline_print.py\", \"r\") as f:\n",
        "          strr = f.read().replace(\"3.7\", \"3.0\")\n",
        "        with open(\"trainset_preprocess_pipeline_print.py\", \"w\") as f:\n",
        "          f.write(strr)\n",
        "      else:\n",
        "        self.gpu_name = None\n",
        "\n",
        "      self.gpu_mem = int(\n",
        "        torch.cuda.get_device_properties(i_device).total_memory\n",
        "        / 1024\n",
        "        / 1024\n",
        "        / 1024\n",
        "        + 0.4\n",
        "      )\n",
        "      if self.gpu_mem <= 4:\n",
        "        with open(\"trainset_preprocess_pipeline_print.py\", \"r\") as f:\n",
        "          strr = f.read().replace(\"3.7\", \"3.0\")\n",
        "        with open(\"trainset_preprocess_pipeline_print.py\", \"w\") as f:\n",
        "          f.write(strr)\n",
        "\n",
        "    elif torch.backends.mps.is_available():\n",
        "      print(\"No supported N-card found, use MPS for inference\")\n",
        "      self.device = \"mps\"\n",
        "    else:\n",
        "      print(\"No supported N-card found, use CPU for inference\")\n",
        "      self.device = \"cpu\"\n",
        "      self.is_half = True\n",
        "\n",
        "    if self.n_cpu == 0:\n",
        "      self.n_cpu = cpu_count()\n",
        "\n",
        "    if self.is_half:\n",
        "      # 6G memory config\n",
        "      x_pad = 3\n",
        "      x_query = 10\n",
        "      x_center = 60\n",
        "      x_max = 65\n",
        "    else:\n",
        "      # 5G memory config\n",
        "      x_pad = 1\n",
        "      x_query = 6\n",
        "      x_center = 38\n",
        "      x_max = 41\n",
        "\n",
        "    if self.gpu_mem != None and self.gpu_mem <= 4:\n",
        "      x_pad = 1\n",
        "      x_query = 5\n",
        "      x_center = 30\n",
        "      x_max = 32\n",
        "\n",
        "    return x_pad, x_query, x_center, x_max\n",
        "\n",
        "\n",
        "CONFIG = Config(DEVICE, True)\n",
        "\n",
        "\n",
        "def load_hubert():\n",
        "  models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task(['hubert_base.pt'], suffix='', )\n",
        "  hubert = models[0]\n",
        "  hubert = hubert.to(DEVICE)\n",
        "\n",
        "  if True:\n",
        "    hubert = hubert.half()\n",
        "  else:\n",
        "    hubert = hubert.float()\n",
        "\n",
        "  hubert.eval()\n",
        "  return hubert\n",
        "\n",
        "HUBERT_MODEL = load_hubert()\n",
        "\n",
        "\n",
        "def get_vc(device, is_half, config):\n",
        "  global cpt, version, net_g, tgt_sr, vc\n",
        "  model_path = BASE_DIR / 'weights' / f'{MODEL_NAME}.pth'\n",
        "  if not model_path.exists():\n",
        "    print(f'The model {model_path} does not exist. Please ensure that you have filled in the proper MODEL_NAME in your .env file.')\n",
        "    return None\n",
        "\n",
        "  model_path = str(model_path)\n",
        "  print(f'loading pth {model_path}')\n",
        "  cpt = torch.load(model_path, map_location='cpu')\n",
        "  tgt_sr = cpt[\"config\"][-1]\n",
        "  cpt[\"config\"][-3] = cpt[\"weight\"][\"emb_g.weight\"].shape[0]\n",
        "  if_f0 = cpt.get(\"f0\", 1)\n",
        "  version = cpt.get(\"version\", \"v1\")\n",
        "\n",
        "  if version == \"v1\":\n",
        "    if if_f0 == 1:\n",
        "      net_g = SynthesizerTrnMs256NSFsid(*cpt[\"config\"], is_half=is_half)\n",
        "    else:\n",
        "      net_g = SynthesizerTrnMs256NSFsid_nono(*cpt[\"config\"])\n",
        "  elif version == \"v2\":\n",
        "    if if_f0 == 1:\n",
        "      net_g = SynthesizerTrnMs768NSFsid(*cpt[\"config\"], is_half=is_half)\n",
        "    else:\n",
        "      net_g = SynthesizerTrnMs768NSFsid_nono(*cpt[\"config\"])\n",
        "\n",
        "  del net_g.enc_q\n",
        "  print(net_g.load_state_dict(cpt[\"weight\"], strict=False))\n",
        "  net_g.eval().to(device)\n",
        "\n",
        "  if is_half:\n",
        "    net_g = net_g.half()\n",
        "  else:\n",
        "    net_g = net_g.float()\n",
        "\n",
        "  vc = VC(tgt_sr, config)\n",
        "\n",
        "\n",
        "def rvc_infer(pitch_change, pitch_extraction_algo, volume_envelope, index_rate):\n",
        "  logs_dir = BASE_DIR / 'logs' / MODEL_NAME\n",
        "  index_path = ''\n",
        "  for file in logs_dir.iterdir():\n",
        "    if file.suffix == '.index':\n",
        "      index_path = str(logs_dir / file.name)\n",
        "      break\n",
        "\n",
        "  # vc single\n",
        "  audio = load_audio(INPUT_VOICE_PATH, 16000)\n",
        "  times = [0, 0, 0]\n",
        "  if_f0 = cpt.get('f0', 1)\n",
        "  audio_opt = vc.pipeline(HUBERT_MODEL, net_g, 0, audio, INPUT_VOICE_PATH, times, pitch_change, pitch_extraction_algo, index_path, index_rate, if_f0, 3, tgt_sr, 0, volume_envelope, version, 0.33, f0_file=None)\n",
        "  wavfile.write(OUTPUT_VOICE_PATH, tgt_sr, audio_opt)\n",
        "\n",
        "\n",
        "app = Flask(__name__)\n",
        "run_with_ngrok(app, auth_token=NGROK_AUTH_TOKEN)\n",
        "\n",
        "@app.route('/', methods=['GET'])\n",
        "def test():\n",
        "  response = {'status':'OK','message':'Test'}\n",
        "  return json.dumps(response)\n",
        "\n",
        "\n",
        "@app.route('/infer', methods=['POST'])\n",
        "def infer():\n",
        "  global MODEL_NAME, cpt\n",
        "  model_name = request.args.get('model')\n",
        "  if MODEL_NAME != model_name:\n",
        "    MODEL_NAME = model_name\n",
        "    if cpt:\n",
        "      del cpt\n",
        "    get_vc(DEVICE, True, CONFIG)\n",
        "\n",
        "  pitch_change = int(request.args.get('pitch'))\n",
        "  pitch_extraction_algo = request.args.get('algo')\n",
        "  volume_envelope = float(request.args.get('volume'))\n",
        "  index_rate = float(request.args.get('index_rate'))\n",
        "  audio_data = request.files['audio_file']\n",
        "  audio_data.save(INPUT_VOICE_PATH)\n",
        "  rvc_infer(pitch_change, pitch_extraction_algo, volume_envelope, index_rate)\n",
        "  return send_file(OUTPUT_VOICE_PATH, mimetype=\"audio/wav\")\n",
        "\n",
        "app.run()"
      ],
      "metadata": {
        "id": "3qcqylOdMDFg",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}