demo_wenet.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "view-in-github"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/adelacvg/ttts/blob/v4/demo_wenet.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WB8vmGyDAPrr",
        "outputId": "fdc175f8-77a7-4270-b1a3-022c32e7facb"
      },
      "outputs": [],
      "source": [
        "# !git clone -b v4 https://github.com/adelacvg/ttts.git"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HKikTDsVA1_g",
        "outputId": "3a662666-15a9-4a44-e1cb-4d23466c1029"
      },
      "outputs": [],
      "source": [
        "!pip install -e ."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_KYD7eZQB4x7",
        "outputId": "4de04733-cdb3-4497-f10e-64f630b3a8e0"
      },
      "outputs": [],
      "source": [
        "!git lfs install\n",
        "!git clone https://huggingface.co/adelacvg/TTTS_v4"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UcOqgsONDGZ3",
        "outputId": "c936ccf1-1baf-4675-ff26-37be70ff9807"
      },
      "outputs": [],
      "source": [
        "import locale\n",
        "locale.getpreferredencoding = lambda: \"UTF-8\"\n",
        "!pip install pypinyin einops omegaconf faiss-cpu==1.8.0 cutlet hangul_romanize unidic\n",
        "!python -m unidic download"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sTZY0ll3c-MK"
      },
      "outputs": [],
      "source": [
        "from pypinyin import lazy_pinyin, Style\n",
        "import torch\n",
        "import numpy as np\n",
        "from ttts.utils import vc_utils\n",
        "import cutlet\n",
        "from hangul_romanize import Transliter\n",
        "from hangul_romanize.rule import academic\n",
        "import torchaudio\n",
        "from pypinyin import pinyin, lazy_pinyin, Style\n",
        "\n",
        "device = 'cuda:0'\n",
        "MODELS = {\n",
        "    # \"vqvae.pth\": \"model-520.pt\",\n",
        "    \"vqvae.pth\": \"model-1060.pt\"\n",
        "}\n",
        "\n",
        "\n",
        "speaker = 'unseen3'\n",
        "cond_audio = \"wenetdemo/Ref/\" + speaker + \"_spk.wav\"\n",
        "lang = 'ZH'\n",
        "variable_texts = {\n",
        "    \"text1\": \"应该给千千万万还在路上的创业者致意。\",\n",
        "    \"text2\": \"准确点说，小森林是一部美食类电影食物佳肴，贯穿了柿子的寒暑交替四十三餐。\",\n",
        "    \"text3\": \"要大力开展全省网吧专项整治工作，加强网络文化内容的整治，深入开展低俗音像制品清查行动。\",\n",
        "    \"text4\": \"因为这是我们法律存在的前提。\",\n",
        "    \"text5\": \"书记旗县市长乡镇苏木长企事业负责人以及国营农牧场的老兵团们，像赶庙会似的你挤我扛，口里说着，手里记着。\",\n",
        "    \"text6\": \"哎，不是有一句话这样说嘛，你永远不知道明天和意外哪个先来。\",\n",
        "    \"text7\": \"湿热天最麻烦的还属家中的煤气，早上起来，连拌果酱的木铲也发霉了。\",\n",
        "    \"text8\": \"这时，朱警官等人才发现小男孩腿脚也异常，根本走不了路。\",\n",
        "    \"text9\": \"他一度梦想参军，但是因没身份而作罢，整日在村内瞎晃。\",\n",
        "    \"text10\": \"当时有点感动还是什么？\",\n",
        "    \"text11\": \"这些飞机仍然由位于华盛顿州埃弗里特的工厂生产。\",\n",
        "    \"text12\": \"派大军前去争抢，赫赫查拉二话不说，喊出了巴顿的名字，结果手套开始传递给斯科特半路遭到灭霸拦截，真正的杀神从天而降。\",\n",
        "    \"text13\": \"而中国却只是将其作为红旗反导系统下的一个补充而已。\",\n",
        "    \"text14\": \"这一段戏同样也表达了亚瑟说的，我原本以为我的人生是一出悲剧，但其实它是一出戏剧。\",\n",
        "    \"text15\": \"常常暴跳如雷，红着眼睛用力攥着洛洛的手，把洛洛的手攥得生疼也不撒手。洛洛吓坏了，她很害怕，那个平日里对她那么好的男友为什么会变得这么可怕？\",\n",
        "}\n",
        "text_choose = \"text1\"\n",
        "text = variable_texts.get(text_choose)\n",
        "\n",
        "\n",
        "def multi_lang_convert(text, lang):\n",
        "    if lang == \"ZH\":\n",
        "        text = ' '.join(lazy_pinyin(text, style=Style.TONE3, neutral_tone_with_five=True))\n",
        "        text = ' '+text+' '\n",
        "    else:\n",
        "        return None\n",
        "    return text\n",
        "\n",
        "\n",
        "from ttts.gpt.voice_tokenizer import VoiceBpeTokenizer\n",
        "import torch.nn.functional as F\n",
        "\n",
        "tok_zh = VoiceBpeTokenizer(\"tokenizers/wenet_tokenizer.json\")\n",
        "text = multi_lang_convert(text, lang)\n",
        "if lang == \"ZH\":\n",
        "    text = tok_zh.encode(text.lower())\n",
        "else:\n",
        "    print('not supprt lang')\n",
        "text = torch.LongTensor(text).unsqueeze(0)\n",
        "if lang == \"ZH\":\n",
        "    text = text + 256*0\n",
        "    lang = 0\n",
        "text_tokens = text.to(device)\n",
        "text_length = torch.LongTensor([text_tokens.shape[1]]).to(device)\n",
        "\n",
        "\n",
        "from ttts.utils.infer_utils import load_model\n",
        "import torchaudio\n",
        "import json\n",
        "import torchaudio.functional as F\n",
        "from ttts.utils.data_utils import spectrogram_torch,HParams\n",
        "hps = HParams(**json.load(open('ttts/vqvae/config_v3_wenet.json')))\n",
        "wav, sr = torchaudio.load(cond_audio)\n",
        "if wav.shape[0] > 1:\n",
        "    wav = wav[0].unsqueeze(0)\n",
        "wav = wav.to(device)\n",
        "wav32k = F.resample(wav, sr, 32000)\n",
        "wav32k = wav32k[:,:int(hps.data.hop_length * (wav32k.shape[-1]//hps.data.hop_length))]\n",
        "wav = torch.clamp(wav32k, min=-1.0, max=1.0)\n",
        "wav_length = torch.LongTensor([wav.shape[1]])\n",
        "spec = spectrogram_torch(wav, hps.data.filter_length,\n",
        "                    hps.data.hop_length, hps.data.win_length, center=False).squeeze(0)\n",
        "spec_length = torch.LongTensor([\n",
        "    x//hps.data.hop_length for x in wav_length]).to(device)\n",
        "vqvae = load_model('vqvae', MODELS['vqvae.pth'], 'ttts/vqvae/config_v3_wenet.json', device)\n",
        "with torch.no_grad():\n",
        "    wav_out = vqvae.infer(text_tokens, text_length, spec.unsqueeze(0), spec_length)\n",
        "\n",
        "save_name = \"wenetdemo/\" + speaker + \"_\" + text_choose + \".wav\"\n",
        "torchaudio.save(save_name, wav_out.squeeze(0).cpu(), 32000)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 74
        },
        "id": "PJ_aewa2c5XP",
        "outputId": "250c3070-2d80-4e14-a7e0-1f7e55bbd7ea"
      },
      "outputs": [],
      "source": [
        "from IPython.display import Audio\n",
        "wav = wav_out.detach().cpu()\n",
        "Audio(wav[0],rate=32000)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "authorship_tag": "ABX9TyOPHZ4W3QVgU4jYMq8Ehlhw",
      "gpuType": "T4",
      "include_colab_link": true,
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.14"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}