plot_vocal_separation_file.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "mimetype": "text/x-python",
      "name": "python",
      "version": "3.5.3",
      "file_extension": ".py",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3"
    },
    "kernelspec": {
      "name": "python3",
      "language": "python",
      "display_name": "Python 3"
    },
    "colab": {
      "name": "plot_vocal_separation.ipynb",
      "provenance": []
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "q3rUsRHWfs3b"
      },
      "source": [
        "%matplotlib inline"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M2ONmnVJfs3e"
      },
      "source": [
        "\n",
        "# Vocal separation\n",
        "\n",
        "\n",
        "This notebook demonstrates a simple technique for separating vocals (and\n",
        "other sporadic foreground signals) from accompanying instrumentation.\n",
        "\n",
        "This is based on the \"REPET-SIM\" method of `Rafii and Pardo, 2012\n",
        "<http://www.cs.northwestern.edu/~zra446/doc/Rafii-Pardo%20-%20Music-Voice%20Separation%20using%20the%20Similarity%20Matrix%20-%20ISMIR%202012.pdf>`_, but includes a couple of modifications and extensions:\n",
        "\n",
        "    - FFT windows overlap by 1/4, instead of 1/2\n",
        "    - Non-local filtering is converted into a soft mask by Wiener filtering.\n",
        "      This is similar in spirit to the soft-masking method used by `Fitzgerald, 2012\n",
        "      <http://arrow.dit.ie/cgi/viewcontent.cgi?article=1086&context=argcon>`_,\n",
        "      but is a bit more numerically stable in practice.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P3hGbZU-JLOZ",
        "outputId": "3f7ec73c-c40a-4283-aa5e-1615600ff750",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!wget -c -A '*.mp3' -r -l 1 -nd https://github.com/librosa/librosa_gallery/blob/master/audio/Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3\n"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-06-28 09:30:18--  https://github.com/librosa/librosa_gallery/blob/master/audio/Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3\n",
            "Resolving github.com (github.com)... 140.82.112.4\n",
            "Connecting to github.com (github.com)|140.82.112.4|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: unspecified [text/html]\n",
            "Saving to: ‘Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3’\n",
            "\n",
            "Cheese_N_Pot-C_-_16     [ <=>                ] 111.33K   615KB/s    in 0.2s    \n",
            "\n",
            "2021-06-28 09:30:19 (615 KB/s) - ‘Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3’ saved [113997]\n",
            "\n",
            "Loading robots.txt; please ignore errors.\n",
            "--2021-06-28 09:30:19--  https://github.com/robots.txt\n",
            "Reusing existing connection to github.com:443.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1481 (1.4K) [text/plain]\n",
            "Saving to: ‘robots.txt.tmp’\n",
            "\n",
            "robots.txt.tmp      100%[===================>]   1.45K  --.-KB/s    in 0s      \n",
            "\n",
            "2021-06-28 09:30:19 (42.8 MB/s) - ‘robots.txt.tmp’ saved [1481/1481]\n",
            "\n",
            "--2021-06-28 09:30:19--  https://github.com/\n",
            "Reusing existing connection to github.com:443.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: unspecified [text/html]\n",
            "Saving to: ‘index.html.tmp’\n",
            "\n",
            "index.html.tmp          [ <=>                ] 219.32K  --.-KB/s    in 0.06s   \n",
            "\n",
            "2021-06-28 09:30:19 (3.44 MB/s) - ‘index.html.tmp’ saved [224583]\n",
            "\n",
            "Removing index.html.tmp since it should be rejected.\n",
            "\n",
            "--2021-06-28 09:30:19--  https://github.com/features/code-review/\n",
            "Reusing existing connection to github.com:443.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: unspecified [text/html]\n",
            "Saving to: ‘index.html.tmp’\n",
            "\n",
            "index.html.tmp          [ <=>                ]  96.59K  --.-KB/s    in 0.002s  \n",
            "\n",
            "2021-06-28 09:30:19 (55.8 MB/s) - ‘index.html.tmp’ saved [98904]\n",
            "\n",
            "Removing index.html.tmp since it should be rejected.\n",
            "\n",
            "--2021-06-28 09:30:19--  https://github.com/features/issues/\n",
            "Reusing existing connection to github.com:443.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: unspecified [text/html]\n",
            "Saving to: ‘index.html.tmp’\n",
            "\n",
            "index.html.tmp          [ <=>                ] 158.28K  --.-KB/s    in 0.002s  \n",
            "\n",
            "2021-06-28 09:30:19 (64.9 MB/s) - ‘index.html.tmp’ saved [162077]\n",
            "\n",
            "Removing index.html.tmp since it should be rejected.\n",
            "\n",
            "--2021-06-28 09:30:19--  https://github.com/librosa/librosa_gallery/blob/0014a56b9f596f9d98e7bb90ce055dd6f02a17bc/audio/Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3\n",
            "Reusing existing connection to github.com:443.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: unspecified [text/html]\n",
            "Saving to: ‘Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3’\n",
            "\n",
            "Cheese_N_Pot-C_-_16     [ <=>                ] 111.60K  --.-KB/s    in 0.002s  \n",
            "\n",
            "2021-06-28 09:30:20 (45.7 MB/s) - ‘Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3’ saved [114279]\n",
            "\n",
            "--2021-06-28 09:30:20--  https://github.com/librosa/librosa_gallery/blob/%7B%7B%20urlEncodedRefName%20%7D%7D/audio/Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3\n",
            "Reusing existing connection to github.com:443.\n",
            "HTTP request sent, awaiting response... 404 Not Found\n",
            "2021-06-28 09:30:20 ERROR 404: Not Found.\n",
            "\n",
            "--2021-06-28 09:30:20--  https://github.com/librosa/librosa_gallery/commits/master/audio/Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3\n",
            "Reusing existing connection to github.com:443.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: unspecified [text/html]\n",
            "Saving to: ‘Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3’\n",
            "\n",
            "Cheese_N_Pot-C_-_16     [ <=>                ]  84.54K  --.-KB/s    in 0.002s  \n",
            "\n",
            "2021-06-28 09:30:20 (53.5 MB/s) - ‘Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3’ saved [114279]\n",
            "\n",
            "--2021-06-28 09:30:20--  https://github.com/librosa/librosa_gallery/raw/master/audio/Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3\n",
            "Reusing existing connection to github.com:443.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://raw.githubusercontent.com/librosa/librosa_gallery/master/audio/Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3 [following]\n",
            "--2021-06-28 09:30:20--  https://raw.githubusercontent.com/librosa/librosa_gallery/master/audio/Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.111.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 206 Partial Content\n",
            "Length: 8470176 (8.1M), 8355897 (8.0M) remaining [application/octet-stream]\n",
            "Saving to: ‘Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3’\n",
            "\n",
            "Cheese_N_Pot-C_-_16 100%[===================>]   8.08M  50.9MB/s    in 0.2s    \n",
            "\n",
            "2021-06-28 09:30:21 (50.9 MB/s) - ‘Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3’ saved [8470176/8470176]\n",
            "\n",
            "--2021-06-28 09:30:21--  https://github.com/librosa/librosa_gallery/blob/master/audio/Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3?raw=true\n",
            "Connecting to github.com (github.com)|140.82.112.4|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://github.com/librosa/librosa_gallery/raw/master/audio/Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3 [following]\n",
            "--2021-06-28 09:30:22--  https://github.com/librosa/librosa_gallery/raw/master/audio/Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3\n",
            "Reusing existing connection to github.com:443.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://raw.githubusercontent.com/librosa/librosa_gallery/master/audio/Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3 [following]\n",
            "--2021-06-28 09:30:22--  https://raw.githubusercontent.com/librosa/librosa_gallery/master/audio/Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 8470176 (8.1M) [application/octet-stream]\n",
            "Saving to: ‘Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3?raw=true.tmp’\n",
            "\n",
            "Cheese_N_Pot-C_-_16 100%[===================>]   8.08M  34.4MB/s    in 0.2s    \n",
            "\n",
            "2021-06-28 09:30:22 (34.4 MB/s) - ‘Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3?raw=true.tmp’ saved [8470176/8470176]\n",
            "\n",
            "Removing Cheese_N_Pot-C_-_16_-_The_Raps_Well_Clean_Album_Version.mp3?raw=true.tmp since it should be rejected.\n",
            "\n",
            "FINISHED --2021-06-28 09:30:22--\n",
            "Total wall clock time: 3.6s\n",
            "Downloaded: 9 files, 17M in 0.6s (26.2 MB/s)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bp1uuF4d-Uks"
      },
      "source": [
        "import IPython\n",
        "IPython.display.Audio('/content/music.mp3')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u-HAAObx9VWv",
        "outputId": "248adbba-3e6a-4e7c-d47b-86d2223dda0e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import numpy as np\n",
        "import librosa.display\n",
        "import soundfile as sf\n",
        "import matplotlib.pyplot as plt\n",
        "import librosa\n",
        "\n",
        "y, sr = librosa.load('/content/music.mp3', sr=None)\n",
        "output_file_path = \"/content/music_margin_10.wav\"\n",
        "\n",
        "#get spectrogram magnitude and phase\n",
        "S_full, phase = librosa.magphase(librosa.stft(y))\n",
        "S_filter = librosa.decompose.nn_filter(S_full,\n",
        "                                       aggregate=np.median,\n",
        "                                       metric='cosine',\n",
        "                                       width=int(librosa.time_to_frames(2, sr=sr)))\n",
        "# S_filter should be minimum - output filter should not be greater than input\n",
        "S_filter = np.minimum(S_full, S_filter)\n",
        "#\n",
        "margin_i, margin_v = 1, 10\n",
        "power = 1\n",
        "\n",
        "mask_i = librosa.util.softmask(S_filter,\n",
        "                               margin_i * (S_full - S_filter),\n",
        "                               power=power)\n",
        "\n",
        "mask_v = librosa.util.softmask(S_full - S_filter,\n",
        "                               margin_v * S_filter,\n",
        "                               power=power)\n",
        "\n",
        "# Once we have the masks, simply multiply them with the input spectrum\n",
        "# to separate the components\n",
        "S_foreground = mask_v * S_full\n",
        "S_background = mask_i * S_full\n",
        "D_foreground = S_foreground * phase\n",
        "y_foreground = librosa.istft(D_foreground)\n",
        "sf.write(output_file_path, y_foreground, samplerate=sr, subtype='PCM_24')"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/librosa/core/audio.py:165: UserWarning: PySoundFile failed. Trying audioread instead.\n",
            "  warnings.warn(\"PySoundFile failed. Trying audioread instead.\")\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AtAIZEYm3fDR"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}