infer.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "REC8UsP7mr0-",
        "outputId": "7611bcd2-37da-4954-b8ed-906c2ba4a56c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Thu Apr  6 18:26:28 2023       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 525.85.12    Driver Version: 525.85.12    CUDA Version: 12.0     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   48C    P0    27W /  70W |      0MiB / 15360MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ],
      "source": [
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oumckS9eRAUb",
        "outputId": "aac38984-ef2b-454d-af91-56100d437958"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "fatal: destination path 'vigogne' already exists and is not an empty directory.\n",
            "/content/vigogne\n"
          ]
        }
      ],
      "source": [
        "!git clone https://github.com/bofenghuang/vigogne.git\n",
        "%cd vigogne"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lTSBldIT7Y96",
        "outputId": "1d0c3dcf-b7cd-43ec-ceb1-8021bd2372f5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting git+https://github.com/huggingface/transformers.git (from -r requirements.txt (line 4))\n",
            "  Cloning https://github.com/huggingface/transformers.git to /tmp/pip-req-build-1zjn7_t2\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/huggingface/transformers.git /tmp/pip-req-build-1zjn7_t2\n",
            "  Resolved https://github.com/huggingface/transformers.git to commit ed67286465c5e9e3d3005de3e21bc3c679d93072\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting git+https://github.com/huggingface/peft.git (from -r requirements.txt (line 7))\n",
            "  Cloning https://github.com/huggingface/peft.git to /tmp/pip-req-build-08qqyy80\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/huggingface/peft.git /tmp/pip-req-build-08qqyy80\n",
            "  Resolved https://github.com/huggingface/peft.git to commit 382b178911edff38c1ff619bbac2ba556bd2276b\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.9/dist-packages (from -r requirements.txt (line 1)) (2.11.0)\n",
            "Requirement already satisfied: loralib in /usr/local/lib/python3.9/dist-packages (from -r requirements.txt (line 2)) (0.1.1)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.9/dist-packages (from -r requirements.txt (line 3)) (0.1.97)\n",
            "Requirement already satisfied: accelerate in /usr/local/lib/python3.9/dist-packages (from -r requirements.txt (line 5)) (0.18.0)\n",
            "Requirement already satisfied: bitsandbytes in /usr/local/lib/python3.9/dist-packages (from -r requirements.txt (line 6)) (0.37.2)\n",
            "Requirement already satisfied: gradio in /usr/local/lib/python3.9/dist-packages (from -r requirements.txt (line 8)) (3.24.1)\n",
            "Requirement already satisfied: appdirs in /usr/local/lib/python3.9/dist-packages (from -r requirements.txt (line 9)) (1.4.4)\n",
            "Requirement already satisfied: fire in /usr/local/lib/python3.9/dist-packages (from -r requirements.txt (line 10)) (0.5.0)\n",
            "Requirement already satisfied: openai in /usr/local/lib/python3.9/dist-packages (from -r requirements.txt (line 11)) (0.27.4)\n",
            "Requirement already satisfied: tiktoken in /usr/local/lib/python3.9/dist-packages (from -r requirements.txt (line 12)) (0.3.3)\n",
            "Requirement already satisfied: tenacity in /usr/local/lib/python3.9/dist-packages (from -r requirements.txt (line 13)) (8.2.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from datasets->-r requirements.txt (line 1)) (6.0)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.9/dist-packages (from datasets->-r requirements.txt (line 1)) (2.27.1)\n",
            "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.9/dist-packages (from datasets->-r requirements.txt (line 1)) (0.18.0)\n",
            "Requirement already satisfied: dill<0.3.7,>=0.3.0 in /usr/local/lib/python3.9/dist-packages (from datasets->-r requirements.txt (line 1)) (0.3.6)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from datasets->-r requirements.txt (line 1)) (1.22.4)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.9/dist-packages (from datasets->-r requirements.txt (line 1)) (9.0.0)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.9/dist-packages (from datasets->-r requirements.txt (line 1)) (0.70.14)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.9/dist-packages (from datasets->-r requirements.txt (line 1)) (4.65.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.9/dist-packages (from datasets->-r requirements.txt (line 1)) (23.0)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.9/dist-packages (from datasets->-r requirements.txt (line 1)) (3.2.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0.0,>=0.11.0 in /usr/local/lib/python3.9/dist-packages (from datasets->-r requirements.txt (line 1)) (0.13.4)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (from datasets->-r requirements.txt (line 1)) (1.4.4)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.9/dist-packages (from datasets->-r requirements.txt (line 1)) (3.8.4)\n",
            "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.9/dist-packages (from datasets->-r requirements.txt (line 1)) (2023.3.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers==4.28.0.dev0->-r requirements.txt (line 4)) (3.10.7)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers==4.28.0.dev0->-r requirements.txt (line 4)) (2022.10.31)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.9/dist-packages (from transformers==4.28.0.dev0->-r requirements.txt (line 4)) (0.13.3)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.9/dist-packages (from accelerate->-r requirements.txt (line 5)) (5.9.4)\n",
            "Requirement already satisfied: torch>=1.4.0 in /usr/local/lib/python3.9/dist-packages (from accelerate->-r requirements.txt (line 5)) (2.0.0+cu118)\n",
            "Requirement already satisfied: pydub in /usr/local/lib/python3.9/dist-packages (from gradio->-r requirements.txt (line 8)) (0.25.1)\n",
            "Requirement already satisfied: markupsafe in /usr/local/lib/python3.9/dist-packages (from gradio->-r requirements.txt (line 8)) (2.1.2)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.9/dist-packages (from gradio->-r requirements.txt (line 8)) (3.7.1)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.9/dist-packages (from gradio->-r requirements.txt (line 8)) (8.4.0)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.9/dist-packages (from gradio->-r requirements.txt (line 8)) (3.1.2)\n",
            "Requirement already satisfied: pydantic in /usr/local/lib/python3.9/dist-packages (from gradio->-r requirements.txt (line 8)) (1.10.7)\n",
            "Requirement already satisfied: python-multipart in /usr/local/lib/python3.9/dist-packages (from gradio->-r requirements.txt (line 8)) (0.0.6)\n",
            "Requirement already satisfied: ffmpy in /usr/local/lib/python3.9/dist-packages (from gradio->-r requirements.txt (line 8)) (0.3.0)\n",
            "Requirement already satisfied: uvicorn in /usr/local/lib/python3.9/dist-packages (from gradio->-r requirements.txt (line 8)) (0.21.1)\n",
            "Requirement already satisfied: semantic-version in /usr/local/lib/python3.9/dist-packages (from gradio->-r requirements.txt (line 8)) (2.10.0)\n",
            "Requirement already satisfied: mdit-py-plugins<=0.3.3 in /usr/local/lib/python3.9/dist-packages (from gradio->-r requirements.txt (line 8)) (0.3.3)\n",
            "Requirement already satisfied: markdown-it-py[linkify]>=2.0.0 in /usr/local/lib/python3.9/dist-packages (from gradio->-r requirements.txt (line 8)) (2.2.0)\n",
            "Requirement already satisfied: gradio-client>=0.0.5 in /usr/local/lib/python3.9/dist-packages (from gradio->-r requirements.txt (line 8)) (0.0.7)\n",
            "Requirement already satisfied: aiofiles in /usr/local/lib/python3.9/dist-packages (from gradio->-r requirements.txt (line 8)) (23.1.0)\n",
            "Requirement already satisfied: fastapi in /usr/local/lib/python3.9/dist-packages (from gradio->-r requirements.txt (line 8)) (0.95.0)\n",
            "Requirement already satisfied: altair>=4.2.0 in /usr/local/lib/python3.9/dist-packages (from gradio->-r requirements.txt (line 8)) (4.2.2)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.9/dist-packages (from gradio->-r requirements.txt (line 8)) (4.5.0)\n",
            "Requirement already satisfied: orjson in /usr/local/lib/python3.9/dist-packages (from gradio->-r requirements.txt (line 8)) (3.8.9)\n",
            "Requirement already satisfied: httpx in /usr/local/lib/python3.9/dist-packages (from gradio->-r requirements.txt (line 8)) (0.23.3)\n",
            "Requirement already satisfied: websockets>=10.0 in /usr/local/lib/python3.9/dist-packages (from gradio->-r requirements.txt (line 8)) (11.0.1)\n",
            "Requirement already satisfied: termcolor in /usr/local/lib/python3.9/dist-packages (from fire->-r requirements.txt (line 10)) (2.2.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.9/dist-packages (from fire->-r requirements.txt (line 10)) (1.16.0)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.9/dist-packages (from altair>=4.2.0->gradio->-r requirements.txt (line 8)) (0.12.0)\n",
            "Requirement already satisfied: jsonschema>=3.0 in /usr/local/lib/python3.9/dist-packages (from altair>=4.2.0->gradio->-r requirements.txt (line 8)) (4.3.3)\n",
            "Requirement already satisfied: entrypoints in /usr/local/lib/python3.9/dist-packages (from altair>=4.2.0->gradio->-r requirements.txt (line 8)) (0.4)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets->-r requirements.txt (line 1)) (1.3.1)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets->-r requirements.txt (line 1)) (1.3.3)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets->-r requirements.txt (line 1)) (22.2.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets->-r requirements.txt (line 1)) (1.8.2)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets->-r requirements.txt (line 1)) (4.0.2)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets->-r requirements.txt (line 1)) (2.0.12)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets->-r requirements.txt (line 1)) (6.0.4)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.9/dist-packages (from markdown-it-py[linkify]>=2.0.0->gradio->-r requirements.txt (line 8)) (0.1.2)\n",
            "Requirement already satisfied: linkify-it-py<3,>=1 in /usr/local/lib/python3.9/dist-packages (from markdown-it-py[linkify]>=2.0.0->gradio->-r requirements.txt (line 8)) (2.0.0)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets->-r requirements.txt (line 1)) (2022.7.1)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/dist-packages (from pandas->datasets->-r requirements.txt (line 1)) (2.8.2)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets->-r requirements.txt (line 1)) (2022.12.7)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets->-r requirements.txt (line 1)) (1.26.15)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->datasets->-r requirements.txt (line 1)) (3.4)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.9/dist-packages (from torch>=1.4.0->accelerate->-r requirements.txt (line 5)) (3.0)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.9/dist-packages (from torch>=1.4.0->accelerate->-r requirements.txt (line 5)) (2.0.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.9/dist-packages (from torch>=1.4.0->accelerate->-r requirements.txt (line 5)) (1.11.1)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.9/dist-packages (from triton==2.0.0->torch>=1.4.0->accelerate->-r requirements.txt (line 5)) (16.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.9/dist-packages (from triton==2.0.0->torch>=1.4.0->accelerate->-r requirements.txt (line 5)) (3.25.2)\n",
            "Requirement already satisfied: starlette<0.27.0,>=0.26.1 in /usr/local/lib/python3.9/dist-packages (from fastapi->gradio->-r requirements.txt (line 8)) (0.26.1)\n",
            "Requirement already satisfied: httpcore<0.17.0,>=0.15.0 in /usr/local/lib/python3.9/dist-packages (from httpx->gradio->-r requirements.txt (line 8)) (0.16.3)\n",
            "Requirement already satisfied: rfc3986[idna2008]<2,>=1.3 in /usr/local/lib/python3.9/dist-packages (from httpx->gradio->-r requirements.txt (line 8)) (1.5.0)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.9/dist-packages (from httpx->gradio->-r requirements.txt (line 8)) (1.3.0)\n",
            "Requirement already satisfied: importlib-resources>=3.2.0 in /usr/local/lib/python3.9/dist-packages (from matplotlib->gradio->-r requirements.txt (line 8)) (5.12.0)\n",
            "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.9/dist-packages (from matplotlib->gradio->-r requirements.txt (line 8)) (1.0.7)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.9/dist-packages (from matplotlib->gradio->-r requirements.txt (line 8)) (4.39.3)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.9/dist-packages (from matplotlib->gradio->-r requirements.txt (line 8)) (3.0.9)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.9/dist-packages (from matplotlib->gradio->-r requirements.txt (line 8)) (0.11.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.9/dist-packages (from matplotlib->gradio->-r requirements.txt (line 8)) (1.4.4)\n",
            "Requirement already satisfied: h11>=0.8 in /usr/local/lib/python3.9/dist-packages (from uvicorn->gradio->-r requirements.txt (line 8)) (0.14.0)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.9/dist-packages (from uvicorn->gradio->-r requirements.txt (line 8)) (8.1.3)\n",
            "Requirement already satisfied: anyio<5.0,>=3.0 in /usr/local/lib/python3.9/dist-packages (from httpcore<0.17.0,>=0.15.0->httpx->gradio->-r requirements.txt (line 8)) (3.6.2)\n",
            "Requirement already satisfied: zipp>=3.1.0 in /usr/local/lib/python3.9/dist-packages (from importlib-resources>=3.2.0->matplotlib->gradio->-r requirements.txt (line 8)) (3.15.0)\n",
            "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.9/dist-packages (from jsonschema>=3.0->altair>=4.2.0->gradio->-r requirements.txt (line 8)) (0.19.3)\n",
            "Requirement already satisfied: uc-micro-py in /usr/local/lib/python3.9/dist-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify]>=2.0.0->gradio->-r requirements.txt (line 8)) (1.0.1)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.9/dist-packages (from sympy->torch>=1.4.0->accelerate->-r requirements.txt (line 5)) (1.3.0)\n"
          ]
        }
      ],
      "source": [
        "# Install requirements\n",
        "!pip install -r requirements.txt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9QF2LHZWYBj8"
      },
      "outputs": [],
      "source": [
        "# Uncomment if necessary\n",
        "# import os\n",
        "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qNQ9YTXmYBj8",
        "outputId": "1e41443c-93cb-4734-ed42-909888b75b5c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "===================================BUG REPORT===================================\n",
            "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
            "================================================================================\n",
            "CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching /usr/local/cuda/lib64...\n",
            "CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\n",
            "CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n",
            "CUDA SETUP: Detected CUDA version 118\n",
            "CUDA SETUP: Loading binary /usr/local/lib/python3.9/dist-packages/bitsandbytes/libbitsandbytes_cuda118.so...\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /usr/lib64-nvidia did not contain libcudart.so as expected! Searching further paths...\n",
            "  warn(msg)\n",
            "/usr/local/lib/python3.9/dist-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/sys/fs/cgroup/memory.events /var/colab/cgroup/jupyter-children/memory.events')}\n",
            "  warn(msg)\n",
            "/usr/local/lib/python3.9/dist-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('http'), PosixPath('8013'), PosixPath('//172.28.0.1')}\n",
            "  warn(msg)\n",
            "/usr/local/lib/python3.9/dist-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('//colab.research.google.com/tun/m/cc48301118ce562b961b3c22d803539adc1e0c19/gpu-t4-s-3lb580sp6u811 --tunnel_background_save_delay=10s --tunnel_periodic_background_save_frequency=30m0s --enable_output_coalescing=true --output_coalescing_required=true'), PosixPath('--listen_host=172.28.0.12 --target_host=172.28.0.12 --tunnel_background_save_url=https')}\n",
            "  warn(msg)\n",
            "/usr/local/lib/python3.9/dist-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/env/python')}\n",
            "  warn(msg)\n",
            "/usr/local/lib/python3.9/dist-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('module'), PosixPath('//ipykernel.pylab.backend_inline')}\n",
            "  warn(msg)\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "from peft import PeftModel\n",
        "from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig, LlamaTokenizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 103,
          "referenced_widgets": [
            "5b6a6b0c997349f1be5bfb4b9e8aedff",
            "b3f0e2d833a8443489cb29282c0a0f00",
            "bc6c9d13781b4cdb96a86f888e1df061",
            "da77ad173e8242638abc0890d3d46a4e",
            "d1f41efda9b14978a7bb054afb7922c7",
            "5577858f268749cd980ae5e309ffc288",
            "6ee02c29232049309f2dba7cb667752c",
            "cc3605d7c1ff48b9a97724d825da23aa",
            "221d054f8a1b4e30bca3acfd70f0b7da",
            "8a008d7f7c3948d495a8947dd95175eb",
            "c52d8200660049ca9f061676e53f75b8"
          ]
        },
        "id": "H3rj68xDYBj8",
        "outputId": "3f2b1dda-69b3-4b3e-a0a9-1b7c2e7fcb21"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The tokenizer class you load from this checkpoint is not the same type as the class this function is called from. It may result in unexpected tokenization. \n",
            "The tokenizer class you load from this checkpoint is 'LLaMATokenizer'. \n",
            "The class this function is called from is 'LlamaTokenizer'.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/33 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5b6a6b0c997349f1be5bfb4b9e8aedff"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "base_model_name_or_path = \"name/or/path/to/hf/llama/7b/model\"\n",
        "lora_model_name_or_path = \"bofenghuang/vigogne-lora-7b\"\n",
        "\n",
        "# base_model_name_or_path = \"name/or/path/to/hf/llama/30b/model\"\n",
        "# lora_model_name_or_path = \"bofenghuang/vigogne-lora-30b\"\n",
        "\n",
        "# base_model_name_or_path = \"bigscience/bloom-7b1\"\n",
        "# lora_model_name_or_path = \"bofenghuang/vigogne-lora-bloom-7b1\"\n",
        "\n",
        "load_8bit = True\n",
        "\n",
        "tokenizer_class = LlamaTokenizer if \"llama\" in base_model_name_or_path else AutoTokenizer\n",
        "tokenizer = tokenizer_class.from_pretrained(base_model_name_or_path)\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    device = \"cuda\"\n",
        "else:\n",
        "    device = \"cpu\"\n",
        "\n",
        "try:\n",
        "    if torch.backends.mps.is_available():\n",
        "        device = \"mps\"\n",
        "except:\n",
        "    pass\n",
        "\n",
        "if device == \"cuda\":\n",
        "    model = AutoModelForCausalLM.from_pretrained(\n",
        "        base_model_name_or_path,\n",
        "        load_in_8bit=load_8bit,\n",
        "        torch_dtype=torch.float16,\n",
        "        device_map=\"auto\",\n",
        "    )\n",
        "    model = PeftModel.from_pretrained(\n",
        "        model,\n",
        "        lora_model_name_or_path,\n",
        "        torch_dtype=torch.float16,\n",
        "    )\n",
        "elif device == \"mps\":\n",
        "    model = AutoModelForCausalLM.from_pretrained(\n",
        "        base_model_name_or_path,\n",
        "        device_map={\"\": device},\n",
        "        torch_dtype=torch.float16,\n",
        "    )\n",
        "    model = PeftModel.from_pretrained(\n",
        "        model,\n",
        "        lora_model_name_or_path,\n",
        "        device_map={\"\": device},\n",
        "        torch_dtype=torch.float16,\n",
        "    )\n",
        "else:\n",
        "    model = AutoModelForCausalLM.from_pretrained(\n",
        "        base_model_name_or_path, device_map={\"\": device}, low_cpu_mem_usage=True\n",
        "    )\n",
        "    model = PeftModel.from_pretrained(\n",
        "        model,\n",
        "        lora_model_name_or_path,\n",
        "        device_map={\"\": device},\n",
        "    )\n",
        "\n",
        "if not load_8bit:\n",
        "    model.half()  # seems to fix bugs for some users.\n",
        "\n",
        "model.eval()\n",
        "if torch.__version__ >= \"2\":\n",
        "    model = torch.compile(model)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "12-zZj3jYBj9"
      },
      "outputs": [],
      "source": [
        "PROMPT_DICT = {\n",
        "    \"prompt_input\": (\n",
        "        \"Ci-dessous se trouve une instruction qui décrit une tâche, associée à une entrée qui fournit un contexte supplémentaire. Écrivez une réponse qui complète correctement la demande.\\n\\n\"\n",
        "        \"### Instruction:\\n{instruction}\\n\\n### Entrée:\\n{input}\\n\\n### Réponse:\\n\"\n",
        "    ),\n",
        "    \"prompt_no_input\": (\n",
        "        \"Ci-dessous se trouve une instruction qui décrit une tâche. Écrivez une réponse qui complète correctement la demande.\\n\\n\"\n",
        "        \"### Instruction:\\n{instruction}\\n\\n### Réponse:\\n\"\n",
        "    ),\n",
        "}\n",
        "\n",
        "\n",
        "def generate_prompt(instruction, input=None):\n",
        "    return (\n",
        "        PROMPT_DICT[\"prompt_input\"].format_map({\"instruction\": instruction, \"input\": input})\n",
        "        if input is not None\n",
        "        else PROMPT_DICT[\"prompt_no_input\"].format_map({\"instruction\": instruction})\n",
        "    )\n",
        "\n",
        "\n",
        "def instruct(\n",
        "    instruction,\n",
        "    input=None,\n",
        "    temperature=0.1,\n",
        "    no_repeat_ngram_size=3,\n",
        "    max_new_tokens=512,\n",
        "    **kwargs,\n",
        "):\n",
        "    prompt = generate_prompt(instruction, input)\n",
        "    input_ids = tokenizer(prompt, return_tensors=\"pt\")[\"input_ids\"].to(device)\n",
        "\n",
        "    generation_config = GenerationConfig(\n",
        "        temperature=temperature,\n",
        "        no_repeat_ngram_size=no_repeat_ngram_size,\n",
        "        **kwargs,\n",
        "    )\n",
        "\n",
        "    generation_output = model.generate(\n",
        "        input_ids=input_ids,\n",
        "        generation_config=generation_config,\n",
        "        # return_dict_in_generate=True,\n",
        "        # output_scores=True,\n",
        "        max_new_tokens=max_new_tokens,\n",
        "    )\n",
        "\n",
        "    output_text = tokenizer.decode(generation_output[0], skip_special_tokens=True)\n",
        "    output_text = output_text.rsplit(\"### Réponse:\", 1)[-1].strip()\n",
        "    return output_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "id": "bAl6_6PrizMb",
        "outputId": "8787e97a-f5f3-44ff-c148-41ba7dfcc78f"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"Le théorême central limites stipule que la moyenne arithmétique d'une série de valeurs aléatoires est la même que la valeur moyenne de la série. Cela signifie que la probabilité qu'une valeur aléatoire soit proche de la moyenne de toutes les valeurs est la plus grande lorsque la série est plus longue.\""
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 8
        }
      ],
      "source": [
        "# instruct\n",
        "instruct(\"Expliquer le théorème central limite.\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "id": "W8hRrkPcO3JR",
        "outputId": "b38e5457-4d3a-45e5-f6b3-0af0b2f92618"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"Les papillons extraient des nutriments qui sont ensuite convertis en papillions. Les gens ont extrait des milliards de nugget de compréhension et GPT4 est l'humanité's papillon.\""
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 9
        }
      ],
      "source": [
        "# instruct + input\n",
        "instruct(\n",
        "    \"Traduisez le texte suivant en français.\",\n",
        "    input=\"Caterpillars extract nutrients which are then converted into butterflies. People have extracted billions of nuggets of understanding and GPT-4 is humanity's butterfly.\",\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "quG-opBuYBj_"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "machine_shape": "hm",
      "provenance": []
    },
    "gpuClass": "premium",
    "kernelspec": {
      "display_name": "Python 3.9.0 ('nlp')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.0"
    },
    "vscode": {
      "interpreter": {
        "hash": "f99034d7e2e0fcf8bb579b4deb12a7e98f13c7bc298256d783a54f6812e2c7dd"
      }
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "5b6a6b0c997349f1be5bfb4b9e8aedff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b3f0e2d833a8443489cb29282c0a0f00",
              "IPY_MODEL_bc6c9d13781b4cdb96a86f888e1df061",
              "IPY_MODEL_da77ad173e8242638abc0890d3d46a4e"
            ],
            "layout": "IPY_MODEL_d1f41efda9b14978a7bb054afb7922c7"
          }
        },
        "b3f0e2d833a8443489cb29282c0a0f00": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5577858f268749cd980ae5e309ffc288",
            "placeholder": "​",
            "style": "IPY_MODEL_6ee02c29232049309f2dba7cb667752c",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "bc6c9d13781b4cdb96a86f888e1df061": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cc3605d7c1ff48b9a97724d825da23aa",
            "max": 33,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_221d054f8a1b4e30bca3acfd70f0b7da",
            "value": 33
          }
        },
        "da77ad173e8242638abc0890d3d46a4e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8a008d7f7c3948d495a8947dd95175eb",
            "placeholder": "​",
            "style": "IPY_MODEL_c52d8200660049ca9f061676e53f75b8",
            "value": " 33/33 [01:13&lt;00:00,  2.12s/it]"
          }
        },
        "d1f41efda9b14978a7bb054afb7922c7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5577858f268749cd980ae5e309ffc288": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6ee02c29232049309f2dba7cb667752c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cc3605d7c1ff48b9a97724d825da23aa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "221d054f8a1b4e30bca3acfd70f0b7da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8a008d7f7c3948d495a8947dd95175eb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c52d8200660049ca9f061676e53f75b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}