From 5c7d8dc4f5d31ec80c2fb7e12512457c3b97de44 Mon Sep 17 00:00:00 2001 From: Siddharth Gupta Date: Sun, 17 Mar 2024 08:59:01 -0400 Subject: [PATCH] Created using Colaboratory --- bert_discord.ipynb | 1534 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1534 insertions(+) create mode 100644 bert_discord.ipynb diff --git a/bert_discord.ipynb b/bert_discord.ipynb new file mode 100644 index 0000000..1c476a9 --- /dev/null +++ b/bert_discord.ipynb @@ -0,0 +1,1534 @@ +{ + "metadata": { + "kernelspec": { + "language": "python", + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.13", + "mimetype": "text/x-python", + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "pygments_lexer": "ipython3", + "nbconvert_exporter": "python", + "file_extension": ".py" + }, + "kaggle": { + "accelerator": "nvidiaTeslaT4", + "dataSources": [ + { + "sourceId": 7837629, + "sourceType": "datasetVersion", + "datasetId": 4594308 + }, + { + "sourceId": 7839317, + "sourceType": "datasetVersion", + "datasetId": 4595521 + } + ], + "dockerImageVersionId": 30665, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook", + "isGpuEnabled": true + }, + "colab": { + "name": "bert discord", + "provenance": [], + "include_colab_link": true + } + }, + "nbformat_minor": 0, + "nbformat": 4, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "source": [ + "\n", + "# IMPORTANT: RUN THIS CELL IN ORDER TO IMPORT YOUR KAGGLE DATA SOURCES\n", + "# TO THE CORRECT LOCATION (/kaggle/input) IN YOUR NOTEBOOK,\n", + "# THEN FEEL FREE TO DELETE THIS CELL.\n", + "# NOTE: THIS NOTEBOOK ENVIRONMENT DIFFERS FROM KAGGLE'S PYTHON\n", + "# ENVIRONMENT SO THERE MAY BE MISSING LIBRARIES USED BY YOUR\n", + "# NOTEBOOK.\n", + "\n", + "import os\n", + "import sys\n", + "from tempfile import NamedTemporaryFile\n", + "from urllib.request import urlopen\n", + "from urllib.parse import unquote, urlparse\n", + "from urllib.error import HTTPError\n", + "from zipfile import ZipFile\n", + "import tarfile\n", + "import shutil\n", + "\n", + "CHUNK_SIZE = 40960\n", + "DATA_SOURCE_MAPPING = 'disc-data:https%3A%2F%2Fstorage.googleapis.com%2Fkaggle-data-sets%2F4594308%2F7837629%2Fbundle%2Farchive.zip%3FX-Goog-Algorithm%3DGOOG4-RSA-SHA256%26X-Goog-Credential%3Dgcp-kaggle-com%2540kaggle-161607.iam.gserviceaccount.com%252F20240315%252Fauto%252Fstorage%252Fgoog4_request%26X-Goog-Date%3D20240315T014941Z%26X-Goog-Expires%3D259200%26X-Goog-SignedHeaders%3Dhost%26X-Goog-Signature%3D5d4f46c137b152ef1c79c43c01802c9edb0951c58127b8d6d8ba51c869adcf060bc8001ee2e359edc935b13e2d0d8b8302a29d19c15e5c041ec287b5a3979e6ce988c1e08162bcca5451a043fe0ee9a09d01edf02bd99eb008ffcf6a6fab0ab782f9d04c9f3977598ad4b9a51d38a5e916cfb28ecc9e4392d91b861d85d8c433179744f289397ae7c7645ce32aeee70810cce55ef4feed78efd0e82d81502c9c8801a9c24a7bb4c5af8b390f8ea4ae70c492d63436ed4f4062241b8f55ec213183be84d5e5d60a4c235ba99e9243c429e78e07560e9fc2f477f08a4645a5ef39802c69b8aeb71197c28ab3216b33bdbc79a367be6d41ad21eb1f09be11101451,all-discord-data:https%3A%2F%2Fstorage.googleapis.com%2Fkaggle-data-sets%2F4595521%2F7839317%2Fbundle%2Farchive.zip%3FX-Goog-Algorithm%3DGOOG4-RSA-SHA256%26X-Goog-Credential%3Dgcp-kaggle-com%2540kaggle-161607.iam.gserviceaccount.com%252F20240315%252Fauto%252Fstorage%252Fgoog4_request%26X-Goog-Date%3D20240315T014941Z%26X-Goog-Expires%3D259200%26X-Goog-SignedHeaders%3Dhost%26X-Goog-Signature%3D166d99ce2ef43be08b25b698531db915fc362e5fe17f05e94d32fe7728af232df4be495fba6c9942ebdfee8894c8b2543eeabc3daa314bf9a27dcec0d0b57768cffd9f625efbcb3976b41ed36a6dab64d34921e0839983117c7f9c52daabdb1a67c595e7553665a10e130d657afcd70f6cdf4800290a94ca73db8f2a957348c1be1ef07b74be3648f0ae2430161304abdfd6d7207a341d5d2c3f6daefa1291641baec78fa9a00b09fd429f6b755be53ea7f73dfdc7d95f37304363da776adf7924029618c674472911c679eaa3933e43ab7664aa28f254f2aec842022cc43308e67404f74346d5519852dff2f3e3519688025381b5eca7a296f59fcfee24a548'\n", + "\n", + "KAGGLE_INPUT_PATH='/kaggle/input'\n", + "KAGGLE_WORKING_PATH='/kaggle/working'\n", + "KAGGLE_SYMLINK='kaggle'\n", + "\n", + "!umount /kaggle/input/ 2> /dev/null\n", + "shutil.rmtree('/kaggle/input', ignore_errors=True)\n", + "os.makedirs(KAGGLE_INPUT_PATH, 0o777, exist_ok=True)\n", + "os.makedirs(KAGGLE_WORKING_PATH, 0o777, exist_ok=True)\n", + "\n", + "try:\n", + " os.symlink(KAGGLE_INPUT_PATH, os.path.join(\"..\", 'input'), target_is_directory=True)\n", + "except FileExistsError:\n", + " pass\n", + "try:\n", + " os.symlink(KAGGLE_WORKING_PATH, os.path.join(\"..\", 'working'), target_is_directory=True)\n", + "except FileExistsError:\n", + " pass\n", + "\n", + "for data_source_mapping in DATA_SOURCE_MAPPING.split(','):\n", + " directory, download_url_encoded = data_source_mapping.split(':')\n", + " download_url = unquote(download_url_encoded)\n", + " filename = urlparse(download_url).path\n", + " destination_path = os.path.join(KAGGLE_INPUT_PATH, directory)\n", + " try:\n", + " with urlopen(download_url) as fileres, NamedTemporaryFile() as tfile:\n", + " total_length = fileres.headers['content-length']\n", + " print(f'Downloading {directory}, {total_length} bytes compressed')\n", + " dl = 0\n", + " data = fileres.read(CHUNK_SIZE)\n", + " while len(data) > 0:\n", + " dl += len(data)\n", + " tfile.write(data)\n", + " done = int(50 * dl / int(total_length))\n", + " sys.stdout.write(f\"\\r[{'=' * done}{' ' * (50-done)}] {dl} bytes downloaded\")\n", + " sys.stdout.flush()\n", + " data = fileres.read(CHUNK_SIZE)\n", + " if filename.endswith('.zip'):\n", + " with ZipFile(tfile) as zfile:\n", + " zfile.extractall(destination_path)\n", + " else:\n", + " with tarfile.open(tfile.name) as tarfile:\n", + " tarfile.extractall(destination_path)\n", + " print(f'\\nDownloaded and uncompressed: {directory}')\n", + " except HTTPError as e:\n", + " print(f'Failed to load (likely expired) {download_url} to path {destination_path}')\n", + " continue\n", + " except OSError as e:\n", + " print(f'Failed to load {download_url} to path {destination_path}')\n", + " continue\n", + "\n", + "print('Data source import complete.')\n" + ], + "metadata": { + "id": "rDLaHAAZfkTS" + }, + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "cell_type": "code", + "source": [ + "# This Python 3 environment comes with many helpful analytics libraries installed\n", + "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n", + "# For example, here's several helpful packages to load\n", + "\n", + "import numpy as np # linear algebra\n", + "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", + "\n", + "# Input data files are available in the read-only \"../input/\" directory\n", + "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n", + "\n", + "import os\n", + "for dirname, _, filenames in os.walk('/kaggle/input'):\n", + " for filename in filenames:\n", + " print(os.path.join(dirname, filename))\n", + "\n", + "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\"\n", + "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session" + ], + "metadata": { + "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5", + "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19", + "execution": { + "iopub.status.busy": "2024-03-15T01:23:32.799915Z", + "iopub.execute_input": "2024-03-15T01:23:32.800237Z", + "iopub.status.idle": "2024-03-15T01:23:33.910083Z", + "shell.execute_reply.started": "2024-03-15T01:23:32.80021Z", + "shell.execute_reply": "2024-03-15T01:23:33.909047Z" + }, + "trusted": true, + "id": "BPpcBJklfkTT", + "outputId": "70a56ff5-1c32-4970-99d4-6f31d2e9d6db" + }, + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "text": "/kaggle/input/disc-data/disc_6.csv\n/kaggle/input/all-discord-data/combined_df.csv\n", + "output_type": "stream" + } + ] + }, + { + "cell_type": "code", + "source": [ + "! pip install -U accelerate\n", + "! pip install datasets transformers\n", + "!apt install git-lfs" + ], + "metadata": { + "execution": { + "iopub.status.busy": "2024-03-15T01:23:33.911745Z", + "iopub.execute_input": "2024-03-15T01:23:33.912224Z", + "iopub.status.idle": "2024-03-15T01:24:04.215466Z", + "shell.execute_reply.started": "2024-03-15T01:23:33.912182Z", + "shell.execute_reply": "2024-03-15T01:24:04.214261Z" + }, + "trusted": true, + "id": "iX3E6gAkfkTU", + "outputId": "f3b6f749-6727-4057-f351-8bc10ff03782" + }, + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "text": "Requirement already satisfied: accelerate in /opt/conda/lib/python3.10/site-packages (0.27.2)\nCollecting accelerate\n Downloading accelerate-0.28.0-py3-none-any.whl.metadata (18 kB)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from accelerate) (1.26.4)\nRequirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from accelerate) (21.3)\nRequirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from accelerate) (5.9.3)\nRequirement already satisfied: pyyaml in /opt/conda/lib/python3.10/site-packages (from accelerate) (6.0.1)\nRequirement already satisfied: torch>=1.10.0 in /opt/conda/lib/python3.10/site-packages (from accelerate) (2.1.2)\nRequirement already satisfied: huggingface-hub in /opt/conda/lib/python3.10/site-packages (from accelerate) (0.20.3)\nRequirement already satisfied: safetensors>=0.3.1 in /opt/conda/lib/python3.10/site-packages (from accelerate) (0.4.2)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging>=20.0->accelerate) (3.1.1)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.13.1)\nRequirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (4.9.0)\nRequirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (1.12)\nRequirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.2.1)\nRequirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.1.2)\nRequirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (2024.2.0)\nRequirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->accelerate) (2.31.0)\nRequirement already satisfied: tqdm>=4.42.1 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub->accelerate) (4.66.1)\nRequirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->accelerate) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->accelerate) (3.6)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->accelerate) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub->accelerate) (2024.2.2)\nRequirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\nDownloading accelerate-0.28.0-py3-none-any.whl (290 kB)\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m290.1/290.1 kB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0mm\n\u001b[?25hInstalling collected packages: accelerate\n Attempting uninstall: accelerate\n Found existing installation: accelerate 0.27.2\n Uninstalling accelerate-0.27.2:\n Successfully uninstalled accelerate-0.27.2\nSuccessfully installed accelerate-0.28.0\nRequirement already satisfied: datasets in /opt/conda/lib/python3.10/site-packages (2.1.0)\nRequirement already satisfied: transformers in /opt/conda/lib/python3.10/site-packages (4.38.1)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from datasets) (1.26.4)\nRequirement already satisfied: pyarrow>=5.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (11.0.0)\nRequirement already satisfied: dill in /opt/conda/lib/python3.10/site-packages (from datasets) (0.3.8)\nRequirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from datasets) (2.1.4)\nRequirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (2.31.0)\nRequirement already satisfied: tqdm>=4.62.1 in /opt/conda/lib/python3.10/site-packages (from datasets) (4.66.1)\nRequirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from datasets) (3.4.1)\nRequirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from datasets) (0.70.16)\nRequirement already satisfied: fsspec>=2021.05.0 in /opt/conda/lib/python3.10/site-packages (from fsspec[http]>=2021.05.0->datasets) (2024.2.0)\nRequirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets) (3.9.1)\nRequirement already satisfied: huggingface-hub<1.0.0,>=0.1.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.20.3)\nRequirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from datasets) (21.3)\nRequirement already satisfied: responses<0.19 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.18.0)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from transformers) (3.13.1)\nRequirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (6.0.1)\nRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers) (2023.12.25)\nRequirement already satisfied: tokenizers<0.19,>=0.14 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.15.2)\nRequirement already satisfied: safetensors>=0.4.1 in /opt/conda/lib/python3.10/site-packages (from transformers) (0.4.2)\nRequirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (23.2.0)\nRequirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (6.0.4)\nRequirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.9.3)\nRequirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.4.1)\nRequirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.1)\nRequirement already satisfied: async-timeout<5.0,>=4.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (4.0.3)\nRequirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (4.9.0)\nRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.10/site-packages (from packaging->datasets) (3.1.1)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (3.6)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (2024.2.2)\nRequirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2.8.2)\nRequirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2023.3.post1)\nRequirement already satisfied: tzdata>=2022.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2023.4)\nRequirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\nReading package lists... Done\nBuilding dependency tree \nReading state information... Done\ngit-lfs is already the newest version (2.9.2-1).\n0 upgraded, 0 newly installed, 0 to remove and 38 not upgraded.\n", + "output_type": "stream" + } + ] + }, + { + "cell_type": "code", + "source": [ + "from huggingface_hub import notebook_login, login\n", + "\n", + "login(token=\"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\")" + ], + "metadata": { + "execution": { + "iopub.status.busy": "2024-03-15T01:24:04.217102Z", + "iopub.execute_input": "2024-03-15T01:24:04.217407Z", + "iopub.status.idle": "2024-03-15T01:24:04.944095Z", + "shell.execute_reply.started": "2024-03-15T01:24:04.21738Z", + "shell.execute_reply": "2024-03-15T01:24:04.943145Z" + }, + "trusted": true, + "id": "GEjhMNwbfkTU", + "outputId": "65aa69bd-a9a7-4ad1-b0ff-b6712d8b7d5d" + }, + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "text": "Token will not been saved to git credential helper. Pass `add_to_git_credential=True` if you want to set the git credential as well.\nToken is valid (permission: write).\nYour token has been saved to /root/.cache/huggingface/token\nLogin successful\n", + "output_type": "stream" + } + ] + }, + { + "cell_type": "code", + "source": [ + "import transformers\n", + "\n", + "print(transformers.__version__)" + ], + "metadata": { + "execution": { + "iopub.status.busy": "2024-03-15T01:24:04.946388Z", + "iopub.execute_input": "2024-03-15T01:24:04.946771Z", + "iopub.status.idle": "2024-03-15T01:24:10.865687Z", + "shell.execute_reply.started": "2024-03-15T01:24:04.946746Z", + "shell.execute_reply": "2024-03-15T01:24:10.864621Z" + }, + "trusted": true, + "id": "0YkqBDmGfkTV", + "outputId": "2f2e3125-0f74-49e0-9471-7f57e732649d" + }, + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "text": "4.38.1\n", + "output_type": "stream" + } + ] + }, + { + "cell_type": "code", + "source": [ + "!pip install wandb" + ], + "metadata": { + "execution": { + "iopub.status.busy": "2024-03-15T01:24:10.86701Z", + "iopub.execute_input": "2024-03-15T01:24:10.867427Z", + "iopub.status.idle": "2024-03-15T01:24:23.106806Z", + "shell.execute_reply.started": "2024-03-15T01:24:10.867402Z", + "shell.execute_reply": "2024-03-15T01:24:23.105567Z" + }, + "trusted": true, + "id": "OZZza02VfkTV", + "outputId": "6b56d7e8-5dd2-4e1b-fe39-37ad3090b000" + }, + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "text": "Requirement already satisfied: wandb in /opt/conda/lib/python3.10/site-packages (0.16.3)\nRequirement already satisfied: Click!=8.0.0,>=7.1 in /opt/conda/lib/python3.10/site-packages (from wandb) (8.1.7)\nRequirement already satisfied: GitPython!=3.1.29,>=1.0.0 in /opt/conda/lib/python3.10/site-packages (from wandb) (3.1.41)\nRequirement already satisfied: requests<3,>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from wandb) (2.31.0)\nRequirement already satisfied: psutil>=5.0.0 in /opt/conda/lib/python3.10/site-packages (from wandb) (5.9.3)\nRequirement already satisfied: sentry-sdk>=1.0.0 in /opt/conda/lib/python3.10/site-packages (from wandb) (1.40.5)\nRequirement already satisfied: docker-pycreds>=0.4.0 in /opt/conda/lib/python3.10/site-packages (from wandb) (0.4.0)\nRequirement already satisfied: PyYAML in /opt/conda/lib/python3.10/site-packages (from wandb) (6.0.1)\nRequirement already satisfied: setproctitle in /opt/conda/lib/python3.10/site-packages (from wandb) (1.3.3)\nRequirement already satisfied: setuptools in /opt/conda/lib/python3.10/site-packages (from wandb) (69.0.3)\nRequirement already satisfied: appdirs>=1.4.3 in /opt/conda/lib/python3.10/site-packages (from wandb) (1.4.4)\nRequirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /opt/conda/lib/python3.10/site-packages (from wandb) (3.20.3)\nRequirement already satisfied: six>=1.4.0 in /opt/conda/lib/python3.10/site-packages (from docker-pycreds>=0.4.0->wandb) (1.16.0)\nRequirement already satisfied: gitdb<5,>=4.0.1 in /opt/conda/lib/python3.10/site-packages (from GitPython!=3.1.29,>=1.0.0->wandb) (4.0.11)\nRequirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2.0.0->wandb) (3.3.2)\nRequirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2.0.0->wandb) (3.6)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2.0.0->wandb) (1.26.18)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests<3,>=2.0.0->wandb) (2024.2.2)\nRequirement already satisfied: smmap<6,>=3.0.1 in /opt/conda/lib/python3.10/site-packages (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb) (5.0.1)\n", + "output_type": "stream" + } + ] + }, + { + "cell_type": "code", + "source": [ + "import wandb" + ], + "metadata": { + "execution": { + "iopub.status.busy": "2024-03-15T01:24:23.108354Z", + "iopub.execute_input": "2024-03-15T01:24:23.108645Z", + "iopub.status.idle": "2024-03-15T01:24:24.173069Z", + "shell.execute_reply.started": "2024-03-15T01:24:23.10862Z", + "shell.execute_reply": "2024-03-15T01:24:24.172013Z" + }, + "trusted": true, + "id": "-X66XkbAfkTV" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "wandb.login(key=\"8ff81dcac301a265e86bec8669d939dcd3b76bdc\")" + ], + "metadata": { + "execution": { + "iopub.status.busy": "2024-03-15T01:24:24.174783Z", + "iopub.execute_input": "2024-03-15T01:24:24.175111Z", + "iopub.status.idle": "2024-03-15T01:24:26.402709Z", + "shell.execute_reply.started": "2024-03-15T01:24:24.175084Z", + "shell.execute_reply": "2024-03-15T01:24:26.401781Z" + }, + "trusted": true, + "id": "iiLC3uO2fkTV", + "outputId": "aad1981a-2aab-4eda-8354-f76e2c801898" + }, + "execution_count": null, + "outputs": [ + { + "name": "stderr", + "text": "\u001b[34m\u001b[1mwandb\u001b[0m: W&B API key is configured. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m If you're specifying your api key in code, ensure this code is not shared publicly.\n\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n", + "output_type": "stream" + }, + { + "execution_count": 7, + "output_type": "execute_result", + "data": { + "text/plain": "True" + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "source": [ + "from datasets import load_dataset, Dataset, DatasetDict\n", + "import pandas as pd\n", + "from sklearn.model_selection import train_test_split" + ], + "metadata": { + "execution": { + "iopub.status.busy": "2024-03-15T01:24:26.404016Z", + "iopub.execute_input": "2024-03-15T01:24:26.405015Z", + "iopub.status.idle": "2024-03-15T01:24:28.381818Z", + "shell.execute_reply.started": "2024-03-15T01:24:26.404974Z", + "shell.execute_reply": "2024-03-15T01:24:28.381051Z" + }, + "trusted": true, + "id": "5AdRYoszfkTW" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "df = pd.read_csv(\"/kaggle/input/all-discord-data/combined_df.csv\").dropna().drop(\"Author_ID\", axis=1)\n", + "# df_2 = pd.read_csv(\"/content/drive/MyDrive/Data/disc_data/disc_7.csv\")\n", + "\n", + "df = df.sample(frac=0.05, replace=True, random_state=1)\n", + "\n", + "train, test = train_test_split(df, test_size=0.2)\n", + "test, val = train_test_split(test, test_size=0.5)" + ], + "metadata": { + "execution": { + "iopub.status.busy": "2024-03-15T01:24:28.383102Z", + "iopub.execute_input": "2024-03-15T01:24:28.383758Z", + "iopub.status.idle": "2024-03-15T01:24:48.478663Z", + "shell.execute_reply.started": "2024-03-15T01:24:28.383721Z", + "shell.execute_reply": "2024-03-15T01:24:48.477731Z" + }, + "trusted": true, + "id": "NgqDqwjlfkTW" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "train.head()" + ], + "metadata": { + "execution": { + "iopub.status.busy": "2024-03-15T01:24:48.482488Z", + "iopub.execute_input": "2024-03-15T01:24:48.48277Z", + "iopub.status.idle": "2024-03-15T01:24:48.495534Z", + "shell.execute_reply.started": "2024-03-15T01:24:48.482746Z", + "shell.execute_reply": "2024-03-15T01:24:48.494614Z" + }, + "trusted": true, + "id": "K5ebhURCfkTW", + "outputId": "bb5f1e27-3051-45fb-d0cf-3ae21ff7b066" + }, + "execution_count": null, + "outputs": [ + { + "execution_count": 10, + "output_type": "execute_result", + "data": { + "text/plain": " Message\n11936715 Tohka\\nSleep\\n..\n11165328 @Khan Mileff fake\n1575371 hi\\nwelcome\n834803 .pik\n14173488 L", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
Message
11936715Tohka\\nSleep\\n..
11165328@Khan Mileff fake
1575371hi\\nwelcome
834803.pik
14173488L
\n
" + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "source": [ + "train.shape" + ], + "metadata": { + "execution": { + "iopub.status.busy": "2024-03-15T01:24:48.496651Z", + "iopub.execute_input": "2024-03-15T01:24:48.497024Z", + "iopub.status.idle": "2024-03-15T01:24:48.503038Z", + "shell.execute_reply.started": "2024-03-15T01:24:48.49699Z", + "shell.execute_reply": "2024-03-15T01:24:48.502158Z" + }, + "trusted": true, + "id": "8wlj32GBfkTX", + "outputId": "cca63965-776d-4994-8a6b-ffb2d9a42958" + }, + "execution_count": null, + "outputs": [ + { + "execution_count": 11, + "output_type": "execute_result", + "data": { + "text/plain": "(678328, 1)" + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "source": [ + "test.head()" + ], + "metadata": { + "execution": { + "iopub.status.busy": "2024-03-15T01:24:48.504251Z", + "iopub.execute_input": "2024-03-15T01:24:48.504792Z", + "iopub.status.idle": "2024-03-15T01:24:48.516892Z", + "shell.execute_reply.started": "2024-03-15T01:24:48.504759Z", + "shell.execute_reply": "2024-03-15T01:24:48.515967Z" + }, + "trusted": true, + "id": "wCpHSssifkTX", + "outputId": "23c3f0e4-3404-4900-dfae-14da7de6d1fe" + }, + "execution_count": null, + "outputs": [ + { + "execution_count": 12, + "output_type": "execute_result", + "data": { + "text/plain": " Message\n10364349 @vic also there is no date wth\n589910 I'm screebshotting everything nora\\n@LunaAmherst\n7184787 toast is 13\n4075592 Idk\n8037680 hirp\\nshut", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
Message
10364349@vic also there is no date wth
589910I'm screebshotting everything nora\\n@LunaAmherst
7184787toast is 13
4075592Idk
8037680hirp\\nshut
\n
" + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "source": [ + "train_ds = Dataset.from_pandas(train)\n", + "test_ds = Dataset.from_pandas(test)\n", + "validation_ds = Dataset.from_pandas(val)\n", + "\n", + "datasets = DatasetDict({\n", + " 'train': train_ds,\n", + " 'test': test_ds,\n", + " 'valid': validation_ds})" + ], + "metadata": { + "execution": { + "iopub.status.busy": "2024-03-15T01:24:48.518195Z", + "iopub.execute_input": "2024-03-15T01:24:48.518484Z", + "iopub.status.idle": "2024-03-15T01:24:48.722376Z", + "shell.execute_reply.started": "2024-03-15T01:24:48.518461Z", + "shell.execute_reply": "2024-03-15T01:24:48.721377Z" + }, + "trusted": true, + "id": "Guvxu1EmfkTX" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "datasets[\"train\"][11]" + ], + "metadata": { + "execution": { + "iopub.status.busy": "2024-03-15T01:24:48.723512Z", + "iopub.execute_input": "2024-03-15T01:24:48.723782Z", + "iopub.status.idle": "2024-03-15T01:24:48.732387Z", + "shell.execute_reply.started": "2024-03-15T01:24:48.723759Z", + "shell.execute_reply": "2024-03-15T01:24:48.731574Z" + }, + "trusted": true, + "id": "dB7AU8qefkTX", + "outputId": "f8022a7c-55a4-4985-e610-27a7d4b6a1f9" + }, + "execution_count": null, + "outputs": [ + { + "execution_count": 14, + "output_type": "execute_result", + "data": { + "text/plain": "{'Message': 'HELLO SOMEONE EXPLAIN PLS', '__index_level_0__': 12770734}" + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "source": [ + "model_checkpoint = \"distilroberta-base\"" + ], + "metadata": { + "execution": { + "iopub.status.busy": "2024-03-15T01:24:48.733693Z", + "iopub.execute_input": "2024-03-15T01:24:48.734048Z", + "iopub.status.idle": "2024-03-15T01:24:48.742518Z", + "shell.execute_reply.started": "2024-03-15T01:24:48.734019Z", + "shell.execute_reply": "2024-03-15T01:24:48.741581Z" + }, + "trusted": true, + "id": "vlUkP9WafkTY" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "from transformers import AutoTokenizer\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, use_fast=True)" + ], + "metadata": { + "execution": { + "iopub.status.busy": "2024-03-15T01:24:48.743459Z", + "iopub.execute_input": "2024-03-15T01:24:48.743752Z", + "iopub.status.idle": "2024-03-15T01:24:50.77152Z", + "shell.execute_reply.started": "2024-03-15T01:24:48.743729Z", + "shell.execute_reply": "2024-03-15T01:24:50.770739Z" + }, + "trusted": true, + "id": "6BYD0SqVfkTY", + "outputId": "d75b62a3-fb1d-4824-c809-8816f35c98b6", + "colab": { + "referenced_widgets": [ + "4fdbf10ddb494ccdab4b9f5cc0ff4872", + "b76a7928fabe494b93c9cd466da7e4b1", + "3b97a4e8ab0e4f29b1cc305e684f95be", + "8eff91b779974e18b748d1051c17abb8", + "2e1ebd07550045dbbc95ae534b435b13" + ] + } + }, + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": "tokenizer_config.json: 0%| | 0.00/25.0 [00:00 512). Running this sequence through the model will result in indexing errors\nToken indices sequence length is longer than the specified maximum sequence length for this model (560 > 512). Running this sequence through the model will result in indexing errors\nToken indices sequence length is longer than the specified maximum sequence length for this model (2673 > 512). Running this sequence through the model will result in indexing errors\nToken indices sequence length is longer than the specified maximum sequence length for this model (568 > 512). Running this sequence through the model will result in indexing errors\n", + "output_type": "stream" + }, + { + "name": "stdout", + "text": " ", + "output_type": "stream" + }, + { + "output_type": "display_data", + "data": { + "text/plain": "#0: 0%| | 0/22 [00:00 512). Running this sequence through the model will result in indexing errors\nToken indices sequence length is longer than the specified maximum sequence length for this model (788 > 512). Running this sequence through the model will result in indexing errors\nToken indices sequence length is longer than the specified maximum sequence length for this model (633 > 512). Running this sequence through the model will result in indexing errors\n", + "output_type": "stream" + }, + { + "name": "stdout", + "text": " ", + "output_type": "stream" + }, + { + "output_type": "display_data", + "data": { + "text/plain": "#0: 0%| | 0/22 [00:00 512). Running this sequence through the model will result in indexing errors\nToken indices sequence length is longer than the specified maximum sequence length for this model (606 > 512). Running this sequence through the model will result in indexing errors\n", + "output_type": "stream" + } + ] + }, + { + "cell_type": "code", + "source": [ + "tokenized_datasets" + ], + "metadata": { + "execution": { + "iopub.status.busy": "2024-03-15T01:25:13.027851Z", + "iopub.execute_input": "2024-03-15T01:25:13.028139Z", + "iopub.status.idle": "2024-03-15T01:25:13.034925Z", + "shell.execute_reply.started": "2024-03-15T01:25:13.028113Z", + "shell.execute_reply": "2024-03-15T01:25:13.034044Z" + }, + "trusted": true, + "id": "MJ9tooWWfkTZ", + "outputId": "7933df51-cf73-4d6e-9c09-e6aa962959fa" + }, + "execution_count": null, + "outputs": [ + { + "execution_count": 19, + "output_type": "execute_result", + "data": { + "text/plain": "DatasetDict({\n train: Dataset({\n features: ['input_ids', 'attention_mask'],\n num_rows: 678328\n })\n test: Dataset({\n features: ['input_ids', 'attention_mask'],\n num_rows: 84791\n })\n valid: Dataset({\n features: ['input_ids', 'attention_mask'],\n num_rows: 84791\n })\n})" + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "source": [ + "block_size = 32\n", + "def group_texts(examples):\n", + " # Concatenate all texts.\n", + " concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}\n", + " total_length = len(concatenated_examples[list(examples.keys())[0]])\n", + " # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can\n", + " # customize this part to your needs.\n", + " total_length = (total_length // block_size) * block_size\n", + " # Split by chunks of max_len.\n", + " result = {\n", + " k: [t[i : i + block_size] for i in range(0, total_length, block_size)]\n", + " for k, t in concatenated_examples.items()\n", + " }\n", + " result[\"labels\"] = result[\"input_ids\"].copy()\n", + " return result" + ], + "metadata": { + "execution": { + "iopub.status.busy": "2024-03-15T01:25:13.036135Z", + "iopub.execute_input": "2024-03-15T01:25:13.036397Z", + "iopub.status.idle": "2024-03-15T01:25:13.047837Z", + "shell.execute_reply.started": "2024-03-15T01:25:13.036374Z", + "shell.execute_reply": "2024-03-15T01:25:13.046951Z" + }, + "trusted": true, + "id": "UhTzeELVfkTZ" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "lm_datasets = tokenized_datasets.map(\n", + " group_texts,\n", + " batched=True,\n", + " batch_size=1000,\n", + " num_proc=4,\n", + ")" + ], + "metadata": { + "execution": { + "iopub.status.busy": "2024-03-15T01:25:13.049026Z", + "iopub.execute_input": "2024-03-15T01:25:13.049607Z", + "iopub.status.idle": "2024-03-15T01:25:44.845138Z", + "shell.execute_reply.started": "2024-03-15T01:25:13.049576Z", + "shell.execute_reply": "2024-03-15T01:25:44.844096Z" + }, + "trusted": true, + "id": "RJDSVHcCfkTZ", + "outputId": "b01bc4a4-ba45-4a8f-ece8-5e7dc3167d03", + "colab": { + "referenced_widgets": [ + "0b11196457b54151ba3edf31218ab4bc", + "4043379796894b1588531a6825d655ac", + "7dbe8053ac6c43d7b38ffb5ceb19d6fc", + "0b607b24407343e0a1bc855316ebdbf3", + "62412c9b5a984e239417ddf1235266bc", + "56ed57da4e6c49c3ac1d0b27573cfe3d", + "8532342477c941cb9ddb8f36069a445b", + "f266afcbef094e03bc14697ff8564abc", + "1e416624903341e3bdf4fe099712cce0", + "2552f69eef794392872f9d1ed3c43aab", + "57bf7329df91440c959c4cbcb8facfe6", + "0b12fd94b84a41b5bd59ce8f9e62c690" + ] + } + }, + "execution_count": null, + "outputs": [ + { + "name": "stdout", + "text": " ", + "output_type": "stream" + }, + { + "output_type": "display_data", + "data": { + "text/plain": "#0: 0%| | 0/170 [00:00God vic@Tornadotracker whatBut you can easily text or call people you know irl. That's what\"" + }, + "metadata": {} + } + ] + }, + { + "cell_type": "code", + "source": [ + "from transformers import AutoModelForCausalLM\n", + "model = AutoModelForCausalLM.from_pretrained(model_checkpoint)" + ], + "metadata": { + "execution": { + "iopub.status.busy": "2024-03-15T01:25:55.378071Z", + "iopub.execute_input": "2024-03-15T01:25:55.378673Z", + "iopub.status.idle": "2024-03-15T01:25:59.490146Z", + "shell.execute_reply.started": "2024-03-15T01:25:55.378645Z", + "shell.execute_reply": "2024-03-15T01:25:59.489013Z" + }, + "trusted": true, + "id": "-W1UZomrfkTZ", + "outputId": "7f20de95-114d-4935-e4e4-cf353b77865f", + "colab": { + "referenced_widgets": [ + "3d979e7a408241a7956f2fd36b52bbbb" + ] + } + }, + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": "model.safetensors: 0%| | 0.00/331M [00:00", + "text/html": "wandb version 0.16.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": "", + "text/html": "Tracking run with wandb version 0.16.3" + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": "", + "text/html": "Run data is saved locally in /kaggle/working/wandb/run-20240315_012600-wluf4l0n" + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": "", + "text/html": "Syncing run snowy-valley-10 to Weights & Biases (docs)
" + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": "", + "text/html": " View project at https://wandb.ai/windshields/huggingface" + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": "", + "text/html": " View run at https://wandb.ai/windshields/huggingface/runs/wluf4l0n" + }, + "metadata": {} + }, + { + "name": "stderr", + "text": "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n", + "output_type": "stream" + }, + { + "output_type": "display_data", + "data": { + "text/plain": "", + "text/html": "\n
\n \n \n [ 9586/37173 23:00 < 1:06:14, 6.94 it/s, Epoch 0.77/3]\n
\n \n \n \n \n \n \n \n \n \n \n
EpochTraining LossValidation Loss

" + }, + "metadata": {} + }, + { + "name": "stderr", + "text": "/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n/opt/conda/lib/python3.10/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n warnings.warn('Was asked to gather along dimension 0, but all '\n", + "output_type": "stream" + } + ] + }, + { + "cell_type": "code", + "source": [ + "import math" + ], + "metadata": { + "trusted": true, + "id": "H0FyuKjXfkTa" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "eval_results = trainer.evaluate()\n", + "print(f\"Perplexity: {math.exp(eval_results['eval_loss']):.2f}\")" + ], + "metadata": { + "trusted": true, + "id": "3nxAeZGffkTa" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "trainer.push_to_hub()" + ], + "metadata": { + "trusted": true, + "id": "BuR_A5-nfkTa" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "tokenizer.push_to_hub(\"RoBERTa_for_Discord\")" + ], + "metadata": { + "jupyter": { + "source_hidden": true + }, + "trusted": true, + "id": "ZQDXNqIgfkTb" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [], + "metadata": { + "id": "vPcccVmYfkTc" + }, + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file