data_preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2844\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "276acc7b049c4d619bbd8626026d724a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b48635278954dda90d91c48718f45f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dfc07484d4404cf7b49ac6ee712ff779",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8db5a53fb0594149a7f1d5b0f8e018ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "058a3833468d4a0cbd89f5fa56414c27",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f959b46e06ec43c39ae81d6fda1b1542",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "627f15ddfcb04048941466759ed858b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md:   0%|          | 0.00/956 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/datasets/Jise/ruletaker/commit/b6e36448e186cde51052e978da0465d84da06f31', commit_message='Upload dataset', commit_description='', oid='b6e36448e186cde51052e978da0465d84da06f31', pr_url=None, repo_url=RepoUrl('https://huggingface.co/datasets/Jise/ruletaker', endpoint='https://huggingface.co', repo_type='dataset', repo_id='Jise/ruletaker'), pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "from datasets import Dataset, DatasetDict\n",
    "import random\n",
    "import os\n",
    "\n",
    "with open('data.json', \"r\") as f:\n",
    "    data = json.load(f)\n",
    "print(len(data))\n",
    "\n",
    "\n",
    "def display_data(data):\n",
    "    for item in data:\n",
    "        print(item + \":\")\n",
    "        print(data[item])\n",
    "\n",
    "for i in range(len(data)):\n",
    "    data[i][\"flag\"] = data[i][\"label\"]\n",
    "    del data[i][\"label\"]\n",
    "    data[i][\"reasoning\"] = \"\\n\".join(data[i][\"reasoning\"])\n",
    "\n",
    "depth_5 = random.sample([i for i in data if i[\"depth\"] > 4], 125)\n",
    "depth_4 = random.sample([i for i in data if i[\"depth\"] == 4], 250)\n",
    "depth_3 = [i for i in data if i[\"depth\"] <= 3]\n",
    "\n",
    "train_index = random.sample(range(len(depth_3)), 1000)\n",
    "test_index = [i for i in range(len(depth_3)) if i not in train_index]\n",
    "\n",
    "train_data = Dataset.from_list([depth_3[i] for i in train_index])\n",
    "test_data = Dataset.from_list([depth_3[i] for i in test_index])\n",
    "ood_data = Dataset.from_list(depth_4 + depth_5)\n",
    "\n",
    "dataset = DatasetDict({\n",
    "    \"train\": train_data,\n",
    "    \"test\": test_data,\n",
    "    \"ood_test\": ood_data\n",
    "})\n",
    "\n",
    "dataset.push_to_hub(\"ruletaker\", token=os.environ[\"HF_TOKEN\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "depth_5 = [i for i in dataset if i[\"depth\"] > 4]\n",
    "depth_4 = [i for i in dataset if i[\"depth\"] == 4]\n",
    "depth_3 = [i for i in dataset if i[\"depth\"] == 3]\n",
    "depth_2 = [i for i in dataset if i[\"depth\"] == 2]\n",
    "depth_1 = [i for i in dataset if i[\"depth\"] == 1]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}