gpt_experiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: openai in ./.venv/lib/python3.12/site-packages (1.56.2)\n",
      "Requirement already satisfied: datasets in ./.venv/lib/python3.12/site-packages (3.1.0)\n",
      "Requirement already satisfied: pygments in ./.venv/lib/python3.12/site-packages (2.18.0)\n",
      "Requirement already satisfied: rouge_score in ./.venv/lib/python3.12/site-packages (0.1.2)\n",
      "Requirement already satisfied: evaluate in ./.venv/lib/python3.12/site-packages (0.4.3)\n",
      "Requirement already satisfied: anyio<5,>=3.5.0 in ./.venv/lib/python3.12/site-packages (from openai) (4.6.2.post1)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in ./.venv/lib/python3.12/site-packages (from openai) (1.9.0)\n",
      "Requirement already satisfied: httpx<1,>=0.23.0 in ./.venv/lib/python3.12/site-packages (from openai) (0.28.0)\n",
      "Requirement already satisfied: jiter<1,>=0.4.0 in ./.venv/lib/python3.12/site-packages (from openai) (0.8.0)\n",
      "Requirement already satisfied: pydantic<3,>=1.9.0 in ./.venv/lib/python3.12/site-packages (from openai) (2.10.3)\n",
      "Requirement already satisfied: sniffio in ./.venv/lib/python3.12/site-packages (from openai) (1.3.1)\n",
      "Requirement already satisfied: tqdm>4 in ./.venv/lib/python3.12/site-packages (from openai) (4.67.1)\n",
      "Requirement already satisfied: typing-extensions<5,>=4.11 in ./.venv/lib/python3.12/site-packages (from openai) (4.12.2)\n",
      "Requirement already satisfied: filelock in ./.venv/lib/python3.12/site-packages (from datasets) (3.16.1)\n",
      "Requirement already satisfied: numpy>=1.17 in ./.venv/lib/python3.12/site-packages (from datasets) (2.1.3)\n",
      "Requirement already satisfied: pyarrow>=15.0.0 in ./.venv/lib/python3.12/site-packages (from datasets) (18.1.0)\n",
      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in ./.venv/lib/python3.12/site-packages (from datasets) (0.3.8)\n",
      "Requirement already satisfied: pandas in ./.venv/lib/python3.12/site-packages (from datasets) (2.2.3)\n",
      "Requirement already satisfied: requests>=2.32.2 in ./.venv/lib/python3.12/site-packages (from datasets) (2.32.3)\n",
      "Requirement already satisfied: xxhash in ./.venv/lib/python3.12/site-packages (from datasets) (3.5.0)\n",
      "Requirement already satisfied: multiprocess<0.70.17 in ./.venv/lib/python3.12/site-packages (from datasets) (0.70.16)\n",
      "Requirement already satisfied: fsspec<=2024.9.0,>=2023.1.0 in ./.venv/lib/python3.12/site-packages (from fsspec[http]<=2024.9.0,>=2023.1.0->datasets) (2024.9.0)\n",
      "Requirement already satisfied: aiohttp in ./.venv/lib/python3.12/site-packages (from datasets) (3.11.9)\n",
      "Requirement already satisfied: huggingface-hub>=0.23.0 in ./.venv/lib/python3.12/site-packages (from datasets) (0.26.3)\n",
      "Requirement already satisfied: packaging in ./.venv/lib/python3.12/site-packages (from datasets) (24.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in ./.venv/lib/python3.12/site-packages (from datasets) (6.0.2)\n",
      "Requirement already satisfied: absl-py in ./.venv/lib/python3.12/site-packages (from rouge_score) (2.1.0)\n",
      "Requirement already satisfied: nltk in ./.venv/lib/python3.12/site-packages (from rouge_score) (3.9.1)\n",
      "Requirement already satisfied: six>=1.14.0 in ./.venv/lib/python3.12/site-packages (from rouge_score) (1.17.0)\n",
      "Requirement already satisfied: idna>=2.8 in ./.venv/lib/python3.12/site-packages (from anyio<5,>=3.5.0->openai) (3.10)\n",
      "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in ./.venv/lib/python3.12/site-packages (from aiohttp->datasets) (2.4.4)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in ./.venv/lib/python3.12/site-packages (from aiohttp->datasets) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in ./.venv/lib/python3.12/site-packages (from aiohttp->datasets) (24.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in ./.venv/lib/python3.12/site-packages (from aiohttp->datasets) (1.5.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in ./.venv/lib/python3.12/site-packages (from aiohttp->datasets) (6.1.0)\n",
      "Requirement already satisfied: propcache>=0.2.0 in ./.venv/lib/python3.12/site-packages (from aiohttp->datasets) (0.2.1)\n",
      "Requirement already satisfied: yarl<2.0,>=1.17.0 in ./.venv/lib/python3.12/site-packages (from aiohttp->datasets) (1.18.3)\n",
      "Requirement already satisfied: certifi in ./.venv/lib/python3.12/site-packages (from httpx<1,>=0.23.0->openai) (2024.8.30)\n",
      "Requirement already satisfied: httpcore==1.* in ./.venv/lib/python3.12/site-packages (from httpx<1,>=0.23.0->openai) (1.0.7)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in ./.venv/lib/python3.12/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai) (0.14.0)\n",
      "Requirement already satisfied: annotated-types>=0.6.0 in ./.venv/lib/python3.12/site-packages (from pydantic<3,>=1.9.0->openai) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.27.1 in ./.venv/lib/python3.12/site-packages (from pydantic<3,>=1.9.0->openai) (2.27.1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in ./.venv/lib/python3.12/site-packages (from requests>=2.32.2->datasets) (3.4.0)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in ./.venv/lib/python3.12/site-packages (from requests>=2.32.2->datasets) (2.2.3)\n",
      "Requirement already satisfied: click in ./.venv/lib/python3.12/site-packages (from nltk->rouge_score) (8.1.7)\n",
      "Requirement already satisfied: joblib in ./.venv/lib/python3.12/site-packages (from nltk->rouge_score) (1.4.2)\n",
      "Requirement already satisfied: regex>=2021.8.3 in ./.venv/lib/python3.12/site-packages (from nltk->rouge_score) (2024.11.6)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in ./.venv/lib/python3.12/site-packages (from pandas->datasets) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in ./.venv/lib/python3.12/site-packages (from pandas->datasets) (2024.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in ./.venv/lib/python3.12/site-packages (from pandas->datasets) (2024.2)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install openai datasets pygments rouge_score evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "from openai import OpenAI\n",
    "from tqdm import tqdm\n",
    "import evaluate\n",
    "import json\n",
    "import random\n",
    "\n",
    "random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset({\n",
      "    features: ['context', 'statement', 'reasoning', 'depth', 'flag'],\n",
      "    num_rows: 1000\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "train_dataset = load_dataset(\"Jise/ruletaker\", split=\"train\")\n",
    "test_dataset = load_dataset(\"Jise/ruletaker\", split=\"test\")\n",
    "ood_dataset = load_dataset(\"Jise/ruletaker\", split=\"ood_test\")\n",
    "print(train_dataset)\n",
    "\n",
    "rouge = evaluate.load(\"rouge\")\n",
    "\n",
    "model = \"gpt-4o\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Zero-shot CoT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = \"\"\"\n",
    "    {}\n",
    "    Assertion: {}\n",
    "\"\"\"\n",
    "\n",
    "instruction = \"\"\"\n",
    "    Based on the facts and rules, give simple reasoning steps citing the rules, and output whether the assertion is true.\n",
    "    You must output in json format: {\"reason\": \"Because {rule1}, {rule2}, ..., {conclusion}\", \"answer\": 1/0}. where rules are copies from the Rule, and conclusion should\n",
    "    be either the assertion or its negation, do not add other texts.\n",
    "\"\"\"\n",
    "\n",
    "client = OpenAI()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 250/250 [04:45<00:00,  1.14s/it]\n"
     ]
    }
   ],
   "source": [
    "test_buffer = []\n",
    "ood_buffer = []\n",
    "\n",
    "for n, i in enumerate(tqdm(test_dataset)):\n",
    "    completion = client.chat.completions.create(\n",
    "        model=model,\n",
    "        messages=[\n",
    "            {\"role\": \"system\", \"content\": instruction},\n",
    "            {\n",
    "                \"role\": \"user\",\n",
    "                \"content\": prompt.format(i[\"context\"], i[\"statement\"])\n",
    "            }\n",
    "        ]\n",
    "    )\n",
    "    test_buffer.append(completion.choices[0].message.content)\n",
    "\n",
    "for n, i in enumerate(ood_dataset):\n",
    "    completion = client.chat.completions.create(\n",
    "        model=model,\n",
    "        messages=[\n",
    "            {\"role\": \"system\", \"content\": instruction},\n",
    "            {\n",
    "                \"role\": \"user\",\n",
    "                \"content\": prompt.format(i[\"context\"], i[\"statement\"])\n",
    "            }\n",
    "        ]\n",
    "    )\n",
    "    ood_buffer.append(completion.choices[0].message.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gpt-4o zeroshot Test Accuracy: 0.8951612903225806\n",
      "gpt-4o zeroshot Output Error: 2\n",
      "gpt-4o zeroshot Test Reason Rouge: {'rouge1': np.float64(0.4744683576298864), 'rouge2': np.float64(0.35779002476575317), 'rougeL': np.float64(0.41083381112160444), 'rougeLsum': np.float64(0.44453047027650283)}\n",
      "gpt-4o zeroshot OOD Accuracy: 0.8506666666666667\n",
      "gpt-4o zeroshot Error count: 0\n",
      "gpt-4o zeroshot Reason Rouge: {'rouge1': np.float64(0.35752862730132395), 'rouge2': np.float64(0.263847634179622), 'rougeL': np.float64(0.2903490586727546), 'rougeLsum': np.float64(0.3376695137594531)}\n"
     ]
    }
   ],
   "source": [
    "acc = 0\n",
    "count = 0\n",
    "err_count = 0\n",
    "mapping = {\"False\": 0, \"True\": 1}\n",
    "test_output_reasons = []\n",
    "ood_output_reasons = []\n",
    "list_to_save = {\"test\": [], \"ood\": []}\n",
    "\n",
    "for i, (o, t) in enumerate(zip(test_buffer, test_dataset)):\n",
    "    try:\n",
    "        output = eval(o.strip(\"```json\").strip(\"```\"))\n",
    "    except:\n",
    "        err_count += 1\n",
    "        test_output_reasons.append(o)  # If there is output format error, using the raw output as reason\n",
    "        continue\n",
    "    list_to_save[\"test\"].append({\"input\": t, \"output\": o})\n",
    "    count += 1\n",
    "    if int(output[\"answer\"]) == mapping[t[\"flag\"]]:\n",
    "        acc += 1\n",
    "    test_output_reasons.append(output[\"reason\"])\n",
    "\n",
    "print(f\"{model} zeroshot Test Accuracy: {acc/count}\")\n",
    "print(f\"{model} zeroshot Output Error: {err_count}\")\n",
    "print(f\"{model} zeroshot Test Reason Rouge: {rouge.compute(predictions=test_output_reasons, references=[i[\"reasoning\"] for i in test_dataset], use_stemmer=True)}\")\n",
    "\n",
    "acc = 0\n",
    "count = 0\n",
    "err_count = 0\n",
    "\n",
    "for i, (o, t) in enumerate(zip(ood_buffer, ood_dataset)):\n",
    "    try:    \n",
    "        output = eval(o.strip(\"```json\").strip(\"```\"))\n",
    "    except:\n",
    "        err_count += 1\n",
    "        ood_output_reasons.append(o)\n",
    "        continue\n",
    "    if int(output[\"answer\"]) == mapping[t[\"flag\"]]:\n",
    "        acc += 1\n",
    "    list_to_save[\"ood\"].append({\"input\": t, \"output\": o})\n",
    "    count += 1\n",
    "    ood_output_reasons.append(output[\"reason\"])\n",
    "print(f\"{model} zeroshot OOD Accuracy: {acc/count}\")\n",
    "print(f\"{model} zeroshot Error count: {err_count}\")\n",
    "print(f\"{model} zeroshot Reason Rouge: {rouge.compute(predictions=ood_output_reasons, references=[i[\"reasoning\"] for i in ood_dataset], use_stemmer=True)}\")\n",
    "with open(f\"{model}-zero-shot.json\", \"w\") as f:\n",
    "    json.dump(list_to_save, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Few-shot CoT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = \"\"\"\n",
    "    {}\n",
    "    Assertion: {}\n",
    "\"\"\"\n",
    "\n",
    "instruction = \"\"\"\n",
    "    Based on the facts and rules, give simple reasoning steps citing the rules, and output whether the assertion is true.\n",
    "    You must output in json format: {{\"reason\": \"Because {{rule1}}, {{rule2}}, ..., {{conclusion}}\", \"answer\": 1/0}}. where rules are copies from the Rule, and conclusion should\n",
    "    be either the assertion or its negation, do not add other texts.\n",
    "    Some examples: {}\n",
    "\"\"\"\n",
    "\n",
    "example_template = \"\"\"\n",
    "    {context}\n",
    "    Assertion: {statement}\n",
    "    Output: {{\"reason\": \"{reasoning}\", \"answer\": {flag}}}\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 250/250 [07:24<00:00,  1.78s/it]\n"
     ]
    }
   ],
   "source": [
    "test_buffer = []\n",
    "ood_buffer = []\n",
    "\n",
    "for n, i in enumerate(tqdm(test_dataset)):\n",
    "    example = []\n",
    "    for j in random.sample(range(len(train_dataset)), 3):\n",
    "        example.append(example_template.format(**train_dataset[j]))\n",
    "    examples = \"\\n\".join(example)\n",
    "    completion = client.chat.completions.create(\n",
    "        model=model,\n",
    "        messages=[\n",
    "            {\"role\": \"system\", \"content\": instruction.format(examples)},\n",
    "            {\n",
    "                \"role\": \"user\",\n",
    "                \"content\": prompt.format(i[\"context\"], i[\"statement\"])\n",
    "            }\n",
    "        ]\n",
    "    )\n",
    "    test_buffer.append({\"output\": completion.choices[0].message.content, \"example\": example})\n",
    "\n",
    "for n, i in enumerate(ood_dataset):\n",
    "    example = []\n",
    "    for j in random.sample(range(len(train_dataset)), 3):\n",
    "        example.append(example_template.format(**train_dataset[j]))\n",
    "    examples = \"\\n\".join(example)\n",
    "    completion = client.chat.completions.create(\n",
    "        model=model,\n",
    "        messages=[\n",
    "            {\"role\": \"system\", \"content\": instruction.format(examples)},\n",
    "            {\n",
    "                \"role\": \"user\",\n",
    "                \"content\": prompt.format(i[\"context\"], i[\"statement\"])\n",
    "            }\n",
    "        ]\n",
    "    )\n",
    "    ood_buffer.append({\"output\": completion.choices[0].message.content, \"example\": example})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test 5\n",
      "test 26\n",
      "test 74\n",
      "test 96\n",
      "test 105\n",
      "test 131\n",
      "gpt-4o fewshot Test Accuracy: 0.930327868852459\n",
      "gpt-4o fewshot Test Output Error: 6\n",
      "gpt-4o fewshot Test Reason Rouge: {'rouge1': np.float64(0.5308854623248525), 'rouge2': np.float64(0.4205108911262426), 'rougeL': np.float64(0.4567073292090176), 'rougeLsum': np.float64(0.4984785261028818)}\n",
      "ood 343\n",
      "gpt-4o fewshot OOD Accuracy: 0.946524064171123\n",
      "gpt-4o fewshot OOD Error count: 1\n",
      "gpt-4o fewshot OOD Reason Rouge: {'rouge1': np.float64(0.4182177225354021), 'rouge2': np.float64(0.3316091253072725), 'rougeL': np.float64(0.33733374379685566), 'rougeLsum': np.float64(0.3946646865691559)}\n"
     ]
    }
   ],
   "source": [
    "acc = 0\n",
    "count = 0\n",
    "err_count = 0\n",
    "mapping = {\"False\": 0, \"True\": 1}\n",
    "test_output_reasons = []\n",
    "ood_output_reasons = []\n",
    "list_to_save = {\"test\": [], \"ood\": []}\n",
    "\n",
    "for i, (o, t) in enumerate(zip(test_buffer, test_dataset)):\n",
    "    try:\n",
    "        output = eval(o[\"output\"].strip(\"```json\").strip(\"```\"))\n",
    "    except:\n",
    "        err_count += 1\n",
    "        print(\"test\", i)\n",
    "        test_output_reasons.append(o[\"output\"])  # If there is output format error, using the raw output as reason\n",
    "        continue\n",
    "    list_to_save[\"test\"].append({\"input\": t, \"output\": o})\n",
    "    count += 1\n",
    "    if int(output[\"answer\"]) == mapping[t[\"flag\"]]:\n",
    "        acc += 1\n",
    "    test_output_reasons.append(output[\"reason\"])\n",
    "\n",
    "print(f\"{model} fewshot Test Accuracy: {acc/count}\")\n",
    "print(f\"{model} fewshot Test Output Error: {err_count}\")\n",
    "print(f\"{model} fewshot Test Reason Rouge: {rouge.compute(predictions=test_output_reasons, references=[i[\"reasoning\"] for i in test_dataset], use_stemmer=True)}\")\n",
    "\n",
    "acc = 0\n",
    "count = 0\n",
    "err_count = 0\n",
    "\n",
    "for i, (o, t) in enumerate(zip(ood_buffer, ood_dataset)):\n",
    "    try:    \n",
    "        output = eval(o[\"output\"].strip(\"```json\").strip(\"```\"))\n",
    "        answer = output[\"answer\"]\n",
    "    except:\n",
    "        err_count += 1\n",
    "        print(\"ood\", i)\n",
    "        ood_output_reasons.append(o[\"output\"])\n",
    "        continue\n",
    "    if int(output[\"answer\"]) == mapping[t[\"flag\"]]:\n",
    "        acc += 1\n",
    "    list_to_save[\"ood\"].append({\"input\": t, \"output\": o})\n",
    "    count += 1\n",
    "    ood_output_reasons.append(output[\"reason\"])\n",
    "print(f\"{model} fewshot OOD Accuracy: {acc/count}\")\n",
    "print(f\"{model} fewshot OOD Error count: {err_count}\")\n",
    "print(f\"{model} fewshot OOD Reason Rouge: {rouge.compute(predictions=ood_output_reasons, references=[i[\"reasoning\"] for i in ood_dataset], use_stemmer=True)}\")\n",
    "with open(f\"{model}-few-shot.json\", \"w\") as f:\n",
    "    json.dump(list_to_save, f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}