1.6 Convert to XtreamUp.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Missing balance with {'sna', 'lug'}\n",
      "Missing confirm_reservation with {'sna', 'lug'}\n",
      "Missing freeze_account with {'sna', 'lug'}\n",
      "Missing restaurant_reservation with {'sna', 'lug'}\n",
      "Missing shopping_list_update with {'sna', 'lug'}\n",
      "Missing time with {'sna', 'lug'}\n",
      "Missing timezone with {'sna', 'lug'}\n",
      "Missing transfer with {'sna', 'lug'}\n",
      "Missing translate with {'sna', 'lug'}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train</th>\n",
       "      <th>dev</th>\n",
       "      <th>test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>amh</th>\n",
       "      <td>2240</td>\n",
       "      <td>320</td>\n",
       "      <td>640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ewe</th>\n",
       "      <td>2240</td>\n",
       "      <td>320</td>\n",
       "      <td>640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hau</th>\n",
       "      <td>2240</td>\n",
       "      <td>320</td>\n",
       "      <td>640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ibo</th>\n",
       "      <td>2240</td>\n",
       "      <td>320</td>\n",
       "      <td>640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kin</th>\n",
       "      <td>2240</td>\n",
       "      <td>320</td>\n",
       "      <td>640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lin</th>\n",
       "      <td>2240</td>\n",
       "      <td>320</td>\n",
       "      <td>640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lug</th>\n",
       "      <td>2240</td>\n",
       "      <td>320</td>\n",
       "      <td>640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>orm</th>\n",
       "      <td>2240</td>\n",
       "      <td>320</td>\n",
       "      <td>640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sna</th>\n",
       "      <td>2240</td>\n",
       "      <td>320</td>\n",
       "      <td>640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sot</th>\n",
       "      <td>2240</td>\n",
       "      <td>320</td>\n",
       "      <td>640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>swa</th>\n",
       "      <td>2240</td>\n",
       "      <td>320</td>\n",
       "      <td>640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>twi</th>\n",
       "      <td>2240</td>\n",
       "      <td>320</td>\n",
       "      <td>640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wol</th>\n",
       "      <td>2240</td>\n",
       "      <td>320</td>\n",
       "      <td>640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xho</th>\n",
       "      <td>2240</td>\n",
       "      <td>320</td>\n",
       "      <td>640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yor</th>\n",
       "      <td>2240</td>\n",
       "      <td>320</td>\n",
       "      <td>640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zul</th>\n",
       "      <td>2240</td>\n",
       "      <td>320</td>\n",
       "      <td>640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eng</th>\n",
       "      <td>1047</td>\n",
       "      <td>110</td>\n",
       "      <td>622</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     train  dev  test\n",
       "amh   2240  320   640\n",
       "ewe   2240  320   640\n",
       "hau   2240  320   640\n",
       "ibo   2240  320   640\n",
       "kin   2240  320   640\n",
       "lin   2240  320   640\n",
       "lug   2240  320   640\n",
       "orm   2240  320   640\n",
       "sna   2240  320   640\n",
       "sot   2240  320   640\n",
       "swa   2240  320   640\n",
       "twi   2240  320   640\n",
       "wol   2240  320   640\n",
       "xho   2240  320   640\n",
       "yor   2240  320   640\n",
       "zul   2240  320   640\n",
       "eng   1047  110   622"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from mlds import data_loader\n",
    "import warnings, re, os\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "manager = data_loader.SlotDataManager(data_folder=\"data/output\")\n",
    "\n",
    "def parse_spans(spans_str: str):\n",
    "    \"\"\"Parse spans string from format like '10:29:SL:ACCOUNT_TYPE' into list of dictionaries.\"\"\"\n",
    "    if not spans_str:\n",
    "        return []\n",
    "    \n",
    "    spans = []\n",
    "    for span in spans_str.split(','):\n",
    "        if ':' not in span:\n",
    "            continue\n",
    "        span = span.strip()\n",
    "        parts = span.split(':')\n",
    "        if len(parts) >= 4:\n",
    "            start, end, _, label = parts[:4]\n",
    "            spans.append({\n",
    "                \"start_byte\": int(start),\n",
    "                \"limit_byte\": int(end),\n",
    "                \"label\": label\n",
    "            })\n",
    "    return spans\n",
    "\n",
    "def parse_logical_form(logical_form: str) -> str:\n",
    "    \"\"\"Convert logical form to target format.\"\"\"\n",
    "    # [IN:balance [SL:ACCOUNT_TYPE àpò ìfowópamọ́ Dorm] ] => ACCOUNT_TYPE: àpò ìfowópamọ́ Dorm\n",
    "    # [IN:translate [SL:DISH_OR_FOOD nemi] [SL:LANGUAGE_NAME fɔ̃gbe] ] => DISH_OR_FOOD: nemi $$ LANGUAGE_NAME: fɔ̃gbe\n",
    "    parts = []\n",
    "    for part in re.findall(r\"\\[SL:([A-Z_]+) ([^\\]]+)\\]\", logical_form):\n",
    "        label, value = part\n",
    "        parts.append(f\"{label}: {value}\")\n",
    "    return \" $$ \".join(parts)\n",
    "\n",
    "summary_information = {}\n",
    "\n",
    "for language in data_loader.LANGUAGES + [\"eng\"]:\n",
    "    data = manager.load_data(language, \"split\")\n",
    "    \n",
    "    summary_information[language] = {\n",
    "        \"train\": len(data[\"train\"]),\n",
    "        \"dev\": len(data[\"dev\"]),\n",
    "        \"test\": len(data[\"test\"]),\n",
    "    }\n",
    "\n",
    "    for split, item in data.items():\n",
    "        # print(item.keys())\n",
    "        item['spans'] = item['spans'].fillna(\"\").apply(parse_spans)\n",
    "        item['target'] = item['logical_form'].apply(parse_logical_form)\n",
    "        # print(item.index)\n",
    "        item['example_id'] = f\"{split}-\" + \\\n",
    "            item.index.map(lambda x: f\"{x[0]}-{x[1]:08d}\" if isinstance(x, tuple) else f\"{x:08d}\")\n",
    "        item.drop(columns=['logical_form'], inplace=True) # , 'split'\n",
    "        os.makedirs(f\"data/cleaned-dataset/{language}\", exist_ok=True)\n",
    "        item.to_json(f\"data/cleaned-dataset/{language}/{split}.jsonl\", orient='records', lines=True, force_ascii=False)\n",
    "        \n",
    "summary_information = pd.DataFrame(summary_information).T\n",
    "summary_information\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rag",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}