cso_benchmark_heavytable.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# Lookup classes\n",
    "# Each key column requires an input class which maps from the source datatype to an integer\n",
    "\n",
    "class IntegerLookup:\n",
    "    def get(self, values):\n",
    "        return values\n",
    "\n",
    "class BoundIntLookup:\n",
    "    def __init__(self, lower, upper):\n",
    "        self.lower = int(lower)\n",
    "        self.upper = int(upper)\n",
    "    \n",
    "    def get(self, numpy_array):\n",
    "        return np.clip(numpy_array, self.lower, self.upper)\n",
    "\n",
    "class BandLookup:\n",
    "    def __init__(self, upper_bounds, labels):\n",
    "        \"\"\"Inputs must be sorted\"\"\"\n",
    "        self.upper_bounds = np.array(upper_bounds)\n",
    "        self.labels = np.array(labels)\n",
    "\n",
    "    def get(self, numpy_array):\n",
    "        \"\"\"get the labels for vector\"\"\"\n",
    "        indices = np.searchsorted(self.upper_bounds, numpy_array)\n",
    "        return self.labels[indices]\n",
    "    \n",
    "    def get_max(self, numpy_array):\n",
    "        \"\"\"get the upper band limit\"\"\"\n",
    "        indices = np.searchsorted(self.upper_bounds, numpy_array)\n",
    "        return self.upper_bounds[indices]\n",
    "    \n",
    "    @classmethod\n",
    "    def from_dataframe(cls, df:pd.DataFrame, band_column_name, integer_column_name):\n",
    "        df_copy = df.sort_values(band_column_name)\n",
    "        band_arr = df_copy[band_column_name].values\n",
    "        band_labels = df_copy[integer_column_name].values\n",
    "        return cls(band_arr, band_labels)\n",
    "    \n",
    "    @classmethod\n",
    "    def from_excel_sheet(cls, workbook_path, sheet_name):\n",
    "        \"\"\"Extract from an excel sheet, there must be two columns, the band column and the integer column\"\"\"\n",
    "        df = pd.read_excel(workbook_path, sheet_name=sheet_name)\n",
    "        assert len(df.columns) == 2, \"must have 2 columns\"\n",
    "        band_col_name, int_col_name = df.columns[0], df.columns[1]\n",
    "        return cls.from_dataframe(df, band_col_name, int_col_name)\n",
    "\n",
    "class IntKeyTable:\n",
    "    \"\"\"Vectorised Table Allowing for multiple integer keys\"\"\"\n",
    "    def __init__(self, df: pd.DataFrame):\n",
    "        self.key_cols = list(df.columns[:-1])\n",
    "        self.value_col = df.columns[-1]\n",
    "\n",
    "        # ensure all key_cols are integers\n",
    "        for col in self.key_cols:\n",
    "            df[col] = df[col].astype(int)\n",
    "\n",
    "        self.df = df.sort_values(by=list(reversed(self.key_cols))) # TODO: don't need to store this in self, this but handy for testing.\n",
    "        self.bases = [min(self.df[col]) for col in self.key_cols]\n",
    "        self.ranges = [max(self.df[col]) - min(self.df[col]) + 1 for col in self.key_cols]\n",
    "\n",
    "        # set up index scalars\n",
    "        total_scalar = 1\n",
    "        self.scalars = []\n",
    "        for col in self.ranges:\n",
    "            self.scalars.append(total_scalar)\n",
    "            total_scalar *= col\n",
    "\n",
    "        self.values = self.df[self.value_col].values\n",
    "\n",
    "        expected_rows = np.prod(self.ranges)\n",
    "        assert expected_rows == len(self.values)\n",
    "\n",
    "    def get_index(self, *keys):\n",
    "        index = 0\n",
    "        for key, base, scalar in zip(keys, self.bases, self.scalars):\n",
    "            index += (key - base) * scalar\n",
    "        return index\n",
    "    \n",
    "    def get_value(self, *keys):\n",
    "        indices = self.get_index(*keys)\n",
    "        return self.values[indices]\n",
    "    \n",
    "    def __getitem__(self, keys):\n",
    "        return self.get_value(*keys)\n",
    "    \n",
    "class Table:\n",
    "    \"\"\"Table provides multi-key compatible high performance table lookup.\n",
    "    \"\"\"\n",
    "    col_types = \"int\", \"int_bound\", \"str\", \"band\", \"float\"\n",
    "\n",
    "    def __init__(self, df:pd.DataFrame):\n",
    "        \"\"\"Initialise a table from a dataframe.\n",
    "        \n",
    "        Tables should be in long format:\n",
    "         - the final column containing the values to look up\n",
    "         - all other columns contain keys to lookup\n",
    "         - tables should be contingous, i.e. no gaps in integer keys.\n",
    "         - tables should be complete if viewed as square matrixes (i.e. all combinations of keys are input).  If not, you should fill any gaps with np.nan or a suitable value.\n",
    "\n",
    "        The type of key is determined by the suffix on the column name:\n",
    "        `|int`: integers (...0, 1, 2, 3...), can start and end anywhere, but must be consecutive\n",
    "        `|int_bound`: as `|int` but any values are constrained to the lowest and highest values.\n",
    "        `|str': keys are interpreted as strings, e.g. 'M' and 'F'\n",
    "        `|band`: key is numeric and treated as the upper bound on a lookup.\n",
    "        `|float`: not currently available due to floating point equality, use int or band depending on use case.\n",
    "        \"\"\"\n",
    "        key_cols = list(df.columns[:-1])\n",
    "        df = df.sort_values(key_cols[::-1]) # sort by reverse order\n",
    "\n",
    "        df_int_keys = df.copy() # this will have keys overridden as we work through mappers\n",
    "\n",
    "        # prepare the mappers\n",
    "        self.mappers = []\n",
    "        for col in key_cols:\n",
    "            col_type = col.split(\"|\")[1] # \"int\", \"str\" etc\n",
    "            if col_type == \"int\":\n",
    "                self.mappers.append(IntegerLookup()) # just so we have .get (a bit inefficient?)\n",
    "            elif col_type == \"int_bound\":\n",
    "                # bound integer forces values to be between the lowest and highest value in the table, for example maximum durations in mortality tables.\n",
    "                lower = df[col].min()\n",
    "                upper = df[col].max()\n",
    "                self.mappers.append(BoundIntLookup(lower=lower, upper=upper))\n",
    "            elif col_type in [\"str\", \"band\"]:\n",
    "                df_col = pd.DataFrame(df[col].unique(), columns=[\"band_name\"]).reset_index().sort_values(\"band_name\")\n",
    "\n",
    "                # add a nan on the end so we get errors if the lookup fails\n",
    "                # as by default the BandLookup will return last item if no earlier matches\n",
    "                # commented out as it converts the datatype - we need these to be integers\n",
    "                # df_col.loc[len(df_col)] = len(df_col), np.nan\n",
    "                band_mapper = BandLookup.from_dataframe(df_col, \"band_name\", \"index\")\n",
    "                self.mappers.append(band_mapper)\n",
    "                df_int_keys[col] = band_mapper.get(df_int_keys[col])\n",
    "            else:\n",
    "                raise NotImplementedError(f\"{col_type} not implemented on {col}\")\n",
    "        \n",
    "        # create an intkeytable\n",
    "\n",
    "        self._int_key_table = IntKeyTable(df_int_keys)\n",
    "        self.df = df # store for referencing TODO: decide if this should be removed.\n",
    "    \n",
    "    def get(self, *keys):\n",
    "        assert len(keys) == len(self.mappers)\n",
    "        # TODO: if just one key then this doesn't work?  (needs fixed throughout)\n",
    "        int_keys = [mapper.get(key) for key, mapper in zip(keys, self.mappers)]\n",
    "        return self._int_key_table.get_value(*int_keys)\n",
    "\n",
    "    def __getitem__(self, keys):\n",
    "        # print(keys, type(keys))\n",
    "        if not isinstance(keys, tuple):\n",
    "            keys = keys, #force to be a tuple\n",
    "        return self.get(*keys)\n",
    "\n",
    "    def __repr__(self):\n",
    "        # TODO: return a nice representation of the table, e.g. head/keys etc.\n",
    "        return repr(self.df)\n",
    "\n",
    "    @classmethod\n",
    "    def read_excel(cls, spreadsheet_path, sheet_name):\n",
    "        \"\"\"Read in a table from an excel sheet\"\"\"\n",
    "        df = pd.read_excel(spreadsheet_path, sheet_name=sheet_name)\n",
    "        return cls(df)\n",
    "    \n",
    "    @classmethod\n",
    "    def read_csv(cls, csv_path):\n",
    "        df = pd.read_csv(csv_path)\n",
    "        return cls(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('assumption_tables/2017_loaded_CSO_mortality_rates_heavytable.csv')\n",
    "# remove the IssueAge column, it is redundant\n",
    "df = df.drop(columns=['IssueAge|int'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "AssertionError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[9], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mTable\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdf\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[0;32mIn[1], line 144\u001b[0m, in \u001b[0;36mTable.__init__\u001b[0;34m(self, df)\u001b[0m\n\u001b[1;32m    140\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mNotImplementedError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcol_type\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m not implemented on \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcol\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    142\u001b[0m \u001b[38;5;66;03m# create an intkeytable\u001b[39;00m\n\u001b[0;32m--> 144\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_int_key_table \u001b[38;5;241m=\u001b[39m \u001b[43mIntKeyTable\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdf_int_keys\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    145\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdf \u001b[38;5;241m=\u001b[39m df\n",
      "Cell \u001b[0;32mIn[1], line 77\u001b[0m, in \u001b[0;36mIntKeyTable.__init__\u001b[0;34m(self, df)\u001b[0m\n\u001b[1;32m     74\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mvalues \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdf[\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mvalue_col]\u001b[38;5;241m.\u001b[39mvalues\n\u001b[1;32m     76\u001b[0m expected_rows \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mprod(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mranges)\n\u001b[0;32m---> 77\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m expected_rows \u001b[38;5;241m==\u001b[39m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mvalues)\n",
      "\u001b[0;31mAssertionError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "Table(df)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}