src/EBI_analysis/analyze_missing_values.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# EBI Metadata - Missing Values Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('../../data/metadata_EBI_preprocessed.csv',\n",
    "                 dtype={'collected_by': object,\n",
    "                        'collection_date': object,\n",
    "                        'culture_collection': object,\n",
    "                        'identified_by': object,\n",
    "                        'isolate': object,\n",
    "                        'isolation_source': object,\n",
    "                        'keywords': object,\n",
    "                        'lab_host': object,\n",
    "                        'location': object,\n",
    "                        'sample_accession': object,\n",
    "                        'strain': object,\n",
    "                        'study_accession': object})"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Dependence on Missing Values to Countries and Institutes"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "df.isna().sum()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# Get names of columns with missing values\n",
    "cols_missing_vals = df.columns[df.isnull().any()]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "# Load number of inhabitants per country (taken from Wikipedia)\n",
    "with open('../helper_dicts/country_inhabitants.json') as f:\n",
    "    country_inhabitants_map = json.load(f)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "if not os.path.isdir('../../plots/missing_vals_per_col'):\n",
    "    os.mkdir('../../plots/missing_vals_per_col')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# Plot country distribution for each column with missing values\n",
    "for col in cols_missing_vals:\n",
    "    df_temp = df[df[col].isna()][[col, 'country']]\n",
    "\n",
    "    fig, ax = plt.subplots(figsize=(42, 22))\n",
    "    sns.countplot(data=df_temp,\n",
    "                  x='country',\n",
    "                  palette=['blue'] * len(df_temp['country']),\n",
    "                  order=df_temp['country'].value_counts().index)\n",
    "\n",
    "    fig.suptitle(f'Distribution of Countries for Rows with Missing Values in \"{col}\"', fontsize=22)\n",
    "    plt.xlabel('Country', fontsize=18)\n",
    "    plt.ylabel('Count Normalized by Number of Inhabitants', fontsize=18)\n",
    "    plt.xticks(fontsize=14, rotation=90)\n",
    "    plt.yticks(fontsize=14)\n",
    "\n",
    "    # Normalize height of bars with number of inhabitants\n",
    "    max_y = 0\n",
    "    for i, patch in enumerate(ax.patches):\n",
    "        country = ax.get_xticklabels()[i].get_text()\n",
    "        if country == 'Unknown':\n",
    "            new_height = 0.0\n",
    "        else:\n",
    "            new_height = patch.get_height() / country_inhabitants_map[country]\n",
    "\n",
    "        max_y = max(max_y, new_height)\n",
    "        patch.set_height(new_height)\n",
    "    plt.gca().set_ylim([0, max_y + (max_y / 20)])\n",
    "\n",
    "    plt.savefig(f'../../plots/missing_vals_per_col/country_count_of_missing_vals_in_{col}_normalized.png', dpi=300)\n",
    "    plt.close()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "df['collected_by'].value_counts()\n",
    "# 385 unique entries\n",
    "# -> cannot create above plots for this column (at least not out-of-the-box, would have to summarize and/or delete entries)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Missing Values Overview per Country"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    },
    {
     "data": {
      "text/plain": "           accession  base_count  collected_by  collection_date  \\\ncountry                                                           \nArgentina          0           0            43                0   \nArmenia            0           0            48                0   \nAustralia          0           0         12977                0   \nAustria            0           0           242                0   \nBahrain            0           0          2884                0   \n...              ...         ...           ...              ...   \nVenezuela          0           0            40                0   \nViet Nam           0           0             8                0   \nWest Bank          0           0            63                0   \nZambia             0           0             1                0   \nZimbabwe           0           0            98                0   \n\n           country_and_region  culture_collection  dataclass  description  \\\ncountry                                                                     \nArgentina                   0                  43          0            0   \nArmenia                     0                  48          0            0   \nAustralia                   0               13311          0            0   \nAustria                     0                 242          0            0   \nBahrain                     0                2884          0            0   \n...                       ...                 ...        ...          ...   \nVenezuela                   0                  40          0            0   \nViet Nam                    0                  11          0            0   \nWest Bank                   0                  63          0            0   \nZambia                      0                   1          0            0   \nZimbabwe                    0                  98          0            0   \n\n           environmental_sample  first_public  ...  sequence_md5  \\\ncountry                                        ...                 \nArgentina                     0             0  ...             0   \nArmenia                       0             0  ...             0   \nAustralia                     0             0  ...             0   \nAustria                       0             0  ...             0   \nBahrain                       0             0  ...             0   \n...                         ...           ...  ...           ...   \nVenezuela                     0             0  ...             0   \nViet Nam                      0             0  ...             0   \nWest Bank                     0             0  ...             0   \nZambia                        0             0  ...             0   \nZimbabwe                      0             0  ...             0   \n\n           sequence_version  strain  study_accession  tax_division  tax_id  \\\ncountry                                                                      \nArgentina                 0      43                0             0       0   \nArmenia                   0      48               48             0       0   \nAustralia                 0   13311             7755             0       0   \nAustria                   0     242              242             0       0   \nBahrain                   0    2884             2884             0       0   \n...                     ...     ...              ...           ...     ...   \nVenezuela                 0      40               40             0       0   \nViet Nam                  0      11               11             0       0   \nWest Bank                 0      63               63             0       0   \nZambia                    0       1                1             0       0   \nZimbabwe                  0      98                0             0       0   \n\n           topology  gender    age  n_inhabitants  \ncountry                                            \nArgentina         0       0     43              0  \nArmenia           0       0     48              0  \nAustralia         0       0  13311              0  \nAustria           0       0    242              0  \nBahrain           0       0   2884              0  \n...             ...     ...    ...            ...  \nVenezuela         0       0     40              0  \nViet Nam          0       0     11              0  \nWest Bank         0       0     63              0  \nZambia            0       0      1              0  \nZimbabwe          0       0     98              0  \n\n[104 rows x 32 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>accession</th>\n      <th>base_count</th>\n      <th>collected_by</th>\n      <th>collection_date</th>\n      <th>country_and_region</th>\n      <th>culture_collection</th>\n      <th>dataclass</th>\n      <th>description</th>\n      <th>environmental_sample</th>\n      <th>first_public</th>\n      <th>...</th>\n      <th>sequence_md5</th>\n      <th>sequence_version</th>\n      <th>strain</th>\n      <th>study_accession</th>\n      <th>tax_division</th>\n      <th>tax_id</th>\n      <th>topology</th>\n      <th>gender</th>\n      <th>age</th>\n      <th>n_inhabitants</th>\n    </tr>\n    <tr>\n      <th>country</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>Argentina</th>\n      <td>0</td>\n      <td>0</td>\n      <td>43</td>\n      <td>0</td>\n      <td>0</td>\n      <td>43</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>43</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>43</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>Armenia</th>\n      <td>0</td>\n      <td>0</td>\n      <td>48</td>\n      <td>0</td>\n      <td>0</td>\n      <td>48</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>48</td>\n      <td>48</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>48</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>Australia</th>\n      <td>0</td>\n      <td>0</td>\n      <td>12977</td>\n      <td>0</td>\n      <td>0</td>\n      <td>13311</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>13311</td>\n      <td>7755</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>13311</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>Austria</th>\n      <td>0</td>\n      <td>0</td>\n      <td>242</td>\n      <td>0</td>\n      <td>0</td>\n      <td>242</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>242</td>\n      <td>242</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>242</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>Bahrain</th>\n      <td>0</td>\n      <td>0</td>\n      <td>2884</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2884</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2884</td>\n      <td>2884</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2884</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>Venezuela</th>\n      <td>0</td>\n      <td>0</td>\n      <td>40</td>\n      <td>0</td>\n      <td>0</td>\n      <td>40</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>40</td>\n      <td>40</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>40</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>Viet Nam</th>\n      <td>0</td>\n      <td>0</td>\n      <td>8</td>\n      <td>0</td>\n      <td>0</td>\n      <td>11</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>11</td>\n      <td>11</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>11</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>West Bank</th>\n      <td>0</td>\n      <td>0</td>\n      <td>63</td>\n      <td>0</td>\n      <td>0</td>\n      <td>63</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>63</td>\n      <td>63</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>63</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>Zambia</th>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>Zimbabwe</th>\n      <td>0</td>\n      <td>0</td>\n      <td>98</td>\n      <td>0</td>\n      <td>0</td>\n      <td>98</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>0</td>\n      <td>0</td>\n      <td>98</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>98</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>104 rows × 32 columns</p>\n</div>"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get absolute amount of missing values per country and column\n",
    "n_missing = df.set_index('country').isna().sum(level=0).sort_index()\n",
    "n_missing"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "data": {
      "text/plain": "{'Argentina': 43,\n 'Armenia': 48,\n 'Australia': 13311,\n 'Austria': 242,\n 'Bahrain': 2884,\n 'Bangladesh': 666,\n 'Belarus': 2,\n 'Belgium': 5,\n 'Belize': 4,\n 'Benin': 12,\n 'Brazil': 322,\n 'Cambodia': 2,\n 'Cameroon': 1,\n 'Canada': 169,\n 'Chile': 383,\n 'China': 297,\n 'Colombia': 9,\n 'Croatia': 1,\n 'Cuba': 2,\n 'Czech Republic': 24,\n 'Denmark': 15,\n 'Djibouti': 288,\n 'Dominican Republic': 9,\n 'Ecuador': 4,\n 'Egypt': 1071,\n 'Estonia': 1681,\n 'Ethiopia': 7,\n 'Finland': 28,\n 'France': 1618,\n 'Gabon': 17,\n 'Gambia': 6,\n 'Georgia': 21,\n 'Germany': 165037,\n 'Ghana': 297,\n 'Greece': 98,\n 'Guam': 3,\n 'Guatemala': 10,\n 'Guinea': 13,\n 'Hong Kong': 347,\n 'Hungary': 36,\n 'Iceland': 5365,\n 'India': 2062,\n 'Indonesia': 9,\n 'Iran': 416,\n 'Iraq': 284,\n 'Israel': 125,\n 'Italy': 333,\n 'Jamaica': 8,\n 'Japan': 880,\n 'Jordan': 28,\n 'Kazakhstan': 18,\n 'Kenya': 2320,\n 'Lebanon': 931,\n 'Libya': 46,\n 'Liechtenstein': 164,\n 'Malaysia': 77,\n 'Mali': 42,\n 'Malta': 12,\n 'Mexico': 2922,\n 'Morocco': 17,\n 'Myanmar': 45,\n 'Nepal': 11,\n 'Netherlands': 1561,\n 'New Zealand': 2427,\n 'Nigeria': 383,\n 'Pakistan': 806,\n 'Palau': 2,\n 'Peru': 179,\n 'Philippines': 84,\n 'Poland': 1219,\n 'Portugal': 22,\n 'Puerto Rico': 304,\n 'Qatar': 6,\n 'Romania': 2,\n 'Russia': 358,\n 'Saudi Arabia': 1090,\n 'Serbia': 153,\n 'Seychelles': 451,\n 'Sierra Leone': 99,\n 'Singapore': 2,\n 'Slovakia': 6175,\n 'Somalia': 10,\n 'South Africa': 5,\n 'South Korea': 35,\n 'Spain': 1184,\n 'Sri Lanka': 6,\n 'Sweden': 10,\n 'Switzerland': 81810,\n 'Taiwan': 61,\n 'Thailand': 1106,\n 'Timor-Leste': 29,\n 'Togo': 11,\n 'Tunisia': 278,\n 'Turkey': 78,\n 'USA': 1257315,\n 'Uganda': 29,\n 'United Kingdom': 957374,\n 'Uruguay': 25,\n 'Uzbekistan': 71,\n 'Venezuela': 40,\n 'Viet Nam': 11,\n 'West Bank': 63,\n 'Zambia': 1,\n 'Zimbabwe': 98}"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get how often each country appears overall\n",
    "country_counts = df.groupby('country').size().to_dict()\n",
    "country_counts"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "data": {
      "text/plain": "           accession  base_count  collected_by  collection_date  \\\ncountry                                                           \nArgentina        0.0         0.0      1.000000              0.0   \nArmenia          0.0         0.0      1.000000              0.0   \nAustralia        0.0         0.0      0.974908              0.0   \nAustria          0.0         0.0      1.000000              0.0   \nBahrain          0.0         0.0      1.000000              0.0   \n...              ...         ...           ...              ...   \nVenezuela        0.0         0.0      1.000000              0.0   \nViet Nam         0.0         0.0      0.727273              0.0   \nWest Bank        0.0         0.0      1.000000              0.0   \nZambia           0.0         0.0      1.000000              0.0   \nZimbabwe         0.0         0.0      1.000000              0.0   \n\n           country_and_region  culture_collection  dataclass  description  \\\ncountry                                                                     \nArgentina                 0.0                 1.0        0.0          0.0   \nArmenia                   0.0                 1.0        0.0          0.0   \nAustralia                 0.0                 1.0        0.0          0.0   \nAustria                   0.0                 1.0        0.0          0.0   \nBahrain                   0.0                 1.0        0.0          0.0   \n...                       ...                 ...        ...          ...   \nVenezuela                 0.0                 1.0        0.0          0.0   \nViet Nam                  0.0                 1.0        0.0          0.0   \nWest Bank                 0.0                 1.0        0.0          0.0   \nZambia                    0.0                 1.0        0.0          0.0   \nZimbabwe                  0.0                 1.0        0.0          0.0   \n\n           environmental_sample  first_public  ...  sequence_md5  \\\ncountry                                        ...                 \nArgentina                   0.0           0.0  ...           0.0   \nArmenia                     0.0           0.0  ...           0.0   \nAustralia                   0.0           0.0  ...           0.0   \nAustria                     0.0           0.0  ...           0.0   \nBahrain                     0.0           0.0  ...           0.0   \n...                         ...           ...  ...           ...   \nVenezuela                   0.0           0.0  ...           0.0   \nViet Nam                    0.0           0.0  ...           0.0   \nWest Bank                   0.0           0.0  ...           0.0   \nZambia                      0.0           0.0  ...           0.0   \nZimbabwe                    0.0           0.0  ...           0.0   \n\n           sequence_version  strain  study_accession  tax_division  tax_id  \\\ncountry                                                                      \nArgentina               0.0     1.0         0.000000           0.0     0.0   \nArmenia                 0.0     1.0         1.000000           0.0     0.0   \nAustralia               0.0     1.0         0.582601           0.0     0.0   \nAustria                 0.0     1.0         1.000000           0.0     0.0   \nBahrain                 0.0     1.0         1.000000           0.0     0.0   \n...                     ...     ...              ...           ...     ...   \nVenezuela               0.0     1.0         1.000000           0.0     0.0   \nViet Nam                0.0     1.0         1.000000           0.0     0.0   \nWest Bank               0.0     1.0         1.000000           0.0     0.0   \nZambia                  0.0     1.0         1.000000           0.0     0.0   \nZimbabwe                0.0     1.0         0.000000           0.0     0.0   \n\n           topology  gender  age  n_inhabitants  \ncountry                                          \nArgentina       0.0     0.0  1.0            0.0  \nArmenia         0.0     0.0  1.0            0.0  \nAustralia       0.0     0.0  1.0            0.0  \nAustria         0.0     0.0  1.0            0.0  \nBahrain         0.0     0.0  1.0            0.0  \n...             ...     ...  ...            ...  \nVenezuela       0.0     0.0  1.0            0.0  \nViet Nam        0.0     0.0  1.0            0.0  \nWest Bank       0.0     0.0  1.0            0.0  \nZambia          0.0     0.0  1.0            0.0  \nZimbabwe        0.0     0.0  1.0            0.0  \n\n[104 rows x 32 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>accession</th>\n      <th>base_count</th>\n      <th>collected_by</th>\n      <th>collection_date</th>\n      <th>country_and_region</th>\n      <th>culture_collection</th>\n      <th>dataclass</th>\n      <th>description</th>\n      <th>environmental_sample</th>\n      <th>first_public</th>\n      <th>...</th>\n      <th>sequence_md5</th>\n      <th>sequence_version</th>\n      <th>strain</th>\n      <th>study_accession</th>\n      <th>tax_division</th>\n      <th>tax_id</th>\n      <th>topology</th>\n      <th>gender</th>\n      <th>age</th>\n      <th>n_inhabitants</th>\n    </tr>\n    <tr>\n      <th>country</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>Argentina</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.000000</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>...</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.000000</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>Armenia</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.000000</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>...</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>1.000000</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>Australia</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.974908</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>...</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.582601</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>Austria</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.000000</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>...</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>1.000000</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>Bahrain</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.000000</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>...</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>1.000000</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>Venezuela</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.000000</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>...</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>1.000000</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>Viet Nam</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.727273</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>...</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>1.000000</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>West Bank</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.000000</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>...</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>1.000000</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>Zambia</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.000000</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>...</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>1.000000</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n    <tr>\n      <th>Zimbabwe</th>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.000000</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>...</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.000000</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n    </tr>\n  </tbody>\n</table>\n<p>104 rows × 32 columns</p>\n</div>"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert absolute numbers into percentages/ fractions\n",
    "percentage_missing = n_missing.div(country_counts, axis=0)\n",
    "percentage_missing"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "if not os.path.isdir('../../plots/missing_vals_per_country'):\n",
    "    os.mkdir('../../plots/missing_vals_per_country')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# For each country, generate a barplot showing the amount of missing values (as fraction) for each column\n",
    "sns.set_style('whitegrid')\n",
    "cols = percentage_missing.columns.tolist()\n",
    "\n",
    "for country, count in country_counts.items():\n",
    "    fig, ax = plt.subplots(figsize=(16, 10))\n",
    "    country_details = percentage_missing[percentage_missing.index == country].T[country].tolist()\n",
    "    plt.bar(cols, country_details)\n",
    "\n",
    "    fig.suptitle(f'Missing Values per Column for {country} ({count} Samples)', fontsize=22)\n",
    "    plt.xlabel('Column Names', fontsize=18)\n",
    "    plt.ylabel('Missing Values', fontsize=18)\n",
    "    plt.xticks(fontsize=14, rotation=90)\n",
    "    plt.yticks(fontsize=14)\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(f'../../plots/missing_vals_per_country/missing_vals_for_{country.replace(\" \", \"_\")}.png', dpi=300)\n",
    "    plt.close()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}