03_preprocessing_2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import sys, os\n",
    "data_dir = '../noaa-runtime/data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dst_labs = pd.read_csv(data_dir+'dst_labels.csv' ,index_col=[\"period\", \"timedelta\"],)\n",
    "sat_pos  = pd.read_csv(data_dir+'satellite_positions.csv',index_col=[\"period\", \"timedelta\"],)\n",
    "sw       = pd.read_csv(data_dir+'solar_wind.csv',index_col=[\"period\", \"timedelta\"],)\n",
    "sunspots = pd.read_csv(data_dir+'sunspots.csv',index_col=[\"period\", \"timedelta\"],)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Interpolate and aggregate other files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# drop the unnecessary columns in sw file\n",
    "cols_to_drop= ['bx_gsm','by_gsm', 'bz_gsm', 'theta_gsm', 'phi_gsm', 'source']\n",
    "sw.drop(cols_to_drop, axis=1, inplace=True)\n",
    "\n",
    "# drop the unnecessary columns in sat_pos file\n",
    "cols_to_drop = ['gse_x_dscovr', 'gse_y_dscovr', 'gse_z_dscovr']\n",
    "sat_pos.drop(cols_to_drop, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>dst</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">train_a</th>\n",
       "      <th>0 days 00:00:00</th>\n",
       "      <td>-7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 01:00:00</th>\n",
       "      <td>-10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         dst\n",
       "period  timedelta           \n",
       "train_a 0 days 00:00:00   -7\n",
       "        0 days 01:00:00  -10"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>gse_x_ace</th>\n",
       "      <th>gse_y_ace</th>\n",
       "      <th>gse_z_ace</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">train_a</th>\n",
       "      <th>0 days</th>\n",
       "      <td>1522376.9</td>\n",
       "      <td>143704.6</td>\n",
       "      <td>149496.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1 days</th>\n",
       "      <td>1525410.9</td>\n",
       "      <td>136108.8</td>\n",
       "      <td>151034.1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   gse_x_ace  gse_y_ace  gse_z_ace\n",
       "period  timedelta                                 \n",
       "train_a 0 days     1522376.9   143704.6   149496.7\n",
       "        1 days     1525410.9   136108.8   151034.1"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>bx_gse</th>\n",
       "      <th>by_gse</th>\n",
       "      <th>bz_gse</th>\n",
       "      <th>theta_gse</th>\n",
       "      <th>phi_gse</th>\n",
       "      <th>bt</th>\n",
       "      <th>density</th>\n",
       "      <th>speed</th>\n",
       "      <th>temperature</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">train_a</th>\n",
       "      <th>0 days 00:00:00</th>\n",
       "      <td>-5.55</td>\n",
       "      <td>3.00</td>\n",
       "      <td>1.25</td>\n",
       "      <td>11.09</td>\n",
       "      <td>153.37</td>\n",
       "      <td>6.80</td>\n",
       "      <td>1.53</td>\n",
       "      <td>383.92</td>\n",
       "      <td>110237.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 00:01:00</th>\n",
       "      <td>-5.58</td>\n",
       "      <td>3.16</td>\n",
       "      <td>1.17</td>\n",
       "      <td>10.10</td>\n",
       "      <td>151.91</td>\n",
       "      <td>6.83</td>\n",
       "      <td>1.69</td>\n",
       "      <td>381.79</td>\n",
       "      <td>123825.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         bx_gse  by_gse  bz_gse  theta_gse  phi_gse    bt  \\\n",
       "period  timedelta                                                           \n",
       "train_a 0 days 00:00:00   -5.55    3.00    1.25      11.09   153.37  6.80   \n",
       "        0 days 00:01:00   -5.58    3.16    1.17      10.10   151.91  6.83   \n",
       "\n",
       "                         density   speed  temperature  \n",
       "period  timedelta                                      \n",
       "train_a 0 days 00:00:00     1.53  383.92     110237.0  \n",
       "        0 days 00:01:00     1.69  381.79     123825.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>smoothed_ssn</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">train_a</th>\n",
       "      <th>0 days 00:00:00</th>\n",
       "      <td>65.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13 days 00:00:00</th>\n",
       "      <td>72.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          smoothed_ssn\n",
       "period  timedelta                     \n",
       "train_a 0 days 00:00:00           65.4\n",
       "        13 days 00:00:00          72.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display ( dst_labs.head(2) )\n",
    "display ( sat_pos.head(2) )\n",
    "display ( sw.head(2) )\n",
    "display ( sunspots.head(2) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# joining one file at a time as it was taking a long time to join when taken multiple at the same time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of all (8392320, 9) (139872, 1) (5828, 3) (192, 1) (8392320, 10)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>bx_gse</th>\n",
       "      <th>by_gse</th>\n",
       "      <th>bz_gse</th>\n",
       "      <th>theta_gse</th>\n",
       "      <th>phi_gse</th>\n",
       "      <th>bt</th>\n",
       "      <th>density</th>\n",
       "      <th>speed</th>\n",
       "      <th>temperature</th>\n",
       "      <th>dst</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">train_a</th>\n",
       "      <th>0 days 00:00:00</th>\n",
       "      <td>-5.55</td>\n",
       "      <td>3.00</td>\n",
       "      <td>1.25</td>\n",
       "      <td>11.09</td>\n",
       "      <td>153.37</td>\n",
       "      <td>6.80</td>\n",
       "      <td>1.53</td>\n",
       "      <td>383.92</td>\n",
       "      <td>110237.0</td>\n",
       "      <td>-7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 00:01:00</th>\n",
       "      <td>-5.58</td>\n",
       "      <td>3.16</td>\n",
       "      <td>1.17</td>\n",
       "      <td>10.10</td>\n",
       "      <td>151.91</td>\n",
       "      <td>6.83</td>\n",
       "      <td>1.69</td>\n",
       "      <td>381.79</td>\n",
       "      <td>123825.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         bx_gse  by_gse  bz_gse  theta_gse  phi_gse    bt  \\\n",
       "period  timedelta                                                           \n",
       "train_a 0 days 00:00:00   -5.55    3.00    1.25      11.09   153.37  6.80   \n",
       "        0 days 00:01:00   -5.58    3.16    1.17      10.10   151.91  6.83   \n",
       "\n",
       "                         density   speed  temperature  dst  \n",
       "period  timedelta                                           \n",
       "train_a 0 days 00:00:00     1.53  383.92     110237.0 -7.0  \n",
       "        0 days 00:01:00     1.69  381.79     123825.0  NaN  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final = sw.join(dst_labs)\n",
    "print (\"shape of all\", sw.shape, dst_labs.shape, sat_pos.shape, sunspots.shape, df_final.shape)\n",
    "df_final.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of all (8392320, 9) (139872, 1) (5828, 3) (192, 1) (8392320, 13)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>bx_gse</th>\n",
       "      <th>by_gse</th>\n",
       "      <th>bz_gse</th>\n",
       "      <th>theta_gse</th>\n",
       "      <th>phi_gse</th>\n",
       "      <th>bt</th>\n",
       "      <th>density</th>\n",
       "      <th>speed</th>\n",
       "      <th>temperature</th>\n",
       "      <th>dst</th>\n",
       "      <th>gse_x_ace</th>\n",
       "      <th>gse_y_ace</th>\n",
       "      <th>gse_z_ace</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">train_a</th>\n",
       "      <th>0 days 00:00:00</th>\n",
       "      <td>-5.55</td>\n",
       "      <td>3.00</td>\n",
       "      <td>1.25</td>\n",
       "      <td>11.09</td>\n",
       "      <td>153.37</td>\n",
       "      <td>6.80</td>\n",
       "      <td>1.53</td>\n",
       "      <td>383.92</td>\n",
       "      <td>110237.0</td>\n",
       "      <td>-7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 00:01:00</th>\n",
       "      <td>-5.58</td>\n",
       "      <td>3.16</td>\n",
       "      <td>1.17</td>\n",
       "      <td>10.10</td>\n",
       "      <td>151.91</td>\n",
       "      <td>6.83</td>\n",
       "      <td>1.69</td>\n",
       "      <td>381.79</td>\n",
       "      <td>123825.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         bx_gse  by_gse  bz_gse  theta_gse  phi_gse    bt  \\\n",
       "period  timedelta                                                           \n",
       "train_a 0 days 00:00:00   -5.55    3.00    1.25      11.09   153.37  6.80   \n",
       "        0 days 00:01:00   -5.58    3.16    1.17      10.10   151.91  6.83   \n",
       "\n",
       "                         density   speed  temperature  dst  gse_x_ace  \\\n",
       "period  timedelta                                                       \n",
       "train_a 0 days 00:00:00     1.53  383.92     110237.0 -7.0        NaN   \n",
       "        0 days 00:01:00     1.69  381.79     123825.0  NaN        NaN   \n",
       "\n",
       "                         gse_y_ace  gse_z_ace  \n",
       "period  timedelta                              \n",
       "train_a 0 days 00:00:00        NaN        NaN  \n",
       "        0 days 00:01:00        NaN        NaN  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final = df_final.join(sat_pos)\n",
    "print (\"shape of all\", sw.shape, dst_labs.shape, sat_pos.shape, sunspots.shape, df_final.shape)\n",
    "df_final.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of all (8392320, 9) (139872, 1) (5828, 3) (192, 1) (8392320, 14)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>bx_gse</th>\n",
       "      <th>by_gse</th>\n",
       "      <th>bz_gse</th>\n",
       "      <th>theta_gse</th>\n",
       "      <th>phi_gse</th>\n",
       "      <th>bt</th>\n",
       "      <th>density</th>\n",
       "      <th>speed</th>\n",
       "      <th>temperature</th>\n",
       "      <th>dst</th>\n",
       "      <th>gse_x_ace</th>\n",
       "      <th>gse_y_ace</th>\n",
       "      <th>gse_z_ace</th>\n",
       "      <th>smoothed_ssn</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">train_a</th>\n",
       "      <th>0 days 00:00:00</th>\n",
       "      <td>-5.55</td>\n",
       "      <td>3.00</td>\n",
       "      <td>1.25</td>\n",
       "      <td>11.09</td>\n",
       "      <td>153.37</td>\n",
       "      <td>6.80</td>\n",
       "      <td>1.53</td>\n",
       "      <td>383.92</td>\n",
       "      <td>110237.0</td>\n",
       "      <td>-7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>65.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 00:01:00</th>\n",
       "      <td>-5.58</td>\n",
       "      <td>3.16</td>\n",
       "      <td>1.17</td>\n",
       "      <td>10.10</td>\n",
       "      <td>151.91</td>\n",
       "      <td>6.83</td>\n",
       "      <td>1.69</td>\n",
       "      <td>381.79</td>\n",
       "      <td>123825.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         bx_gse  by_gse  bz_gse  theta_gse  phi_gse    bt  \\\n",
       "period  timedelta                                                           \n",
       "train_a 0 days 00:00:00   -5.55    3.00    1.25      11.09   153.37  6.80   \n",
       "        0 days 00:01:00   -5.58    3.16    1.17      10.10   151.91  6.83   \n",
       "\n",
       "                         density   speed  temperature  dst  gse_x_ace  \\\n",
       "period  timedelta                                                       \n",
       "train_a 0 days 00:00:00     1.53  383.92     110237.0 -7.0        NaN   \n",
       "        0 days 00:01:00     1.69  381.79     123825.0  NaN        NaN   \n",
       "\n",
       "                         gse_y_ace  gse_z_ace  smoothed_ssn  \n",
       "period  timedelta                                            \n",
       "train_a 0 days 00:00:00        NaN        NaN          65.4  \n",
       "        0 days 00:01:00        NaN        NaN           NaN  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final = df_final.join(sunspots)\n",
    "print (\"shape of all\", sw.shape, dst_labs.shape, sat_pos.shape, sunspots.shape, df_final.shape)\n",
    "df_final.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>gse_x_ace</th>\n",
       "      <th>gse_y_ace</th>\n",
       "      <th>gse_z_ace</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">train_a</th>\n",
       "      <th>0 days</th>\n",
       "      <td>1522376.9</td>\n",
       "      <td>143704.6</td>\n",
       "      <td>149496.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1 days</th>\n",
       "      <td>1525410.9</td>\n",
       "      <td>136108.8</td>\n",
       "      <td>151034.1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   gse_x_ace  gse_y_ace  gse_z_ace\n",
       "period  timedelta                                 \n",
       "train_a 0 days     1522376.9   143704.6   149496.7\n",
       "        1 days     1525410.9   136108.8   151034.1"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sat_pos.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>gse_x_ace</th>\n",
       "      <th>gse_y_ace</th>\n",
       "      <th>gse_z_ace</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">train_a</th>\n",
       "      <th>0 days</th>\n",
       "      <td>1522376.9</td>\n",
       "      <td>143704.6</td>\n",
       "      <td>149496.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1 days</th>\n",
       "      <td>1525410.9</td>\n",
       "      <td>136108.8</td>\n",
       "      <td>151034.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2 days</th>\n",
       "      <td>1528484.9</td>\n",
       "      <td>128470.5</td>\n",
       "      <td>152387.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3 days</th>\n",
       "      <td>1531570.3</td>\n",
       "      <td>120818.4</td>\n",
       "      <td>153561.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4 days</th>\n",
       "      <td>1534633.1</td>\n",
       "      <td>113178.5</td>\n",
       "      <td>154534.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">train_c</th>\n",
       "      <th>2431 days</th>\n",
       "      <td>1413526.2</td>\n",
       "      <td>218154.6</td>\n",
       "      <td>141105.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2432 days</th>\n",
       "      <td>1416712.3</td>\n",
       "      <td>222471.2</td>\n",
       "      <td>141261.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2433 days</th>\n",
       "      <td>1420050.2</td>\n",
       "      <td>226690.3</td>\n",
       "      <td>141247.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2434 days</th>\n",
       "      <td>1423478.8</td>\n",
       "      <td>230817.3</td>\n",
       "      <td>141070.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2435 days</th>\n",
       "      <td>1426937.9</td>\n",
       "      <td>234813.7</td>\n",
       "      <td>140701.8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5828 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   gse_x_ace  gse_y_ace  gse_z_ace\n",
       "period  timedelta                                 \n",
       "train_a 0 days     1522376.9   143704.6   149496.7\n",
       "        1 days     1525410.9   136108.8   151034.1\n",
       "        2 days     1528484.9   128470.5   152387.7\n",
       "        3 days     1531570.3   120818.4   153561.4\n",
       "        4 days     1534633.1   113178.5   154534.6\n",
       "...                      ...        ...        ...\n",
       "train_c 2431 days  1413526.2   218154.6   141105.6\n",
       "        2432 days  1416712.3   222471.2   141261.8\n",
       "        2433 days  1420050.2   226690.3   141247.2\n",
       "        2434 days  1423478.8   230817.3   141070.4\n",
       "        2435 days  1426937.9   234813.7   140701.8\n",
       "\n",
       "[5828 rows x 3 columns]"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sat_pos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "Cannot convert input [0 days 00:00:00] of type <class 'pandas._libs.tslibs.timedeltas.Timedelta'> to Timestamp",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-77-0e35f27ba0a6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdates\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdate_range\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdff\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdff\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfreq\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'M'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/indexes/datetimes.py\u001b[0m in \u001b[0;36mdate_range\u001b[0;34m(start, end, periods, freq, tz, normalize, name, closed, **kwargs)\u001b[0m\n\u001b[1;32m   1170\u001b[0m         \u001b[0mfreq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"D\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1171\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1172\u001b[0;31m     dtarr = DatetimeArray._generate_range(\n\u001b[0m\u001b[1;32m   1173\u001b[0m         \u001b[0mstart\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1174\u001b[0m         \u001b[0mend\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/arrays/datetimes.py\u001b[0m in \u001b[0;36m_generate_range\u001b[0;34m(cls, start, end, periods, freq, tz, normalize, ambiguous, nonexistent, closed)\u001b[0m\n\u001b[1;32m    363\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    364\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mstart\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 365\u001b[0;31m             \u001b[0mstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTimestamp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    366\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    367\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mend\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/tslibs/timestamps.pyx\u001b[0m in \u001b[0;36mpandas._libs.tslibs.timestamps.Timestamp.__new__\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/tslibs/conversion.pyx\u001b[0m in \u001b[0;36mpandas._libs.tslibs.conversion.convert_to_tsobject\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: Cannot convert input [0 days 00:00:00] of type <class 'pandas._libs.tslibs.timedeltas.Timedelta'> to Timestamp"
     ]
    }
   ],
   "source": [
    "dates = pd.date_range(dff.index[0], dff.index[-1], freq='M')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "dff = sat_pos.reset_index(level='period')\n",
    "dff.index = pd.to_timedelta(dff.index)\n",
    "#dff = dff.groupby('period').resample('H').interpolate(method='linear', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "Cannot convert input [0 days 00:00:00] of type <class 'pandas._libs.tslibs.timedeltas.Timedelta'> to Timestamp",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-79-be12e8c922db>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdates\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdate_range\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdff\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdff\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfreq\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'M'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mdates\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/indexes/datetimes.py\u001b[0m in \u001b[0;36mdate_range\u001b[0;34m(start, end, periods, freq, tz, normalize, name, closed, **kwargs)\u001b[0m\n\u001b[1;32m   1170\u001b[0m         \u001b[0mfreq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"D\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1171\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1172\u001b[0;31m     dtarr = DatetimeArray._generate_range(\n\u001b[0m\u001b[1;32m   1173\u001b[0m         \u001b[0mstart\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1174\u001b[0m         \u001b[0mend\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/arrays/datetimes.py\u001b[0m in \u001b[0;36m_generate_range\u001b[0;34m(cls, start, end, periods, freq, tz, normalize, ambiguous, nonexistent, closed)\u001b[0m\n\u001b[1;32m    363\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    364\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mstart\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 365\u001b[0;31m             \u001b[0mstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTimestamp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    366\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    367\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mend\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/tslibs/timestamps.pyx\u001b[0m in \u001b[0;36mpandas._libs.tslibs.timestamps.Timestamp.__new__\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/tslibs/conversion.pyx\u001b[0m in \u001b[0;36mpandas._libs.tslibs.conversion.convert_to_tsobject\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: Cannot convert input [0 days 00:00:00] of type <class 'pandas._libs.tslibs.timedeltas.Timedelta'> to Timestamp"
     ]
    }
   ],
   "source": [
    "dates = pd.date_range(dff.index[0], dff.index[-1], freq='M')\n",
    "dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.resample('D').interpolate('cubic')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "Cannot convert input [0 days 00:00:00] of type <class 'pandas._libs.tslibs.timedeltas.Timedelta'> to Timestamp",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-70-c1764d28982f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m#dates = pd.date_range(start, end, freq='D')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m#ts_daily = ts_mon.reindex(dates, method='ffill')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mdates\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdate_range\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdff\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdff\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfreq\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'M'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0;31m#dff_ = dff.reindex(dates).ffill()#, method='ffill')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/indexes/datetimes.py\u001b[0m in \u001b[0;36mdate_range\u001b[0;34m(start, end, periods, freq, tz, normalize, name, closed, **kwargs)\u001b[0m\n\u001b[1;32m   1170\u001b[0m         \u001b[0mfreq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"D\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1171\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1172\u001b[0;31m     dtarr = DatetimeArray._generate_range(\n\u001b[0m\u001b[1;32m   1173\u001b[0m         \u001b[0mstart\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1174\u001b[0m         \u001b[0mend\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/arrays/datetimes.py\u001b[0m in \u001b[0;36m_generate_range\u001b[0;34m(cls, start, end, periods, freq, tz, normalize, ambiguous, nonexistent, closed)\u001b[0m\n\u001b[1;32m    363\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    364\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mstart\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 365\u001b[0;31m             \u001b[0mstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTimestamp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    366\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    367\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mend\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/tslibs/timestamps.pyx\u001b[0m in \u001b[0;36mpandas._libs.tslibs.timestamps.Timestamp.__new__\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/tslibs/conversion.pyx\u001b[0m in \u001b[0;36mpandas._libs.tslibs.conversion.convert_to_tsobject\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: Cannot convert input [0 days 00:00:00] of type <class 'pandas._libs.tslibs.timedeltas.Timedelta'> to Timestamp"
     ]
    }
   ],
   "source": [
    "#dates = pd.date_range(start, end, freq='D')\n",
    "#ts_daily = ts_mon.reindex(dates, method='ffill')\n",
    "dates = pd.date_range(dff.index[0], dff.index[-1], freq='M')\n",
    "#dff_ = dff.reindex(dates).ffill()#, method='ffill')\n",
    "\n",
    "dff_ = dff.resample('M').ffill()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "<MonthEnd> is a non-fixed frequency",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-58-a1ec84ce40be>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mdff\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msat_pos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'period'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mdff\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_timedelta\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdff\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mdff\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdff\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroupby\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'period'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'M'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mffill\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0mdff\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/resample.py\u001b[0m in \u001b[0;36mpad\u001b[0;34m(self, limit)\u001b[0m\n\u001b[1;32m    451\u001b[0m         \u001b[0mDataFrame\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfillna\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    452\u001b[0m         \"\"\"\n\u001b[0;32m--> 453\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_upsample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"pad\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlimit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlimit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    454\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    455\u001b[0m     \u001b[0mffill\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpad\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/resample.py\u001b[0m in \u001b[0;36m_apply\u001b[0;34m(self, f, grouper, *args, **kwargs)\u001b[0m\n\u001b[1;32m    989\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    990\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 991\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_groupby\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    992\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wrap_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    993\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/groupby/groupby.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, func, *args, **kwargs)\u001b[0m\n\u001b[1;32m    734\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0moption_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"mode.chained_assignment\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    735\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 736\u001b[0;31m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_python_apply_general\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    737\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    738\u001b[0m                 \u001b[0;31m# gh-20949\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/groupby/groupby.py\u001b[0m in \u001b[0;36m_python_apply_general\u001b[0;34m(self, f)\u001b[0m\n\u001b[1;32m    750\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    751\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_python_apply_general\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 752\u001b[0;31m         \u001b[0mkeys\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmutated\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrouper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_selected_obj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    753\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    754\u001b[0m         return self._wrap_applied_output(\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/groupby/ops.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, f, data, axis)\u001b[0m\n\u001b[1;32m    204\u001b[0m             \u001b[0;31m# group might be modified\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    205\u001b[0m             \u001b[0mgroup_axes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgroup\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxes\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 206\u001b[0;31m             \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    207\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0m_is_indexed_like\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mres\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroup_axes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m                 \u001b[0mmutated\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/resample.py\u001b[0m in \u001b[0;36mfunc\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m    985\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    986\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 987\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    988\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    989\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/resample.py\u001b[0m in \u001b[0;36mpad\u001b[0;34m(self, limit)\u001b[0m\n\u001b[1;32m    451\u001b[0m         \u001b[0mDataFrame\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfillna\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    452\u001b[0m         \"\"\"\n\u001b[0;32m--> 453\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_upsample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"pad\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlimit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlimit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    454\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    455\u001b[0m     \u001b[0mffill\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpad\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/resample.py\u001b[0m in \u001b[0;36m_upsample\u001b[0;34m(self, method, limit, fill_value)\u001b[0m\n\u001b[1;32m   1072\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1073\u001b[0m         \"\"\"\n\u001b[0;32m-> 1074\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_binner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1075\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1076\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mAssertionError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"axis must be 0\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/resample.py\u001b[0m in \u001b[0;36m_set_binner\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    176\u001b[0m         \"\"\"\n\u001b[1;32m    177\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbinner\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 178\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbinner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrouper\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_binner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    179\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    180\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_get_binner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/resample.py\u001b[0m in \u001b[0;36m_get_binner\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    184\u001b[0m         \"\"\"\n\u001b[1;32m    185\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 186\u001b[0;31m         \u001b[0mbinner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbinlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_binner_for_time\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    187\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbins\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbinlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    188\u001b[0m         \u001b[0mbin_grouper\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBinGrouper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbinlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroupby\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/resample.py\u001b[0m in \u001b[0;36m_get_binner_for_time\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1241\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1242\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_get_binner_for_time\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1243\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroupby\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_time_delta_bins\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1244\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1245\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_adjust_binner_for_upsample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbinner\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/resample.py\u001b[0m in \u001b[0;36m_get_time_delta_bins\u001b[0;34m(self, ax)\u001b[0m\n\u001b[1;32m   1501\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1502\u001b[0m         \u001b[0mstart\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1503\u001b[0;31m         labels = binner = timedelta_range(\n\u001b[0m\u001b[1;32m   1504\u001b[0m             \u001b[0mstart\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfreq\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfreq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1505\u001b[0m         )\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/indexes/timedeltas.py\u001b[0m in \u001b[0;36mtimedelta_range\u001b[0;34m(start, end, periods, freq, name, closed)\u001b[0m\n\u001b[1;32m    506\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    507\u001b[0m     \u001b[0mfreq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfreq_infer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdtl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmaybe_infer_freq\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfreq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 508\u001b[0;31m     \u001b[0mtdarr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTimedeltaArray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_generate_range\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperiods\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfreq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclosed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    509\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mTimedeltaIndex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_simple_new\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtdarr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfreq\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtdarr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfreq\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/arrays/timedeltas.py\u001b[0m in \u001b[0;36m_generate_range\u001b[0;34m(cls, start, end, periods, freq, closed)\u001b[0m\n\u001b[1;32m    259\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    260\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mfreq\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 261\u001b[0;31m             \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_generate_regular_range\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperiods\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfreq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    262\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    263\u001b[0m             \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinspace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperiods\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"i8\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/arrays/timedeltas.py\u001b[0m in \u001b[0;36m_generate_regular_range\u001b[0;34m(start, end, periods, offset)\u001b[0m\n\u001b[1;32m   1071\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1072\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_generate_regular_range\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mperiods\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moffset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1073\u001b[0;31m     \u001b[0mstride\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moffset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnanos\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1074\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mperiods\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1075\u001b[0m         \u001b[0mb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTimedelta\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/tseries/offsets.py\u001b[0m in \u001b[0;36mnanos\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    486\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    487\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mnanos\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 488\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{self} is a non-fixed frequency\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    489\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    490\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: <MonthEnd> is a non-fixed frequency"
     ]
    }
   ],
   "source": [
    "#df_bfill = df.groupby('house').resample('D').bfill().drop('house', axis=1)\n",
    "\n",
    "dff = sat_pos.reset_index(level='period')\n",
    "dff.index = pd.to_timedelta(dff.index)\n",
    "dff = dff.groupby('period').resample('M').ffill()\n",
    "dff.head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method NDFrame.head of             period  gse_x_ace  gse_y_ace  gse_z_ace\n",
       "timedelta                                          \n",
       "0 days     train_a  1522376.9   143704.6   149496.7\n",
       "1 days     train_a  1525410.9   136108.8   151034.1\n",
       "2 days     train_a  1528484.9   128470.5   152387.7\n",
       "3 days     train_a  1531570.3   120818.4   153561.4\n",
       "4 days     train_a  1534633.1   113178.5   154534.6\n",
       "...            ...        ...        ...        ...\n",
       "2431 days  train_c  1413526.2   218154.6   141105.6\n",
       "2432 days  train_c  1416712.3   222471.2   141261.8\n",
       "2433 days  train_c  1420050.2   226690.3   141247.2\n",
       "2434 days  train_c  1423478.8   230817.3   141070.4\n",
       "2435 days  train_c  1426937.9   234813.7   140701.8\n",
       "\n",
       "[5828 rows x 4 columns]>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#df.loc[(\"baz\", \"two\"):(\"qux\", \"one\")]\n",
    "dff = sat_pos.reset_index(level='period')\n",
    "dff.index = pd.to_timedelta(dff.index)\n",
    "dff.groupby('period').resample('M')\n",
    "dff.head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def using_reset_index(df):\n",
    "    df = df.reset_index(level='period')\n",
    "    df.index=pd.to_timedelta(df.index)\n",
    "    return df.groupby('period').resample('H').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "bx_gse           325888\n",
       "by_gse           325888\n",
       "bz_gse           325888\n",
       "theta_gse        325888\n",
       "phi_gse          326388\n",
       "bt               325888\n",
       "density          684890\n",
       "speed            689555\n",
       "temperature      811768\n",
       "dst             8252448\n",
       "gse_x_ace       8392320\n",
       "gse_y_ace       8392320\n",
       "gse_z_ace       8392320\n",
       "smoothed_ssn    8392128\n",
       "dtype: int64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "since the first values are `NaN`s we can't use interpolate. So let's try bfill. this can be done two ways\n",
    "\n",
    "- using `df.bfill`\n",
    "- using `df.fillna(method= 'bfill')`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>bx_gse</th>\n",
       "      <th>by_gse</th>\n",
       "      <th>bz_gse</th>\n",
       "      <th>theta_gse</th>\n",
       "      <th>phi_gse</th>\n",
       "      <th>bt</th>\n",
       "      <th>density</th>\n",
       "      <th>speed</th>\n",
       "      <th>temperature</th>\n",
       "      <th>dst</th>\n",
       "      <th>gse_x_ace</th>\n",
       "      <th>gse_y_ace</th>\n",
       "      <th>gse_z_ace</th>\n",
       "      <th>smoothed_ssn</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">train_a</th>\n",
       "      <th>0 days 00:00:00</th>\n",
       "      <td>-5.55</td>\n",
       "      <td>3.00</td>\n",
       "      <td>1.25</td>\n",
       "      <td>11.09</td>\n",
       "      <td>153.37</td>\n",
       "      <td>6.80</td>\n",
       "      <td>1.53</td>\n",
       "      <td>383.92</td>\n",
       "      <td>110237.0</td>\n",
       "      <td>-7.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>65.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 00:01:00</th>\n",
       "      <td>-5.58</td>\n",
       "      <td>3.16</td>\n",
       "      <td>1.17</td>\n",
       "      <td>10.10</td>\n",
       "      <td>151.91</td>\n",
       "      <td>6.83</td>\n",
       "      <td>1.69</td>\n",
       "      <td>381.79</td>\n",
       "      <td>123825.0</td>\n",
       "      <td>-10.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>72.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 00:02:00</th>\n",
       "      <td>-5.15</td>\n",
       "      <td>3.66</td>\n",
       "      <td>0.85</td>\n",
       "      <td>7.87</td>\n",
       "      <td>146.04</td>\n",
       "      <td>6.77</td>\n",
       "      <td>1.97</td>\n",
       "      <td>389.11</td>\n",
       "      <td>82548.0</td>\n",
       "      <td>-10.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>72.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 00:03:00</th>\n",
       "      <td>-5.20</td>\n",
       "      <td>3.68</td>\n",
       "      <td>0.68</td>\n",
       "      <td>6.17</td>\n",
       "      <td>146.17</td>\n",
       "      <td>6.74</td>\n",
       "      <td>1.97</td>\n",
       "      <td>389.11</td>\n",
       "      <td>82548.0</td>\n",
       "      <td>-10.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>72.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 00:04:00</th>\n",
       "      <td>-5.12</td>\n",
       "      <td>3.68</td>\n",
       "      <td>0.49</td>\n",
       "      <td>4.62</td>\n",
       "      <td>145.72</td>\n",
       "      <td>6.65</td>\n",
       "      <td>1.77</td>\n",
       "      <td>384.26</td>\n",
       "      <td>94269.0</td>\n",
       "      <td>-10.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>72.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">train_c</th>\n",
       "      <th>2435 days 23:55:00</th>\n",
       "      <td>-1.18</td>\n",
       "      <td>2.00</td>\n",
       "      <td>-3.92</td>\n",
       "      <td>-59.28</td>\n",
       "      <td>120.51</td>\n",
       "      <td>4.56</td>\n",
       "      <td>4.64</td>\n",
       "      <td>348.73</td>\n",
       "      <td>23368.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2435 days 23:56:00</th>\n",
       "      <td>-1.34</td>\n",
       "      <td>2.12</td>\n",
       "      <td>-3.83</td>\n",
       "      <td>-56.78</td>\n",
       "      <td>122.23</td>\n",
       "      <td>4.58</td>\n",
       "      <td>4.27</td>\n",
       "      <td>346.36</td>\n",
       "      <td>26497.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2435 days 23:57:00</th>\n",
       "      <td>-1.62</td>\n",
       "      <td>2.33</td>\n",
       "      <td>-3.63</td>\n",
       "      <td>-51.98</td>\n",
       "      <td>124.76</td>\n",
       "      <td>4.61</td>\n",
       "      <td>3.95</td>\n",
       "      <td>344.12</td>\n",
       "      <td>27050.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2435 days 23:58:00</th>\n",
       "      <td>-2.27</td>\n",
       "      <td>2.48</td>\n",
       "      <td>-3.22</td>\n",
       "      <td>-43.81</td>\n",
       "      <td>132.50</td>\n",
       "      <td>4.65</td>\n",
       "      <td>2.81</td>\n",
       "      <td>338.52</td>\n",
       "      <td>33257.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2435 days 23:59:00</th>\n",
       "      <td>-2.15</td>\n",
       "      <td>2.41</td>\n",
       "      <td>-3.30</td>\n",
       "      <td>-45.55</td>\n",
       "      <td>131.76</td>\n",
       "      <td>4.62</td>\n",
       "      <td>2.30</td>\n",
       "      <td>342.31</td>\n",
       "      <td>32267.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8392320 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                            bx_gse  by_gse  bz_gse  theta_gse  phi_gse    bt  \\\n",
       "period  timedelta                                                              \n",
       "train_a 0 days 00:00:00      -5.55    3.00    1.25      11.09   153.37  6.80   \n",
       "        0 days 00:01:00      -5.58    3.16    1.17      10.10   151.91  6.83   \n",
       "        0 days 00:02:00      -5.15    3.66    0.85       7.87   146.04  6.77   \n",
       "        0 days 00:03:00      -5.20    3.68    0.68       6.17   146.17  6.74   \n",
       "        0 days 00:04:00      -5.12    3.68    0.49       4.62   145.72  6.65   \n",
       "...                            ...     ...     ...        ...      ...   ...   \n",
       "train_c 2435 days 23:55:00   -1.18    2.00   -3.92     -59.28   120.51  4.56   \n",
       "        2435 days 23:56:00   -1.34    2.12   -3.83     -56.78   122.23  4.58   \n",
       "        2435 days 23:57:00   -1.62    2.33   -3.63     -51.98   124.76  4.61   \n",
       "        2435 days 23:58:00   -2.27    2.48   -3.22     -43.81   132.50  4.65   \n",
       "        2435 days 23:59:00   -2.15    2.41   -3.30     -45.55   131.76  4.62   \n",
       "\n",
       "                            density   speed  temperature   dst  gse_x_ace  \\\n",
       "period  timedelta                                                           \n",
       "train_a 0 days 00:00:00        1.53  383.92     110237.0  -7.0        NaN   \n",
       "        0 days 00:01:00        1.69  381.79     123825.0 -10.0        NaN   \n",
       "        0 days 00:02:00        1.97  389.11      82548.0 -10.0        NaN   \n",
       "        0 days 00:03:00        1.97  389.11      82548.0 -10.0        NaN   \n",
       "        0 days 00:04:00        1.77  384.26      94269.0 -10.0        NaN   \n",
       "...                             ...     ...          ...   ...        ...   \n",
       "train_c 2435 days 23:55:00     4.64  348.73      23368.0   NaN        NaN   \n",
       "        2435 days 23:56:00     4.27  346.36      26497.0   NaN        NaN   \n",
       "        2435 days 23:57:00     3.95  344.12      27050.0   NaN        NaN   \n",
       "        2435 days 23:58:00     2.81  338.52      33257.0   NaN        NaN   \n",
       "        2435 days 23:59:00     2.30  342.31      32267.0   NaN        NaN   \n",
       "\n",
       "                            gse_y_ace  gse_z_ace  smoothed_ssn  \n",
       "period  timedelta                                               \n",
       "train_a 0 days 00:00:00           NaN        NaN          65.4  \n",
       "        0 days 00:01:00           NaN        NaN          72.0  \n",
       "        0 days 00:02:00           NaN        NaN          72.0  \n",
       "        0 days 00:03:00           NaN        NaN          72.0  \n",
       "        0 days 00:04:00           NaN        NaN          72.0  \n",
       "...                               ...        ...           ...  \n",
       "train_c 2435 days 23:55:00        NaN        NaN           NaN  \n",
       "        2435 days 23:56:00        NaN        NaN           NaN  \n",
       "        2435 days 23:57:00        NaN        NaN           NaN  \n",
       "        2435 days 23:58:00        NaN        NaN           NaN  \n",
       "        2435 days 23:59:00        NaN        NaN           NaN  \n",
       "\n",
       "[8392320 rows x 14 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final.fillna(method='bfill')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-a52156e9ebf5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdf_sw_dst_sat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf_sw_dst\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0msat_pos\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msunspots\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mdf_sw_dst_sat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36mjoin\u001b[0;34m(self, other, on, how, lsuffix, rsuffix, sort)\u001b[0m\n\u001b[1;32m   7206\u001b[0m         \u001b[0;36m5\u001b[0m  \u001b[0mK5\u001b[0m  \u001b[0mA5\u001b[0m  \u001b[0mNaN\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   7207\u001b[0m         \"\"\"\n\u001b[0;32m-> 7208\u001b[0;31m         return self._join_compat(\n\u001b[0m\u001b[1;32m   7209\u001b[0m             \u001b[0mother\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mon\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mon\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhow\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlsuffix\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlsuffix\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrsuffix\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrsuffix\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msort\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msort\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   7210\u001b[0m         )\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m_join_compat\u001b[0;34m(self, other, on, how, lsuffix, rsuffix, sort)\u001b[0m\n\u001b[1;32m   7245\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mcan_concat\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   7246\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mhow\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"left\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 7247\u001b[0;31m                     res = concat(\n\u001b[0m\u001b[1;32m   7248\u001b[0m                         \u001b[0mframes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjoin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"outer\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverify_integrity\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msort\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msort\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   7249\u001b[0m                     )\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/reshape/concat.py\u001b[0m in \u001b[0;36mconcat\u001b[0;34m(objs, axis, join, ignore_index, keys, levels, names, verify_integrity, sort, copy)\u001b[0m\n\u001b[1;32m    282\u001b[0m     )\n\u001b[1;32m    283\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 284\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    285\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    286\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/reshape/concat.py\u001b[0m in \u001b[0;36mget_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    494\u001b[0m                 \u001b[0mmgrs_indexers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindexers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 496\u001b[0;31m             new_data = concatenate_block_managers(\n\u001b[0m\u001b[1;32m    497\u001b[0m                 \u001b[0mmgrs_indexers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnew_axes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconcat_axis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    498\u001b[0m             )\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/internals/managers.py\u001b[0m in \u001b[0;36mconcatenate_block_managers\u001b[0;34m(mgrs_indexers, axes, concat_axis, copy)\u001b[0m\n\u001b[1;32m   2020\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2021\u001b[0m             b = make_block(\n\u001b[0;32m-> 2022\u001b[0;31m                 \u001b[0mconcatenate_join_units\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjoin_units\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconcat_axis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2023\u001b[0m                 \u001b[0mplacement\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mplacement\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2024\u001b[0m             )\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/internals/concat.py\u001b[0m in \u001b[0;36mconcatenate_join_units\u001b[0;34m(join_units, concat_axis, copy)\u001b[0m\n\u001b[1;32m    244\u001b[0m     \u001b[0mempty_dtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mupcasted_na\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_get_empty_dtype_and_na\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjoin_units\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    245\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 246\u001b[0;31m     to_concat = [\n\u001b[0m\u001b[1;32m    247\u001b[0m         \u001b[0mju\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_reindexed_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mempty_dtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mempty_dtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mupcasted_na\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mupcasted_na\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    248\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mju\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mjoin_units\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/internals/concat.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    245\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    246\u001b[0m     to_concat = [\n\u001b[0;32m--> 247\u001b[0;31m         \u001b[0mju\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_reindexed_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mempty_dtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mempty_dtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mupcasted_na\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mupcasted_na\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    248\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mju\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mjoin_units\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    249\u001b[0m     ]\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/internals/concat.py\u001b[0m in \u001b[0;36mget_reindexed_values\u001b[0;34m(self, empty_dtype, upcasted_na)\u001b[0m\n\u001b[1;32m    229\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    230\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindexer\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindexers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 231\u001b[0;31m                 \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0malgos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtake_nd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfill_value\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfill_value\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    232\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mvalues\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.8/site-packages/pandas/core/algorithms.py\u001b[0m in \u001b[0;36mtake_nd\u001b[0;34m(arr, indexer, axis, out, fill_value, allow_fill)\u001b[0m\n\u001b[1;32m   1660\u001b[0m         \u001b[0marr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask_info\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmask_info\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1661\u001b[0m     )\n\u001b[0;32m-> 1662\u001b[0;31m     \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfill_value\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1663\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1664\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mflip_order\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "df_sw_dst_sat = df_sw_dst.join([sat_pos, sunspots])\n",
    "df_sw_dst_sat.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final = df_sw_dst_sat.join(sunspots)\n",
    "df_final.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Interpolation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The Null values in the data:\n",
      " bx_gse         325888\n",
      "by_gse         325888\n",
      "bz_gse         325888\n",
      "theta_gse      325888\n",
      "phi_gse        326388\n",
      "bt             325888\n",
      "density        684890\n",
      "speed          689555\n",
      "temperature    811768\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print (\"The Null values in the data:\\n\", sw.isnull().sum() )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "sw.interpolate(method='linear', limit_direction='forward', axis=0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of the data shape: (8392320, 9)\n",
      "Null values after interpolation: 0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>bx_gse</th>\n",
       "      <th>by_gse</th>\n",
       "      <th>bz_gse</th>\n",
       "      <th>theta_gse</th>\n",
       "      <th>phi_gse</th>\n",
       "      <th>bt</th>\n",
       "      <th>density</th>\n",
       "      <th>speed</th>\n",
       "      <th>temperature</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">train_a</th>\n",
       "      <th>0 days 00:00:00</th>\n",
       "      <td>-5.55</td>\n",
       "      <td>3.00</td>\n",
       "      <td>1.25</td>\n",
       "      <td>11.09</td>\n",
       "      <td>153.37</td>\n",
       "      <td>6.80</td>\n",
       "      <td>1.53</td>\n",
       "      <td>383.92</td>\n",
       "      <td>110237.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 00:01:00</th>\n",
       "      <td>-5.58</td>\n",
       "      <td>3.16</td>\n",
       "      <td>1.17</td>\n",
       "      <td>10.10</td>\n",
       "      <td>151.91</td>\n",
       "      <td>6.83</td>\n",
       "      <td>1.69</td>\n",
       "      <td>381.79</td>\n",
       "      <td>123825.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 00:02:00</th>\n",
       "      <td>-5.15</td>\n",
       "      <td>3.66</td>\n",
       "      <td>0.85</td>\n",
       "      <td>7.87</td>\n",
       "      <td>146.04</td>\n",
       "      <td>6.77</td>\n",
       "      <td>1.97</td>\n",
       "      <td>389.11</td>\n",
       "      <td>82548.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 00:03:00</th>\n",
       "      <td>-5.20</td>\n",
       "      <td>3.68</td>\n",
       "      <td>0.68</td>\n",
       "      <td>6.17</td>\n",
       "      <td>146.17</td>\n",
       "      <td>6.74</td>\n",
       "      <td>1.97</td>\n",
       "      <td>389.11</td>\n",
       "      <td>82548.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 00:04:00</th>\n",
       "      <td>-5.12</td>\n",
       "      <td>3.68</td>\n",
       "      <td>0.49</td>\n",
       "      <td>4.62</td>\n",
       "      <td>145.72</td>\n",
       "      <td>6.65</td>\n",
       "      <td>1.77</td>\n",
       "      <td>384.26</td>\n",
       "      <td>94269.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         bx_gse  by_gse  bz_gse  theta_gse  phi_gse    bt  \\\n",
       "period  timedelta                                                           \n",
       "train_a 0 days 00:00:00   -5.55    3.00    1.25      11.09   153.37  6.80   \n",
       "        0 days 00:01:00   -5.58    3.16    1.17      10.10   151.91  6.83   \n",
       "        0 days 00:02:00   -5.15    3.66    0.85       7.87   146.04  6.77   \n",
       "        0 days 00:03:00   -5.20    3.68    0.68       6.17   146.17  6.74   \n",
       "        0 days 00:04:00   -5.12    3.68    0.49       4.62   145.72  6.65   \n",
       "\n",
       "                         density   speed  temperature  \n",
       "period  timedelta                                      \n",
       "train_a 0 days 00:00:00     1.53  383.92     110237.0  \n",
       "        0 days 00:01:00     1.69  381.79     123825.0  \n",
       "        0 days 00:02:00     1.97  389.11      82548.0  \n",
       "        0 days 00:03:00     1.97  389.11      82548.0  \n",
       "        0 days 00:04:00     1.77  384.26      94269.0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print (\"shape of the data shape:\", sw.shape)\n",
    "print (\"Null values after interpolation:\", sw.isnull().sum().sum())\n",
    "sw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def using_reset_index(df):\n",
    "    df = df.reset_index(level='period')\n",
    "    df.index=pd.to_timedelta(df.index)\n",
    "    return df.groupby('period').resample('H').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of the data shape: (139872, 9)\n",
      "Null values : 0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>bx_gse</th>\n",
       "      <th>by_gse</th>\n",
       "      <th>bz_gse</th>\n",
       "      <th>theta_gse</th>\n",
       "      <th>phi_gse</th>\n",
       "      <th>bt</th>\n",
       "      <th>density</th>\n",
       "      <th>speed</th>\n",
       "      <th>temperature</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">train_a</th>\n",
       "      <th>00:00:00</th>\n",
       "      <td>-6.149500</td>\n",
       "      <td>1.6455</td>\n",
       "      <td>0.844667</td>\n",
       "      <td>7.103667</td>\n",
       "      <td>165.420333</td>\n",
       "      <td>7.140000</td>\n",
       "      <td>1.174167</td>\n",
       "      <td>355.278250</td>\n",
       "      <td>68880.341667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>01:00:00</th>\n",
       "      <td>-6.693833</td>\n",
       "      <td>0.7665</td>\n",
       "      <td>1.265167</td>\n",
       "      <td>10.183333</td>\n",
       "      <td>174.267833</td>\n",
       "      <td>7.284833</td>\n",
       "      <td>0.815917</td>\n",
       "      <td>330.080583</td>\n",
       "      <td>52203.225000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     bx_gse  by_gse    bz_gse  theta_gse     phi_gse  \\\n",
       "period  timedelta                                                      \n",
       "train_a 00:00:00  -6.149500  1.6455  0.844667   7.103667  165.420333   \n",
       "        01:00:00  -6.693833  0.7665  1.265167  10.183333  174.267833   \n",
       "\n",
       "                         bt   density       speed   temperature  \n",
       "period  timedelta                                                \n",
       "train_a 00:00:00   7.140000  1.174167  355.278250  68880.341667  \n",
       "        01:00:00   7.284833  0.815917  330.080583  52203.225000  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sw_ipt_agg_ml=using_reset_index(sw)\n",
    "print (\"shape of the data shape:\", sw_ipt_agg_ml.shape)\n",
    "print (\"Null values :\", sw_ipt_agg_ml.isnull().sum().sum())\n",
    "sw_ipt_agg_ml.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## join `dst labels` to the original solar wind data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of the data dst_labs: (139872, 1)\n",
      "Null values: 0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>dst</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">train_a</th>\n",
       "      <th>0 days 00:00:00</th>\n",
       "      <td>-7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 01:00:00</th>\n",
       "      <td>-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 02:00:00</th>\n",
       "      <td>-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 03:00:00</th>\n",
       "      <td>-6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0 days 04:00:00</th>\n",
       "      <td>-2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         dst\n",
       "period  timedelta           \n",
       "train_a 0 days 00:00:00   -7\n",
       "        0 days 01:00:00  -10\n",
       "        0 days 02:00:00  -10\n",
       "        0 days 03:00:00   -6\n",
       "        0 days 04:00:00   -2"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dst_labs = pd.read_csv(data_dir+'dst_labels.csv' ,index_col=[\"period\", \"timedelta\"],)\n",
    "print (\"shape of the data dst_labs:\", dst_labs.shape)\n",
    "print (\"Null values:\", dst_labs.isnull().sum().sum())\n",
    "dst_labs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sample only hourly data\n",
    "dst_labs = using_reset_index(dst_labs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>bx_gse</th>\n",
       "      <th>by_gse</th>\n",
       "      <th>bz_gse</th>\n",
       "      <th>theta_gse</th>\n",
       "      <th>phi_gse</th>\n",
       "      <th>bt</th>\n",
       "      <th>density</th>\n",
       "      <th>speed</th>\n",
       "      <th>temperature</th>\n",
       "      <th>dst</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">train_a</th>\n",
       "      <th>00:00:00</th>\n",
       "      <td>-6.149500</td>\n",
       "      <td>1.6455</td>\n",
       "      <td>0.844667</td>\n",
       "      <td>7.103667</td>\n",
       "      <td>165.420333</td>\n",
       "      <td>7.140000</td>\n",
       "      <td>1.174167</td>\n",
       "      <td>355.278250</td>\n",
       "      <td>68880.341667</td>\n",
       "      <td>-7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>01:00:00</th>\n",
       "      <td>-6.693833</td>\n",
       "      <td>0.7665</td>\n",
       "      <td>1.265167</td>\n",
       "      <td>10.183333</td>\n",
       "      <td>174.267833</td>\n",
       "      <td>7.284833</td>\n",
       "      <td>0.815917</td>\n",
       "      <td>330.080583</td>\n",
       "      <td>52203.225000</td>\n",
       "      <td>-10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     bx_gse  by_gse    bz_gse  theta_gse     phi_gse  \\\n",
       "period  timedelta                                                      \n",
       "train_a 00:00:00  -6.149500  1.6455  0.844667   7.103667  165.420333   \n",
       "        01:00:00  -6.693833  0.7665  1.265167  10.183333  174.267833   \n",
       "\n",
       "                         bt   density       speed   temperature  dst  \n",
       "period  timedelta                                                     \n",
       "train_a 00:00:00   7.140000  1.174167  355.278250  68880.341667   -7  \n",
       "        01:00:00   7.284833  0.815917  330.080583  52203.225000  -10  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sw_dst=sw_ipt_agg_ml.join(dst_labs)\n",
    "df_sw_dst.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(139872, 9) (139872, 1) (139872, 10)\n"
     ]
    }
   ],
   "source": [
    "print(sw_ipt_agg_ml.shape , dst_labs.shape, df_sw_dst.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## join `Sunspots data` to the dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of the data sunspots: (192, 1)\n",
      "Null values: 0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>smoothed_ssn</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">train_a</th>\n",
       "      <th>00:00:00</th>\n",
       "      <td>65.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>01:00:00</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02:00:00</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>03:00:00</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>04:00:00</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   smoothed_ssn\n",
       "period  timedelta              \n",
       "train_a 00:00:00           65.4\n",
       "        01:00:00            NaN\n",
       "        02:00:00            NaN\n",
       "        03:00:00            NaN\n",
       "        04:00:00            NaN"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sunspots = pd.read_csv(data_dir+'sunspots.csv',index_col=[\"period\", \"timedelta\"],)\n",
    "print (\"shape of the data sunspots:\", sunspots.shape)\n",
    "print (\"Null values:\", sunspots.isnull().sum().sum())\n",
    "sunspots = using_reset_index(sunspots)\n",
    "sunspots.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(139872, 9) (139872, 1) (139872, 10) (137643, 1)\n"
     ]
    }
   ],
   "source": [
    "print(sw_ipt_agg_ml.shape, dst_labs.shape, df_sw_dst.shape, sunspots.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "the rows are slightly different so, let's join the dataframe anyway and then replace the NULL data by interpolation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sw_dst_ss=df_sw_dst.join(sunspots)\n",
    "df_sw_dst_ss['smoothed_ssn']=df_sw_dst_ss[['smoothed_ssn']].interpolate(method='linear', limit_direction='forward', axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of all: (139872, 9) (139872, 1) (139872, 10) (137643, 1)\n",
      "Total Null values: 0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>bx_gse</th>\n",
       "      <th>by_gse</th>\n",
       "      <th>bz_gse</th>\n",
       "      <th>theta_gse</th>\n",
       "      <th>phi_gse</th>\n",
       "      <th>bt</th>\n",
       "      <th>density</th>\n",
       "      <th>speed</th>\n",
       "      <th>temperature</th>\n",
       "      <th>dst</th>\n",
       "      <th>smoothed_ssn</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">train_c</th>\n",
       "      <th>2435 days 22:00:00</th>\n",
       "      <td>-2.3850</td>\n",
       "      <td>2.860333</td>\n",
       "      <td>-2.221667</td>\n",
       "      <td>-29.645667</td>\n",
       "      <td>130.601500</td>\n",
       "      <td>4.716667</td>\n",
       "      <td>2.148917</td>\n",
       "      <td>343.488167</td>\n",
       "      <td>41803.641667</td>\n",
       "      <td>-15</td>\n",
       "      <td>42.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2435 days 23:00:00</th>\n",
       "      <td>-2.2515</td>\n",
       "      <td>2.619333</td>\n",
       "      <td>-2.821000</td>\n",
       "      <td>-39.064000</td>\n",
       "      <td>130.277667</td>\n",
       "      <td>4.554833</td>\n",
       "      <td>2.283583</td>\n",
       "      <td>343.848750</td>\n",
       "      <td>36191.908333</td>\n",
       "      <td>-14</td>\n",
       "      <td>42.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            bx_gse    by_gse    bz_gse  theta_gse     phi_gse  \\\n",
       "period  timedelta                                                               \n",
       "train_c 2435 days 22:00:00 -2.3850  2.860333 -2.221667 -29.645667  130.601500   \n",
       "        2435 days 23:00:00 -2.2515  2.619333 -2.821000 -39.064000  130.277667   \n",
       "\n",
       "                                  bt   density       speed   temperature  dst  \\\n",
       "period  timedelta                                                               \n",
       "train_c 2435 days 22:00:00  4.716667  2.148917  343.488167  41803.641667  -15   \n",
       "        2435 days 23:00:00  4.554833  2.283583  343.848750  36191.908333  -14   \n",
       "\n",
       "                            smoothed_ssn  \n",
       "period  timedelta                         \n",
       "train_c 2435 days 22:00:00          42.5  \n",
       "        2435 days 23:00:00          42.5  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print (\"shape of all:\", sw_ipt_agg_ml.shape, dst_labs.shape, df_sw_dst.shape, sunspots.shape)\n",
    "print (\"Total Null values:\", df_sw_dst_ss.isnull().sum().sum())\n",
    "df_sw_dst_ss.tail(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## join `Satellite Position` to the dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of the data sunspots: (5828, 6)\n",
      "Null values: 14382\n",
      "Columns: Index(['gse_x_ace', 'gse_y_ace', 'gse_z_ace', 'gse_x_dscovr', 'gse_y_dscovr',\n",
      "       'gse_z_dscovr'],\n",
      "      dtype='object')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>gse_x_ace</th>\n",
       "      <th>gse_y_ace</th>\n",
       "      <th>gse_z_ace</th>\n",
       "      <th>gse_x_dscovr</th>\n",
       "      <th>gse_y_dscovr</th>\n",
       "      <th>gse_z_dscovr</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">train_a</th>\n",
       "      <th>00:00:00</th>\n",
       "      <td>1522376.9</td>\n",
       "      <td>143704.6</td>\n",
       "      <td>149496.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>01:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   gse_x_ace  gse_y_ace  gse_z_ace  gse_x_dscovr  \\\n",
       "period  timedelta                                                  \n",
       "train_a 00:00:00   1522376.9   143704.6   149496.7           NaN   \n",
       "        01:00:00         NaN        NaN        NaN           NaN   \n",
       "\n",
       "                   gse_y_dscovr  gse_z_dscovr  \n",
       "period  timedelta                              \n",
       "train_a 00:00:00            NaN           NaN  \n",
       "        01:00:00            NaN           NaN  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sat_pos  = pd.read_csv(data_dir+'satellite_positions.csv',index_col=[\"period\", \"timedelta\"],)\n",
    "\n",
    "\n",
    "print (\"shape of the data sunspots:\", sat_pos.shape)\n",
    "print (\"Null values:\", sat_pos.isnull().sum().sum())\n",
    "sat_pos = using_reset_index(sat_pos)\n",
    "print (\"Columns:\", sat_pos.columns)\n",
    "sat_pos.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Droping the columns: ['gse_x_dscovr', 'gse_y_dscovr', 'gse_z_dscovr']\n",
      "remaining columns: Index(['gse_x_ace', 'gse_y_ace', 'gse_z_ace'], dtype='object')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>gse_x_ace</th>\n",
       "      <th>gse_y_ace</th>\n",
       "      <th>gse_z_ace</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">train_a</th>\n",
       "      <th>00:00:00</th>\n",
       "      <td>1522376.9</td>\n",
       "      <td>143704.6</td>\n",
       "      <td>149496.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>01:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>03:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>04:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   gse_x_ace  gse_y_ace  gse_z_ace\n",
       "period  timedelta                                 \n",
       "train_a 00:00:00   1522376.9   143704.6   149496.7\n",
       "        01:00:00         NaN        NaN        NaN\n",
       "        02:00:00         NaN        NaN        NaN\n",
       "        03:00:00         NaN        NaN        NaN\n",
       "        04:00:00         NaN        NaN        NaN"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop the unnecessary columns\n",
    "cols_to_drop = ['gse_x_dscovr', 'gse_y_dscovr', 'gse_z_dscovr']\n",
    "sat_pos.drop(cols_to_drop, axis=1, inplace=True)\n",
    "\n",
    "print ('Droping the columns:', cols_to_drop)\n",
    "print ('remaining columns:', sat_pos.columns )\n",
    "sat_pos.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final=df_sw_dst_ss.join(sat_pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Null values: bx_gse               0\n",
      "by_gse               0\n",
      "bz_gse               0\n",
      "theta_gse            0\n",
      "phi_gse              0\n",
      "bt                   0\n",
      "density              0\n",
      "speed                0\n",
      "temperature          0\n",
      "dst                  0\n",
      "smoothed_ssn         0\n",
      "gse_x_ace       134044\n",
      "gse_y_ace       134044\n",
      "gse_z_ace       134044\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print (\"Total Null values:\", df_final.isnull().sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Interpolate the above three columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "final_shape: (139872, 14)\n",
      "Number of nulls: 0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>bx_gse</th>\n",
       "      <th>by_gse</th>\n",
       "      <th>bz_gse</th>\n",
       "      <th>theta_gse</th>\n",
       "      <th>phi_gse</th>\n",
       "      <th>bt</th>\n",
       "      <th>density</th>\n",
       "      <th>speed</th>\n",
       "      <th>temperature</th>\n",
       "      <th>dst</th>\n",
       "      <th>smoothed_ssn</th>\n",
       "      <th>gse_x_ace</th>\n",
       "      <th>gse_y_ace</th>\n",
       "      <th>gse_z_ace</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">train_a</th>\n",
       "      <th>00:00:00</th>\n",
       "      <td>-6.149500</td>\n",
       "      <td>1.6455</td>\n",
       "      <td>0.844667</td>\n",
       "      <td>7.103667</td>\n",
       "      <td>165.420333</td>\n",
       "      <td>7.140000</td>\n",
       "      <td>1.174167</td>\n",
       "      <td>355.278250</td>\n",
       "      <td>68880.341667</td>\n",
       "      <td>-7</td>\n",
       "      <td>65.400000</td>\n",
       "      <td>1.522377e+06</td>\n",
       "      <td>143704.600000</td>\n",
       "      <td>149496.700000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>01:00:00</th>\n",
       "      <td>-6.693833</td>\n",
       "      <td>0.7665</td>\n",
       "      <td>1.265167</td>\n",
       "      <td>10.183333</td>\n",
       "      <td>174.267833</td>\n",
       "      <td>7.284833</td>\n",
       "      <td>0.815917</td>\n",
       "      <td>330.080583</td>\n",
       "      <td>52203.225000</td>\n",
       "      <td>-10</td>\n",
       "      <td>65.421154</td>\n",
       "      <td>1.522503e+06</td>\n",
       "      <td>143388.108333</td>\n",
       "      <td>149560.758333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     bx_gse  by_gse    bz_gse  theta_gse     phi_gse  \\\n",
       "period  timedelta                                                      \n",
       "train_a 00:00:00  -6.149500  1.6455  0.844667   7.103667  165.420333   \n",
       "        01:00:00  -6.693833  0.7665  1.265167  10.183333  174.267833   \n",
       "\n",
       "                         bt   density       speed   temperature  dst  \\\n",
       "period  timedelta                                                      \n",
       "train_a 00:00:00   7.140000  1.174167  355.278250  68880.341667   -7   \n",
       "        01:00:00   7.284833  0.815917  330.080583  52203.225000  -10   \n",
       "\n",
       "                   smoothed_ssn     gse_x_ace      gse_y_ace      gse_z_ace  \n",
       "period  timedelta                                                            \n",
       "train_a 00:00:00      65.400000  1.522377e+06  143704.600000  149496.700000  \n",
       "        01:00:00      65.421154  1.522503e+06  143388.108333  149560.758333  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cols_to_interp=['gse_x_ace', 'gse_y_ace', 'gse_z_ace']\n",
    "df_final[cols_to_interp]=df_final[cols_to_interp].interpolate(method='linear', limit_direction='forward', axis=0)\n",
    "print ('final_shape:', df_final.shape)\n",
    "print ('Number of nulls:', df_sw_dst_ss.isnull().sum().sum() )\n",
    "df_final.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of all: (139872, 9) (139872, 1) (139872, 10) (139803, 3) (139872, 14)\n"
     ]
    }
   ],
   "source": [
    "print (\"shape of all:\", sw_ipt_agg_ml.shape, dst_labs.shape, df_sw_dst.shape, sat_pos.shape, df_final.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adding the Standard Deviation columns for `by_gse`, `bz_gse`, `bt`, `density`, `speed`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def agg_std(df_final, df_sw):\n",
    "    cols_to_keep = ['by_gse', 'bz_gse', 'bt', 'density', 'speed']\n",
    "    df_sw = df_sw[cols_to_keep]\n",
    "    \n",
    "    df_sw = df_sw.reset_index(level='period')\n",
    "    df_sw.index=pd.to_timedelta(df_sw.index)\n",
    "    df_sw = df_sw.groupby('period').resample('H').std()\n",
    "    \n",
    "    # rename the columns for the std\n",
    "    columns = [c+'_std' for c in df_sw.columns]\n",
    "    df_sw.columns = columns\n",
    "\n",
    "    df = df_final.join(df_sw)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "df_final.shape: (139872, 19)\n",
      "Number of nulls: 0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>bx_gse</th>\n",
       "      <th>by_gse</th>\n",
       "      <th>bz_gse</th>\n",
       "      <th>theta_gse</th>\n",
       "      <th>phi_gse</th>\n",
       "      <th>bt</th>\n",
       "      <th>density</th>\n",
       "      <th>speed</th>\n",
       "      <th>temperature</th>\n",
       "      <th>dst</th>\n",
       "      <th>smoothed_ssn</th>\n",
       "      <th>gse_x_ace</th>\n",
       "      <th>gse_y_ace</th>\n",
       "      <th>gse_z_ace</th>\n",
       "      <th>by_gse_std</th>\n",
       "      <th>bz_gse_std</th>\n",
       "      <th>bt_std</th>\n",
       "      <th>density_std</th>\n",
       "      <th>speed_std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>period</th>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">train_a</th>\n",
       "      <th>00:00:00</th>\n",
       "      <td>-6.149500</td>\n",
       "      <td>1.6455</td>\n",
       "      <td>0.844667</td>\n",
       "      <td>7.103667</td>\n",
       "      <td>165.420333</td>\n",
       "      <td>7.140000</td>\n",
       "      <td>1.174167</td>\n",
       "      <td>355.278250</td>\n",
       "      <td>68880.341667</td>\n",
       "      <td>-7</td>\n",
       "      <td>65.400000</td>\n",
       "      <td>1.522377e+06</td>\n",
       "      <td>143704.600000</td>\n",
       "      <td>149496.700000</td>\n",
       "      <td>1.224754</td>\n",
       "      <td>0.580028</td>\n",
       "      <td>1.607043</td>\n",
       "      <td>0.479647</td>\n",
       "      <td>15.025364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>01:00:00</th>\n",
       "      <td>-6.693833</td>\n",
       "      <td>0.7665</td>\n",
       "      <td>1.265167</td>\n",
       "      <td>10.183333</td>\n",
       "      <td>174.267833</td>\n",
       "      <td>7.284833</td>\n",
       "      <td>0.815917</td>\n",
       "      <td>330.080583</td>\n",
       "      <td>52203.225000</td>\n",
       "      <td>-10</td>\n",
       "      <td>65.421154</td>\n",
       "      <td>1.522503e+06</td>\n",
       "      <td>143388.108333</td>\n",
       "      <td>149560.758333</td>\n",
       "      <td>0.907098</td>\n",
       "      <td>0.731278</td>\n",
       "      <td>0.325293</td>\n",
       "      <td>0.698365</td>\n",
       "      <td>17.203299</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     bx_gse  by_gse    bz_gse  theta_gse     phi_gse  \\\n",
       "period  timedelta                                                      \n",
       "train_a 00:00:00  -6.149500  1.6455  0.844667   7.103667  165.420333   \n",
       "        01:00:00  -6.693833  0.7665  1.265167  10.183333  174.267833   \n",
       "\n",
       "                         bt   density       speed   temperature  dst  \\\n",
       "period  timedelta                                                      \n",
       "train_a 00:00:00   7.140000  1.174167  355.278250  68880.341667   -7   \n",
       "        01:00:00   7.284833  0.815917  330.080583  52203.225000  -10   \n",
       "\n",
       "                   smoothed_ssn     gse_x_ace      gse_y_ace      gse_z_ace  \\\n",
       "period  timedelta                                                             \n",
       "train_a 00:00:00      65.400000  1.522377e+06  143704.600000  149496.700000   \n",
       "        01:00:00      65.421154  1.522503e+06  143388.108333  149560.758333   \n",
       "\n",
       "                   by_gse_std  bz_gse_std    bt_std  density_std  speed_std  \n",
       "period  timedelta                                                            \n",
       "train_a 00:00:00     1.224754    0.580028  1.607043     0.479647  15.025364  \n",
       "        01:00:00     0.907098    0.731278  0.325293     0.698365  17.203299  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final = agg_std(df_final, sw)\n",
    "print ('df_final.shape:', df_final.shape)\n",
    "print ('Number of nulls:', df_final.isnull().sum().sum() )\n",
    "df_final.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>period</th>\n",
       "      <th>bx_gse</th>\n",
       "      <th>by_gse</th>\n",
       "      <th>bz_gse</th>\n",
       "      <th>theta_gse</th>\n",
       "      <th>phi_gse</th>\n",
       "      <th>bt</th>\n",
       "      <th>density</th>\n",
       "      <th>speed</th>\n",
       "      <th>temperature</th>\n",
       "      <th>dst</th>\n",
       "      <th>smoothed_ssn</th>\n",
       "      <th>gse_x_ace</th>\n",
       "      <th>gse_y_ace</th>\n",
       "      <th>gse_z_ace</th>\n",
       "      <th>by_gse_std</th>\n",
       "      <th>bz_gse_std</th>\n",
       "      <th>bt_std</th>\n",
       "      <th>density_std</th>\n",
       "      <th>speed_std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>00:00:00</th>\n",
       "      <td>train_a</td>\n",
       "      <td>-6.149500</td>\n",
       "      <td>1.6455</td>\n",
       "      <td>0.844667</td>\n",
       "      <td>7.103667</td>\n",
       "      <td>165.420333</td>\n",
       "      <td>7.140000</td>\n",
       "      <td>1.174167</td>\n",
       "      <td>355.278250</td>\n",
       "      <td>68880.341667</td>\n",
       "      <td>-7</td>\n",
       "      <td>65.400000</td>\n",
       "      <td>1.522377e+06</td>\n",
       "      <td>143704.600000</td>\n",
       "      <td>149496.700000</td>\n",
       "      <td>1.224754</td>\n",
       "      <td>0.580028</td>\n",
       "      <td>1.607043</td>\n",
       "      <td>0.479647</td>\n",
       "      <td>15.025364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>01:00:00</th>\n",
       "      <td>train_a</td>\n",
       "      <td>-6.693833</td>\n",
       "      <td>0.7665</td>\n",
       "      <td>1.265167</td>\n",
       "      <td>10.183333</td>\n",
       "      <td>174.267833</td>\n",
       "      <td>7.284833</td>\n",
       "      <td>0.815917</td>\n",
       "      <td>330.080583</td>\n",
       "      <td>52203.225000</td>\n",
       "      <td>-10</td>\n",
       "      <td>65.421154</td>\n",
       "      <td>1.522503e+06</td>\n",
       "      <td>143388.108333</td>\n",
       "      <td>149560.758333</td>\n",
       "      <td>0.907098</td>\n",
       "      <td>0.731278</td>\n",
       "      <td>0.325293</td>\n",
       "      <td>0.698365</td>\n",
       "      <td>17.203299</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            period    bx_gse  by_gse    bz_gse  theta_gse     phi_gse  \\\n",
       "timedelta                                                               \n",
       "00:00:00   train_a -6.149500  1.6455  0.844667   7.103667  165.420333   \n",
       "01:00:00   train_a -6.693833  0.7665  1.265167  10.183333  174.267833   \n",
       "\n",
       "                 bt   density       speed   temperature  dst  smoothed_ssn  \\\n",
       "timedelta                                                                    \n",
       "00:00:00   7.140000  1.174167  355.278250  68880.341667   -7     65.400000   \n",
       "01:00:00   7.284833  0.815917  330.080583  52203.225000  -10     65.421154   \n",
       "\n",
       "              gse_x_ace      gse_y_ace      gse_z_ace  by_gse_std  bz_gse_std  \\\n",
       "timedelta                                                                       \n",
       "00:00:00   1.522377e+06  143704.600000  149496.700000    1.224754    0.580028   \n",
       "01:00:00   1.522503e+06  143388.108333  149560.758333    0.907098    0.731278   \n",
       "\n",
       "             bt_std  density_std  speed_std  \n",
       "timedelta                                    \n",
       "00:00:00   1.607043     0.479647  15.025364  \n",
       "01:00:00   0.325293     0.698365  17.203299  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_to_save = df_final.reset_index(level='period')\n",
    "df_to_save.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>period</th>\n",
       "      <th>bx_gse</th>\n",
       "      <th>by_gse</th>\n",
       "      <th>bz_gse</th>\n",
       "      <th>theta_gse</th>\n",
       "      <th>phi_gse</th>\n",
       "      <th>bt</th>\n",
       "      <th>density</th>\n",
       "      <th>speed</th>\n",
       "      <th>temperature</th>\n",
       "      <th>dst</th>\n",
       "      <th>smoothed_ssn</th>\n",
       "      <th>gse_x_ace</th>\n",
       "      <th>gse_y_ace</th>\n",
       "      <th>gse_z_ace</th>\n",
       "      <th>by_gse_std</th>\n",
       "      <th>bz_gse_std</th>\n",
       "      <th>bt_std</th>\n",
       "      <th>density_std</th>\n",
       "      <th>speed_std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>timedelta</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2435 days 22:00:00</th>\n",
       "      <td>train_c</td>\n",
       "      <td>-2.3850</td>\n",
       "      <td>2.860333</td>\n",
       "      <td>-2.221667</td>\n",
       "      <td>-29.645667</td>\n",
       "      <td>130.601500</td>\n",
       "      <td>4.716667</td>\n",
       "      <td>2.148917</td>\n",
       "      <td>343.488167</td>\n",
       "      <td>41803.641667</td>\n",
       "      <td>-15</td>\n",
       "      <td>42.5</td>\n",
       "      <td>1426937.9</td>\n",
       "      <td>234813.7</td>\n",
       "      <td>140701.8</td>\n",
       "      <td>0.876128</td>\n",
       "      <td>1.328527</td>\n",
       "      <td>0.202373</td>\n",
       "      <td>0.680983</td>\n",
       "      <td>5.550519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2435 days 23:00:00</th>\n",
       "      <td>train_c</td>\n",
       "      <td>-2.2515</td>\n",
       "      <td>2.619333</td>\n",
       "      <td>-2.821000</td>\n",
       "      <td>-39.064000</td>\n",
       "      <td>130.277667</td>\n",
       "      <td>4.554833</td>\n",
       "      <td>2.283583</td>\n",
       "      <td>343.848750</td>\n",
       "      <td>36191.908333</td>\n",
       "      <td>-14</td>\n",
       "      <td>42.5</td>\n",
       "      <td>1426937.9</td>\n",
       "      <td>234813.7</td>\n",
       "      <td>140701.8</td>\n",
       "      <td>0.488619</td>\n",
       "      <td>0.599527</td>\n",
       "      <td>0.049076</td>\n",
       "      <td>0.918391</td>\n",
       "      <td>2.906941</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     period  bx_gse    by_gse    bz_gse  theta_gse  \\\n",
       "timedelta                                                            \n",
       "2435 days 22:00:00  train_c -2.3850  2.860333 -2.221667 -29.645667   \n",
       "2435 days 23:00:00  train_c -2.2515  2.619333 -2.821000 -39.064000   \n",
       "\n",
       "                       phi_gse        bt   density       speed   temperature  \\\n",
       "timedelta                                                                      \n",
       "2435 days 22:00:00  130.601500  4.716667  2.148917  343.488167  41803.641667   \n",
       "2435 days 23:00:00  130.277667  4.554833  2.283583  343.848750  36191.908333   \n",
       "\n",
       "                    dst  smoothed_ssn  gse_x_ace  gse_y_ace  gse_z_ace  \\\n",
       "timedelta                                                                \n",
       "2435 days 22:00:00  -15          42.5  1426937.9   234813.7   140701.8   \n",
       "2435 days 23:00:00  -14          42.5  1426937.9   234813.7   140701.8   \n",
       "\n",
       "                    by_gse_std  bz_gse_std    bt_std  density_std  speed_std  \n",
       "timedelta                                                                     \n",
       "2435 days 22:00:00    0.876128    1.328527  0.202373     0.680983   5.550519  \n",
       "2435 days 23:00:00    0.488619    0.599527  0.049076     0.918391   2.906941  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_to_save.tail(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_to_save.to_csv('processed_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}