examples/Gordo-Workflow-Semi-Low-Level.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running a Gordo workflow locally:\n",
    "\n",
    "This demonstrates the basic workflow of gordo, running locally.\n",
    "\n",
    "---\n",
    "\n",
    "### Import and initialize a Gordo dataset\n",
    "In this case we shall be using the `DataLakeProvider` where `InfluxDataProvider` is also available."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dateutil.parser\n",
    "import yaml\n",
    "\n",
    "from datetime import datetime\n",
    "\n",
    "from gordo.machine.dataset.datasets import TimeSeriesDataset\n",
    "from gordo.machine.dataset.data_provider.providers import DataLakeProvider\n",
    "from gordo import serializer\n",
    "\n",
    "data_provider = DataLakeProvider(storename=\"dataplatformdlsprod\", interactive=True)\n",
    "dataset = TimeSeriesDataset(train_start_date=dateutil.parser.isoparse('2016-07-01T00:10:00+00:00'),\n",
    "    train_end_date=dateutil.parser.isoparse('2017-01-01T00:00:00+00:00'),\n",
    "    tag_list=[\n",
    "        'asgb.19ZT3950%2FY%2FPRIM',\n",
    "        'asgb.19PST3925%2FDispMeasOut%2FPRIM'\n",
    "    ],\n",
    "    data_provider=data_provider)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### We'll need to login to Azure to authenticate the ability load data from the Data Lake"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = dataset.get_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>asgb.19ZT3950%2FY%2FPRIM</th>\n",
       "      <th>asgb.19PST3925%2FDispMeasOut%2FPRIM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2016-07-01 07:40:00+00:00</th>\n",
       "      <td>99.989201</td>\n",
       "      <td>46.329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-07-01 07:50:00+00:00</th>\n",
       "      <td>99.989201</td>\n",
       "      <td>46.329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-07-01 08:00:00+00:00</th>\n",
       "      <td>99.989201</td>\n",
       "      <td>46.329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-07-01 08:10:00+00:00</th>\n",
       "      <td>99.989201</td>\n",
       "      <td>46.329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-07-01 08:20:00+00:00</th>\n",
       "      <td>99.989201</td>\n",
       "      <td>46.329</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           asgb.19ZT3950%2FY%2FPRIM  \\\n",
       "2016-07-01 07:40:00+00:00                 99.989201   \n",
       "2016-07-01 07:50:00+00:00                 99.989201   \n",
       "2016-07-01 08:00:00+00:00                 99.989201   \n",
       "2016-07-01 08:10:00+00:00                 99.989201   \n",
       "2016-07-01 08:20:00+00:00                 99.989201   \n",
       "\n",
       "                           asgb.19PST3925%2FDispMeasOut%2FPRIM  \n",
       "2016-07-01 07:40:00+00:00                               46.329  \n",
       "2016-07-01 07:50:00+00:00                               46.329  \n",
       "2016-07-01 08:00:00+00:00                               46.329  \n",
       "2016-07-01 08:10:00+00:00                               46.329  \n",
       "2016-07-01 08:20:00+00:00                               46.329  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define a pipeline for model building"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('step_0', MinMaxScaler(copy=True, feature_range=(0, 1))), ('step_1', <gordo.machine.model.models.KerasAutoEncoder object at 0x7f64bd6de7f0>)])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config = yaml.load(\n",
    "    \"\"\" \n",
    "    sklearn.pipeline.Pipeline:\n",
    "        steps:\n",
    "          - sklearn.preprocessing.data.MinMaxScaler\n",
    "          - gordo.machine.model.models.KerasAutoEncoder:\n",
    "              kind: feedforward_hourglass\n",
    "    \"\"\"\n",
    ")\n",
    "pipe = serializer.pipeline_from_definition(config)\n",
    "pipe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AutoEncoders were agreed to meet the specifications of a `Transformer`. Therefore, they do not implement a `predict` method.\n",
    "\n",
    "We shall then call `fit_transform` or `fit` -> `transform` if desired to treat datasets separately. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/private/milg/Projects/gordo/venv/lib/python3.6/site-packages/sklearn/preprocessing/data.py:323: DataConversionWarning: Data with input dtype float32 were all converted to float64 by MinMaxScaler.\n",
      "  return self.partial_fit(X, y)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "26411/26411 [==============================] - 3s 110us/step - loss: 0.0015 - acc: 0.9846\n"
     ]
    }
   ],
   "source": [
    "pipe.fit(X, y=X.copy())  # Our target is just X\n",
    "xhat = pipe.predict(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "### `xhat` is now the auto-encoded result*\n",
    "\n",
    "*where the first half of each resulting sample was the _input_ to the model and secondhalf is the _output_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.32019111, 0.29786357, 0.32511118, 0.30381536],\n",
       "       [0.32019111, 0.29786357, 0.32511118, 0.30381536],\n",
       "       [0.32019111, 0.29786357, 0.32511118, 0.30381536],\n",
       "       ...,\n",
       "       [0.32429197, 0.26762422, 0.32392147, 0.27555713],\n",
       "       [0.32429197, 0.26762422, 0.32392147, 0.27555713],\n",
       "       [0.32450209, 0.26762422, 0.32392147, 0.27555713]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xhat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using custom or multiple aggregation methods\n",
    "TimeSeriesDataset supports customization of the aggregation method used for the resampled buckets, and it can even use multiple aggregation methods. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Custom aggregation method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>asgb.19ZT3950%2FY%2FPRIM</th>\n",
       "      <th>asgb.19PST3925%2FDispMeasOut%2FPRIM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2016-07-01 07:40:00+00:00</th>\n",
       "      <td>100.032417</td>\n",
       "      <td>46.330772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-07-01 07:50:00+00:00</th>\n",
       "      <td>100.032417</td>\n",
       "      <td>46.330772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-07-01 08:00:00+00:00</th>\n",
       "      <td>100.032417</td>\n",
       "      <td>46.330772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-07-01 08:10:00+00:00</th>\n",
       "      <td>100.032417</td>\n",
       "      <td>46.330772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-07-01 08:20:00+00:00</th>\n",
       "      <td>100.032417</td>\n",
       "      <td>46.330772</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           asgb.19ZT3950%2FY%2FPRIM  \\\n",
       "2016-07-01 07:40:00+00:00                100.032417   \n",
       "2016-07-01 07:50:00+00:00                100.032417   \n",
       "2016-07-01 08:00:00+00:00                100.032417   \n",
       "2016-07-01 08:10:00+00:00                100.032417   \n",
       "2016-07-01 08:20:00+00:00                100.032417   \n",
       "\n",
       "                           asgb.19PST3925%2FDispMeasOut%2FPRIM  \n",
       "2016-07-01 07:40:00+00:00                            46.330772  \n",
       "2016-07-01 07:50:00+00:00                            46.330772  \n",
       "2016-07-01 08:00:00+00:00                            46.330772  \n",
       "2016-07-01 08:10:00+00:00                            46.330772  \n",
       "2016-07-01 08:20:00+00:00                            46.330772  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remember to load the first cell to have the required imports\n",
    "dataset = TimeSeriesDataset(train_start_date=dateutil.parser.isoparse('2016-07-01T00:10:00+00:00'),\n",
    "    train_end_date=dateutil.parser.isoparse('2017-01-01T00:00:00+00:00'),\n",
    "    tag_list=[\n",
    "        'asgb.19ZT3950%2FY%2FPRIM',\n",
    "        'asgb.19PST3925%2FDispMeasOut%2FPRIM'\n",
    "    ],\n",
    "    aggregation_methods=\"max\",\n",
    "    data_provider=data_provider)\n",
    "X, y = dataset.get_data()\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multiple aggregation methods\n",
    "When using multiple aggregation methods the returned dataframe will have multi-level columns, with the tag-name as top-level  and aggregation method as the second level. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>tag</th>\n",
       "      <th colspan=\"3\" halign=\"left\">asgb.19ZT3950%2FY%2FPRIM</th>\n",
       "      <th colspan=\"3\" halign=\"left\">asgb.19PST3925%2FDispMeasOut%2FPRIM</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aggregation_method</th>\n",
       "      <th>max</th>\n",
       "      <th>min</th>\n",
       "      <th>mean</th>\n",
       "      <th>max</th>\n",
       "      <th>min</th>\n",
       "      <th>mean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2016-07-01 07:40:00+00:00</th>\n",
       "      <td>100.032417</td>\n",
       "      <td>99.945984</td>\n",
       "      <td>99.989201</td>\n",
       "      <td>46.330772</td>\n",
       "      <td>46.327229</td>\n",
       "      <td>46.329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-07-01 07:50:00+00:00</th>\n",
       "      <td>100.032417</td>\n",
       "      <td>99.945984</td>\n",
       "      <td>99.989201</td>\n",
       "      <td>46.330772</td>\n",
       "      <td>46.327229</td>\n",
       "      <td>46.329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-07-01 08:00:00+00:00</th>\n",
       "      <td>100.032417</td>\n",
       "      <td>99.945984</td>\n",
       "      <td>99.989201</td>\n",
       "      <td>46.330772</td>\n",
       "      <td>46.327229</td>\n",
       "      <td>46.329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-07-01 08:10:00+00:00</th>\n",
       "      <td>100.032417</td>\n",
       "      <td>99.945984</td>\n",
       "      <td>99.989201</td>\n",
       "      <td>46.330772</td>\n",
       "      <td>46.327229</td>\n",
       "      <td>46.329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-07-01 08:20:00+00:00</th>\n",
       "      <td>100.032417</td>\n",
       "      <td>99.945984</td>\n",
       "      <td>99.989201</td>\n",
       "      <td>46.330772</td>\n",
       "      <td>46.327229</td>\n",
       "      <td>46.329</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "tag                       asgb.19ZT3950%2FY%2FPRIM                        \\\n",
       "aggregation_method                             max        min       mean   \n",
       "2016-07-01 07:40:00+00:00               100.032417  99.945984  99.989201   \n",
       "2016-07-01 07:50:00+00:00               100.032417  99.945984  99.989201   \n",
       "2016-07-01 08:00:00+00:00               100.032417  99.945984  99.989201   \n",
       "2016-07-01 08:10:00+00:00               100.032417  99.945984  99.989201   \n",
       "2016-07-01 08:20:00+00:00               100.032417  99.945984  99.989201   \n",
       "\n",
       "tag                       asgb.19PST3925%2FDispMeasOut%2FPRIM             \\\n",
       "aggregation_method                                        max        min   \n",
       "2016-07-01 07:40:00+00:00                           46.330772  46.327229   \n",
       "2016-07-01 07:50:00+00:00                           46.330772  46.327229   \n",
       "2016-07-01 08:00:00+00:00                           46.330772  46.327229   \n",
       "2016-07-01 08:10:00+00:00                           46.330772  46.327229   \n",
       "2016-07-01 08:20:00+00:00                           46.330772  46.327229   \n",
       "\n",
       "tag                                \n",
       "aggregation_method           mean  \n",
       "2016-07-01 07:40:00+00:00  46.329  \n",
       "2016-07-01 07:50:00+00:00  46.329  \n",
       "2016-07-01 08:00:00+00:00  46.329  \n",
       "2016-07-01 08:10:00+00:00  46.329  \n",
       "2016-07-01 08:20:00+00:00  46.329  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remember to load the first cell to have the required imports\n",
    "dataset = TimeSeriesDataset(train_start_date=dateutil.parser.isoparse('2016-07-01T00:10:00+00:00'),\n",
    "    train_end_date=dateutil.parser.isoparse('2017-01-01T00:00:00+00:00'),\n",
    "    tag_list=[\n",
    "        'asgb.19ZT3950%2FY%2FPRIM',\n",
    "        'asgb.19PST3925%2FDispMeasOut%2FPRIM'\n",
    "    ],\n",
    "    aggregation_methods=[\"max\",\"min\",\"mean\"],\n",
    "    data_provider=data_provider)\n",
    "X, y = dataset.get_data()\n",
    "X.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}