DL_well_4to1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4028054f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Restored 'well/log4tutorial.xlsx'\n",
      "Checked out revision 52.\n"
     ]
    }
   ],
   "source": [
    "## A tutorial script for well log prediction \n",
    "# This tutorial is for the most general case:\n",
    "# inputing 4 values from four existing well logs and predicting the missing S-wave log (DT)\n",
    "# \n",
    "# INPUT: P-wave sonic log (DTCO), Gamma-ray (ECGR), Density (RHOB), Total porosity (PHIT)\n",
    "# OUTPUT: S-wave sonic log (DTSM)\n",
    "\n",
    "# first download the data from\n",
    "!svn co https://github.com/chenyk1990/mldata/trunk/well ./well"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "925b5119",
   "metadata": {},
   "outputs": [],
   "source": [
    "!cp well/log4tutorial.xlsx ./"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "35fcf703",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-04-06 22:22:42.195391: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F AVX512_VNNI FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-04-06 22:22:46.931598: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F AVX512_VNNI FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " input_layer (InputLayer)       [(None, 4, 1)]       0           []                               \n",
      "                                                                                                  \n",
      " bidirectional (Bidirectional)  (None, 4, 16)        640         ['input_layer[0][0]']            \n",
      "                                                                                                  \n",
      " global_average_pooling1d (Glob  (None, 16)          0           ['bidirectional[0][0]']          \n",
      " alAveragePooling1D)                                                                              \n",
      "                                                                                                  \n",
      " reshape (Reshape)              (None, 1, 16)        0           ['global_average_pooling1d[0][0]'\n",
      "                                                                 ]                                \n",
      "                                                                                                  \n",
      " dense (Dense)                  (None, 1, 2)         32          ['reshape[0][0]']                \n",
      "                                                                                                  \n",
      " dense_1 (Dense)                (None, 1, 16)        32          ['dense[0][0]']                  \n",
      "                                                                                                  \n",
      " multiply (Multiply)            (None, 4, 16)        0           ['bidirectional[0][0]',          \n",
      "                                                                  'dense_1[0][0]']                \n",
      "                                                                                                  \n",
      " add (Add)                      (None, 4, 16)        0           ['multiply[0][0]',               \n",
      "                                                                  'bidirectional[0][0]']          \n",
      "                                                                                                  \n",
      " bidirectional_1 (Bidirectional  (None, 4, 32)       4224        ['add[0][0]']                    \n",
      " )                                                                                                \n",
      "                                                                                                  \n",
      " global_average_pooling1d_1 (Gl  (None, 32)          0           ['bidirectional_1[0][0]']        \n",
      " obalAveragePooling1D)                                                                            \n",
      "                                                                                                  \n",
      " reshape_1 (Reshape)            (None, 1, 32)        0           ['global_average_pooling1d_1[0][0\n",
      "                                                                 ]']                              \n",
      "                                                                                                  \n",
      " dense_2 (Dense)                (None, 1, 4)         128         ['reshape_1[0][0]']              \n",
      "                                                                                                  \n",
      " dense_3 (Dense)                (None, 1, 32)        128         ['dense_2[0][0]']                \n",
      "                                                                                                  \n",
      " multiply_1 (Multiply)          (None, 4, 32)        0           ['bidirectional_1[0][0]',        \n",
      "                                                                  'dense_3[0][0]']                \n",
      "                                                                                                  \n",
      " add_1 (Add)                    (None, 4, 32)        0           ['multiply_1[0][0]',             \n",
      "                                                                  'bidirectional_1[0][0]']        \n",
      "                                                                                                  \n",
      " bidirectional_2 (Bidirectional  (None, 4, 64)       16640       ['add_1[0][0]']                  \n",
      " )                                                                                                \n",
      "                                                                                                  \n",
      " global_average_pooling1d_2 (Gl  (None, 64)          0           ['bidirectional_2[0][0]']        \n",
      " obalAveragePooling1D)                                                                            \n",
      "                                                                                                  \n",
      " reshape_2 (Reshape)            (None, 1, 64)        0           ['global_average_pooling1d_2[0][0\n",
      "                                                                 ]']                              \n",
      "                                                                                                  \n",
      " dense_4 (Dense)                (None, 1, 8)         512         ['reshape_2[0][0]']              \n",
      "                                                                                                  \n",
      " dense_5 (Dense)                (None, 1, 64)        512         ['dense_4[0][0]']                \n",
      "                                                                                                  \n",
      " multiply_2 (Multiply)          (None, 4, 64)        0           ['bidirectional_2[0][0]',        \n",
      "                                                                  'dense_5[0][0]']                \n",
      "                                                                                                  \n",
      " add_2 (Add)                    (None, 4, 64)        0           ['multiply_2[0][0]',             \n",
      "                                                                  'bidirectional_2[0][0]']        \n",
      "                                                                                                  \n",
      " bidirectional_3 (Bidirectional  (None, 128)         66048       ['add_2[0][0]']                  \n",
      " )                                                                                                \n",
      "                                                                                                  \n",
      " dense_6 (Dense)                (None, 1)            129         ['bidirectional_3[0][0]']        \n",
      "                                                                                                  \n",
      " output_layer (Activation)      (None, 1)            0           ['dense_6[0][0]']                \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 89,025\n",
      "Trainable params: 89,025\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Here we introduce a self-attention BiLSTM network for training a model\n",
    "\n",
    "import numpy as np\n",
    "from keras.callbacks import Callback, EarlyStopping, ModelCheckpoint\n",
    "from keras.layers import Activation, Add, Bidirectional, Conv1D, Dense, Dropout, Embedding, Flatten, Reshape, multiply\n",
    "from keras.layers import concatenate, GRU, Input, LSTM, MaxPooling1D\n",
    "from keras.layers import GlobalAveragePooling1D,  GlobalMaxPooling1D, SpatialDropout1D\n",
    "from keras.models import Model\n",
    "from keras.preprocessing import text, sequence\n",
    "from sklearn.metrics import accuracy_score, roc_auc_score, log_loss\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras import initializers, regularizers, constraints, optimizers, layers, callbacks\n",
    "from keras import backend as K\n",
    "from keras.models import Model\n",
    "\n",
    "def SE_Block(tensor, ratio=8):\n",
    "    init = tensor\n",
    "    channel_axis = 1 if K.image_data_format() == \"channels_first\" else -1\n",
    "    filters = int(init.shape[-1])\n",
    "    se_shape = (1, filters)\n",
    "    se = GlobalAveragePooling1D()(init)\n",
    "    se = Reshape(se_shape)(se)\n",
    "    se = Dense(filters // ratio, activation='relu', kernel_initializer='he_normal', use_bias=False)(se)\n",
    "    se = Dense(filters, activation='sigmoid', kernel_initializer='he_normal', use_bias=False)(se)\n",
    "    x = multiply([init, se])\n",
    "    x = Add()([x,init])\n",
    "    return x\n",
    "\n",
    "inp1 = layers.Input(shape=(4,1),name='input_layer')\n",
    "D1 = 8\n",
    "D2 = int(2*D1)\n",
    "D3 = int(2*D2)\n",
    "D4 = int(2*D3)\n",
    "D5 = int(2*D4)\n",
    "\n",
    "features = Bidirectional(LSTM(D1, return_sequences=True))(inp1)\n",
    "features = SE_Block(features)\n",
    "features = Bidirectional(LSTM(D2, return_sequences=True))(features)\n",
    "features = SE_Block(features)\n",
    "features = Bidirectional(LSTM(D3, return_sequences=True))(features)\n",
    "features = SE_Block(features)\n",
    "features = Bidirectional(LSTM(D4, return_sequences=False))(features)\n",
    "\n",
    "#features = Dense(D4,activation='relu')(features)\n",
    "#features = Dense(D3,activation='relu')(features)\n",
    "#features = Dense(D2,activation='relu')(features)\n",
    "\n",
    "e = Dense(1)(features)\n",
    "o = Activation('linear', name='output_layer')(e)\n",
    "model = Model(inputs=[inp1], outputs=o)\n",
    "model.summary()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "36848944",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/chenyk/opt/anaconda3/lib/python3.9/site-packages/keras/optimizers/optimizer_v2/adam.py:114: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super().__init__(name, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/200\n",
      "346/349 [============================>.] - ETA: 0s - loss: 6288.6826 - mse: 6288.6826\n",
      "Epoch 1: val_loss improved from inf to 3165.15283, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 17s 21ms/step - loss: 6267.0356 - mse: 6267.0356 - val_loss: 3165.1528 - val_mse: 3165.1528 - lr: 0.0010\n",
      "Epoch 2/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 1830.0050 - mse: 1830.0050\n",
      "Epoch 2: val_loss improved from 3165.15283 to 888.92865, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 7s 20ms/step - loss: 1826.0298 - mse: 1826.0298 - val_loss: 888.9286 - val_mse: 888.9286 - lr: 0.0010\n",
      "Epoch 3/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 541.7598 - mse: 541.7598\n",
      "Epoch 3: val_loss improved from 888.92865 to 320.88644, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 7s 19ms/step - loss: 540.6859 - mse: 540.6859 - val_loss: 320.8864 - val_mse: 320.8864 - lr: 0.0010\n",
      "Epoch 4/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 269.1735 - mse: 269.1735\n",
      "Epoch 4: val_loss improved from 320.88644 to 235.50363, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 8s 23ms/step - loss: 268.9944 - mse: 268.9944 - val_loss: 235.5036 - val_mse: 235.5036 - lr: 0.0010\n",
      "Epoch 5/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 234.9391 - mse: 234.9391\n",
      "Epoch 5: val_loss improved from 235.50363 to 229.05476, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 12s 34ms/step - loss: 234.8103 - mse: 234.8103 - val_loss: 229.0548 - val_mse: 229.0548 - lr: 0.0010\n",
      "Epoch 6/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 232.4166 - mse: 232.4166\n",
      "Epoch 6: val_loss improved from 229.05476 to 228.90469, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 9s 25ms/step - loss: 232.4406 - mse: 232.4406 - val_loss: 228.9047 - val_mse: 228.9047 - lr: 0.0010\n",
      "Epoch 7/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 232.3543 - mse: 232.3543\n",
      "Epoch 7: val_loss did not improve from 228.90469\n",
      "349/349 [==============================] - 11s 31ms/step - loss: 232.3543 - mse: 232.3543 - val_loss: 228.9199 - val_mse: 228.9199 - lr: 0.0010\n",
      "Epoch 8/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 232.3587 - mse: 232.3587\n",
      "Epoch 8: val_loss did not improve from 228.90469\n",
      "349/349 [==============================] - 9s 27ms/step - loss: 232.3519 - mse: 232.3519 - val_loss: 228.9397 - val_mse: 228.9397 - lr: 0.0010\n",
      "Epoch 9/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 232.3852 - mse: 232.3852\n",
      "Epoch 9: val_loss did not improve from 228.90469\n",
      "349/349 [==============================] - 10s 30ms/step - loss: 232.3489 - mse: 232.3489 - val_loss: 228.9221 - val_mse: 228.9221 - lr: 0.0010\n",
      "Epoch 10/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 232.3492 - mse: 232.3492\n",
      "Epoch 10: val_loss did not improve from 228.90469\n",
      "349/349 [==============================] - 10s 29ms/step - loss: 232.3492 - mse: 232.3492 - val_loss: 228.9144 - val_mse: 228.9144 - lr: 0.0010\n",
      "Epoch 11/200\n",
      "345/349 [============================>.] - ETA: 0s - loss: 232.4306 - mse: 232.4306\n",
      "Epoch 11: val_loss did not improve from 228.90469\n",
      "349/349 [==============================] - 9s 25ms/step - loss: 232.3590 - mse: 232.3590 - val_loss: 228.9197 - val_mse: 228.9197 - lr: 0.0010\n",
      "Epoch 12/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 232.3519 - mse: 232.3519\n",
      "Epoch 12: val_loss did not improve from 228.90469\n",
      "349/349 [==============================] - 11s 31ms/step - loss: 232.3566 - mse: 232.3566 - val_loss: 228.9744 - val_mse: 228.9744 - lr: 0.0010\n",
      "Epoch 13/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 232.3570 - mse: 232.3570\n",
      "Epoch 13: val_loss did not improve from 228.90469\n",
      "349/349 [==============================] - 10s 28ms/step - loss: 232.3570 - mse: 232.3570 - val_loss: 229.0295 - val_mse: 229.0295 - lr: 0.0010\n",
      "Epoch 14/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 232.4054 - mse: 232.4054\n",
      "Epoch 14: val_loss did not improve from 228.90469\n",
      "349/349 [==============================] - 10s 27ms/step - loss: 232.3700 - mse: 232.3700 - val_loss: 228.9879 - val_mse: 228.9879 - lr: 0.0010\n",
      "Epoch 15/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 232.3731 - mse: 232.3731\n",
      "Epoch 15: val_loss improved from 228.90469 to 228.65192, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 8s 22ms/step - loss: 232.3152 - mse: 232.3152 - val_loss: 228.6519 - val_mse: 228.6519 - lr: 0.0010\n",
      "Epoch 16/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 87.7690 - mse: 87.7690\n",
      "Epoch 16: val_loss improved from 228.65192 to 52.99467, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 9s 24ms/step - loss: 87.7118 - mse: 87.7118 - val_loss: 52.9947 - val_mse: 52.9947 - lr: 0.0010\n",
      "Epoch 17/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 44.0779 - mse: 44.0779\n",
      "Epoch 17: val_loss improved from 52.99467 to 41.20025, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 8s 24ms/step - loss: 44.0570 - mse: 44.0570 - val_loss: 41.2003 - val_mse: 41.2003 - lr: 0.0010\n",
      "Epoch 18/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 37.7529 - mse: 37.7529\n",
      "Epoch 18: val_loss improved from 41.20025 to 34.77895, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 7s 21ms/step - loss: 37.7999 - mse: 37.7999 - val_loss: 34.7789 - val_mse: 34.7789 - lr: 0.0010\n",
      "Epoch 19/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 35.6197 - mse: 35.6197\n",
      "Epoch 19: val_loss improved from 34.77895 to 34.23876, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 10s 27ms/step - loss: 35.6046 - mse: 35.6046 - val_loss: 34.2388 - val_mse: 34.2388 - lr: 0.0010\n",
      "Epoch 20/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 34.8898 - mse: 34.8898\n",
      "Epoch 20: val_loss did not improve from 34.23876\n",
      "349/349 [==============================] - 11s 32ms/step - loss: 34.8898 - mse: 34.8898 - val_loss: 37.5935 - val_mse: 37.5935 - lr: 0.0010\n",
      "Epoch 21/200\n",
      "345/349 [============================>.] - ETA: 0s - loss: 34.2978 - mse: 34.2978\n",
      "Epoch 21: val_loss improved from 34.23876 to 31.85963, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 8s 23ms/step - loss: 34.5632 - mse: 34.5632 - val_loss: 31.8596 - val_mse: 31.8596 - lr: 0.0010\n",
      "Epoch 22/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 33.4656 - mse: 33.4656\n",
      "Epoch 22: val_loss improved from 31.85963 to 30.80693, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 6s 18ms/step - loss: 33.4476 - mse: 33.4476 - val_loss: 30.8069 - val_mse: 30.8069 - lr: 0.0010\n",
      "Epoch 23/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 32.7994 - mse: 32.7994\n",
      "Epoch 23: val_loss improved from 30.80693 to 30.12290, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 9s 25ms/step - loss: 32.8003 - mse: 32.8003 - val_loss: 30.1229 - val_mse: 30.1229 - lr: 0.0010\n",
      "Epoch 24/200\n",
      "344/349 [============================>.] - ETA: 0s - loss: 32.1565 - mse: 32.1565\n",
      "Epoch 24: val_loss did not improve from 30.12290\n",
      "349/349 [==============================] - 7s 21ms/step - loss: 32.1173 - mse: 32.1173 - val_loss: 30.1313 - val_mse: 30.1313 - lr: 0.0010\n",
      "Epoch 25/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 30.9812 - mse: 30.9812\n",
      "Epoch 25: val_loss improved from 30.12290 to 30.06101, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 6s 19ms/step - loss: 31.0353 - mse: 31.0353 - val_loss: 30.0610 - val_mse: 30.0610 - lr: 0.0010\n",
      "Epoch 26/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 30.9931 - mse: 30.9931\n",
      "Epoch 26: val_loss improved from 30.06101 to 28.96081, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 10s 29ms/step - loss: 30.9730 - mse: 30.9730 - val_loss: 28.9608 - val_mse: 28.9608 - lr: 0.0010\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 27/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 30.7486 - mse: 30.7486\n",
      "Epoch 27: val_loss did not improve from 28.96081\n",
      "349/349 [==============================] - 9s 26ms/step - loss: 30.7486 - mse: 30.7486 - val_loss: 29.1100 - val_mse: 29.1100 - lr: 0.0010\n",
      "Epoch 28/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 29.6868 - mse: 29.6868\n",
      "Epoch 28: val_loss improved from 28.96081 to 28.41645, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 8s 21ms/step - loss: 29.7947 - mse: 29.7947 - val_loss: 28.4164 - val_mse: 28.4164 - lr: 0.0010\n",
      "Epoch 29/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 28.9081 - mse: 28.9081\n",
      "Epoch 29: val_loss improved from 28.41645 to 27.49255, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 9s 26ms/step - loss: 28.8994 - mse: 28.8994 - val_loss: 27.4925 - val_mse: 27.4925 - lr: 0.0010\n",
      "Epoch 30/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 29.0400 - mse: 29.0400\n",
      "Epoch 30: val_loss improved from 27.49255 to 27.11111, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 10s 29ms/step - loss: 29.0400 - mse: 29.0400 - val_loss: 27.1111 - val_mse: 27.1111 - lr: 0.0010\n",
      "Epoch 31/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 28.5891 - mse: 28.5891\n",
      "Epoch 31: val_loss improved from 27.11111 to 26.58004, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 11s 30ms/step - loss: 28.5488 - mse: 28.5488 - val_loss: 26.5800 - val_mse: 26.5800 - lr: 0.0010\n",
      "Epoch 32/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 28.0484 - mse: 28.0484\n",
      "Epoch 32: val_loss did not improve from 26.58004\n",
      "349/349 [==============================] - 10s 28ms/step - loss: 28.0461 - mse: 28.0461 - val_loss: 27.0459 - val_mse: 27.0459 - lr: 0.0010\n",
      "Epoch 33/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 27.4683 - mse: 27.4683\n",
      "Epoch 33: val_loss improved from 26.58004 to 26.29309, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 9s 27ms/step - loss: 27.4683 - mse: 27.4683 - val_loss: 26.2931 - val_mse: 26.2931 - lr: 0.0010\n",
      "Epoch 34/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 27.2156 - mse: 27.2156\n",
      "Epoch 34: val_loss did not improve from 26.29309\n",
      "349/349 [==============================] - 12s 33ms/step - loss: 27.2248 - mse: 27.2248 - val_loss: 26.8726 - val_mse: 26.8726 - lr: 0.0010\n",
      "Epoch 35/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 27.1812 - mse: 27.1812\n",
      "Epoch 35: val_loss improved from 26.29309 to 25.86149, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 11s 33ms/step - loss: 27.1812 - mse: 27.1812 - val_loss: 25.8615 - val_mse: 25.8615 - lr: 0.0010\n",
      "Epoch 36/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 26.9053 - mse: 26.9053\n",
      "Epoch 36: val_loss did not improve from 25.86149\n",
      "349/349 [==============================] - 12s 35ms/step - loss: 26.9053 - mse: 26.9053 - val_loss: 29.0515 - val_mse: 29.0515 - lr: 0.0010\n",
      "Epoch 37/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 26.8836 - mse: 26.8836\n",
      "Epoch 37: val_loss did not improve from 25.86149\n",
      "349/349 [==============================] - 10s 30ms/step - loss: 26.9129 - mse: 26.9129 - val_loss: 26.2953 - val_mse: 26.2953 - lr: 0.0010\n",
      "Epoch 38/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 26.4566 - mse: 26.4566\n",
      "Epoch 38: val_loss did not improve from 25.86149\n",
      "349/349 [==============================] - 13s 36ms/step - loss: 26.4566 - mse: 26.4566 - val_loss: 25.9703 - val_mse: 25.9703 - lr: 0.0010\n",
      "Epoch 39/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 26.3671 - mse: 26.3671\n",
      "Epoch 39: val_loss improved from 25.86149 to 25.58713, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 11s 33ms/step - loss: 26.3725 - mse: 26.3725 - val_loss: 25.5871 - val_mse: 25.5871 - lr: 0.0010\n",
      "Epoch 40/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 26.3502 - mse: 26.3502\n",
      "Epoch 40: val_loss improved from 25.58713 to 25.44579, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 11s 30ms/step - loss: 26.3373 - mse: 26.3373 - val_loss: 25.4458 - val_mse: 25.4458 - lr: 0.0010\n",
      "Epoch 41/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 26.0298 - mse: 26.0298\n",
      "Epoch 41: val_loss did not improve from 25.44579\n",
      "349/349 [==============================] - 12s 33ms/step - loss: 26.0298 - mse: 26.0298 - val_loss: 27.2789 - val_mse: 27.2789 - lr: 0.0010\n",
      "Epoch 42/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 26.9020 - mse: 26.9020\n",
      "Epoch 42: val_loss improved from 25.44579 to 24.98439, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 10s 30ms/step - loss: 26.8767 - mse: 26.8767 - val_loss: 24.9844 - val_mse: 24.9844 - lr: 0.0010\n",
      "Epoch 43/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 25.9493 - mse: 25.9493\n",
      "Epoch 43: val_loss did not improve from 24.98439\n",
      "349/349 [==============================] - 10s 29ms/step - loss: 25.9569 - mse: 25.9569 - val_loss: 27.3884 - val_mse: 27.3884 - lr: 0.0010\n",
      "Epoch 44/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 26.4439 - mse: 26.4439\n",
      "Epoch 44: val_loss did not improve from 24.98439\n",
      "349/349 [==============================] - 10s 29ms/step - loss: 26.4467 - mse: 26.4467 - val_loss: 26.2516 - val_mse: 26.2516 - lr: 0.0010\n",
      "Epoch 45/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 25.9926 - mse: 25.9926\n",
      "Epoch 45: val_loss did not improve from 24.98439\n",
      "349/349 [==============================] - 8s 23ms/step - loss: 25.9908 - mse: 25.9908 - val_loss: 26.6383 - val_mse: 26.6383 - lr: 0.0010\n",
      "Epoch 46/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 25.7724 - mse: 25.7724\n",
      "Epoch 46: val_loss did not improve from 24.98439\n",
      "349/349 [==============================] - 8s 23ms/step - loss: 25.7735 - mse: 25.7735 - val_loss: 28.2550 - val_mse: 28.2550 - lr: 0.0010\n",
      "Epoch 47/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 26.0481 - mse: 26.0481\n",
      "Epoch 47: val_loss did not improve from 24.98439\n",
      "349/349 [==============================] - 12s 36ms/step - loss: 26.0464 - mse: 26.0464 - val_loss: 29.2267 - val_mse: 29.2267 - lr: 0.0010\n",
      "Epoch 48/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 25.8363 - mse: 25.8363\n",
      "Epoch 48: val_loss improved from 24.98439 to 24.57892, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 9s 27ms/step - loss: 25.8363 - mse: 25.8363 - val_loss: 24.5789 - val_mse: 24.5789 - lr: 0.0010\n",
      "Epoch 49/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 25.6771 - mse: 25.6771\n",
      "Epoch 49: val_loss did not improve from 24.57892\n",
      "349/349 [==============================] - 10s 30ms/step - loss: 25.6771 - mse: 25.6771 - val_loss: 25.4307 - val_mse: 25.4307 - lr: 0.0010\n",
      "Epoch 50/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 25.7704 - mse: 25.7704\n",
      "Epoch 50: val_loss did not improve from 24.57892\n",
      "349/349 [==============================] - 11s 32ms/step - loss: 25.7369 - mse: 25.7369 - val_loss: 24.6235 - val_mse: 24.6235 - lr: 0.0010\n",
      "Epoch 51/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 27.0299 - mse: 27.0299\n",
      "Epoch 51: val_loss did not improve from 24.57892\n",
      "349/349 [==============================] - 12s 35ms/step - loss: 27.0176 - mse: 27.0176 - val_loss: 25.5098 - val_mse: 25.5098 - lr: 0.0010\n",
      "Epoch 52/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 26.0517 - mse: 26.0517\n",
      "Epoch 52: val_loss did not improve from 24.57892\n",
      "349/349 [==============================] - 11s 31ms/step - loss: 26.0441 - mse: 26.0441 - val_loss: 24.8210 - val_mse: 24.8210 - lr: 0.0010\n",
      "Epoch 53/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 25.7254 - mse: 25.7254\n",
      "Epoch 53: val_loss did not improve from 24.57892\n",
      "349/349 [==============================] - 11s 32ms/step - loss: 25.7282 - mse: 25.7282 - val_loss: 24.7580 - val_mse: 24.7580 - lr: 0.0010\n",
      "Epoch 54/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "348/349 [============================>.] - ETA: 0s - loss: 25.5605 - mse: 25.5605\n",
      "Epoch 54: val_loss did not improve from 24.57892\n",
      "349/349 [==============================] - 13s 37ms/step - loss: 25.5572 - mse: 25.5572 - val_loss: 25.1644 - val_mse: 25.1644 - lr: 0.0010\n",
      "Epoch 55/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 25.4436 - mse: 25.4436\n",
      "Epoch 55: val_loss did not improve from 24.57892\n",
      "349/349 [==============================] - 10s 30ms/step - loss: 25.4232 - mse: 25.4232 - val_loss: 24.5984 - val_mse: 24.5984 - lr: 0.0010\n",
      "Epoch 56/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 25.3604 - mse: 25.3604\n",
      "Epoch 56: val_loss did not improve from 24.57892\n",
      "349/349 [==============================] - 12s 36ms/step - loss: 25.3519 - mse: 25.3519 - val_loss: 27.9582 - val_mse: 27.9582 - lr: 0.0010\n",
      "Epoch 57/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 26.4418 - mse: 26.4418\n",
      "Epoch 57: val_loss did not improve from 24.57892\n",
      "349/349 [==============================] - 9s 25ms/step - loss: 26.5486 - mse: 26.5486 - val_loss: 29.2800 - val_mse: 29.2800 - lr: 0.0010\n",
      "Epoch 58/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 25.5921 - mse: 25.5921\n",
      "Epoch 58: val_loss did not improve from 24.57892\n",
      "349/349 [==============================] - 10s 30ms/step - loss: 25.5948 - mse: 25.5948 - val_loss: 25.9186 - val_mse: 25.9186 - lr: 0.0010\n",
      "Epoch 59/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 25.2803 - mse: 25.2803\n",
      "Epoch 59: val_loss did not improve from 24.57892\n",
      "349/349 [==============================] - 13s 38ms/step - loss: 25.3718 - mse: 25.3718 - val_loss: 26.6507 - val_mse: 26.6507 - lr: 0.0010\n",
      "Epoch 60/200\n",
      "346/349 [============================>.] - ETA: 0s - loss: 25.0762 - mse: 25.0762\n",
      "Epoch 60: val_loss improved from 24.57892 to 24.51437, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 11s 33ms/step - loss: 25.1071 - mse: 25.1071 - val_loss: 24.5144 - val_mse: 24.5144 - lr: 0.0010\n",
      "Epoch 61/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 25.3954 - mse: 25.3954\n",
      "Epoch 61: val_loss improved from 24.51437 to 24.31793, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 11s 32ms/step - loss: 25.4031 - mse: 25.4031 - val_loss: 24.3179 - val_mse: 24.3179 - lr: 0.0010\n",
      "Epoch 62/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 25.3966 - mse: 25.3966\n",
      "Epoch 62: val_loss improved from 24.31793 to 24.25948, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 7s 20ms/step - loss: 25.3966 - mse: 25.3966 - val_loss: 24.2595 - val_mse: 24.2595 - lr: 0.0010\n",
      "Epoch 63/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 25.1101 - mse: 25.1101\n",
      "Epoch 63: val_loss did not improve from 24.25948\n",
      "349/349 [==============================] - 9s 26ms/step - loss: 25.1101 - mse: 25.1101 - val_loss: 24.3553 - val_mse: 24.3553 - lr: 0.0010\n",
      "Epoch 64/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 25.2299 - mse: 25.2299\n",
      "Epoch 64: val_loss improved from 24.25948 to 24.06650, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 16s 45ms/step - loss: 25.2129 - mse: 25.2129 - val_loss: 24.0665 - val_mse: 24.0665 - lr: 0.0010\n",
      "Epoch 65/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 24.7332 - mse: 24.7332\n",
      "Epoch 65: val_loss did not improve from 24.06650\n",
      "349/349 [==============================] - 10s 28ms/step - loss: 24.7147 - mse: 24.7147 - val_loss: 24.8940 - val_mse: 24.8940 - lr: 0.0010\n",
      "Epoch 66/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 25.3272 - mse: 25.3272\n",
      "Epoch 66: val_loss did not improve from 24.06650\n",
      "349/349 [==============================] - 16s 47ms/step - loss: 25.3356 - mse: 25.3356 - val_loss: 25.4907 - val_mse: 25.4907 - lr: 0.0010\n",
      "Epoch 67/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 25.0208 - mse: 25.0208\n",
      "Epoch 67: val_loss did not improve from 24.06650\n",
      "349/349 [==============================] - 8s 23ms/step - loss: 25.0208 - mse: 25.0208 - val_loss: 24.7357 - val_mse: 24.7357 - lr: 0.0010\n",
      "Epoch 68/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 24.7097 - mse: 24.7097\n",
      "Epoch 68: val_loss did not improve from 24.06650\n",
      "349/349 [==============================] - 7s 19ms/step - loss: 24.7097 - mse: 24.7097 - val_loss: 24.1964 - val_mse: 24.1964 - lr: 0.0010\n",
      "Epoch 69/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 24.8876 - mse: 24.8876\n",
      "Epoch 69: val_loss improved from 24.06650 to 23.85963, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 12s 33ms/step - loss: 24.8857 - mse: 24.8857 - val_loss: 23.8596 - val_mse: 23.8596 - lr: 0.0010\n",
      "Epoch 70/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 24.8412 - mse: 24.8412\n",
      "Epoch 70: val_loss did not improve from 23.85963\n",
      "349/349 [==============================] - 11s 30ms/step - loss: 24.8412 - mse: 24.8412 - val_loss: 24.1130 - val_mse: 24.1130 - lr: 0.0010\n",
      "Epoch 71/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 24.7406 - mse: 24.7406\n",
      "Epoch 71: val_loss improved from 23.85963 to 23.85962, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 7s 21ms/step - loss: 24.7829 - mse: 24.7829 - val_loss: 23.8596 - val_mse: 23.8596 - lr: 0.0010\n",
      "Epoch 72/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 24.4654 - mse: 24.4654\n",
      "Epoch 72: val_loss did not improve from 23.85962\n",
      "349/349 [==============================] - 11s 31ms/step - loss: 24.4654 - mse: 24.4654 - val_loss: 24.5440 - val_mse: 24.5440 - lr: 0.0010\n",
      "Epoch 73/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 24.7428 - mse: 24.7428\n",
      "Epoch 73: val_loss did not improve from 23.85962\n",
      "349/349 [==============================] - 13s 38ms/step - loss: 24.7428 - mse: 24.7428 - val_loss: 24.2863 - val_mse: 24.2863 - lr: 0.0010\n",
      "Epoch 74/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 24.3314 - mse: 24.3314\n",
      "Epoch 74: val_loss did not improve from 23.85962\n",
      "349/349 [==============================] - 9s 27ms/step - loss: 24.3414 - mse: 24.3414 - val_loss: 25.7179 - val_mse: 25.7179 - lr: 0.0010\n",
      "Epoch 75/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 24.8055 - mse: 24.8055\n",
      "Epoch 75: val_loss did not improve from 23.85962\n",
      "349/349 [==============================] - 10s 30ms/step - loss: 24.8055 - mse: 24.8055 - val_loss: 24.3754 - val_mse: 24.3754 - lr: 0.0010\n",
      "Epoch 76/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 24.1198 - mse: 24.1198\n",
      "Epoch 76: val_loss did not improve from 23.85962\n",
      "349/349 [==============================] - 7s 20ms/step - loss: 24.1388 - mse: 24.1388 - val_loss: 24.3430 - val_mse: 24.3430 - lr: 0.0010\n",
      "Epoch 77/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 24.7244 - mse: 24.7244\n",
      "Epoch 77: val_loss did not improve from 23.85962\n",
      "349/349 [==============================] - 10s 28ms/step - loss: 24.7255 - mse: 24.7255 - val_loss: 25.8614 - val_mse: 25.8614 - lr: 0.0010\n",
      "Epoch 78/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 24.5546 - mse: 24.5546\n",
      "Epoch 78: val_loss did not improve from 23.85962\n",
      "349/349 [==============================] - 15s 43ms/step - loss: 24.5546 - mse: 24.5546 - val_loss: 24.3072 - val_mse: 24.3072 - lr: 0.0010\n",
      "Epoch 79/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 24.5636 - mse: 24.5636\n",
      "Epoch 79: val_loss improved from 23.85962 to 23.53287, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 13s 37ms/step - loss: 24.5636 - mse: 24.5636 - val_loss: 23.5329 - val_mse: 23.5329 - lr: 0.0010\n",
      "Epoch 80/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 24.3001 - mse: 24.3001\n",
      "Epoch 80: val_loss did not improve from 23.53287\n",
      "349/349 [==============================] - 13s 36ms/step - loss: 24.3001 - mse: 24.3001 - val_loss: 25.4797 - val_mse: 25.4797 - lr: 0.0010\n",
      "Epoch 81/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 24.3675 - mse: 24.3675\n",
      "Epoch 81: val_loss improved from 23.53287 to 23.41517, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 12s 34ms/step - loss: 24.3195 - mse: 24.3195 - val_loss: 23.4152 - val_mse: 23.4152 - lr: 0.0010\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 82/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 24.2100 - mse: 24.2100\n",
      "Epoch 82: val_loss did not improve from 23.41517\n",
      "349/349 [==============================] - 13s 36ms/step - loss: 24.1988 - mse: 24.1988 - val_loss: 24.4282 - val_mse: 24.4282 - lr: 0.0010\n",
      "Epoch 83/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 24.8916 - mse: 24.8916\n",
      "Epoch 83: val_loss did not improve from 23.41517\n",
      "349/349 [==============================] - 13s 37ms/step - loss: 24.8605 - mse: 24.8605 - val_loss: 26.5323 - val_mse: 26.5323 - lr: 0.0010\n",
      "Epoch 84/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 23.9583 - mse: 23.9583\n",
      "Epoch 84: val_loss did not improve from 23.41517\n",
      "349/349 [==============================] - 14s 41ms/step - loss: 24.0487 - mse: 24.0487 - val_loss: 23.9174 - val_mse: 23.9174 - lr: 0.0010\n",
      "Epoch 85/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 24.3710 - mse: 24.3710\n",
      "Epoch 85: val_loss improved from 23.41517 to 23.40747, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 14s 39ms/step - loss: 24.4270 - mse: 24.4270 - val_loss: 23.4075 - val_mse: 23.4075 - lr: 0.0010\n",
      "Epoch 86/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 24.2297 - mse: 24.2297\n",
      "Epoch 86: val_loss did not improve from 23.40747\n",
      "349/349 [==============================] - 14s 41ms/step - loss: 24.2297 - mse: 24.2297 - val_loss: 23.8709 - val_mse: 23.8709 - lr: 0.0010\n",
      "Epoch 87/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 24.5219 - mse: 24.5219\n",
      "Epoch 87: val_loss did not improve from 23.40747\n",
      "349/349 [==============================] - 14s 39ms/step - loss: 24.5260 - mse: 24.5260 - val_loss: 23.6684 - val_mse: 23.6684 - lr: 0.0010\n",
      "Epoch 88/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 24.0367 - mse: 24.0367\n",
      "Epoch 88: val_loss did not improve from 23.40747\n",
      "349/349 [==============================] - 13s 37ms/step - loss: 24.0310 - mse: 24.0310 - val_loss: 23.4422 - val_mse: 23.4422 - lr: 0.0010\n",
      "Epoch 89/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 24.9823 - mse: 24.9823\n",
      "Epoch 89: val_loss did not improve from 23.40747\n",
      "349/349 [==============================] - 17s 48ms/step - loss: 24.9823 - mse: 24.9823 - val_loss: 24.0148 - val_mse: 24.0148 - lr: 0.0010\n",
      "Epoch 90/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 24.3713 - mse: 24.3713\n",
      "Epoch 90: val_loss improved from 23.40747 to 23.34069, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 12s 35ms/step - loss: 24.3713 - mse: 24.3713 - val_loss: 23.3407 - val_mse: 23.3407 - lr: 0.0010\n",
      "Epoch 91/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 23.9373 - mse: 23.9373\n",
      "Epoch 91: val_loss did not improve from 23.34069\n",
      "349/349 [==============================] - 14s 40ms/step - loss: 23.9408 - mse: 23.9408 - val_loss: 25.2364 - val_mse: 25.2364 - lr: 0.0010\n",
      "Epoch 92/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 24.3933 - mse: 24.3933\n",
      "Epoch 92: val_loss did not improve from 23.34069\n",
      "349/349 [==============================] - 17s 49ms/step - loss: 24.3957 - mse: 24.3957 - val_loss: 24.1640 - val_mse: 24.1640 - lr: 0.0010\n",
      "Epoch 93/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 23.9564 - mse: 23.9564\n",
      "Epoch 93: val_loss did not improve from 23.34069\n",
      "349/349 [==============================] - 18s 51ms/step - loss: 23.9564 - mse: 23.9564 - val_loss: 23.8701 - val_mse: 23.8701 - lr: 0.0010\n",
      "Epoch 94/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 24.5120 - mse: 24.5120\n",
      "Epoch 94: val_loss did not improve from 23.34069\n",
      "349/349 [==============================] - 17s 48ms/step - loss: 24.5004 - mse: 24.5004 - val_loss: 24.4359 - val_mse: 24.4359 - lr: 0.0010\n",
      "Epoch 95/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 24.4505 - mse: 24.4505\n",
      "Epoch 95: val_loss improved from 23.34069 to 23.31415, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 17s 48ms/step - loss: 24.4462 - mse: 24.4462 - val_loss: 23.3142 - val_mse: 23.3142 - lr: 0.0010\n",
      "Epoch 96/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 24.1563 - mse: 24.1563\n",
      "Epoch 96: val_loss did not improve from 23.31415\n",
      "349/349 [==============================] - 19s 54ms/step - loss: 24.1411 - mse: 24.1411 - val_loss: 23.3341 - val_mse: 23.3341 - lr: 0.0010\n",
      "Epoch 97/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 23.8914 - mse: 23.8914\n",
      "Epoch 97: val_loss did not improve from 23.31415\n",
      "349/349 [==============================] - 16s 45ms/step - loss: 23.8937 - mse: 23.8937 - val_loss: 24.8136 - val_mse: 24.8136 - lr: 0.0010\n",
      "Epoch 98/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 23.8705 - mse: 23.8705\n",
      "Epoch 98: val_loss improved from 23.31415 to 22.97460, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 19s 54ms/step - loss: 23.8705 - mse: 23.8705 - val_loss: 22.9746 - val_mse: 22.9746 - lr: 0.0010\n",
      "Epoch 99/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 23.8347 - mse: 23.8347\n",
      "Epoch 99: val_loss did not improve from 22.97460\n",
      "349/349 [==============================] - 13s 39ms/step - loss: 23.8558 - mse: 23.8558 - val_loss: 23.1808 - val_mse: 23.1808 - lr: 0.0010\n",
      "Epoch 100/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 24.3132 - mse: 24.3132\n",
      "Epoch 100: val_loss did not improve from 22.97460\n",
      "349/349 [==============================] - 18s 52ms/step - loss: 24.3132 - mse: 24.3132 - val_loss: 23.1869 - val_mse: 23.1869 - lr: 0.0010\n",
      "Epoch 101/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 23.8396 - mse: 23.8396\n",
      "Epoch 101: val_loss did not improve from 22.97460\n",
      "349/349 [==============================] - 13s 36ms/step - loss: 23.8262 - mse: 23.8262 - val_loss: 23.0410 - val_mse: 23.0410 - lr: 0.0010\n",
      "Epoch 102/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 24.0116 - mse: 24.0116\n",
      "Epoch 102: val_loss did not improve from 22.97460\n",
      "349/349 [==============================] - 15s 44ms/step - loss: 24.0293 - mse: 24.0293 - val_loss: 31.4729 - val_mse: 31.4729 - lr: 0.0010\n",
      "Epoch 103/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 23.9419 - mse: 23.9419\n",
      "Epoch 103: val_loss did not improve from 22.97460\n",
      "349/349 [==============================] - 13s 37ms/step - loss: 23.9497 - mse: 23.9497 - val_loss: 23.6638 - val_mse: 23.6638 - lr: 0.0010\n",
      "Epoch 104/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 23.8453 - mse: 23.8453\n",
      "Epoch 104: val_loss did not improve from 22.97460\n",
      "349/349 [==============================] - 16s 47ms/step - loss: 23.8453 - mse: 23.8453 - val_loss: 24.7996 - val_mse: 24.7996 - lr: 0.0010\n",
      "Epoch 105/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 23.9657 - mse: 23.9657\n",
      "Epoch 105: val_loss did not improve from 22.97460\n",
      "349/349 [==============================] - 14s 39ms/step - loss: 23.9657 - mse: 23.9657 - val_loss: 23.1807 - val_mse: 23.1807 - lr: 0.0010\n",
      "Epoch 106/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 24.0839 - mse: 24.0839\n",
      "Epoch 106: val_loss did not improve from 22.97460\n",
      "349/349 [==============================] - 19s 55ms/step - loss: 24.0868 - mse: 24.0868 - val_loss: 23.3915 - val_mse: 23.3915 - lr: 0.0010\n",
      "Epoch 107/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 23.8208 - mse: 23.8208\n",
      "Epoch 107: val_loss did not improve from 22.97460\n",
      "349/349 [==============================] - 13s 38ms/step - loss: 23.8028 - mse: 23.8028 - val_loss: 23.9589 - val_mse: 23.9589 - lr: 0.0010\n",
      "Epoch 108/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 23.5286 - mse: 23.5286\n",
      "Epoch 108: val_loss did not improve from 22.97460\n",
      "349/349 [==============================] - 14s 39ms/step - loss: 23.5286 - mse: 23.5286 - val_loss: 23.5040 - val_mse: 23.5040 - lr: 0.0010\n",
      "Epoch 109/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 23.7160 - mse: 23.7160\n",
      "Epoch 109: val_loss did not improve from 22.97460\n",
      "349/349 [==============================] - 10s 27ms/step - loss: 23.7122 - mse: 23.7122 - val_loss: 23.9838 - val_mse: 23.9838 - lr: 0.0010\n",
      "Epoch 110/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "348/349 [============================>.] - ETA: 0s - loss: 23.9758 - mse: 23.9758\n",
      "Epoch 110: val_loss did not improve from 22.97460\n",
      "349/349 [==============================] - 11s 32ms/step - loss: 23.9673 - mse: 23.9673 - val_loss: 24.4712 - val_mse: 24.4712 - lr: 0.0010\n",
      "Epoch 111/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 23.5774 - mse: 23.5774\n",
      "Epoch 111: val_loss did not improve from 22.97460\n",
      "349/349 [==============================] - 12s 34ms/step - loss: 23.5703 - mse: 23.5703 - val_loss: 24.3840 - val_mse: 24.3840 - lr: 0.0010\n",
      "Epoch 112/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 23.5341 - mse: 23.5341\n",
      "Epoch 112: val_loss did not improve from 22.97460\n",
      "349/349 [==============================] - 12s 35ms/step - loss: 23.5341 - mse: 23.5341 - val_loss: 23.0420 - val_mse: 23.0420 - lr: 0.0010\n",
      "Epoch 113/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 23.4154 - mse: 23.4154\n",
      "Epoch 113: val_loss did not improve from 22.97460\n",
      "349/349 [==============================] - 10s 28ms/step - loss: 23.4092 - mse: 23.4092 - val_loss: 24.4172 - val_mse: 24.4172 - lr: 0.0010\n",
      "Epoch 114/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 23.4188 - mse: 23.4188\n",
      "Epoch 114: val_loss did not improve from 22.97460\n",
      "349/349 [==============================] - 9s 25ms/step - loss: 23.4188 - mse: 23.4188 - val_loss: 23.5394 - val_mse: 23.5394 - lr: 0.0010\n",
      "Epoch 115/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 23.6252 - mse: 23.6252\n",
      "Epoch 115: val_loss did not improve from 22.97460\n",
      "349/349 [==============================] - 10s 30ms/step - loss: 23.6252 - mse: 23.6252 - val_loss: 24.8991 - val_mse: 24.8991 - lr: 0.0010\n",
      "Epoch 116/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 23.5468 - mse: 23.5468\n",
      "Epoch 116: val_loss improved from 22.97460 to 22.70744, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 9s 26ms/step - loss: 23.5382 - mse: 23.5382 - val_loss: 22.7074 - val_mse: 22.7074 - lr: 0.0010\n",
      "Epoch 117/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 23.1765 - mse: 23.1765\n",
      "Epoch 117: val_loss did not improve from 22.70744\n",
      "349/349 [==============================] - 13s 36ms/step - loss: 23.1765 - mse: 23.1765 - val_loss: 23.0110 - val_mse: 23.0110 - lr: 0.0010\n",
      "Epoch 118/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 23.5802 - mse: 23.5802\n",
      "Epoch 118: val_loss did not improve from 22.70744\n",
      "349/349 [==============================] - 9s 25ms/step - loss: 23.5802 - mse: 23.5802 - val_loss: 22.9425 - val_mse: 22.9425 - lr: 0.0010\n",
      "Epoch 119/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 23.2022 - mse: 23.2022\n",
      "Epoch 119: val_loss did not improve from 22.70744\n",
      "349/349 [==============================] - 10s 29ms/step - loss: 23.2038 - mse: 23.2038 - val_loss: 22.9000 - val_mse: 22.9000 - lr: 0.0010\n",
      "Epoch 120/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 23.4843 - mse: 23.4843\n",
      "Epoch 120: val_loss did not improve from 22.70744\n",
      "349/349 [==============================] - 8s 22ms/step - loss: 23.4808 - mse: 23.4808 - val_loss: 23.7907 - val_mse: 23.7907 - lr: 0.0010\n",
      "Epoch 121/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 23.3528 - mse: 23.3528\n",
      "Epoch 121: val_loss did not improve from 22.70744\n",
      "349/349 [==============================] - 9s 27ms/step - loss: 23.3739 - mse: 23.3739 - val_loss: 24.4944 - val_mse: 24.4944 - lr: 0.0010\n",
      "Epoch 122/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 23.7389 - mse: 23.7389\n",
      "Epoch 122: val_loss improved from 22.70744 to 22.70617, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 10s 28ms/step - loss: 23.7404 - mse: 23.7404 - val_loss: 22.7062 - val_mse: 22.7062 - lr: 0.0010\n",
      "Epoch 123/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 23.6459 - mse: 23.6459\n",
      "Epoch 123: val_loss did not improve from 22.70617\n",
      "349/349 [==============================] - 6s 19ms/step - loss: 23.6005 - mse: 23.6005 - val_loss: 23.5499 - val_mse: 23.5499 - lr: 0.0010\n",
      "Epoch 124/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 23.1567 - mse: 23.1567\n",
      "Epoch 124: val_loss did not improve from 22.70617\n",
      "349/349 [==============================] - 11s 32ms/step - loss: 23.1567 - mse: 23.1567 - val_loss: 24.0376 - val_mse: 24.0376 - lr: 0.0010\n",
      "Epoch 125/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 23.3616 - mse: 23.3616\n",
      "Epoch 125: val_loss improved from 22.70617 to 22.39085, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 9s 26ms/step - loss: 23.3296 - mse: 23.3296 - val_loss: 22.3908 - val_mse: 22.3908 - lr: 0.0010\n",
      "Epoch 126/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 22.9407 - mse: 22.9407\n",
      "Epoch 126: val_loss did not improve from 22.39085\n",
      "349/349 [==============================] - 11s 31ms/step - loss: 22.9268 - mse: 22.9268 - val_loss: 23.2254 - val_mse: 23.2254 - lr: 0.0010\n",
      "Epoch 127/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 23.6877 - mse: 23.6877\n",
      "Epoch 127: val_loss did not improve from 22.39085\n",
      "349/349 [==============================] - 9s 26ms/step - loss: 23.6877 - mse: 23.6877 - val_loss: 24.0325 - val_mse: 24.0325 - lr: 0.0010\n",
      "Epoch 128/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 23.2787 - mse: 23.2787\n",
      "Epoch 128: val_loss did not improve from 22.39085\n",
      "349/349 [==============================] - 11s 31ms/step - loss: 23.2787 - mse: 23.2787 - val_loss: 23.1338 - val_mse: 23.1338 - lr: 0.0010\n",
      "Epoch 129/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 23.4702 - mse: 23.4702\n",
      "Epoch 129: val_loss did not improve from 22.39085\n",
      "349/349 [==============================] - 7s 21ms/step - loss: 23.4601 - mse: 23.4601 - val_loss: 22.9233 - val_mse: 22.9233 - lr: 0.0010\n",
      "Epoch 130/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 23.2274 - mse: 23.2274\n",
      "Epoch 130: val_loss did not improve from 22.39085\n",
      "349/349 [==============================] - 9s 24ms/step - loss: 23.2303 - mse: 23.2303 - val_loss: 23.6232 - val_mse: 23.6232 - lr: 0.0010\n",
      "Epoch 131/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 23.7312 - mse: 23.7312\n",
      "Epoch 131: val_loss did not improve from 22.39085\n",
      "349/349 [==============================] - 9s 27ms/step - loss: 23.7099 - mse: 23.7099 - val_loss: 23.2003 - val_mse: 23.2003 - lr: 0.0010\n",
      "Epoch 132/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 23.0042 - mse: 23.0042\n",
      "Epoch 132: val_loss did not improve from 22.39085\n",
      "349/349 [==============================] - 9s 25ms/step - loss: 23.0042 - mse: 23.0042 - val_loss: 24.7778 - val_mse: 24.7778 - lr: 0.0010\n",
      "Epoch 133/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 22.9498 - mse: 22.9498\n",
      "Epoch 133: val_loss did not improve from 22.39085\n",
      "349/349 [==============================] - 6s 17ms/step - loss: 22.9136 - mse: 22.9136 - val_loss: 22.4923 - val_mse: 22.4923 - lr: 0.0010\n",
      "Epoch 134/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 22.9261 - mse: 22.9261\n",
      "Epoch 134: val_loss did not improve from 22.39085\n",
      "349/349 [==============================] - 9s 25ms/step - loss: 22.9205 - mse: 22.9205 - val_loss: 22.6074 - val_mse: 22.6074 - lr: 0.0010\n",
      "Epoch 135/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 23.4423 - mse: 23.4423\n",
      "Epoch 135: val_loss did not improve from 22.39085\n",
      "349/349 [==============================] - 8s 23ms/step - loss: 23.4423 - mse: 23.4423 - val_loss: 23.7641 - val_mse: 23.7641 - lr: 0.0010\n",
      "Epoch 136/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 22.7003 - mse: 22.7003\n",
      "Epoch 136: val_loss did not improve from 22.39085\n",
      "349/349 [==============================] - 9s 25ms/step - loss: 22.6911 - mse: 22.6911 - val_loss: 22.4798 - val_mse: 22.4798 - lr: 0.0010\n",
      "Epoch 137/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 22.9985 - mse: 22.9985\n",
      "Epoch 137: val_loss did not improve from 22.39085\n",
      "349/349 [==============================] - 10s 28ms/step - loss: 23.0069 - mse: 23.0069 - val_loss: 22.4943 - val_mse: 22.4943 - lr: 0.0010\n",
      "Epoch 138/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "349/349 [==============================] - ETA: 0s - loss: 22.8887 - mse: 22.8887\n",
      "Epoch 138: val_loss did not improve from 22.39085\n",
      "349/349 [==============================] - 6s 19ms/step - loss: 22.8887 - mse: 22.8887 - val_loss: 23.3770 - val_mse: 23.3770 - lr: 0.0010\n",
      "Epoch 139/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 23.1544 - mse: 23.1544\n",
      "Epoch 139: val_loss did not improve from 22.39085\n",
      "349/349 [==============================] - 6s 18ms/step - loss: 23.1544 - mse: 23.1544 - val_loss: 22.4309 - val_mse: 22.4309 - lr: 0.0010\n",
      "Epoch 140/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 22.9970 - mse: 22.9970\n",
      "Epoch 140: val_loss did not improve from 22.39085\n",
      "349/349 [==============================] - 11s 31ms/step - loss: 23.0309 - mse: 23.0309 - val_loss: 23.0311 - val_mse: 23.0311 - lr: 0.0010\n",
      "Epoch 141/200\n",
      "346/349 [============================>.] - ETA: 0s - loss: 23.2140 - mse: 23.2140\n",
      "Epoch 141: val_loss did not improve from 22.39085\n",
      "349/349 [==============================] - 6s 18ms/step - loss: 23.1810 - mse: 23.1810 - val_loss: 22.5268 - val_mse: 22.5268 - lr: 0.0010\n",
      "Epoch 142/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 22.5589 - mse: 22.5589\n",
      "Epoch 142: val_loss did not improve from 22.39085\n",
      "349/349 [==============================] - 9s 27ms/step - loss: 22.5589 - mse: 22.5589 - val_loss: 22.6338 - val_mse: 22.6338 - lr: 0.0010\n",
      "Epoch 143/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 23.1508 - mse: 23.1508\n",
      "Epoch 143: val_loss did not improve from 22.39085\n",
      "349/349 [==============================] - 9s 27ms/step - loss: 23.1508 - mse: 23.1508 - val_loss: 22.9176 - val_mse: 22.9176 - lr: 0.0010\n",
      "Epoch 144/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 23.0902 - mse: 23.0902\n",
      "Epoch 144: val_loss did not improve from 22.39085\n",
      "349/349 [==============================] - 9s 27ms/step - loss: 23.0850 - mse: 23.0850 - val_loss: 23.0362 - val_mse: 23.0362 - lr: 0.0010\n",
      "Epoch 145/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 22.4804 - mse: 22.4804\n",
      "Epoch 145: val_loss improved from 22.39085 to 22.18015, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 8s 24ms/step - loss: 22.4804 - mse: 22.4804 - val_loss: 22.1802 - val_mse: 22.1802 - lr: 0.0010\n",
      "Epoch 146/200\n",
      "346/349 [============================>.] - ETA: 0s - loss: 22.8814 - mse: 22.8814\n",
      "Epoch 146: val_loss did not improve from 22.18015\n",
      "349/349 [==============================] - 7s 21ms/step - loss: 22.8550 - mse: 22.8550 - val_loss: 22.8947 - val_mse: 22.8947 - lr: 0.0010\n",
      "Epoch 147/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 22.8416 - mse: 22.8416\n",
      "Epoch 147: val_loss improved from 22.18015 to 22.08293, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 9s 27ms/step - loss: 22.8317 - mse: 22.8317 - val_loss: 22.0829 - val_mse: 22.0829 - lr: 0.0010\n",
      "Epoch 148/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 23.6052 - mse: 23.6052\n",
      "Epoch 148: val_loss did not improve from 22.08293\n",
      "349/349 [==============================] - 8s 22ms/step - loss: 23.6129 - mse: 23.6129 - val_loss: 22.9041 - val_mse: 22.9041 - lr: 0.0010\n",
      "Epoch 149/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 23.3221 - mse: 23.3221\n",
      "Epoch 149: val_loss did not improve from 22.08293\n",
      "349/349 [==============================] - 9s 26ms/step - loss: 23.3203 - mse: 23.3203 - val_loss: 23.1982 - val_mse: 23.1982 - lr: 0.0010\n",
      "Epoch 150/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 22.8549 - mse: 22.8549\n",
      "Epoch 150: val_loss did not improve from 22.08293\n",
      "349/349 [==============================] - 6s 17ms/step - loss: 22.8473 - mse: 22.8473 - val_loss: 22.6916 - val_mse: 22.6916 - lr: 0.0010\n",
      "Epoch 151/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 22.3993 - mse: 22.3993\n",
      "Epoch 151: val_loss did not improve from 22.08293\n",
      "349/349 [==============================] - 6s 18ms/step - loss: 22.3878 - mse: 22.3878 - val_loss: 22.8132 - val_mse: 22.8132 - lr: 0.0010\n",
      "Epoch 152/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 22.4546 - mse: 22.4546\n",
      "Epoch 152: val_loss did not improve from 22.08293\n",
      "349/349 [==============================] - 8s 24ms/step - loss: 22.4453 - mse: 22.4453 - val_loss: 22.5718 - val_mse: 22.5718 - lr: 0.0010\n",
      "Epoch 153/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 22.9110 - mse: 22.9110\n",
      "Epoch 153: val_loss did not improve from 22.08293\n",
      "349/349 [==============================] - 5s 16ms/step - loss: 22.8841 - mse: 22.8841 - val_loss: 23.5730 - val_mse: 23.5730 - lr: 0.0010\n",
      "Epoch 154/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 22.1782 - mse: 22.1782\n",
      "Epoch 154: val_loss did not improve from 22.08293\n",
      "349/349 [==============================] - 10s 30ms/step - loss: 22.1782 - mse: 22.1782 - val_loss: 22.4783 - val_mse: 22.4783 - lr: 0.0010\n",
      "Epoch 155/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 22.5627 - mse: 22.5627\n",
      "Epoch 155: val_loss did not improve from 22.08293\n",
      "349/349 [==============================] - 13s 36ms/step - loss: 22.5492 - mse: 22.5492 - val_loss: 22.3369 - val_mse: 22.3369 - lr: 0.0010\n",
      "Epoch 156/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 22.3570 - mse: 22.3570\n",
      "Epoch 156: val_loss improved from 22.08293 to 22.00205, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 5s 13ms/step - loss: 22.3432 - mse: 22.3432 - val_loss: 22.0021 - val_mse: 22.0021 - lr: 0.0010\n",
      "Epoch 157/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 22.2457 - mse: 22.2457\n",
      "Epoch 157: val_loss did not improve from 22.00205\n",
      "349/349 [==============================] - 8s 22ms/step - loss: 22.2362 - mse: 22.2362 - val_loss: 23.5790 - val_mse: 23.5790 - lr: 0.0010\n",
      "Epoch 158/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 23.0605 - mse: 23.0605\n",
      "Epoch 158: val_loss did not improve from 22.00205\n",
      "349/349 [==============================] - 8s 22ms/step - loss: 23.1893 - mse: 23.1893 - val_loss: 22.2267 - val_mse: 22.2267 - lr: 0.0010\n",
      "Epoch 159/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 22.1488 - mse: 22.1488\n",
      "Epoch 159: val_loss did not improve from 22.00205\n",
      "349/349 [==============================] - 11s 31ms/step - loss: 22.1750 - mse: 22.1750 - val_loss: 22.1953 - val_mse: 22.1953 - lr: 0.0010\n",
      "Epoch 160/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 22.5670 - mse: 22.5670\n",
      "Epoch 160: val_loss did not improve from 22.00205\n",
      "349/349 [==============================] - 10s 30ms/step - loss: 22.5670 - mse: 22.5670 - val_loss: 25.4474 - val_mse: 25.4474 - lr: 0.0010\n",
      "Epoch 161/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 22.2587 - mse: 22.2587\n",
      "Epoch 161: val_loss did not improve from 22.00205\n",
      "349/349 [==============================] - 10s 29ms/step - loss: 22.2507 - mse: 22.2507 - val_loss: 22.9631 - val_mse: 22.9631 - lr: 0.0010\n",
      "Epoch 162/200\n",
      "346/349 [============================>.] - ETA: 0s - loss: 22.1342 - mse: 22.1342\n",
      "Epoch 162: val_loss did not improve from 22.00205\n",
      "349/349 [==============================] - 9s 25ms/step - loss: 22.2436 - mse: 22.2436 - val_loss: 22.2489 - val_mse: 22.2489 - lr: 0.0010\n",
      "Epoch 163/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 22.5262 - mse: 22.5262\n",
      "Epoch 163: val_loss did not improve from 22.00205\n",
      "349/349 [==============================] - 8s 22ms/step - loss: 22.5262 - mse: 22.5262 - val_loss: 22.3580 - val_mse: 22.3580 - lr: 0.0010\n",
      "Epoch 164/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 23.8255 - mse: 23.8255\n",
      "Epoch 164: val_loss did not improve from 22.00205\n",
      "349/349 [==============================] - 12s 35ms/step - loss: 23.8248 - mse: 23.8248 - val_loss: 23.2716 - val_mse: 23.2716 - lr: 0.0010\n",
      "Epoch 165/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 22.3597 - mse: 22.3597\n",
      "Epoch 165: val_loss did not improve from 22.00205\n",
      "349/349 [==============================] - 11s 30ms/step - loss: 22.3525 - mse: 22.3525 - val_loss: 22.7244 - val_mse: 22.7244 - lr: 0.0010\n",
      "Epoch 166/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "349/349 [==============================] - ETA: 0s - loss: 23.0540 - mse: 23.0540\n",
      "Epoch 166: val_loss did not improve from 22.00205\n",
      "349/349 [==============================] - 9s 26ms/step - loss: 23.0540 - mse: 23.0540 - val_loss: 22.4595 - val_mse: 22.4595 - lr: 0.0010\n",
      "Epoch 167/200\n",
      "346/349 [============================>.] - ETA: 0s - loss: 22.1520 - mse: 22.1520\n",
      "Epoch 167: val_loss did not improve from 22.00205\n",
      "349/349 [==============================] - 9s 26ms/step - loss: 22.1728 - mse: 22.1728 - val_loss: 22.0898 - val_mse: 22.0898 - lr: 0.0010\n",
      "Epoch 168/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 22.4189 - mse: 22.4189\n",
      "Epoch 168: val_loss did not improve from 22.00205\n",
      "349/349 [==============================] - 8s 23ms/step - loss: 22.4160 - mse: 22.4160 - val_loss: 22.1667 - val_mse: 22.1667 - lr: 0.0010\n",
      "Epoch 169/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 21.9722 - mse: 21.9722\n",
      "Epoch 169: val_loss did not improve from 22.00205\n",
      "349/349 [==============================] - 10s 28ms/step - loss: 21.9702 - mse: 21.9702 - val_loss: 22.4674 - val_mse: 22.4674 - lr: 0.0010\n",
      "Epoch 170/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 23.0252 - mse: 23.0252\n",
      "Epoch 170: val_loss did not improve from 22.00205\n",
      "349/349 [==============================] - 10s 28ms/step - loss: 23.1715 - mse: 23.1715 - val_loss: 22.2984 - val_mse: 22.2984 - lr: 0.0010\n",
      "Epoch 171/200\n",
      "345/349 [============================>.] - ETA: 0s - loss: 22.1664 - mse: 22.1664\n",
      "Epoch 171: val_loss did not improve from 22.00205\n",
      "349/349 [==============================] - 11s 32ms/step - loss: 22.1605 - mse: 22.1605 - val_loss: 25.5106 - val_mse: 25.5106 - lr: 0.0010\n",
      "Epoch 172/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 23.3391 - mse: 23.3391\n",
      "Epoch 172: val_loss did not improve from 22.00205\n",
      "349/349 [==============================] - 10s 29ms/step - loss: 23.3391 - mse: 23.3391 - val_loss: 22.3676 - val_mse: 22.3676 - lr: 0.0010\n",
      "Epoch 173/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 22.3662 - mse: 22.3662\n",
      "Epoch 173: val_loss did not improve from 22.00205\n",
      "349/349 [==============================] - 10s 29ms/step - loss: 22.3662 - mse: 22.3662 - val_loss: 22.2635 - val_mse: 22.2635 - lr: 0.0010\n",
      "Epoch 174/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 22.5118 - mse: 22.5118\n",
      "Epoch 174: val_loss did not improve from 22.00205\n",
      "349/349 [==============================] - 11s 32ms/step - loss: 22.5080 - mse: 22.5080 - val_loss: 25.5329 - val_mse: 25.5329 - lr: 0.0010\n",
      "Epoch 175/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 22.0858 - mse: 22.0858\n",
      "Epoch 175: val_loss improved from 22.00205 to 21.79279, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 12s 34ms/step - loss: 22.0847 - mse: 22.0847 - val_loss: 21.7928 - val_mse: 21.7928 - lr: 0.0010\n",
      "Epoch 176/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 22.0101 - mse: 22.0101\n",
      "Epoch 176: val_loss did not improve from 21.79279\n",
      "349/349 [==============================] - 11s 30ms/step - loss: 21.9789 - mse: 21.9789 - val_loss: 22.2061 - val_mse: 22.2061 - lr: 0.0010\n",
      "Epoch 177/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 22.0339 - mse: 22.0339\n",
      "Epoch 177: val_loss did not improve from 21.79279\n",
      "349/349 [==============================] - 15s 44ms/step - loss: 22.0258 - mse: 22.0258 - val_loss: 22.4267 - val_mse: 22.4267 - lr: 0.0010\n",
      "Epoch 178/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 22.1907 - mse: 22.1907\n",
      "Epoch 178: val_loss did not improve from 21.79279\n",
      "349/349 [==============================] - 11s 31ms/step - loss: 22.1907 - mse: 22.1907 - val_loss: 22.5736 - val_mse: 22.5736 - lr: 0.0010\n",
      "Epoch 179/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 22.2802 - mse: 22.2802\n",
      "Epoch 179: val_loss did not improve from 21.79279\n",
      "349/349 [==============================] - 14s 40ms/step - loss: 22.2804 - mse: 22.2804 - val_loss: 23.5458 - val_mse: 23.5458 - lr: 0.0010\n",
      "Epoch 180/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 22.5533 - mse: 22.5533\n",
      "Epoch 180: val_loss did not improve from 21.79279\n",
      "349/349 [==============================] - 12s 35ms/step - loss: 22.5112 - mse: 22.5112 - val_loss: 22.0759 - val_mse: 22.0759 - lr: 0.0010\n",
      "Epoch 181/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 21.8696 - mse: 21.8696\n",
      "Epoch 181: val_loss did not improve from 21.79279\n",
      "349/349 [==============================] - 14s 40ms/step - loss: 21.8588 - mse: 21.8588 - val_loss: 22.3347 - val_mse: 22.3347 - lr: 0.0010\n",
      "Epoch 182/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 21.8372 - mse: 21.8372\n",
      "Epoch 182: val_loss did not improve from 21.79279\n",
      "349/349 [==============================] - 11s 33ms/step - loss: 21.8372 - mse: 21.8372 - val_loss: 22.5472 - val_mse: 22.5472 - lr: 0.0010\n",
      "Epoch 183/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 21.7091 - mse: 21.7091\n",
      "Epoch 183: val_loss did not improve from 21.79279\n",
      "349/349 [==============================] - 11s 32ms/step - loss: 21.7091 - mse: 21.7091 - val_loss: 22.0381 - val_mse: 22.0381 - lr: 0.0010\n",
      "Epoch 184/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 21.8396 - mse: 21.8396\n",
      "Epoch 184: val_loss improved from 21.79279 to 21.52619, saving model to best_model_log4tutorial.h5\n",
      "349/349 [==============================] - 13s 36ms/step - loss: 21.8265 - mse: 21.8265 - val_loss: 21.5262 - val_mse: 21.5262 - lr: 0.0010\n",
      "Epoch 185/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 21.9594 - mse: 21.9594\n",
      "Epoch 185: val_loss did not improve from 21.52619\n",
      "349/349 [==============================] - 11s 31ms/step - loss: 21.9539 - mse: 21.9539 - val_loss: 22.6144 - val_mse: 22.6144 - lr: 0.0010\n",
      "Epoch 186/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 21.9519 - mse: 21.9519\n",
      "Epoch 186: val_loss did not improve from 21.52619\n",
      "349/349 [==============================] - 11s 31ms/step - loss: 21.9426 - mse: 21.9426 - val_loss: 22.8576 - val_mse: 22.8576 - lr: 0.0010\n",
      "Epoch 187/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 21.9791 - mse: 21.9791\n",
      "Epoch 187: val_loss did not improve from 21.52619\n",
      "349/349 [==============================] - 15s 42ms/step - loss: 21.9791 - mse: 21.9791 - val_loss: 24.2490 - val_mse: 24.2490 - lr: 0.0010\n",
      "Epoch 188/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 21.9889 - mse: 21.9889\n",
      "Epoch 188: val_loss did not improve from 21.52619\n",
      "349/349 [==============================] - 11s 32ms/step - loss: 21.9929 - mse: 21.9929 - val_loss: 22.3935 - val_mse: 22.3935 - lr: 0.0010\n",
      "Epoch 189/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 22.0862 - mse: 22.0862\n",
      "Epoch 189: val_loss did not improve from 21.52619\n",
      "349/349 [==============================] - 12s 34ms/step - loss: 22.0862 - mse: 22.0862 - val_loss: 23.0418 - val_mse: 23.0418 - lr: 0.0010\n",
      "Epoch 190/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 22.1711 - mse: 22.1711\n",
      "Epoch 190: val_loss did not improve from 21.52619\n",
      "349/349 [==============================] - 12s 36ms/step - loss: 22.1597 - mse: 22.1597 - val_loss: 21.8203 - val_mse: 21.8203 - lr: 0.0010\n",
      "Epoch 191/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 21.8931 - mse: 21.8931\n",
      "Epoch 191: val_loss did not improve from 21.52619\n",
      "349/349 [==============================] - 11s 32ms/step - loss: 21.8931 - mse: 21.8931 - val_loss: 22.7726 - val_mse: 22.7726 - lr: 0.0010\n",
      "Epoch 192/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 21.8551 - mse: 21.8551\n",
      "Epoch 192: val_loss did not improve from 21.52619\n",
      "349/349 [==============================] - 12s 34ms/step - loss: 21.8449 - mse: 21.8449 - val_loss: 22.4281 - val_mse: 22.4281 - lr: 0.0010\n",
      "Epoch 193/200\n",
      "346/349 [============================>.] - ETA: 0s - loss: 21.6889 - mse: 21.6889\n",
      "Epoch 193: val_loss did not improve from 21.52619\n",
      "349/349 [==============================] - 8s 22ms/step - loss: 21.6738 - mse: 21.6738 - val_loss: 22.5404 - val_mse: 22.5404 - lr: 0.0010\n",
      "Epoch 194/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 22.2454 - mse: 22.2454\n",
      "Epoch 194: val_loss did not improve from 21.52619\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "349/349 [==============================] - 10s 29ms/step - loss: 22.2304 - mse: 22.2304 - val_loss: 22.1998 - val_mse: 22.1998 - lr: 0.0010\n",
      "Epoch 195/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 21.8947 - mse: 21.8947\n",
      "Epoch 195: val_loss did not improve from 21.52619\n",
      "349/349 [==============================] - 9s 27ms/step - loss: 21.8886 - mse: 21.8886 - val_loss: 22.7632 - val_mse: 22.7632 - lr: 0.0010\n",
      "Epoch 196/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 21.8754 - mse: 21.8754\n",
      "Epoch 196: val_loss did not improve from 21.52619\n",
      "349/349 [==============================] - 12s 34ms/step - loss: 21.8747 - mse: 21.8747 - val_loss: 23.1713 - val_mse: 23.1713 - lr: 0.0010\n",
      "Epoch 197/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 22.7242 - mse: 22.7242\n",
      "Epoch 197: val_loss did not improve from 21.52619\n",
      "349/349 [==============================] - 13s 37ms/step - loss: 22.7153 - mse: 22.7153 - val_loss: 22.7634 - val_mse: 22.7634 - lr: 0.0010\n",
      "Epoch 198/200\n",
      "349/349 [==============================] - ETA: 0s - loss: 21.8317 - mse: 21.8317\n",
      "Epoch 198: val_loss did not improve from 21.52619\n",
      "349/349 [==============================] - 14s 40ms/step - loss: 21.8317 - mse: 21.8317 - val_loss: 21.9868 - val_mse: 21.9868 - lr: 0.0010\n",
      "Epoch 199/200\n",
      "347/349 [============================>.] - ETA: 0s - loss: 22.0422 - mse: 22.0422\n",
      "Epoch 199: val_loss did not improve from 21.52619\n",
      "349/349 [==============================] - 10s 29ms/step - loss: 22.0285 - mse: 22.0285 - val_loss: 22.0897 - val_mse: 22.0897 - lr: 0.0010\n",
      "Epoch 200/200\n",
      "348/349 [============================>.] - ETA: 0s - loss: 22.2236 - mse: 22.2236\n",
      "Epoch 200: val_loss did not improve from 21.52619\n",
      "349/349 [==============================] - 8s 23ms/step - loss: 22.2254 - mse: 22.2254 - val_loss: 21.7032 - val_mse: 21.7032 - lr: 0.0010\n"
     ]
    }
   ],
   "source": [
    "# Bebig the training\n",
    "\n",
    "import pandas as pd\n",
    "import tensorflow.keras\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n",
    "import keras\n",
    "sgd = tensorflow.keras.optimizers.Adam(lr=1e-3)\n",
    "lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),\n",
    "                                cooldown= 0,\n",
    "                                patience= 50,\n",
    "                                min_lr=0.5e-8)\n",
    "\n",
    "early_stopping_monitor = EarlyStopping(monitor= 'val_loss', patience = 100)\n",
    "\n",
    "for fname in ['log4tutorial.xlsx']:\n",
    "    df = pd.read_excel(fname)\n",
    "    df_raw_input = df[['DTCO', 'ECGR', 'RHOB', 'PHIT']]\n",
    "    X = df_raw_input\n",
    "    y = df['DTSM']\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state = 0)\n",
    "    checkpoint = ModelCheckpoint(filepath='best_model_%s.h5'%(fname.split('.')[0]),\n",
    "                             monitor='val_loss',\n",
    "                             mode = 'auto',\n",
    "                             verbose=1,\n",
    "                             save_best_only=True)\n",
    "    callbacks = [lr_reducer, early_stopping_monitor, checkpoint]\n",
    "    model = Model(inputs=[inp1], outputs=o)\n",
    "    model.compile(optimizer=sgd, loss='mse', metrics=['mse'])\n",
    "    history = model.fit([X_train], y_train, epochs=200, validation_data=([X_test],y_test), shuffle=True, batch_size=128, callbacks = callbacks)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "97098290",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "598/598 [==============================] - 5s 4ms/step\n",
      "log4tutorial R2: 0.9059577804227912  RMSE: 4.639632443577757\n"
     ]
    }
   ],
   "source": [
    "## Below is to test\n",
    "import pandas as pd\n",
    "from keras.models import load_model\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import r2_score, mean_squared_error\n",
    "\n",
    "for fname in ['log4tutorial.xlsx']:\n",
    "    df = pd.read_excel(fname)\n",
    "\n",
    "    df_raw_input = df[['DTCO', 'ECGR', 'RHOB', 'PHIT']]\n",
    "    X = df_raw_input\n",
    "    y = df['DTSM']\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state = 0)\n",
    "\n",
    "    model = load_model('best_model_%s.h5'%(fname.split('.')[0].split('_')[-1])) #current one:  0.8361\n",
    "    y_pred = model.predict(X_test)\n",
    "    r2 = r2_score(y_test, y_pred)\n",
    "    rmse = mean_squared_error(y_test, y_pred, squared=False)\n",
    "    print(fname.split('.')[0]+' R2:', r2,' RMSE:',rmse)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3170669e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "log4tutorial R2: 0.8929411242623861  RMSE: 4.950322357831745\n"
     ]
    }
   ],
   "source": [
    "## Comparison with random forest\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from keras.models import load_model\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import r2_score, mean_squared_error\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "for fname in ['log4tutorial.xlsx']:\n",
    "    df = pd.read_excel(fname)\n",
    "\n",
    "    df_raw_input = df[['DTCO', 'ECGR', 'RHOB', 'PHIT']]\n",
    "    X = df_raw_input\n",
    "    y = df['DTSM']\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state = 0)\n",
    "\n",
    "model = RandomForestRegressor(max_depth=5, random_state=0, n_estimators=5)\n",
    "model.fit(X_train, y_train)\n",
    "y_pred = model.predict(X_test)\n",
    "r2 = r2_score(y_test, y_pred)\n",
    "rmse = mean_squared_error(y_test, y_pred, squared=False)\n",
    "\n",
    "print(fname.split('.')[0]+' R2:', r2,' RMSE:',rmse)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8983ef57",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}