(GP)CreateComparisons.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "#Boiler Plate"
      ],
      "metadata": {
        "id": "MGylX7Ii9A6c"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z6Zt9FhSgq5S",
        "outputId": "11f87a1f-a49f-480c-e2e8-3593b13d6ffe"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pickle\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import gc"
      ],
      "metadata": {
        "id": "sKgKBg6fhDS1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Seed value\n",
        "# Apparently you may use different seed values at each stage\n",
        "seed_value= 0\n",
        "\n",
        "# 1. Set the `PYTHONHASHSEED` environment variable at a fixed value\n",
        "import os\n",
        "os.environ['PYTHONHASHSEED']=str(seed_value)\n",
        "\n",
        "# 2. Set the `python` built-in pseudo-random generator at a fixed value\n",
        "import random\n",
        "random.seed(seed_value)\n",
        "\n",
        "# 3. Set the `numpy` pseudo-random generator at a fixed value\n",
        "np.random.seed(seed_value)\n",
        "\n",
        "# 4. Set the `tensorflow` pseudo-random generator at a fixed value\n",
        "import tensorflow as tf\n",
        "tf.random.set_seed(seed_value)\n",
        "# for later versions: \n",
        "# tf.compat.v1.set_random_seed(seed_value)\n",
        "\n",
        "# 5. Configure a new global `tensorflow` session\n",
        "from keras import backend as K\n",
        "# session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)\n",
        "# sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)\n",
        "# K.set_session(sess)\n",
        "# for later versions:\n",
        "session_conf = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)\n",
        "sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph(), config=session_conf)\n",
        "tf.compat.v1.keras.backend.set_session(sess)"
      ],
      "metadata": {
        "id": "ubePkDG1utZW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Load Data"
      ],
      "metadata": {
        "id": "Rx8zZhqN9FUg"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "All Data can be found at this anonymous repo https://anonymous.4open.science/r/CHILData-4164/"
      ],
      "metadata": {
        "id": "AAn02GRUlKam"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "o = open(\"/content/drive/My Drive/ECMOProj/GaussianProcessForArtData/trainingPerfusionCloseClin.pkl\", \"rb\")\n",
        "trainingPerfusion = pickle.load(o)\n",
        "o = open(\"/content/drive/My Drive/ECMOProj/GaussianProcessForArtData/validationPerfusionCloseClin.pkl\", \"rb\")\n",
        "validationPerfusion = pickle.load(o)\n",
        "o = open(\"/content/drive/My Drive/ECMOProj/GaussianProcessForArtData/testPerfusionCloseClin.pkl\", \"rb\")\n",
        "testPerfusion = pickle.load(o)\n",
        "\n",
        "o = open(\"/content/drive/My Drive/ECMOProj/GaussianProcessForArtData/trainingClinicalCloseClin.pkl\", \"rb\")\n",
        "trainingClinical = pickle.load(o)\n",
        "o = open(\"/content/drive/My Drive/ECMOProj/GaussianProcessForArtData/validationClinicalCloseClin.pkl\", \"rb\")\n",
        "validationClinical = pickle.load(o)\n",
        "o = open(\"/content/drive/My Drive/ECMOProj/GaussianProcessForArtData/testClinicalCloseClin.pkl\", \"rb\")\n",
        "testClinical = pickle.load(o)\n"
      ],
      "metadata": {
        "id": "4ZZ-vKTXizVZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "labelTraining = np.append(np.ones(54), np.zeros(54))\n",
        "labelValTest = np.append(np.ones(18), np.zeros(18))"
      ],
      "metadata": {
        "id": "P6iaKqUUqtSt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Compare Methods"
      ],
      "metadata": {
        "id": "zVv7vr10ToAF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.metrics import roc_auc_score"
      ],
      "metadata": {
        "id": "V11MN8xIBYwO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def bootstrapAUC(true, preds, bootstrap = 5000):\n",
        "  AUCTotal = np.zeros([0])\n",
        "  if((true.sum()<1) or (true.shape[0]-true.sum() <1)):\n",
        "    print(\"Less than 1 true or false\")\n",
        "    return AUCTotal\n",
        "  for x in range(bootstrap):\n",
        "    np.random.seed(x)\n",
        "    randInt = np.random.randint(0,true.shape[0], true.shape[0])\n",
        "    trueSub = true[randInt]\n",
        "    predSub = preds[randInt]\n",
        "\n",
        "    while((trueSub.sum()<1) or (trueSub.shape[0]-trueSub.sum() <1)):\n",
        "      randInt = np.random.randint(0,true.shape[0], true.shape[0])\n",
        "      trueSub = true[randInt]\n",
        "      predSub = preds[randInt]\n",
        "\n",
        "    AUC = roc_auc_score(trueSub, predSub)\n",
        "    AUCTotal = np.append(AUCTotal, AUC)\n",
        "\n",
        "  return AUCTotal"
      ],
      "metadata": {
        "id": "5hXIRP4cKidW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Logistic Regression"
      ],
      "metadata": {
        "id": "N55Zsa4ZBxDn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def getAllPredsLR(modelFunc, perf, clin, testPerf, testClin):\n",
        "  samplePredictions = np.zeros([0])\n",
        "  sampleLabels = np.zeros([0])\n",
        "\n",
        "\n",
        "  for j in range(12):\n",
        "    for k in range(5):\n",
        "      print(\"{}-{}\".format(j,k))\n",
        "      #Create Dataset\n",
        "      X = np.append(perf[j,k].reshape(108,-1), clin[j,k], axis = 1) #The first column is the label\n",
        "      Y = labelTraining\n",
        "\n",
        "      #Create TEST\n",
        "      TEST = np.append(testPerf[j,k].reshape(36,-1), testClin[j,k], axis = 1) #The first column is the label\n",
        "\n",
        "      model = modelFunc(random_state = 0, max_iter = 10000)\n",
        "      model.fit(X,Y)\n",
        "      print(\"Training: \" + str(model.score(X,Y)) + \" Test: \" + str(model.score(TEST,labelValTest)))\n",
        "      prediction = model.predict_proba(TEST)[:,1]\n",
        "      samplePredictions = np.append(samplePredictions, prediction)\n",
        "      sampleLabels = np.append(sampleLabels, testClin[j,k])\n",
        "\n",
        "  \n",
        "  return samplePredictions, sampleLabels\n",
        "\n"
      ],
      "metadata": {
        "id": "kpOxHYslGdBK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "LRPreds, LRLabel = getAllPredsLR(LogisticRegression, trainingPerfusion, trainingClinical,testPerfusion, testClinical )"
      ],
      "metadata": {
        "id": "Wt0ty6e6HcZq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Save the generated predictions for later analysis\n",
        "f = open(\"/content/drive/My Drive/ECMOProj/GaussianProcessForArtData/LRPredsCloseClin.pkl\",\"wb\")\n",
        "pickle.dump(LRPreds,f)\n",
        "f.close()"
      ],
      "metadata": {
        "id": "7v5vXE-yNmlN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Open the saved predictions\n",
        "o = open(\"/content/drive/My Drive/ECMOProj/GaussianProcessForArtData/LRPredsCloseClin.pkl\", \"rb\")\n",
        "LRPreds = pickle.load(o)\n",
        "LRLabel = np.tile(np.append(np.ones([18]), np.zeros([18])), 60)"
      ],
      "metadata": {
        "id": "4MVmWkNs9nSD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "LRAUCTotal = bootstrapAUC(LRLabel, LRPreds)"
      ],
      "metadata": {
        "id": "BmwjtuWlNuiY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "LRAUCTotal.mean()\n",
        "#00.80103312655945"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TbjWdQZHN26M",
        "outputId": "6763cc98-cca6-4417-b917-cf8b9262b525"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8015514124671221"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"[{:.4f} - {:.4f}]\".format(np.percentile(LRAUCTotal, 2.5),np.percentile(LRAUCTotal, 97.5)))\n",
        "#[0.7807 - 0.8206]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jqel_1sS3B5q",
        "outputId": "9c1c4f14-505f-44ba-8f1c-255d7f5e12bc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0.7813 - 0.8213]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Decision Tree"
      ],
      "metadata": {
        "id": "KDyBgeQwTPFi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.tree import DecisionTreeClassifier"
      ],
      "metadata": {
        "id": "E1v1bKM6TRCD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def getAllPredsDT(perf, clin, testPerf, testClin):\n",
        "  samplePredictions = np.zeros([0])\n",
        "  sampleLabels = np.zeros([0])\n",
        "\n",
        "\n",
        "  for j in range(12):\n",
        "    for k in range(5):\n",
        "      print(\"{}-{}\".format(j,k))\n",
        "      #Create Dataset\n",
        "      X = np.append(perf[j,k].reshape(108,-1), clin[j,k], axis = 1) \n",
        "      Y = labelTraining\n",
        "\n",
        "      #Create TEST\n",
        "      TEST = np.append(testPerf[j,k].reshape(36,-1), testClin[j,k], axis = 1) #The first column is the label\n",
        "\n",
        "      model = DecisionTreeClassifier(random_state = 0)\n",
        "      model.fit(X,Y)\n",
        "      print(\"Training: \" + str(model.score(X,Y)) + \" Test: \" + str(model.score(TEST,labelValTest)))\n",
        "      prediction = model.predict_proba(TEST)[:,1]\n",
        "      samplePredictions = np.append(samplePredictions, prediction)\n",
        "      sampleLabels = np.append(sampleLabels, labelValTest)\n",
        "\n",
        "  \n",
        "  return samplePredictions, sampleLabels\n",
        "\n"
      ],
      "metadata": {
        "id": "8vTJnlKdTxM7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "DTPreds, DTLabel = getAllPredsDT(trainingPerfusion, trainingClinical,testPerfusion, testClinical )"
      ],
      "metadata": {
        "id": "AkoykYAQUEB2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "f = open(\"/content/drive/My Drive/ECMOProj/GaussianProcessForArtData/DTPredsCloseClin.pkl\",\"wb\")\n",
        "pickle.dump(DTPreds,f)\n",
        "f.close()"
      ],
      "metadata": {
        "id": "AibMADN0UEB2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "o = open(\"/content/drive/My Drive/ECMOProj/GaussianProcessForArtData/DTPredsCloseClin.pkl\", \"rb\")\n",
        "DTPreds = pickle.load(o)"
      ],
      "metadata": {
        "id": "JVRUJX5u9t6v"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "DTAUCTotal = bootstrapAUC(LRLabel, DTPreds)"
      ],
      "metadata": {
        "id": "666inEf-UEB2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "DTAUCTotal.mean()\n",
        "#0.7577265261291511"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hM5rUkG9UEB2",
        "outputId": "6a1df048-ad06-47ed-848b-5e65922f426e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7577265261291511"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"[{:.4f} - {:.4f}]\".format(np.percentile(DTAUCTotal, 2.5),np.percentile(DTAUCTotal, 97.5)))\n",
        "#[0.7410 - 0.7741]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nuLSmL9r4phY",
        "outputId": "78158708-f49d-4276-f89d-a7b028938afb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0.7410 - 0.7741]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Native Bayes"
      ],
      "metadata": {
        "id": "TRRVRn5NUUjD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.naive_bayes import GaussianNB"
      ],
      "metadata": {
        "id": "8FxuFcLDVkgn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def getAllPredsNB(perf, clin, testPerf, testClin):\n",
        "  samplePredictions = np.zeros([0])\n",
        "  sampleLabels = np.zeros([0])\n",
        "\n",
        "\n",
        "  for j in range(12):\n",
        "    for k in range(5):\n",
        "      print(\"{}-{}\".format(j,k))\n",
        "      #Create Dataset\n",
        "      X = np.append(perf[j,k].reshape(108,-1), clin[j,k], axis = 1) \n",
        "      Y = labelTraining\n",
        "\n",
        "      #Create TEST\n",
        "      TEST = np.append(testPerf[j,k].reshape(36,-1), testClin[j,k], axis = 1) \n",
        "\n",
        "      model = GaussianNB()\n",
        "      model.fit(X,Y)\n",
        "      print(\"Training: \" + str(model.score(X,Y)) + \" Test: \" + str(model.score(TEST,labelValTest)))\n",
        "      prediction = model.predict_proba(TEST)[:,1]\n",
        "      samplePredictions = np.append(samplePredictions, prediction)\n",
        "      sampleLabels = np.append(sampleLabels, labelValTest)\n",
        "\n",
        "  \n",
        "  return samplePredictions, sampleLabels\n",
        "\n"
      ],
      "metadata": {
        "id": "7Ounz8zeVkgn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "NBPreds, NBLabel = getAllPredsNB(trainingPerfusion, trainingClinical,testPerfusion, testClinical)"
      ],
      "metadata": {
        "id": "yQeVrSDUVkgo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "f = open(\"/content/drive/My Drive/ECMOProj/GaussianProcessForArtData/NBPredsCloseClin.pkl\",\"wb\")\n",
        "pickle.dump(NBPreds,f)\n",
        "f.close()"
      ],
      "metadata": {
        "id": "TgJ2sGFeVy9U"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "o = open(\"/content/drive/My Drive/ECMOProj/GaussianProcessForArtData/NBPredsCloseClin.pkl\", \"rb\")\n",
        "NBPreds = pickle.load(o)"
      ],
      "metadata": {
        "id": "hNbOmq7l_Dxo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "NBAUCTotal = bootstrapAUC(LRLabel, NBPreds)"
      ],
      "metadata": {
        "id": "RGm8tqPJV4E0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "NBAUCTotal.mean()\n",
        "#0.770665747161304"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LmMftCMJV4E0",
        "outputId": "3f820ef9-007b-49b6-b939-2abd34857775"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.770665747161304"
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"[{:.4f} - {:.4f}]\".format(np.percentile(NBAUCTotal, 2.5),np.percentile(NBAUCTotal, 97.5)))\n",
        "#[0.7537 - 0.7873]"
      ],
      "metadata": {
        "id": "WT6RRKBQV7ZS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8365bf92-15e1-4ca8-a6ae-c8a97c7fc55d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0.7537 - 0.7873]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ERYYm7iB4y2C"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# MLPClassifier"
      ],
      "metadata": {
        "id": "gz8hbWKqTtDC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
        "!pip install keras-tuner\n",
        "import kerastuner as kt\n",
        "from kerastuner import HyperParameters as hp"
      ],
      "metadata": {
        "id": "xJvmVqaaXwdT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def returnModel(hp):\n",
        "  model = tf.keras.Sequential()\n",
        "  units = hp.Int(\"units\", 1,16435, 64)\n",
        "  model.add(tf.keras.layers.Dense(units, activation = \"selu\", input_shape=(16435,))) #16464\n",
        "  model.add(tf.keras.layers.Dense(1, activation = \"sigmoid\"))\n",
        "  opt = tf.keras.optimizers.Adam(learning_rate=0.00001)\n",
        "  model.compile(optimizer=opt, loss='binary_crossentropy')\n",
        "  return model"
      ],
      "metadata": {
        "id": "bWD-Mpc_Xwgp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tuner = kt.BayesianOptimization(\n",
        "  returnModel,                   \n",
        "  objective= \"val_loss\",\n",
        "  max_trials=60,\n",
        "  seed =0, \n",
        "  overwrite=True,\n",
        "  beta = 100          \n",
        ")\n",
        "XTrain = np.append(trainingPerfusion[0,0].reshape(108,-1), trainingClinical[0,0], axis = 1) \n",
        "YTrain = labelTraining\n",
        "\n",
        "XVal = np.append(validationPerfusion[0,0].reshape(36,-1), validationClinical[0,0], axis = 1)\n",
        "YVal = labelValTest\n",
        "\n",
        "es = EarlyStopping(monitor='val_loss', mode='min', verbose=0,  patience=20, min_delta = 0.001)\n",
        "tuner.search(                                                           \n",
        "    XTrain,\n",
        "    YTrain,\n",
        "    epochs=10000,                                                         \n",
        "    validation_data = (XVal, YVal),\n",
        "    callbacks=[es],\n",
        "    verbose=1,\n",
        ")\n",
        "print(tuner.get_best_hyperparameters(1)[0].values)"
      ],
      "metadata": {
        "id": "fDa37Lq5uDz9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def returnTrueModel():\n",
        "  model = tf.keras.Sequential()\n",
        "  model.add(tf.keras.layers.Dense(1, activation = \"selu\", input_shape=(16435,))) \n",
        "  model.add(tf.keras.layers.Dense(1, activation = \"sigmoid\"))\n",
        "  opt = tf.keras.optimizers.Adam(learning_rate=0.00001)\n",
        "  model.compile(optimizer=opt, loss='binary_crossentropy')\n",
        "  return model"
      ],
      "metadata": {
        "id": "X4gkwZDzuD2d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def getAllPredsMLP(modelFunc, perf, clin, valPerf, valClin, testPerf, testClin):\n",
        "  samplePredictions = np.zeros([0])\n",
        "  sampleLabels = np.zeros([0])\n",
        "\n",
        "\n",
        "  for j in range(12):\n",
        "    for k in range(5):\n",
        "      print(\"{}-{}\".format(j,k))\n",
        "      #Create Dataset\n",
        "      XTrain = np.append(perf[j,k].reshape(108,-1), clin[j,k], axis = 1) \n",
        "      YTrain = labelTraining\n",
        "\n",
        "      XVal = np.append(valPerf[j,k].reshape(36,-1), valClin[j,k], axis = 1) \n",
        "      YVal = labelValTest\n",
        "\n",
        "      #Create TEST\n",
        "      TEST = np.append(testPerf[j,k].reshape(36,-1), testClin[j,k], axis = 1)\n",
        "      es = EarlyStopping(monitor='val_loss', mode='min', verbose=0,  patience=20, min_delta = 0.001)\n",
        "      mc = ModelCheckpoint(\"/content/drive/My Drive/ECMOProj/GaussianProcessForArtData/DENSE/CloseClinModelK{}Fold{}.h5\".format(j, k), monitor='val_loss', mode='min', verbose=0, save_best_only=True)\n",
        "\n",
        "      model = modelFunc()\n",
        "      model.fit(\n",
        "          XTrain,\n",
        "          YTrain,\n",
        "          epochs = 10000,\n",
        "          validation_data = (XVal,YVal),\n",
        "          verbose = 1,\n",
        "          callbacks=[es, mc]\n",
        "          )\n",
        "      \n",
        "      prediction = model.predict(TEST)\n",
        "      samplePredictions = np.append(samplePredictions, prediction)\n",
        "      sampleLabels = np.append(sampleLabels, labelValTest)\n",
        "\n",
        "  \n",
        "  return samplePredictions, sampleLabels\n",
        "\n"
      ],
      "metadata": {
        "id": "r7sM5R40TXJA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "MLPPreds, MLPLabel = getAllPredsMLP(returnTrueModel,trainingPerfusion, trainingClinical,validationPerfusion, validationClinical,testPerfusion, testClinical )"
      ],
      "metadata": {
        "id": "_IH64Mk7rxPT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "f = open(\"/content/drive/My Drive/ECMOProj/GaussianProcessForArtData/MLPPredsCloseClin.pkl\",\"wb\")\n",
        "pickle.dump(MLPPreds,f)\n",
        "f.close()"
      ],
      "metadata": {
        "id": "Io7eFwCvr9yT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "o = open(\"/content/drive/My Drive/ECMOProj/GaussianProcessForArtData/MLPPredsCloseClin.pkl\", \"rb\")\n",
        "MLPPreds = pickle.load(o)"
      ],
      "metadata": {
        "id": "iP2Ma7vV_PrZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "MLPAUCTotal = bootstrapAUC(LRLabel, MLPPreds)"
      ],
      "metadata": {
        "id": "JxDQ7AKw1ki4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "MLPAUCTotal.mean()\n",
        "#0.7299333876487715"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IhISOBPK1ki5",
        "outputId": "a4e3b762-1479-4227-fc35-0c0151bff778"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7299333876487715"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"[{:.4f} - {:.4f}]\".format(np.percentile(MLPAUCTotal, 2.5),np.percentile(MLPAUCTotal, 97.5)))\n",
        "#[0.7080 - 0.7513]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f6YpiIAz4hJJ",
        "outputId": "87bfdab4-3132-41ff-dcc4-59930567781f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0.7080 - 0.7513]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "DyKyUsmhI0he"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Comparison with L = 100"
      ],
      "metadata": {
        "id": "1w1gEKShI00D"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def bootstrapComaprisonAUC(trueOne, predsOne,trueTwo, predsTwo, bootstrap = 5000):\n",
        "  diffAUCTotal = np.zeros([0])\n",
        "\n",
        "  if((trueOne.sum()<1) or (trueOne.shape[0]-trueOne.sum() <1)):\n",
        "    print(\"Less than 1 true or false\")\n",
        "    return diffAUCTotal\n",
        "  if((trueTwo.sum()<1) or (trueTwo.shape[0]-trueTwo.sum() <1)):\n",
        "    print(\"Less than 1 true or false\")\n",
        "    return diffAUCTotal\n",
        "\n",
        "  intialTestStat = roc_auc_score(trueOne, predsOne) - roc_auc_score(trueTwo, predsTwo)\n",
        "  lengthOne = trueOne.shape[0]\n",
        "  lengthTwo = trueTwo.shape[0]\n",
        "  totalTrueGroup = np.append(trueOne, trueTwo)\n",
        "  totalPredGroup = np.append(predsOne, predsTwo)\n",
        "  \n",
        "  for x in range(bootstrap):\n",
        "    np.random.seed(x)\n",
        "    randInt = np.random.randint(0,lengthOne+lengthTwo, lengthOne)\n",
        "    leftOvers = np.setdiff1d(np.arange(lengthOne+lengthTwo), randInt)\n",
        "    trueOneSub = totalTrueGroup[randInt]\n",
        "    predOneSub = totalPredGroup[randInt]\n",
        "    trueTwoSub = totalTrueGroup[leftOvers]\n",
        "    predTwoSub = totalPredGroup[leftOvers]\n",
        "\n",
        "    while((trueOneSub.sum()<1) or (trueOneSub.shape[0]-trueOneSub.sum() <1) or (trueTwoSub.sum()<1) or (trueTwoSub.shape[0]-trueTwoSub.sum() <1)):\n",
        "      randInt = np.random.randint(0,lengthOne+lengthTwo, lengthOne)\n",
        "      leftOvers = np.setdiff1d(np.arange(lengthOne+lengthTwo), randInt)\n",
        "      trueOneSub = totalTrueGroup[randInt]\n",
        "      predOneSub = totalPredGroup[randInt]\n",
        "      trueTwoSub = totalTrueGroup[leftOvers]\n",
        "      predTwoSub = totalPredGroup[leftOvers]\n",
        "\n",
        "    AUCOne = roc_auc_score(trueOneSub, predOneSub)\n",
        "    AUCTwo = roc_auc_score(trueTwoSub, predTwoSub)\n",
        "    diff = AUCOne - AUCTwo\n",
        "    diffAUCTotal = np.append(diffAUCTotal, diff)\n",
        "  \n",
        "  pValue = (diffAUCTotal>intialTestStat).sum()/diffAUCTotal.shape[0]\n",
        "\n",
        "  return pValue"
      ],
      "metadata": {
        "id": "-bYz2CiRDRbv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Open up prediction from the models trained on the L=1 GPR\n",
        "o = open(\"/content/drive/My Drive/ECMOProj/GaussianProcessForArtData/predictionsCloseClin\", \"rb\")\n",
        "predictions = pickle.load(o)\n",
        "o = open(\"/content/drive/My Drive/ECMOProj/GaussianProcessForArtData/predictionsLargeCloseClin.pkl\", \"rb\")\n",
        "predictionsLarge = pickle.load(o)\n",
        "\n",
        "o = open(\"/content/drive/My Drive/ECMOProj/GaussianProcessForArtData/LRPredsLargeCloseClin.pkl\", \"rb\")\n",
        "LRPredsLarge = pickle.load(o)\n",
        "o = open(\"/content/drive/My Drive/ECMOProj/GaussianProcessForArtData/MLPPredsLargeCloseClin.pkl\", \"rb\")\n",
        "MLPPredsLarge = pickle.load(o)"
      ],
      "metadata": {
        "id": "TrzqgpjTI11c"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "LSTMAUCDiff = bootstrapComaprisonAUC(LRLabel, predictions.flatten(),LRLabel, predictionsLarge.flatten() , 5000)\n",
        "LRAUCDiff = bootstrapComaprisonAUC(LRLabel, LRPreds.flatten(),LRLabel, LRPredsLarge.flatten(), 5000)\n",
        "MLPAUCDiff = bootstrapComaprisonAUC(LRLabel, MLPPreds.flatten(),LRLabel, MLPPredsLarge.flatten(), 5000)"
      ],
      "metadata": {
        "id": "MylKuUDzJGdc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(LSTMAUCDiff)\n",
        "print(LRAUCDiff)\n",
        "print(MLPAUCDiff)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r0rBAc31BQU-",
        "outputId": "5727271c-16f2-4a08-c7c9-f1b6e9ce8d3b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.0\n",
            "0.4626\n",
            "0.17\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "80k3aqjcL5zy"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}