google_colab.py

# -*- coding: utf-8 -*-
"""Bank-Marketing.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/15N9CGzdzawun-6kiJLtzuFH47gATao8p

<h2>PREDICTION OF NEXT DEPOSIT </H2>
<p><strong>Abstract</strong></p>
<p>The increasingly vast number of marketing campaigns
over time has reduced its effect on the general public.
Furthermore, economical pressures and competition has
led marketing managers to invest on directed campaigns
with a strict and rigorous selection of contacts. Such direct
campaigns can be enhanced through the use of Business
Intelligence (BI) and Data Mining (DM) techniques.</p>
<br>
<p>This study describes an implementation of a DM project
based on the CRISP-DM methodology. Real-world data
were collected from a Portuguese marketing campaign
related with bank deposit subscription. The business goal
is to find a model that can explain success of a contact, i.e.
if the client subscribes the deposit. </p>
<br>
<p>The marketing campaigns were based on phone calls. Often, more than one contact to the same client was required, in order to access if the product (bank term deposit) would be ('yes') or not ('no') subscribed.</p>
<br>

<h4><b> <font color='blue'>DICTIONARIES / METHODS WHICH WE USED IN THIS STUDY</b><H4>
"""

# Commented out IPython magic to ensure Python compatibility.
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings("ignore")
# %matplotlib inline
import seaborn as sns
from sklearn import datasets, linear_model

"""<h4><b> <font color='blue'>Collection of Raw Data</b><H4>"""

bank=pd.read_csv("https://drive.google.com/uc?id=1-dNlI2AM09WHDXaf8onFxyD9anVowRZL", error_bad_lines=False, sep=";",na_values='unknown') #convert 'unknown' to np.nan values
df=bank.copy()

df.head(5)

df.shape #dataframe number of rows and columns

df.info() #dataframe feature's types, number of values

df.describe() #dataframe statistical informations

"""<P><b> <font color='RED'>TARGET VALUE DISTRIBUTION</b></P>"""

fig,ax=plt.subplots()
fig.set_size_inches(22,6)
sns.countplot(x="y",data=df,order=df["y"].unique())
ax.set(ylabel="Count",xlabel="TARGET VALUE",title='TARGET VALUE Y - DISTRIBUTION')
title='TARGET VALUE Y - Distribution'
ax.tick_params(labelsize=15)
sns.despine()
for p in ax.patches:
    ax.annotate('{}'.format(p.get_height()), (p.get_x()+0.1, p.get_height()+50),weight="bold", horizontalalignment='center', size='xx-large', color='blue')

"""<h3><b><font color='blue'> Data Preparation </b></h3>


<p> <li> In there we solve data type problems, importance problems and etc. </li></p>
"""

#change the columns' names
df.columns=['age', 'job', 'marital', 'education', 'default', 'housing', 'loan',
       'contact', 'month', 'day_of_week', 'duration', 'campaign', 'pdays',
       'previous', 'poutcome', 'emp_var_rate', 'cons_price_idx',
       'cons_conf_idx', 'euribor3m', 'nr_employed', 'y']

# transforming some feature's values to 0 and 1
df["housing"]=df["housing"].replace("yes", 1)
df["housing"]=df["housing"].replace("no", 0)

df["loan"]=df["loan"].replace("yes", 1)
df["loan"]=df["loan"].replace("no", 0)

#data type convertion
df["job"]=df["job"].astype("category")
df["marital"]=df["marital"].astype("category")
df["default"]=df["default"].astype("category")
df["housing"]=df["housing"].astype("category")
df["loan"]=df["loan"].astype("category")
df["contact"]=df["contact"].astype("category")
df["month"]=df["month"].astype("category")
df["day_of_week"]=df["day_of_week"].astype("category")
#convert object to categorical variable
from pandas.api.types import CategoricalDtype
educationOrder=CategoricalDtype(categories=["illiterate","basic.4y","basic.6y","basic.9y","high.school","university.degree","professional.course"],ordered=True)
df["education"]=df["education"].astype(educationOrder)
poutcomeOrder=CategoricalDtype(categories=["nonexistent","failure","success"],ordered=True)
df["poutcome"]=df["poutcome"].astype(poutcomeOrder)

# converting floats of number of employers to int bcs of meaningless
df['nr_employed']=[int(x) for x in df['nr_employed']]

# feature classification
numericalColumns=["age", "duration", "campaign", "pdays", "previous", "emp_var_rate", "cons_price_idx",
                   "cons_conf_idx", "nr_employed"]
categoricalColumns=["job","marital","education","default","housing","loan","contact","month","day_of_week","poutcome"]
categoricNominal=["job","marital","default","housing","loan","contact","month","day_of_week"]
categoricKardinal=["education","poutcome"]

def distributionPlot(df,column,info=True):
  '''
  Params
  df:dataframe to use as a main data
  column: feature of dataframe to see distribution
  info: İf you want to see number of feauture True, otherwise False 
  NOTE: Default value of ınfo is equal True
  '''
  fig,ax=plt.subplots()
  fig.set_size_inches(22,6)
  sns.countplot(x=column,data=df,order=df[column].unique())
  ax.set_xlabel(column,fontsize=15)
  ax.set_ylabel("Count",fontsize=15)
  title='{} Count Distribution'.format(column.capitalize())
  ax.set_title(title,fontsize=15)
  ax.tick_params(labelsize=15)
  sns.despine()
  if info:
    for p in ax.patches:
        ax.annotate('{}'.format(p.get_height()), (p.get_x()+0.1, p.get_height()+50),weight="bold")

#df.pdays visualization before change
distributionPlot(df,"pdays")

#95% of this line has a value of 999. The remainder has different values. So we converted them to binary values 999 ->1 others ->0
df.loc[(df.pdays==999),"pdays"]=1 # Main DF changed
df.loc[(df.pdays!=1),"pdays"]=0 # Main DF changed

#df.pdays visualization after change
distributionPlot(df,"pdays",False)

"""<h5><font color='DARKRED'>VISUALIZATION FOR NUMERICAL FEATURES </H5>

"""

#fig, axs = plt.subplots(3,3,figsize=(22,14),subplot_kw=dict(projection='polar')) mollweide
fig, axs = plt.subplots(3,3,figsize=(22,8),subplot_kw=dict(projection='rectilinear'),gridspec_kw={'hspace':0.6, 'wspace': 0.2})
fig.suptitle('NUMERICAL VARIABLES')
axs[0][0].plot(df["age"], 'tab:red')
axs[0, 0].set_title('AGE')
axs[0][1].plot(df["duration"], 'tab:cyan')
axs[0, 1].set_title('DURATION')
axs[0][2].plot(df["campaign"], 'tab:orange')
axs[0, 2].set_title('CAMPAIGN')
axs[1][0].plot(df["pdays"], 'tab:red')
axs[1, 0].set_title('PDAYS')
axs[1][1].plot(df["previous"], 'tab:cyan')
axs[1, 1].set_title('PREVIOUS')
axs[1][2].plot(df["emp_var_rate"], 'tab:orange')
axs[1, 2].set_title('EMP_VAR_RATE')
axs[2][0].plot(df["cons_price_idx"], 'tab:red')
axs[2, 0].set_title('CONS_PRICE_IDX')
axs[2][1].plot(df["cons_conf_idx"], 'tab:cyan')
axs[2, 1].set_title('CONS_CONF_IDX')
axs[2][2].plot(df["nr_employed"], 'tab:orange')
axs[2, 2].set_title('NR_EMPLOYED')
for ax in axs.flat:
    ax.set(xlabel='Number of Value', ylabel='Distribution')

def dataframeInformations(df):
  totalEntryList=[]
  totalMissingValueList=[]
  missingValRateList=[]
  dataTypeList=[]
  uniqueValuesList=[]
  totalUniqueValList=[]
  variableNameList=[]
  
  for element in df.columns:
    missingValRate=round((df[element].isna().sum()/len(df[element]))*100,2) #to show correct decimal and float part of number.
    totalEntryList.append(len(df[element]))
    totalMissingValueList.append(df[element].isna().sum())
    missingValRateList.append(missingValRate)
    dataTypeList.append(df[element].dtype)
    uniqueValuesList.append(list(df[element].unique()))
    totalUniqueValList.append(len(df[element].unique()))
    variableNameList.append(element)
  #create a dataframe to show all informations together
  dataInfoDf=pd.DataFrame({'Variable':variableNameList,'#_Total_Entry':totalEntryList,\
                           '#_Missing_Value':totalMissingValueList,'%_Missing_Value':missingValRateList,\
                           'Data_Type':dataTypeList,'Unique_Values':uniqueValuesList,\
                           '#_Uniques_Values':totalUniqueValList})
  return dataInfoDf.sort_values(by="Variable")

dataInfo=dataframeInformations(df)
variableList=[element for element in dataInfo['Variable'] ]
dataInfo=dataInfo.set_index('Variable')
dataInfo

featuresInformations=pd.read_csv("https://drive.google.com/uc?id=12uck97ajxRSA4YkpaKQsfZr1OtLO5TdQ")
featuresInformations.drop(featuresInformations.index)
featuresInformations.head(5)

#add columns explanation and columns type in dataInfo
variableDefinition=list(featuresInformations['Variable_Definition'])
variableStructure=(featuresInformations['Variable_Structure'])
variableStructure=[x.lower() for x in variableStructure] #uppercase not working well in dataFrame therefore we converted all elements to lowercase
dataInfo['Variable_Definition']=variableDefinition
dataInfo['Variable_Structure']=variableStructure
dataInfo

#this function find imputation technique using by missing rate
def findMethod(df,variableList):
  df['Imputation_Technique']=""
  for element in variableList:
      missingRate=float(dataInfo['%_Missing_Value'][element])
      if missingRate == 0:
          df['Imputation_Technique'][element]='No Missing Value'
      elif missingRate <= 5:
          df['Imputation_Technique'][element]='Simple'
      elif missingRate < 25:
          df['Imputation_Technique'][element]='Tree-based'
      elif missingRate < 50 :
          df['Imputation_Technique'][element]='Model'

findMethod(dataInfo,variableList)

dataInfo

"""<h5> CORRELATION MATRIX OF THE DATASET </H5>
<p><li>A correlation matrix is a table showing correlation coefficients between variables</li></p>
<p><li>A correlation matrix is used to summarize data, as an input into a more advanced analysis, and as a diagnostic for advanced analyses.</li></p>
"""

# to show relation between features
plt.subplots(figsize=(20,10))
sns.heatmap(df.corr(), annot=True)
plt.show()

"""<h3><b> TRAIN AND TEST SPLIT </b></h3>"""

from sklearn.model_selection import train_test_split
X = df.drop("y", axis=1)
y=df.y
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
X_train.shape , X_test.shape , y_train.shape, y_test.shape

#y_train ve x_train target göster

"""<h5><b><font color='DARKRED'>VISUALIZE ALL FEATURES TO SEE DISTRIBUTION ON TRAIN DATA </B></H5>"""

def visualizeAllFeatures(df):
  #fig, axs = plt.subplots(3,3,figsize=(22,14),subplot_kw=dict(projection='polar')) mollweide
  fig, axs = plt.subplots(5,4,figsize=(22,15),subplot_kw=dict(projection='rectilinear'),gridspec_kw={'hspace':0.5, 'wspace': 0.3})
  sns.countplot(df["age"],ax=axs[0][0])
  sns.countplot(df["job"],ax=axs[0][1])
  sns.countplot(df["marital"],ax=axs[0][2])
  sns.countplot(df["education"],ax=axs[0][3])
  #
  sns.countplot(df["default"],ax=axs[1][0])
  sns.countplot(df["housing"],ax=axs[1][1])
  sns.countplot(df["loan"],ax=axs[1][2])
  sns.countplot(df["contact"],ax=axs[1][3])
   #
  sns.countplot(df["month"],ax=axs[2][0])
  sns.countplot(df["day_of_week"],ax=axs[2][1])
  sns.countplot(df["duration"],ax=axs[2][2])
  sns.countplot(df["campaign"],ax=axs[2][3])
  #
  sns.countplot(df["pdays"],ax=axs[3][0])
  sns.countplot(df["previous"],ax=axs[3][1])
  sns.countplot(df["poutcome"],ax=axs[3][2])
  sns.countplot(df["emp_var_rate"],ax=axs[3][3])
  #
  sns.countplot(df["cons_price_idx"],ax=axs[4][0])
  sns.countplot(df["cons_conf_idx"],ax=axs[4][1])
  sns.countplot(df["euribor3m"],ax=axs[4][2])
  sns.countplot(df["nr_employed"],ax=axs[4][3])
  fig.show()

visualizeAllFeatures(X_train)

"""<h3><b> HANDLE OUTLIERS </b></h3>





"""

#for numerical features
def boxPlotForNumerical(df):
  fig,ax=plt.subplots(nrows = 3, ncols = 3, figsize=(18,10),subplot_kw=dict(projection='rectilinear'),gridspec_kw={'hspace':0.5, 'wspace': 0.3})
  sns.boxplot(x = "age", data = df, orient = 'v', ax = ax[0][0])
  ax[0][0].set_title('Distribution', fontsize=15)
  ax[0][0].tick_params(labelsize=15)

  sns.boxplot(x = "duration", data = df, orient = 'v', ax = ax[0][1])
  ax[0][1].set_title('"Distribution"', fontsize=15)
  ax[0][1].tick_params(labelsize=15)

  sns.boxplot(x = "campaign", data = df, orient = 'v', ax = ax[0][2])
  ax[0][2].set_title('Distribution', fontsize=15)
  ax[0][2].tick_params(labelsize=15)

  sns.boxplot(x = "pdays", data = df, orient = 'v', ax = ax[1][0])
  ax[1][0].set_title('Distribution', fontsize=15)
  ax[1][0].tick_params(labelsize=15)

  sns.boxplot(x = "previous", data = df, orient = 'v', ax = ax[1][1])
  ax[1][1].set_title('Distribution', fontsize=15)
  ax[1][1].tick_params(labelsize=15)

  sns.boxplot(x = "emp_var_rate", data = df, orient = 'v', ax = ax[1][2])
  ax[1][2].set_title('Distribution', fontsize=15)
  ax[1][2].tick_params(labelsize=15)

  sns.boxplot(x = "cons_price_idx", data = df, orient = 'v', ax = ax[2][0])
  ax[2][0].set_title('Distribution', fontsize=15)
  ax[2][0].tick_params(labelsize=15)

  sns.boxplot(x = "cons_conf_idx", data = df, orient = 'v', ax = ax[2][1])
  ax[2][1].set_title('Distribution', fontsize=15)
  ax[2][1].tick_params(labelsize=15)

  sns.boxplot(x = "nr_employed", data = df, orient = 'v', ax = ax[2][2])
  ax[2][2].set_title('Distribution', fontsize=15)
  ax[2][2].tick_params(labelsize=15)

boxPlotForNumerical(df)

#this function will return for numerical values number of outlier if it's exist
def findOutlierForNumerical(df,column,coefficient=1.5):
    import matplotlib.pyplot as plt
    numberOfValue=len(df[column])
    sorted(df[column])
    Q1,Q3=np.percentile(df[column], [25,75])
    IQR = Q3 - Q1
    lowerbound = Q1 - (coefficient * IQR)
    upperbound = Q3 + (coefficient * IQR)
    lowerbound, upperbound
    dfOutlier=df[(df[column] < lowerbound) | (df[column] > upperbound)]
    numberOfOutlier=len(dfOutlier[column])
    print(column,'s outlier number is: ',numberOfOutlier)   
    # which values are outlier
    fig1,ax1=plt.subplots()
    ax1.pie([numberOfValue,numberOfOutlier],labels=["Normal Values","Outlier Values"],autopct='%1.1f%%',shadow=True, startangle=90,colors=["#900C3F","#00FFFF"],explode = (0.1,0.1))
    #draw circle
    centre_circle = plt.Circle((0,0),0.70,fc='white')
    fig = plt.gcf()
    fig.gca().add_artist(centre_circle)
    ax1.axis('equal')  
    plt.tight_layout()
    plt.show()
    return df

for column in numericalColumns:
  X_train[column]=findOutlierForNumerical(X_train,column,coefficient=1.5)

X_train.shape,y_train.shape

"""<h3><b> HANDLE MISSING VALUE</b></h3>"""

import missingno as msno
msno.matrix(df,sparkline=True,color=(0.9,0.4,0.1))

msno.bar(df,color='#F08102')

"""<h6>SIMPLE IMPUTATION FOR CATEGORICAL VALUES</H6>"""

#if missing rate between 0-5 and categorical data use this method
def missingvalImputeMF(df,column):
  valueList=list(df[column])
  mostFrequentVal=max(set(valueList), key = valueList.count)
  df[column]=df[column].fillna(mostFrequentVal)

def missingImpute(df):
  for col in df.columns:
    infoStr=""
    missingRatioRate=int(round((df[col].isna().sum()/len(df[col]))*100,2))
    if missingRatioRate==0:
      infoStr+="{} - There is no missing value".format(col)
      if str(df[col].dtype)=="category":
        infoStr+=" - Category"
        print(infoStr)
      else:
        infoStr+=" - Numerical"
        print(infoStr)
      print("--------------------------"),
      infoStr+=" {} - Ratio:{}% 0-5".format(col,missingRatioRate)
      if str(df[col].dtype)=="category":
        #if column's missing rate between 0% - 5% and categorical variable
        missingvalImputeMF(df,col)
        missingRatioRate=round((df[col].isna().sum()/len(df[col]))*100,2)
        infoStr+="{} -  There is no missing value".format(col)
        infoStr+=" - Category"
        print("DONE {} ".format(infoStr))
      else:
        infoStr+=" - Numerical"
        print(infoStr)
      print("--------------------------")
    elif missingRatioRate<=25:
      infoStr+="{} - Ratio:{}% 5-25".format(col,missingRatioRate)
      if str(df[col].dtype)=="category":
        infoStr+=" - Category"
        print(infoStr)
      else:
        infoStr+=" - Numerical"
        print(infoStr)
      print("--------------------------")
    elif missingRatioRate<=50:
      infoStr+="{} - Ratio:{}% 25-50".format(col,missingRatioRate)
      if str(df[col].dtype)=="category":
        infoStr+=" - Category"
        print(infoStr)
      else:
        infoStr+=" - Numerical"
        print(infoStr)
      print("--------------------------")
    elif missingRatioRate<=95:
      infoStr+="{} - Ratio:{}% 50-95".format(col,missingRatioRate)
      if str(df[col].dtype)=="category":
        infoStr+=" - Category"
        print(infoStr)
      else:
        infoStr+=" - Numerical"
        print(infoStr)
      print("--------------------------")
    else:
      infoStr+="{} - Ratio:{}% 95-100".format(col,missingRatioRate)
      if str(df[col].dtype)=="category":
        infoStr+=" - Category"
        print(infoStr)
      else:
        infoStr+=" - Numerical"
        print(infoStr)
      print("--------------------------")

missingImpute(X_train)

#default feature's missing rate is equal 26% so we asssigned na as an new category for X_train
X_train['default'] = X_train['default'].cat.add_categories('unknown')
X_train['default'].fillna('unknown', inplace =True)

#default feature's missing rate is equal 26% so we asssigned na as an new category for X_test
X_test['default'] = X_test['default'].cat.add_categories('unknown')
X_test['default'].fillna('unknown', inplace =True)

y_test.replace('no',0,inplace=True)
y_test.replace('yes',1,inplace=True)
y_train.replace('no',0,inplace=True)
y_train.replace('yes',1,inplace=True)

"""<H5 style="align:center;">CORELATION MATRIX BETWEEN FEATURES</H5>"""

# to show relation between features
plt.subplots(figsize=(20,10))
sns.heatmap(X_train.corr(), annot=True)
plt.show()

"""<h3><b> ENCODING</b></h3>"""

!pip install category_encoders

"""<h5><b> Encoding for Categorical Variables </b></h5>"""

def binaryEncoding(df,column):
  from category_encoders import BinaryEncoder
  encoder=BinaryEncoder(cols=[column])
  df=encoder.fit_transform(df)
  return df
def oneHotEncoding(df,column):
  from category_encoders import OneHotEncoder
  encoder=OneHotEncoder(cols=[column])
  df = encoder.fit_transform(df)
  return df
def encodingForCategoricalFeature(df,categoricalColumns):
  for element in categoricalColumns:
    if len(df[element].unique())<=5:
      df=oneHotEncoding(df,element)
    else:
      df=binaryEncoding(df,element)
  return df

X_train=encodingForCategoricalFeature(X_train,categoricalColumns)

X_test=encodingForCategoricalFeature(X_test,categoricalColumns)

"""<h5><b> Scaling for Numerical Variables </b></h5>"""

from sklearn import preprocessing
#X_train
X_train.loc[:,numericalColumns]=preprocessing.normalize(X_train.loc[:,numericalColumns])
#X_test
X_test.loc[:,numericalColumns]=preprocessing.normalize(X_test.loc[:,numericalColumns])

"""<p>FEATURE IMPORTANCE GRAPHIC </P>"""

from sklearn.ensemble import RandomForestClassifier
import matplotlib.pyplot as plt
model = RandomForestClassifier()
model.fit(X_train,y_train)
print(sorted(model.feature_importances_)) #use inbuilt class feature_importances of tree based classifiers
#plot graph of feature importances for better visualization
feat_importances = pd.Series(model.feature_importances_, index=X_train.columns)
plt.figure(figsize=(16,6))
plt.xlabel('Feature Importance Value',fontsize=15)
plt.ylabel("Most Important 10 Features",fontsize=15)
plt.title("FEATURE IMPORTANCE",fontsize=15)
feat_importances.nlargest(10).plot(kind='barh')
plt.show();

def rocCurveGraphic(model,X_test,y_test):
    from sklearn.metrics import plot_roc_curve
    plt.figure(figsize=(12,6))
    plt.title('Receiver Operating Characteristic Curve')
    ax = plt.gca()
    disp = plot_roc_curve(model, X_test, y_test, ax=ax, alpha=0.8)

"""<h3> <b> MODELLING </B></H3>

<h6><font color='DARKBLUE'>LOGISTIC REGRESSION MODEL<H6>
"""

def logisticRegressionModel(X_train,Y_train):
    #use logistic regression
    from sklearn.linear_model import LogisticRegression
    log=LogisticRegression(random_state=0)
    log.fit(X_train,Y_train)
    print('Logistic Regression Training Accuracy:',log.score(X_train,Y_train))
    return log,log.score(X_train,Y_train)

log,logScore=logisticRegressionModel(X_train,y_train)

rocCurveGraphic(log,X_test,y_test)

"""<h6><font color='DARKBLUE'>KNeıghbors MODEL <H6>"""

def knnNeighborsModel(X_train,Y_train):
    #use KNeighbors
    from sklearn.neighbors import KNeighborsClassifier
    knn=KNeighborsClassifier(n_neighbors=5,metric="minkowski",p=2)
    knn.fit(X_train,Y_train)
    print('\nK Neighbors Training Accuracy:',knn.score(X_train,Y_train))
    return knn,knn.score(X_train,Y_train)

knn,knnScore=knnNeighborsModel(X_train,y_train)

rocCurveGraphic(knn,X_test,y_test)

"""<h6><font color='DARKBLUE'>SUPPORT VECTOR MACHINE LINEAR MODEL <H6>"""

def svcLinearModel(X_train,Y_train):
    #use SVC (linear kernel)
    from sklearn.svm import SVC
    svc_lin=SVC(kernel="linear",random_state=0,probability=True)
    svc_lin.fit(X_train,Y_train)
    print('\n[2]SVC Linear Training Accuracy:',svc_lin.score(X_train,Y_train))
    return svc_lin,svc_lin.score(X_train,Y_train)

svc_lin,svc_linScore=svcLinearModel(X_train,y_train)

rocCurveGraphic(svc_lin,X_test,y_test)

"""<h6><font color='DARKBLUE'>SUPPORT VECTOR MACHINE RBF MODEL <H6>"""

def svcRBFModel(X_train,Y_train):
    from sklearn.svm import SVC
    svc_rbf=SVC(kernel="rbf",random_state=0,probability=True)
    svc_rbf.fit(X_train,Y_train)
    print('\nSVC RBF Training Accuracy:',svc_rbf.score(X_train,Y_train))
    return svc_rbf,svc_rbf.score(X_train,Y_train)

svc_rbf,svc_rbfScore=svcRBFModel(X_train,y_train)

rocCurveGraphic(svc_rbf,X_test,y_test)

"""<h6><font color='DARKBLUE'>GAUSSIAN NAIVE BAYES MODEL <H6>"""

def gaussianBayesModel(X_train,Y_train):
    #use GaussianNB
    from sklearn.naive_bayes import GaussianNB
    gauss=GaussianNB()
    gauss.fit(X_train,Y_train)
    print('\nGaussian Training Accuracy:',gauss.score(X_train,Y_train))
    return gauss,gauss.score(X_train,Y_train)

gauss,gaussScore=gaussianBayesModel(X_train,y_train)

rocCurveGraphic(gauss,X_test,y_test)

"""<h6><font color='DARKBLUE'>DECISION TREE MODEL <H6>"""

def decisionTreeModel(X_train,Y_train):
    #use Decision Tree
    from sklearn.tree import DecisionTreeClassifier
    tree=DecisionTreeClassifier(criterion="entropy",random_state=0)
    tree.fit(X_train,Y_train)
    print('\nDecision Tree Training Accuracy:',tree.score(X_train,Y_train))
    return tree,tree.score(X_train,Y_train)

tree,treeScore=decisionTreeModel(X_train,y_train)

rocCurveGraphic(tree,X_test,y_test)

"""<h6><font color='DARKBLUE'>RANDOM FOREST MODEL <H6>"""

def randomForestModel(X_train,Y_train):
    #use Random Forest Classifier
    from sklearn.ensemble import RandomForestClassifier
    forest=RandomForestClassifier(n_estimators=10,criterion="entropy",random_state=0)
    forest.fit(X_train,Y_train)
    print('\nRandom Forest Training Accuracy:',forest.score(X_train,Y_train))
    return forest,forest.score(X_train,Y_train)

forest,forestScore=randomForestModel(X_train,y_train)

rocCurveGraphic(forest,X_test,y_test)

"""<h6><font color='DARKBLUE'>HIST GRADIENT BOOSTING MODEL <H6>"""

def histGradientModel(X_train,Y_train):
    # use Hist Gradient Boosting Classifier
    from sklearn.experimental import enable_hist_gradient_boosting
    from sklearn.ensemble import HistGradientBoostingClassifier
    histgrad=HistGradientBoostingClassifier()
    histgrad.fit(X_train,y_train)
    print('\nHist Gradient Boosting Training Score:',histgrad.score(X_train,Y_train))
    return histgrad,histgrad.score(X_train,Y_train)

histgrad,histgradScore=histGradientModel(X_train,y_train)

rocCurveGraphic(histgrad,X_test,y_test)

"""<h6><font color='DARKBLUE'>XGBM BOOSTING MODEL <H6>"""

def xgbmBoostingModel(X_train,Y_train):
    # use GBM
    from sklearn.ensemble import GradientBoostingClassifier
    gbm=GradientBoostingClassifier()
    gbm.fit(X_train,y_train)
    print('\nGradient Boosting Training Score:',gbm.score(X_train,Y_train))   
    return gbm,gbm.score(X_train,Y_train)

gbm,gbmScore=xgbmBoostingModel(X_train,y_train)

rocCurveGraphic(gbm,X_test,y_test)

"""<h6><font color='DARKBLUE'>XGBOOST MODEL <H6>"""

def xgboostModel(X_train,Y_train):
    # use XGBoost
    #!pip install xgboost
    from xgboost import XGBClassifier
    xgboost=XGBClassifier()
    xgboost.fit(X_train,y_train)
    print('\nXGBoost Training Score:',xgboost.score(X_train,Y_train))
    return xgboost,xgboost.score(X_train,Y_train)

xgboost,xgboostScore=xgboostModel(X_train,y_train)

rocCurveGraphic(xgboost,X_test,y_test)

"""<h6><font color='DARKBLUE'>LIGHT GBM MODEL <H6>"""

def lightGbmModel(X_train,Y_train):
    # use LightGBM
    #!conda install -c conda-forge lightgbm
    from lightgbm import LGBMClassifier
    lightgbm=LGBMClassifier()
    lightgbm.fit(X_train,y_train)
    print('\nLight GBM Training Score:',lightgbm.score(X_train,Y_train))   
    return lightgbm,lightgbm.score(X_train,Y_train)

lightgbm,lightgbmScore=lightGbmModel(X_train,y_train)

rocCurveGraphic(lightgbm,X_test,y_test)

"""<h3><B>EVALUATION</B></H3>

<h6><font color='DARKBLUE'>COMPARING MODEL SCORES  <H6>
"""

scoreList=[logScore,knnScore,svc_linScore,svc_rbfScore,gaussScore,treeScore,forestScore,histgradScore,gbmScore,xgboostScore,lightgbmScore]
bars=("LOG","KNN","SVC_Linear","SVC_RBF","GAUSS","Decision_Tree","Random_Forest","Hist_Gradient","GRADIENT_BOOSTING","XGBOOST","LIGHT_GBM")
y_pos = np.arange(len(bars))
plt.figure(figsize=(20,8))
plt.bar(y_pos, scoreList, color=['black', 'red', 'green', 'blue', 'cyan','black', 'red', 'green', 'blue', 'cyan',"black"])
plt.ylabel('TRAINING SCORE')
plt.xlabel("MODELS")
plt.title('TRAINING SCORE FOR EACH MODEL')
plt.xticks(y_pos, bars)
plt.show()

"""<h6><font color='DARKBLUE'>RECEIVER OPERATING CHARACTERISTIC FOR ALL MODELS<H6>"""

from sklearn.metrics import plot_roc_curve
plt.figure(figsize=(15,10))
ax = plt.gca()
log_disp = plot_roc_curve(log, X_test, y_test, ax=ax, alpha=0.8)
knn_disp = plot_roc_curve(knn, X_test, y_test, ax=ax, alpha=0.8)
svc_lin_disp = plot_roc_curve(svc_lin, X_test, y_test, ax=ax, alpha=0.8)
svc_rbf_disp = plot_roc_curve(svc_rbf, X_test, y_test, ax=ax, alpha=0.8)
gauss_disp = plot_roc_curve(gauss, X_test, y_test, ax=ax, alpha=0.8)
tree_disp = plot_roc_curve(tree, X_test, y_test, ax=ax, alpha=0.8)
forest_disp = plot_roc_curve(forest, X_test, y_test, ax=ax, alpha=0.8)
histgrad_disp = plot_roc_curve(histgrad, X_test, y_test, ax=ax, alpha=0.8)
gbm_disp = plot_roc_curve(gbm, X_test, y_test, ax=ax, alpha=0.8)
xgboost_disp = plot_roc_curve(xgboost, X_test, y_test, ax=ax, alpha=0.8)
lightgbm_disp = plot_roc_curve(lightgbm, X_test, y_test, ax=ax, alpha=0.8)
plt.legend(loc = 'best', prop={'size': 16})
plt.show()

roc_curve_values=dict()
from sklearn.metrics import roc_auc_score
from lightgbm import LGBMClassifier
lgbm=LGBMClassifier(learning_rate= 0.02 , max_depth= 4, subsample= 0.6, n_estimators= 1000, min_child_samples= 5)
lgbm_tuned=lgbm.fit(X_train,y_train)
y_pred=lgbm_tuned.predict(X_test)
roc_curve_values["Light GBM Classifier"]=roc_auc_score(y_test,y_pred)
roc_auc_score(y_test,y_pred)

from xgboost import XGBClassifier
xgb=XGBClassifier()
xgb_tuned=xgb.fit(X_train,y_train)
y_pred=xgb_tuned.predict(X_test)
roc_curve_values["XGBoost Classifier"]=roc_auc_score(y_test,y_pred)
roc_auc_score(y_test,y_pred)

from sklearn.experimental import enable_hist_gradient_boosting
from sklearn.ensemble import HistGradientBoostingClassifier

histgrad=HistGradientBoostingClassifier(max_depth=8,max_leaf_nodes=14,learning_rate=0.1)
histgrad_tuned=xgb.fit(X_train,y_train)
y_pred=histgrad_tuned.predict(X_test)
roc_curve_values["Histogram Gradient Boosting"]=roc_auc_score(y_test,y_pred)
roc_auc_score(y_test,y_pred)

from sklearn.ensemble import GradientBoostingClassifier
#gbm=GradientBoostingClassifier(learning_rate= , max_depth= , min_samples_split= , n_estimators= )
gbm=GradientBoostingClassifier()
gbm_tuned=gbm.fit(X_train,y_train)
y_pred=gbm_tuned.predict(X_test)
roc_curve_values["Gradient Boosting Classifier"]=roc_auc_score(y_test,y_pred)
roc_auc_score(y_test,y_pred)

roc_curve_values1=pd.DataFrame.from_dict(roc_curve_values, orient='index',columns=["Roc_Auc_Score"])
roc_curve_values1 = roc_curve_values1.sort_values(by=['Roc_Auc_Score'])
roc_curve_values1.plot(kind='barh', y='Roc_Auc_Score', figsize=(18,6), title="Model's Roc Scores");