FInal_Project_Dash.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# **Machine Learning Applications Final Project Dash**\n",
    "\n",
    "Date: May 9, 2023\n",
    "\n",
    "Authors: Óscar Montoya Amores (100451858), Javier Serrano Sánchez-Ocaña (100451673), Miguel Ángel Ponce Martínez (100451309)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### **LIBRARIES IMPORT**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import joblib\n",
    "\n",
    "from jupyter_dash import JupyterDash\n",
    "from dash import dash, dcc, html, Input, Output\n",
    "import dash_bootstrap_components as dbc\n",
    "import plotly.graph_objs as go\n",
    "import matplotlib.pyplot as plt\n",
    "import plotly.figure_factory as ff\n",
    "\n",
    "\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn import metrics\n",
    "import plotly.express as px\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# import gensim\n",
    "import pyLDAvis.gensim as gensimvis\n",
    "import pyLDAvis\n",
    "\n",
    "import random\n",
    "from io import BytesIO\n",
    "\n",
    "from wordcloud import WordCloud\n",
    "import base64"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "PLOTS A IMPLEMENTAR, BORRAR CUANDO ESTE ACABADO:\n",
    "\n",
    "-LDA CON LOS TÓPICOS DEPENDIENDO DE CADA GENERO, ES DECIR PODER ELEGIR QUE GENERO PLOTEAR Y QUE SALGAN LAS PALABRAS Q CONFORMAN EL TOPICO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('clean_songs.csv')\n",
    "# print(df.columns)\n",
    "\n",
    "ldag = joblib.load(\"lda_model.joblib\")\n",
    "\n",
    "from gensim.corpora import Dictionary\n",
    "\n",
    "dictionary_file = \"dictionary.gensim\"\n",
    "D = Dictionary.load(dictionary_file)\n",
    "from gensim import corpora\n",
    "# load the corpus from file\n",
    "corpus = corpora.MmCorpus('corpus.mm')\n",
    "\n",
    "corpus_words = [[D[word_id] for word_id, count in doc] for doc in corpus]\n",
    "\n",
    "\n",
    "# Visualize topics using pyLDAvis\n",
    "vocab = {D[i]: i for i in range(len(D))}\n",
    "vocab2 = {i: D[i] for i in range(len(D))}\n",
    "term_frequency = {D[i]: freq for i, freq in D.cfs.items()}\n",
    "term_frequency = pd.Series(term_frequency)\n",
    "vis = gensimvis.prepare(ldag, corpus, D, mds='tsne', vocab=vocab2, sort_topics=False, term_frequency=term_frequency)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Open probs df for roc curves, and Y\n",
    "probs_df = joblib.load(\"probs_df.joblib\")\n",
    "Y = joblib.load(\"Y.joblib\")\n",
    "\n",
    "random.seed(33)\n",
    "encoder = LabelEncoder()\n",
    "Y = encoder.fit_transform(Y)\n",
    "\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(Y, Y, test_size=0.3, random_state=33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#0. Obtain genres\n",
    "genres = df['type'].unique().tolist()\n",
    "#1. Obtain the Indices of each song genre\n",
    "typeSer = df['type']\n",
    "rock_idx  = typeSer[typeSer == 'rock'].index.tolist()\n",
    "country_idx  = typeSer[typeSer == 'country'].index.tolist()\n",
    "rb_idx  = typeSer[typeSer == 'R&B'].index.tolist()\n",
    "rap_idx  = typeSer[typeSer == 'rap & hip hop'].index.tolist()\n",
    "#2. Generate list including all the words per genre\n",
    "rock_w = []\n",
    "for s in rock_idx:\n",
    "    rock_w += corpus_words[s]\n",
    "country_w = []\n",
    "for s in country_idx:\n",
    "    country_w += corpus_words[s]\n",
    "rb_w = []\n",
    "for s in rb_idx:\n",
    "    rb_w += corpus_words[s]\n",
    "rap_w = []\n",
    "for s in rap_idx:\n",
    "    rap_w += corpus_words[s]\n",
    "\n",
    "#3. Generate Frequence Dataframe\n",
    "rock_freq = {}\n",
    "# iterating over the list\n",
    "for item in rock_w:\n",
    "   # checking the element in dictionary\n",
    "   if item in rock_freq:\n",
    "      # incrementing the counr\n",
    "      rock_freq[item] += 1\n",
    "   else:\n",
    "      # initializing the count\n",
    "      rock_freq[item] = 1 \n",
    "\n",
    "df_rock = pd.DataFrame({'word': list(rock_freq.keys()), 'freq': list(rock_freq.values())})\n",
    "\n",
    "country_freq = {}\n",
    "# iterating over the list\n",
    "for item in country_w:\n",
    "   # checking the element in dictionary\n",
    "   if item in country_freq:\n",
    "      # incrementing the counr\n",
    "      country_freq[item] += 1\n",
    "   else:\n",
    "      # initializing the count\n",
    "      country_freq[item] = 1 \n",
    "\n",
    "df_country = pd.DataFrame({'word': list(country_freq.keys()), 'freq': list(country_freq.values())})\n",
    "\n",
    "rb_freq = {}\n",
    "# iterating over the list\n",
    "for item in rb_w:\n",
    "   # checking the element in dictionary\n",
    "   if item in rb_freq:\n",
    "      # incrementing the counr\n",
    "      rb_freq[item] += 1\n",
    "   else:\n",
    "      # initializing the count\n",
    "      rb_freq[item] = 1 \n",
    "\n",
    "df_rb = pd.DataFrame({'word': list(rb_freq.keys()), 'freq': list(rb_freq.values())})\n",
    "\n",
    "rap_freq = {}\n",
    "# iterating over the list\n",
    "for item in rap_w:\n",
    "   # checking the element in dictionary\n",
    "   if item in rap_freq:\n",
    "      # incrementing the counr\n",
    "      rap_freq[item] += 1\n",
    "   else:\n",
    "      # initializing the count\n",
    "      rap_freq[item] = 1 \n",
    "\n",
    "df_rap = pd.DataFrame({'word': list(rap_freq.keys()), 'freq': list(rap_freq.values())})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize the Dash app\n",
    "app = JupyterDash(__name__)\n",
    "\n",
    "server = app.server\n",
    "\n",
    "# Define the layout of the Dash app\n",
    "app.layout = html.Div([\n",
    "    html.H1(\"Song Lyrics Wordcloud by Genre\"),\n",
    "    dcc.Dropdown(\n",
    "        id='dropdown_words',\n",
    "        options=[{'label': f'Genre: {i}', 'value': i} for i in genres],\n",
    "        value=genres[0]\n",
    "    ),\n",
    "    html.Img(id=\"image_wc\"),\n",
    "    html.H1(\"Song Genre Topic Modeling\"),\n",
    "    dbc.Row([\n",
    "    dcc.Dropdown(\n",
    "        id='dropdown',\n",
    "        options=[{'label': 'Topic {}'.format(i), 'value': i} for i in range(ldag.num_topics)],\n",
    "        value=0),\n",
    "    dbc.Label(\"Introduce the number of words:\"),\n",
    "    dbc.Input(\n",
    "            id='inputNum',\n",
    "            type='number',\n",
    "            min=2, max=20, step=1,\n",
    "            value=2, style={'height': '80%', 'marginLeft':'10px', 'display': 'inline-block'})\n",
    "    ]),\n",
    "\n",
    "    dcc.Graph(id='topic-type-plot'),\n",
    "    html.Iframe(\n",
    "        id='pyldavis',\n",
    "        srcDoc=pyLDAvis.prepared_data_to_html(vis),\n",
    "        style={'width': '100%', 'height': '800px'}\n",
    "    ),\n",
    "\n",
    "    html.H1(\"Different ROC Curves and Confussion Matrix by Genre depending on the Model and Classifier used\"),\n",
    "    dbc.Row([dcc.Dropdown(\n",
    "        id = 'roc_model',\n",
    "        options=[{'label': i, 'value' : i} for i in list(set(probs_df[\"MODELS\"]))], multi = False, value=list(set(probs_df[\"MODELS\"]))[0]),\n",
    "    dcc.Dropdown(id = 'roc_classifier',\n",
    "        options=[{'label': i, 'value' : i} for i in list(set(probs_df[\"CLASSIFIER\"]))], multi = False, value=list(set(probs_df[\"CLASSIFIER\"]))[0])\n",
    "        ]),\n",
    "    html.Div([\n",
    "    dcc.Graph(id='roc-curves',  style={'height': '40%', 'width': '50%', 'display': 'inline-block'}),\n",
    "    dcc.Graph(id = 'confusion-matrix',  style={'height': '40%', 'width': '40%', 'display': 'inline-block'})\n",
    "    ])\n",
    "])\n",
    "\n",
    "#Creation of WordCloud\n",
    "@app.callback(\n",
    "    Output('image_wc', 'src'), \n",
    "    [Input('image_wc', 'id'), \n",
    "     Input('dropdown_words', 'value')]\n",
    ")\n",
    "\n",
    "def make_image(b, gr):\n",
    "    img = BytesIO()\n",
    "    if gr == 'rock':\n",
    "        plot_wordcloud(data=df_rock).save(img, format='PNG')\n",
    "    elif gr == 'country':\n",
    "        plot_wordcloud(data=df_country).save(img, format='PNG')\n",
    "    elif gr == 'R&B':\n",
    "        plot_wordcloud(data=df_rb).save(img, format='PNG')\n",
    "    else:\n",
    "        plot_wordcloud(data=df_rap).save(img, format='PNG')\n",
    "    return 'data:image/png;base64,{}'.format(base64.b64encode(img.getvalue()).decode())\n",
    "\n",
    "def plot_wordcloud(data):\n",
    "    d = {a: x for a, x in data.values}\n",
    "    wc = WordCloud(colormap='prism', background_color='black', width=1640, height=520)#.generate(' '.join(data))\n",
    "    wc.fit_words(d)\n",
    "    return wc.to_image()\n",
    "\n",
    "# Define the update functions for the Dash app\n",
    "@app.callback(\n",
    "    Output('pyldavis', 'srcDoc'),\n",
    "    Input('dropdown', 'value')\n",
    ")\n",
    "def update_pyldavis(selected_topic):\n",
    "    return pyLDAvis.prepared_data_to_html(vis)\n",
    "\n",
    "@app.callback(\n",
    "    Output('topic-type-plot', 'figure'),\n",
    "    [Input('dropdown', 'value'),\n",
    "     Input('inputNum', 'value')]\n",
    ")\n",
    "def update_topic_type_plot(selected_topic,sel_num):\n",
    "    # Get the top 20 words for the selected topic\n",
    "    top_words = ldag.show_topic(selected_topic, topn=sel_num)\n",
    "    top_words = [word[0] for word in top_words]\n",
    "    if \"wa\" in top_words:\n",
    "        top_words.remove(\"wa\")\n",
    "    # Find the indices of the songs in the corpus that contain any of the top 20 words for the selected topic\n",
    "    indices = []\n",
    "\n",
    "    for i in range(len(corpus_words)):\n",
    "        j=0\n",
    "        for word in top_words:\n",
    "            if word in corpus_words[i]:\n",
    "                j+=1\n",
    "        if j==len(top_words):\n",
    "            indices.append(i)\n",
    "            \n",
    "    # Filter the dataframe to include only songs with indices in the list of indices\n",
    "    topic_df = df[df.index.isin(indices)]\n",
    "    # Check if any songs match the selected topic\n",
    "    if topic_df.empty:\n",
    "        return {'data': [], 'layout': {}}\n",
    "    # Get the genre of each song in the selected topic\n",
    "    genres = [genre for genre in topic_df['type']]\n",
    "    # Count the number of songs in each genre for the selected topic\n",
    "    genre_counts = pd.Series(genres).value_counts().sort_index()\n",
    "    # Create the plot\n",
    "    fig = {\n",
    "        'data': [{'x': genre_counts.index, 'y': genre_counts.values, 'type': 'bar', 'marker': {'color': ['red', 'green', 'blue', 'orange']}}],\n",
    "        'layout': {'title': f'Topic {selected_topic} {top_words} Song Genres by LDA' ,'xaxis': {'title': 'Genre'}, 'yaxis': {'title': 'Count'}}\n",
    "    }\n",
    "    return fig\n",
    "\n",
    "@app.callback(\n",
    "    Output('roc-curves', 'figure'),\n",
    "    [Input('roc_model', 'value'),\n",
    "     Input('roc_classifier', 'value')])\n",
    "def draw_roc(model, classifier):\n",
    "    model_df = probs_df.index[probs_df['MODELS']==model].tolist()\n",
    "    classifier_df = probs_df.index[probs_df['CLASSIFIER']==classifier].tolist()\n",
    "    for i in model_df:\n",
    "        if i in classifier_df:\n",
    "            index = i\n",
    "    selected_roc = probs_df.iloc[index,:]\n",
    "    prob = selected_roc[\"PROBS\"]\n",
    "\n",
    "    fpr0, recall0, thresholds0 = metrics.roc_curve(y_test, prob[:,0], pos_label=0) \n",
    "    fpr1, recall1, thresholds1 = metrics.roc_curve(y_test, prob[:,1], pos_label=1) \n",
    "    fpr2, recall2, thresholds2 = metrics.roc_curve(y_test, prob[:,2], pos_label=2) \n",
    "    fpr3, recall3, thresholds3 = metrics.roc_curve(y_test, prob[:,3], pos_label=3) \n",
    "\n",
    "    Y_decoded = encoder.inverse_transform([0,1,2,3]) \n",
    "\n",
    "    # Create a subplot\n",
    "    fig, ax = plt.subplots()\n",
    "\n",
    "    # Plot the ROC curves\n",
    "    fpr_list = [fpr0, fpr1, fpr2, fpr3]\n",
    "    recall_list = [recall0, recall1, recall2, recall3]\n",
    "    color_list = ['red', 'blue', 'green', 'black']\n",
    "    label_list = [f'ROC curve {Y_decoded[i]}' for i in range(len(Y_decoded))]\n",
    "\n",
    "    for fpr, recall, color, label in zip(fpr_list, recall_list, color_list, label_list):\n",
    "        ax.plot(fpr, recall, lw=2.5, label=label, color=color)\n",
    "\n",
    "    # Convert the plot to a plotly figure\n",
    "    plotly_fig = go.Figure()\n",
    "    for trace in fig.get_axes()[0].lines:\n",
    "        plotly_fig.add_trace(\n",
    "            go.Scatter(x=trace.get_xdata(), y=trace.get_ydata(), mode='lines', name=trace.get_label(), line=dict(color=trace.get_color()))\n",
    "    )\n",
    "    # Add a title to the plot\n",
    "    tit = \"ROC Curve of \"+model+\" using \"+ classifier+ \" as Classification method\"\n",
    "    plotly_fig.update_layout(title=tit)\n",
    "    plotly_fig.update_xaxes(title_text='False Positive Rate')\n",
    "    plotly_fig.update_yaxes(title_text='True Positive Rate(Recall)')\n",
    "\n",
    "    return plotly_fig\n",
    "\n",
    "\n",
    "@app.callback(\n",
    "    Output('confusion-matrix', 'figure'),\n",
    "    [Input('roc_model', 'value'),\n",
    "     Input('roc_classifier', 'value')])\n",
    "def draw_confussion(model, classifier):\n",
    "    model_df = probs_df.index[probs_df['MODELS']==model].tolist()\n",
    "    classifier_df = probs_df.index[probs_df['CLASSIFIER']==classifier].tolist()\n",
    "    for i in model_df:\n",
    "        if i in classifier_df:\n",
    "            index = i\n",
    "    selected_roc = probs_df.iloc[index,:]\n",
    "    prob = selected_roc[\"PROBS\"].tolist()\n",
    "    pred = []\n",
    "    for i in range(len(prob)):\n",
    "        max_item = prob[i].index(max(prob[i]))\n",
    "        pred.append(max_item)\n",
    "\n",
    "\n",
    "    Y_decoded = encoder.inverse_transform([0,1,2,3]) \n",
    "\n",
    "    #https://stackoverflow.com/questions/60860121/plotly-how-to-make-an-annotated-confusion-matrix-using-a-heatmap\n",
    "    conf_matrix = metrics.confusion_matrix(y_test, pred)\n",
    "\n",
    "    data = go.Heatmap(z=conf_matrix, y=Y_decoded, x=Y_decoded)\n",
    "    annotations = []\n",
    "    for i, row in enumerate(conf_matrix):\n",
    "        for j, value in enumerate(row):\n",
    "            annotations.append(\n",
    "                {\n",
    "                    \"x\": Y_decoded[i],\n",
    "                    \"y\": Y_decoded[j],\n",
    "                    \"font\": {\"color\": \"white\"},\n",
    "                    \"text\": str(value),\n",
    "                    \"xref\": \"x1\",\n",
    "                    \"yref\": \"y1\",\n",
    "                    \"showarrow\": False\n",
    "                }\n",
    "            )\n",
    "    layout = {\n",
    "        \"title\": \"Confussion Matrix of \"+model+\" using \"+ classifier+ \" as Classification method\",\n",
    "        \"xaxis\": {\"title\": \"Predicted value\"},\n",
    "        \"yaxis\": {\"title\": \"Real value\"},\n",
    "        \"annotations\": annotations\n",
    "    }\n",
    "    fig = go.Figure(data=data, layout=layout)\n",
    "    return fig\n",
    "\n",
    "# Run the app\n",
    "if __name__ == '__main__':\n",
    "    app.run_server(debug=True)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.10 ('finalML')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "9ffb13643f58659a366a8d1a886476c8d650c48df7782f3d3477a7c406b6c188"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}