tfidf word embedding.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyOFgcyPyV1DwB9FgTjFFwHp",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/p-ai-org/p-colleges/blob/main/tfidf%20word%20embedding.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# important imports\n",
        "!pip install kneed\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from scipy import spatial\n",
        "import string\n",
        "import nltk\n",
        "nltk.download('stopwords')\n",
        "from nltk.corpus import stopwords\n",
        "\n",
        "from kneed import KneeLocator\n",
        "from sklearn.cluster import KMeans\n",
        "from sklearn.metrics import silhouette_score\n",
        "\n",
        "#import word embedding \n",
        "!wget http://nlp.stanford.edu/data/glove.42B.300d.zip\n",
        "!unzip glove.42B.300d.zip\n",
        "\n",
        "#import csv and dropping rows without mission statements\n",
        "url = 'https://raw.githubusercontent.com/p-ai-org/p-colleges/main/Brian/Mission%20Statement/Data_2-14-2023.csv'\n",
        "df1 = pd.read_csv(url)\n",
        "df1['Mission statement (IC2020mission)'].replace('', np.nan, inplace=True)\n",
        "df1.dropna(subset=['Mission statement (IC2020mission)'], inplace=True)\n",
        "len(df1)"
      ],
      "metadata": {
        "id": "j2EKRlPCSaAl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#previous functions that we need\n",
        "!head -n 1000 glove.42B.300d.txt > top_1000.txt\n",
        "\n",
        "embeddings = {}\n",
        "with open('top_1000.txt', 'r') as f:\n",
        "    for line in f:\n",
        "        values = line.split()\n",
        "        word = values[0]\n",
        "        vector = np.asarray(values[1:], \"float32\")\n",
        "        embeddings[word] = vector\n",
        "        \n",
        "words_with_embeddings = set([w for w in embeddings])\n",
        "\n",
        "# function 1: changing text to a list of words, removing punctuation and stop words\n",
        "\n",
        "def clean(text):\n",
        "     # clean description\n",
        "    text = text.translate(str.maketrans('', '', string.punctuation))\n",
        "    description = text.lower().strip()\n",
        "    words = description.split(\" \")\n",
        "    stops = set(stopwords.words('english'))\n",
        "    \n",
        "    # filter out stop words\n",
        "    words = [w for w in words if not w in stops]\n",
        "\n",
        "    return words"
      ],
      "metadata": {
        "id": "Gb1CG6IzSZ7x"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QbH3-Ox1RyZG"
      },
      "outputs": [],
      "source": [
        "def word_frequency_in_all_docs(listOfWords):\n",
        "  \"\"\"This function calculates the term frequency (how many documents the word appears in) and returns a dictionary with each word that appears in all the documents and the number of times it appear as its corresponding key.\"\"\"\n",
        "  newList = list(set(listOfWords))\n",
        "\n",
        "  # listOfWords should be: df1[\"Mission statement (IC2020mission)\"]\n",
        "  wordDic = {}\n",
        "  for sentence in newList:\n",
        "    cleanedSentence = set(clean(sentence))\n",
        "    for word in cleanedSentence:\n",
        "      # adds word in as key with value 1\n",
        "      if (word not in wordDic):\n",
        "        wordDic[word] = 1 \n",
        "      # increase value if sees word again\n",
        "      else:\n",
        "        wordDic[word] += 1\n",
        "    \n",
        "  wordDic.pop('') \n",
        "  return wordDic"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "wordDic = word_frequency_in_all_docs(df1[\"Mission statement (IC2020mission)\"])"
      ],
      "metadata": {
        "id": "cLDeCisNSL3O"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def word_idf(dicOfWords, numberOfDocs):\n",
        "  \"\"\"This function takes in a dictonary of words and its frequency, calculates its IDF, and return it a dictionary with each word as the key and tf-idf values as the value.\"\"\"\n",
        "  # dicOfWords should be: wordDic from the word_frequency function\n",
        "  \n",
        "  # calculate inverse of each word and makes a new dictionary with it\n",
        "  idfDic = {}\n",
        "  for w in dicOfWords:\n",
        "    value = dicOfWords[w]\n",
        "    inverse = np.log(numberOfDocs/value)\n",
        "    idfDic[w] = inverse\n",
        "  \n",
        "  #returns dictionary of each word and their new weighted embeddings\n",
        "  print(len(idfDic))\n",
        "  return idfDic"
      ],
      "metadata": {
        "id": "3exKU-5DSQB0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "idf = word_idf(wordDic, 389)"
      ],
      "metadata": {
        "id": "8mn6YK7VSTIR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def calculate_tfidf_weighted_embedding(listOfWords, idfDic):\n",
        "  \"\"\"This function calculates the word embedding of the listOfWords (mission statement) and returns the average word embedding using the weighted word average list.\"\"\"\n",
        "  # listOfWords should be the individual mission statements\n",
        "  #idfDic should be the dictionary with each word's respective idf\n",
        "\n",
        "  #calculate the tf (how many times the term appears in the mission statement) for mission statement\n",
        "  wordDic = {}\n",
        "  for words in listOfWords:\n",
        "    # adds word in as key with value 1\n",
        "    if (words not in wordDic):\n",
        "      wordDic[words] = 1 \n",
        "    # increase value if sees word again\n",
        "    else:\n",
        "      wordDic[words] += 1\n",
        "\n",
        "  #calculate and return a dictionary with each word in the mission statement and their tf-idf values\n",
        "  tfidfDic = {}\n",
        "  for w in idfDic:\n",
        "    if w in wordDic:\n",
        "      idf = idfDic[w]\n",
        "      tf = wordDic[w]\n",
        "      tfidf = idf*tf\n",
        "      tfidfDic[w] = tfidf\n",
        "  \n",
        "  #calculated weighted embeddings\n",
        "  newList = []\n",
        "  for w in listOfWords:\n",
        "    #checks if w is in both newlist and tfidfdic \n",
        "    if (w in words_with_embeddings) and w in tfidfDic:\n",
        "      newList += [w]\n",
        "        \n",
        "  if len(listOfWords) == 0:\n",
        "      return None\n",
        "  \n",
        "  return sum([embeddings[w]*tfidfDic[w] for w in newList])/len(listOfWords)"
      ],
      "metadata": {
        "id": "LDDs2YS7SVFB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mission_statement = 'Boston College was founded in 1863 by the Society of Jesus (the Jesuits) to educate Boston’s predominantly Irish, Catholic immigrant community. It opened its doors on September 5, 1864, in a building on Harrison Avenue in Boston’s South End, a small streetcar college for commuting students. When it outgrew the limitations of the space, then-president Rev. Thomas I. Gasson, S.J., bought 31 acres of the former Lawrence Farm in Chestnut Hill, Massachusetts, and broke ground in 1909 on a new campus, today fondly known as the Heights. BC began as an undergraduate liberal arts college, but as its aspirations grew, it added graduate programs and professional schools fulfilling its charter as a university.'"
      ],
      "metadata": {
        "id": "uSLd9bTGV58f"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "calculate_tfidf_weighted_embedding(clean(mission_statement), idf)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lC4sIHwUSXdr",
        "outputId": "f60b795e-c933-47df-9728-1692e0ab9153"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 1.33040160e-01,  7.43449330e-02,  4.90354374e-02,  1.01876922e-01,\n",
              "       -9.72299054e-02, -1.13919787e-02, -2.00068951e+00, -2.40835324e-02,\n",
              "       -1.00045286e-01,  1.51011229e-01,  1.06512308e-01,  3.78893353e-02,\n",
              "       -2.01741196e-02, -1.30857840e-01,  5.71837686e-02, -1.57147229e-01,\n",
              "        4.30384874e-02, -3.33257169e-02,  2.95615103e-02,  1.78992637e-02,\n",
              "        6.34853616e-02,  3.38694490e-02, -1.38834253e-01,  1.40734136e-01,\n",
              "       -1.21012079e-02, -9.03032813e-03,  1.18442707e-01,  5.97769860e-03,\n",
              "       -1.20611191e-01, -8.55118334e-02, -1.68977425e-01, -9.01536122e-02,\n",
              "       -3.06323986e-04, -4.31827269e-02, -8.60500857e-02,  7.45417103e-02,\n",
              "        3.16357687e-02, -5.04090730e-03, -1.16233908e-01, -1.39256492e-01,\n",
              "        4.49653342e-03, -1.70729727e-01,  1.46325901e-01,  4.80137430e-02,\n",
              "       -4.53421175e-02, -5.97847253e-02,  1.54624939e-01, -7.62414336e-02,\n",
              "       -8.50594863e-02,  7.03085912e-04,  1.08925058e-02,  1.52528942e-01,\n",
              "        7.67351538e-02,  3.76672484e-02,  7.31157362e-02, -1.55426383e-01,\n",
              "        8.25181305e-02, -1.02942385e-01, -2.14707538e-01,  7.21293688e-02,\n",
              "       -5.39588276e-03,  3.01193558e-02, -2.06565112e-02, -3.60755846e-02,\n",
              "       -5.41045815e-02, -8.05998296e-02,  4.44508642e-02, -1.11589916e-02,\n",
              "        1.44895747e-01,  1.02771837e-02, -2.92641521e-01, -5.38252853e-02,\n",
              "        2.73763873e-02, -2.22667381e-01,  6.27428666e-02, -2.04043955e-01,\n",
              "       -1.33689910e-01,  3.95580083e-02,  5.36727393e-03, -3.03786788e-02,\n",
              "       -2.26382865e-03, -6.35565102e-01, -9.93010104e-02, -8.36701691e-02,\n",
              "        3.40787740e-03,  6.87079458e-03,  1.77380741e-01, -7.46594891e-02,\n",
              "        4.93785590e-02,  1.39627993e-01, -2.72069499e-02, -8.38626921e-02,\n",
              "        1.61028635e-02, -9.46052670e-02, -1.95933171e-02, -1.02505237e-01,\n",
              "       -1.99046183e+00,  9.89603400e-02,  6.19178563e-02,  6.74311519e-02,\n",
              "        3.38582173e-02,  9.68200713e-02,  6.15553595e-02, -1.06841244e-01,\n",
              "       -2.74302764e-03,  7.46712536e-02,  7.69882500e-02, -3.18637341e-02,\n",
              "        5.02032936e-02,  7.70637617e-02, -1.25535414e-01,  1.50673762e-01,\n",
              "        7.96332806e-02,  9.52648893e-02, -1.36378594e-02, -2.81316508e-02,\n",
              "        7.41330441e-03,  1.30821466e-01,  1.62449464e-01,  2.24807430e-02,\n",
              "       -1.47413224e-01,  5.98003604e-02, -1.01221597e-03, -1.05907200e-02,\n",
              "        6.95456564e-02,  2.69400496e-02,  8.44217241e-02, -2.48066410e-02,\n",
              "        1.94191597e-02,  1.35626093e-01,  1.22456878e-01,  8.91607329e-02,\n",
              "        6.50446117e-02,  1.29597053e-01,  2.23190233e-01, -5.15743420e-02,\n",
              "        2.13165939e-01,  1.35328723e-02,  5.63500971e-02,  3.71270478e-01,\n",
              "       -6.62296116e-02,  2.68042902e-04, -1.17237702e-01,  9.05845687e-03,\n",
              "        5.35467565e-02, -2.09217407e-02,  9.41150039e-02, -1.71447806e-02,\n",
              "        2.32521389e-02,  7.24415667e-03, -1.15552470e-02, -1.14864387e-01,\n",
              "        1.17404433e-02,  3.35117280e-02, -8.45737979e-02, -7.44143426e-02,\n",
              "       -1.35981925e-02, -2.39126422e-02, -1.97459772e-01, -7.16575831e-02,\n",
              "       -6.65016323e-02,  1.23204663e-02, -6.52875602e-02, -9.55577865e-02,\n",
              "       -8.06514919e-02,  6.25991300e-02,  8.68066549e-02, -3.27745825e-02,\n",
              "        2.07624838e-01, -1.87870473e-01,  6.70422167e-02,  1.45163685e-01,\n",
              "       -4.84855957e-02, -9.15250704e-02, -1.11733447e-03,  9.23172478e-03,\n",
              "       -9.97592807e-02,  2.66851038e-02,  8.56785178e-02, -1.31305950e-02,\n",
              "       -1.00591242e-01,  2.42258102e-01,  1.90636218e-01, -2.13916879e-02,\n",
              "       -1.15778156e-01, -1.04815168e-02, -9.78790224e-04, -2.95655336e-02,\n",
              "       -2.56869961e-02,  2.26571560e-02,  5.39907105e-02, -3.80107127e-02,\n",
              "        1.67662263e-01, -9.04042050e-02,  4.07020412e-02, -1.21042222e-01,\n",
              "       -1.26254275e-01,  1.12861805e-01, -7.33285099e-02,  1.34286299e-01,\n",
              "        3.20601789e-03,  3.48425657e-02,  1.55355394e-01,  1.14549890e-01,\n",
              "        5.51992245e-02,  5.84598258e-03,  1.25771880e-01,  9.63536426e-02,\n",
              "        7.13296467e-03,  4.07806523e-02, -3.91172618e-02, -1.20103821e-01,\n",
              "       -1.44095197e-01, -1.95003301e-01, -3.80457528e-02,  1.56133860e-01,\n",
              "       -1.55652419e-01,  1.02075994e-01, -1.21932536e-01,  1.33655053e-02,\n",
              "       -1.04950741e-01,  2.92023029e-02, -1.30446568e-01, -1.28315836e-01,\n",
              "       -2.54993725e+00, -6.63986616e-03,  1.30393729e-03,  7.39491805e-02,\n",
              "        5.26624545e-02, -8.84257723e-03, -2.48527601e-02,  1.30989864e-01,\n",
              "        9.57476050e-02, -5.16633801e-02, -3.85705195e-02,  1.07580252e-01,\n",
              "        4.65655439e-02,  2.83710286e-02, -5.32151712e-03,  1.72872152e-02,\n",
              "       -1.12587418e-02,  1.00746602e-01, -1.20067187e-01,  2.47115582e-01,\n",
              "       -1.06545113e-01,  1.17048606e-01,  1.96261425e-02, -2.19511449e-01,\n",
              "       -1.95134617e-02, -7.68502653e-02, -1.62410103e-02, -1.77716166e-02,\n",
              "        1.52030990e-01, -3.11892424e-02, -3.07304692e-02, -9.88654792e-02,\n",
              "       -4.90236133e-02, -1.13096200e-01,  5.17830849e-02, -1.40574891e-02,\n",
              "        9.80708655e-03,  4.34412025e-02,  3.73311341e-02,  4.25700173e-02,\n",
              "        7.26084709e-02, -3.51791233e-02, -1.48339450e-01,  3.03679332e-03,\n",
              "        3.35812792e-02,  5.51480986e-02,  8.12820643e-02, -8.26587342e-03,\n",
              "        2.46032532e-02,  9.20327008e-02,  7.43419230e-02,  1.93087548e-01,\n",
              "        5.63174337e-02, -1.72406286e-02, -1.56886782e-02,  1.59392700e-01,\n",
              "        1.61560252e-02,  5.36072766e-03, -5.49672693e-02, -2.63110902e-02,\n",
              "       -1.30516365e-01,  1.11163564e-01, -1.37478625e-02,  1.03753479e-02,\n",
              "       -4.65620272e-02,  1.68157056e-01,  6.42730519e-02, -1.14124060e-01,\n",
              "       -1.08831674e-01, -7.71167055e-02,  5.64812496e-02, -5.66379493e-03,\n",
              "        4.77630347e-02, -4.16934304e-02,  1.67326331e-01, -1.05105594e-01],\n",
              "      dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    }
  ]
}