ElasticSearch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "93731aab-00fa-41e9-ac3c-43fd5440c535",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\sumeet\\AppData\\Local\\Temp\\ipykernel_23192\\1380276048.py:18: DeprecationWarning: The 'timeout' parameter is deprecated in favor of 'request_timeout'\n",
      "  es = Elasticsearch(hosts=[\"http://localhost:9200\"], timeout=60, retry_on_timeout=True)\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "import os\n",
    "from elasticsearch import Elasticsearch, helpers\n",
    "\n",
    "\n",
    "import csv\n",
    "from os.path import abspath, join, dirname, exists\n",
    "import tqdm\n",
    "import urllib3\n",
    "urllib3.disable_warnings()\n",
    "from elasticsearch import Elasticsearch\n",
    "from elasticsearch.helpers import streaming_bulk\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Connect to ES instance\n",
    "es = Elasticsearch(hosts=[\"http://localhost:9200\"], timeout=60, retry_on_timeout=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4826382c-90a7-4b69-87b0-5a52b54853e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install elasticsearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f2dc2e7b-4d36-4ac5-9bb9-465a33bfd56a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from subprocess import Popen, PIPE, STDOUT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25f1620c-efef-49b0-ba7b-a5ce02c1da67",
   "metadata": {},
   "outputs": [],
   "source": [
    "server = Popen(['C:/apps/elasticsearch-8.4.3/bin/elasticsearch.bat'], stdout=PIPE, stderr=STDOUT)\n",
    "!sleep 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01e475f1-6af3-4a0f-b20f-6800ae1f9c37",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "from elasticsearch import Elasticsearch, helpers\n",
    "\n",
    "# Connect to ES instance\n",
    "es = Elasticsearch(['https://localhost:9200'], http_auth=('elastic', 'P_YhQNcMjc36xSELpgzJ'), verify_certs=False)\n",
    "\n",
    "# Load HF dataset\n",
    "dataset = load_dataset(\"ag_news\", split=\"train\")[\"text\"][:50000]\n",
    "\n",
    "# Elasticsearch bulk buffer\n",
    "buffer = []\n",
    "rows = 0\n",
    "\n",
    "for x, text in enumerate(dataset):\n",
    "  # Article record\n",
    "  article = {\"_id\": x, \"_index\": \"articles\", \"title\": text}\n",
    "\n",
    "  # Buffer article\n",
    "  buffer.append(article)\n",
    "\n",
    "  # Increment number of articles processed\n",
    "  rows += 1\n",
    "\n",
    "  # Bulk load every 1000 records\n",
    "  if rows % 1000 == 0:\n",
    "    helpers.bulk(es, buffer)\n",
    "    buffer = []\n",
    "\n",
    "    print(\"Inserted {} articles\".format(rows), end=\"\\r\")\n",
    "\n",
    "if buffer:\n",
    "  helpers.bulk(es, buffer)\n",
    "\n",
    "print(\"Total articles inserted: {}\".format(rows))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "113cbad1-4063-4061-8847-4a39df42b501",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f213077-7cb3-457e-87b8-63049829a7df",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e80bf005-db60-450d-ba7a-00bca0c14a94",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "import csv\n",
    "from os.path import abspath, join, dirname, exists\n",
    "import tqdm\n",
    "import urllib3\n",
    "from elasticsearch import Elasticsearch\n",
    "from elasticsearch.helpers import streaming_bulk\n",
    "\n",
    "\n",
    "NYC_RESTAURANTS = (\n",
    "    \"https://data.cityofnewyork.us/api/views/43nn-pn8j/rows.csv?accessType=DOWNLOAD\"\n",
    ")\n",
    "DATASET_PATH = join(\"C:\\\\apps\\\\\", \"nyc-restaurants.csv\")\n",
    "CHUNK_SIZE = 16384\n",
    "\n",
    "\n",
    "def download_dataset():\n",
    "    \"\"\"Downloads the public dataset if not locally downlaoded\n",
    "    and returns the number of rows are in the .csv file.\n",
    "    \"\"\"\n",
    "    if not exists(DATASET_PATH):\n",
    "        http = urllib3.PoolManager()\n",
    "        resp = http.request(\"GET\", NYC_RESTAURANTS, preload_content=False)\n",
    "\n",
    "        if resp.status != 200:\n",
    "            raise RuntimeError(\"Could not download dataset\")\n",
    "\n",
    "        with open(DATASET_PATH, mode=\"wb\", encoding=\"utf8\") as f:\n",
    "            chunk = resp.read(CHUNK_SIZE)\n",
    "            while chunk:\n",
    "                f.write(chunk)\n",
    "                chunk = resp.read(CHUNK_SIZE)\n",
    "\n",
    "    with open(DATASET_PATH, encoding=\"utf8\") as f:\n",
    "        return sum([1 for _ in f]) - 1\n",
    "\n",
    "\n",
    "def create_index(client):\n",
    "    \"\"\"Creates an index in Elasticsearch if one isn't already there.\"\"\"\n",
    "    client.indices.create(\n",
    "        index=\"nyc-restaurants\",\n",
    "        body={\n",
    "            \"settings\": {\"number_of_shards\": 1},\n",
    "            \"mappings\": {\n",
    "                \"properties\": {\n",
    "                    \"name\": {\"type\": \"text\"},\n",
    "                    \"borough\": {\"type\": \"keyword\"},\n",
    "                    \"cuisine\": {\"type\": \"keyword\"},\n",
    "                    \"grade\": {\"type\": \"keyword\"},\n",
    "                    \"location\": {\"type\": \"geo_point\"},\n",
    "                }\n",
    "            },\n",
    "        },\n",
    "        ignore=400,\n",
    "    )\n",
    "\n",
    "\n",
    "def generate_actions():\n",
    "    \"\"\"Reads the file through csv.DictReader() and for each row\n",
    "    yields a single document. This function is passed into the bulk()\n",
    "    helper to create many documents in sequence.\n",
    "    \"\"\"\n",
    "    with open(DATASET_PATH, mode=\"r\", encoding=\"utf8\") as f:\n",
    "        reader = csv.DictReader(f)\n",
    "\n",
    "        for row in reader:\n",
    "            doc = {\n",
    "                \"_id\": row[\"CAMIS\"],\n",
    "                \"name\": row[\"DBA\"],\n",
    "                \"borough\": row[\"BORO\"],\n",
    "                \"cuisine\": row[\"CUISINE DESCRIPTION\"],\n",
    "                \"grade\": row[\"GRADE\"] or None,\n",
    "            }\n",
    "\n",
    "            lat = row[\"Latitude\"]\n",
    "            lon = row[\"Longitude\"]\n",
    "            if lat not in (\"\", \"0\") and lon not in (\"\", \"0\"):\n",
    "                doc[\"location\"] = {\"lat\": float(lat), \"lon\": float(lon)}\n",
    "            yield doc\n",
    "\n",
    "\n",
    "def main():\n",
    "    print(\"Loading dataset...\")\n",
    "    number_of_docs = download_dataset()\n",
    "\n",
    "    client = Elasticsearch(['https://localhost:9200'], http_auth=('elastic', 'P_YhQNcMjc36xSELpgzJ'), verify_certs=False)\n",
    "    print(\"Creating an index...\")\n",
    "    create_index(client)\n",
    "\n",
    "    print(\"Indexing documents...\")\n",
    "    progress = tqdm.tqdm(unit=\"docs\", total=number_of_docs)\n",
    "    successes = 0\n",
    "    for ok, action in streaming_bulk(\n",
    "        client=client, index=\"nyc-restaurants\", actions=generate_actions(),\n",
    "    ):\n",
    "        progress.update(1)\n",
    "        successes += ok\n",
    "    print(\"Indexed %d/%d documents\" % (successes, number_of_docs))\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "781a9468-483f-4050-a625-99ff08c85162",
   "metadata": {},
   "outputs": [],
   "source": [
    "client = Elasticsearch(['https://localhost:9200'], http_auth=('elastic', 'P_YhQNcMjc36xSELpgzJ'), verify_certs=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "582f158c-bf8c-4f1e-85ec-a4ef1ec0578f",
   "metadata": {},
   "outputs": [],
   "source": [
    "a = client.search(index='nyc-restaurants', body={\n",
    "  'query': {\n",
    "    'match': {\n",
    "      '_id': '50127304',\n",
    "     }\n",
    "  }\n",
    "})\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "849d3e63-e274-4af3-a165-fcaf8e2f8c4f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a8ac88c0-4b58-4d8d-bf7f-b4a7b1b6e431",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa7565a4-864b-4acd-b5bf-ea5b9eea472e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c6a11a8-4291-49d1-aacf-63c7d4ef5414",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11f8ff91-da1c-4eb7-93ff-a3fddbbf85cd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae96e931-0733-4a55-9807-0aa3f08f9fce",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ac704f2-4795-4e39-a560-461df4d51ea1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1a9bfd56-1add-452a-b8c3-df046e30b46d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import os\n",
    "import uuid\n",
    "\n",
    "\n",
    "from sentence_transformers import SentenceTransformer, util\n",
    "model = SentenceTransformer('all-MiniLM-L6-v2')\n",
    "\n",
    "import elasticsearch\n",
    "from elasticsearch import Elasticsearch\n",
    "from elasticsearch import helpers\n",
    "\n",
    "from tqdm.auto import tqdm\n",
    "pd.set_option('display.notebook_repr_html', False)\n",
    "tqdm.pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "dee08a24-f6d3-4e95-a7a5-32949d8e24f7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                             jobpost          date  \\\n",
       "0  AMERIA Investment Consulting Company\\r\\nJOB TI...   Jan 5, 2004   \n",
       "1  International Research & Exchanges Board (IREX...   Jan 7, 2004   \n",
       "2  Caucasus Environmental NGO Network (CENN)\\r\\nJ...   Jan 7, 2004   \n",
       "3  Manoff Group\\r\\nJOB TITLE:  BCC Specialist\\r\\n...   Jan 7, 2004   \n",
       "4  Yerevan Brandy Company\\r\\nJOB TITLE:  Software...  Jan 10, 2004   \n",
       "5  Boutique \"Appollo\"\\r\\nJOB TITLE:  Saleswoman\\r...  Jan 10, 2004   \n",
       "6  OSI Assistance Foundation - Armenian Branch Of...  Jan 11, 2004   \n",
       "7  International Research & Exchanges Board (IREX...  Jan 13, 2004   \n",
       "8  Yerevan Brandy Company \\r\\nJOB TITLE:  Assista...  Jan 13, 2004   \n",
       "9  American Embassy Yerevan\\r\\nANNOUNCEMENT NUMBE...  Jan 13, 2004   \n",
       "\n",
       "                                               Title  \\\n",
       "0                            Chief Financial Officer   \n",
       "1  Full-time Community Connections Intern (paid i...   \n",
       "2                                Country Coordinator   \n",
       "3                                     BCC Specialist   \n",
       "4                                 Software Developer   \n",
       "5                                         Saleswoman   \n",
       "6                Chief Accountant/ Finance Assistant   \n",
       "7     Non-paid part or full time Programmatic Intern   \n",
       "8                     Assistant to Managing Director   \n",
       "9              Program Assistant (INL), FSN-8; FP-6*   \n",
       "\n",
       "                                             Company AnnouncementCode Term  \\\n",
       "0               AMERIA Investment Consulting Company              NaN  NaN   \n",
       "1    International Research & Exchanges Board (IREX)              NaN  NaN   \n",
       "2          Caucasus Environmental NGO Network (CENN)              NaN  NaN   \n",
       "3                                       Manoff Group              NaN  NaN   \n",
       "4                             Yerevan Brandy Company              NaN  NaN   \n",
       "5                                 Boutique \"Appollo\"              NaN  NaN   \n",
       "6  OSI Assistance Foundation - Armenian Branch Of...              NaN  NaN   \n",
       "7    International Research & Exchanges Board (IREX)              NaN  NaN   \n",
       "8                             Yerevan Brandy Company              NaN  NaN   \n",
       "9  American Embassy Yerevan\\r\\nANNOUNCEMENT NUMBE...              NaN  NaN   \n",
       "\n",
       "  Eligibility Audience StartDate                               Duration  ...  \\\n",
       "0         NaN      NaN       NaN                                    NaN  ...   \n",
       "1         NaN      NaN       NaN                               3 months  ...   \n",
       "2         NaN      NaN       NaN  Renewable annual contract\\r\\nPOSITION  ...   \n",
       "3         NaN      NaN       NaN                                    NaN  ...   \n",
       "4         NaN      NaN       NaN                                    NaN  ...   \n",
       "5         NaN      NaN       NaN                                    NaN  ...   \n",
       "6         NaN      NaN       NaN                                    NaN  ...   \n",
       "7         NaN      NaN       NaN                               6 months  ...   \n",
       "8         NaN      NaN       NaN                                    NaN  ...   \n",
       "9         NaN      NaN       NaN                                    NaN  ...   \n",
       "\n",
       "  Salary                                       ApplicationP OpeningDate  \\\n",
       "0    NaN  To apply for this position, please submit a\\r\\...         NaN   \n",
       "1    NaN  Please submit a cover letter and resume to:\\r\\...         NaN   \n",
       "2    NaN  Please send resume or CV toursula.kazarian@......         NaN   \n",
       "3    NaN  Please send cover letter and resume to Amy\\r\\n...         NaN   \n",
       "4    NaN  Successful candidates should submit\\r\\n- CV; \\...         NaN   \n",
       "5    NaN  For further information, please contact Irina\\...         NaN   \n",
       "6    NaN  For submission of applications/ CVs, please\\r\\...         NaN   \n",
       "7    NaN  To apply, please download and submit the\\r\\nap...         NaN   \n",
       "8    NaN  Successful candidates should submit\\r\\n- CV; \\...         NaN   \n",
       "9    NaN  Interested candidates for this position should...         NaN   \n",
       "\n",
       "                                            Deadline Notes  \\\n",
       "0                                    26 January 2004   NaN   \n",
       "1                                    12 January 2004   NaN   \n",
       "2      20 January 2004\\r\\nSTART DATE:  February 2004   NaN   \n",
       "3          23 January 2004\\r\\nSTART DATE:  Immediate   NaN   \n",
       "4                             20 January 2004, 18:00   NaN   \n",
       "5                                   01 February 2004   NaN   \n",
       "6                          16 January 2004, 6:00 pm.   NaN   \n",
       "7                                    16 January 2004   NaN   \n",
       "8                             27 January 2004, 18:00   NaN   \n",
       "9  26 January 2004  \\r\\nDrafted:   GSargsyan\\r\\nC...   NaN   \n",
       "\n",
       "                                              AboutC Attach  Year Month     IT  \n",
       "0                                                NaN    NaN  2004     1  False  \n",
       "1  The International Research & Exchanges Board (...    NaN  2004     1  False  \n",
       "2  The Caucasus Environmental NGO Network is a\\r\\...    NaN  2004     1  False  \n",
       "3                                                NaN    NaN  2004     1  False  \n",
       "4                                                NaN    NaN  2004     1   True  \n",
       "5                                                NaN    NaN  2004     1  False  \n",
       "6                                                NaN    NaN  2004     1  False  \n",
       "7  The International Research & Exchanges Board (...    NaN  2004     1  False  \n",
       "8                                                NaN    NaN  2004     1  False  \n",
       "9                                                NaN    NaN  2004     1  False  \n",
       "\n",
       "[10 rows x 24 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df  = pd.read_csv(\"job_posts.csv\")\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0879d09e-256f-4136-81e1-ba26eefa1cf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Tokenizer(object):\n",
    "    def __init__(self):\n",
    "        self.model = SentenceTransformer('all-MiniLM-L6-v2')\n",
    "\n",
    "    def get_token(self, documents):\n",
    "        sentences  = [documents]\n",
    "        sentence_embeddings = self.model.encode(sentences)\n",
    "        encod_np_array = np.array(sentence_embeddings)\n",
    "        encod_list = encod_np_array.tolist()\n",
    "        return encod_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "272501bf-4032-4780-855a-ff37337c8fe0",
   "metadata": {},
   "outputs": [],
   "source": [
    "token_instance = Tokenizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0c0e90cb-c541-4139-b434-24117e2b1b99",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.head(5000)\n",
    "df = df.dropna(how='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e389b64d-b84f-44c8-8d29-b074f54c07dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f9aa807c86d84815964ceb3af95af245",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/5000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df['vector'] = df['jobpost'].progress_apply(token_instance.get_token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19c80748-745b-4afb-9651-e4f786dbc10c",
   "metadata": {},
   "outputs": [],
   "source": [
    "PUT posting \n",
    "{\n",
    "   \"settings\":{\n",
    "      \"index\":{\n",
    "         \"number_of_shards\":1,\n",
    "         \"number_of_replicas\":0,\n",
    "         \"knn\":{\n",
    "            \"algo_param\":{\n",
    "               \"ef_search\":40,\n",
    "               \"ef_construction\":40,\n",
    "               \"m\":\"4\"\n",
    "            }\n",
    "         }\n",
    "      },\n",
    "      \"knn\":true\n",
    "   },\n",
    "   \"mappings\":{\n",
    "      \"properties\":{\n",
    "         \"vector\":{\n",
    "            \"type\":\"knn_vector\",\n",
    "            \"dimension\":384\n",
    "         },\n",
    "         \"title\":{\n",
    "            \"type\":\"text\",\n",
    "            \"fields\":{\n",
    "               \"keyword\":{\n",
    "                  \"type\":\"keyword\",\n",
    "                   \"index\":false\n",
    "               }\n",
    "            }\n",
    "         },\n",
    "         \"company\":{\n",
    "            \"type\":\"keyword\",\n",
    "            \"index\":false\n",
    "         },\n",
    "         \"location\":{\n",
    "            \"type\":\"keyword\",\n",
    "            \"index\":false\n",
    "         },\n",
    "         \"salary\":{\n",
    "            \"type\":\"keyword\",\n",
    "            \"index\":false\n",
    "         },\n",
    "         \"job_description\":{\n",
    "            \"type\":\"keyword\",\n",
    "            \"index\":false\n",
    "         }\n",
    "      }\n",
    "   }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75332fe0-4c70-4b35-880a-6ad75913f068",
   "metadata": {},
   "outputs": [],
   "source": [
    "elk_data = df.to_dict(\"records\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "364be483-5b29-48b4-8533-0fd49e9224eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "es = Elasticsearch(['https://localhost:9200'], http_auth=('elastic', 'P_YhQNcMjc36xSELpgzJ'), verify_certs=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c422339-ecca-4660-86a5-e2a8e6051871",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "for x in elk_data:\n",
    "    try:\n",
    "        _={\n",
    "           \"title\":x.get(\"Title\", \"\"),\n",
    "            \"company\":x.get(\"Company\", \"\"),\n",
    "            \"location\":x.get(\"Location\", \"\"),\n",
    "            \"salary\":x.get(\"Salary\", \"\"),\n",
    "            \"vector\":x.get(\"vector\", \"\"),\n",
    "            \"job_description\":x.get(\"JobDescription\", \"\"),\n",
    "\n",
    "        }\n",
    "        es.index(index='posting',body=_)\n",
    "    except Exception as e:pass\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76fb0141-31d8-4fcb-a7e9-73c66dbddf44",
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT = input(\"Enter the Input Query \")\n",
    "token_vector = token_instance.get_token(INPUT)\n",
    "\n",
    "query  ={\n",
    "  \"_source\": [\"title\", \"job_description\"], \n",
    "   \"size\":50,\n",
    "   \"query\":{\n",
    "      \"bool\":{\n",
    "         \"must\":[\n",
    "            {\n",
    "               \"knn\":{\n",
    "                  \"vector\":{\n",
    "                     \"vector\":token_vector,\n",
    "                     \"k\":20\n",
    "                  }\n",
    "               }\n",
    "            }\n",
    "         ]\n",
    "      }\n",
    "   }\n",
    "}\n",
    "\n",
    "\n",
    "res = es.search(index='posting1',\n",
    "                size=50,\n",
    "                body=query,\n",
    "                request_timeout=55)\n",
    "\n",
    "title = [x['_source']  for x in res['hits']['hits']] \n",
    "title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99ebce93-b6fe-4eaf-96b4-b499e8e00b8b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c8a9dbf-20c9-4c13-8758-ec7504b85d55",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "200ec56e-b638-42b6-8387-bad9f3763010",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1cf79538-3a21-4248-9f33-6b8779179200",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5774e86-91e9-483f-85e4-f857380d655d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "f3e159eb-193e-428c-acf1-91b8d1e66e68",
   "metadata": {},
   "source": [
    "## FInal Code Starts from here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ce5c3a54-d2f3-4775-8a80-24f6422c828a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.notebook import tqdm\n",
    "import os\n",
    "import json\n",
    "import pandas as pd\n",
    "import time\n",
    "import numpy as np\n",
    "from tqdm.auto import tqdm\n",
    "from elasticsearch import Elasticsearch\n",
    "from elasticsearch.helpers import streaming_bulk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1b85d1f8-961f-41a4-ba92-cbc5a911a0ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pd.read_csv(\"D:\\\\arxiv dump\\\\Embeddings\\\\merged_out_0.csv\")\n",
    "\n",
    "def convert(item):\n",
    "    item = item.strip()  # remove spaces at the end\n",
    "    item = item[1:-1]    # remove `[ ]`\n",
    "    item = np.fromstring(item, sep=' ')  # convert string to `numpy.array`\n",
    "    return item\n",
    "\n",
    "dataset['vector'] = dataset['embeddings'].apply(convert)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8b8f8007-963d-48ff-9317-3afc83b6f51b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order</th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>abstract</th>\n",
       "      <th>update_date</th>\n",
       "      <th>authors</th>\n",
       "      <th>Categories</th>\n",
       "      <th>embeddings</th>\n",
       "      <th>vector</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>704.0001</td>\n",
       "      <td>Calculation of prompt diphoton production cros...</td>\n",
       "      <td>A fully differential calculation in perturba...</td>\n",
       "      <td>2008-11-26</td>\n",
       "      <td>[['Balázs', 'C.', ''], ['Berger', 'E. L.', '']...</td>\n",
       "      <td>['High Energy Physics - Phenomenology']</td>\n",
       "      <td>[-1.33643940e-01  7.99820647e-02 -2.59015679e-...</td>\n",
       "      <td>[-0.13364394, 0.0799820647, -0.259015679, -0.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>704.0002</td>\n",
       "      <td>Sparsity-certifying Graph Decompositions</td>\n",
       "      <td>We describe a new algorithm, the $(k,\\ell)$-...</td>\n",
       "      <td>2008-12-13</td>\n",
       "      <td>[['Streinu', 'Ileana', ''], ['Theran', 'Louis'...</td>\n",
       "      <td>['Combinatorics', 'Computational Geometry']</td>\n",
       "      <td>[ 1.40859067e-01  2.30508655e-01  1.20633028e-...</td>\n",
       "      <td>[0.140859067, 0.230508655, 0.120633028, 0.0760...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>704.0003</td>\n",
       "      <td>The evolution of the Earth-Moon system based o...</td>\n",
       "      <td>The evolution of Earth-Moon system is descri...</td>\n",
       "      <td>2008-01-13</td>\n",
       "      <td>[['Pan', 'Hongjun', '']]</td>\n",
       "      <td>['General Physics']</td>\n",
       "      <td>[ 8.67411792e-02  1.22752758e-02 -6.91849440e-...</td>\n",
       "      <td>[0.0867411792, 0.0122752758, -0.069184944, 0.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>704.0004</td>\n",
       "      <td>A determinant of Stirling cycle numbers counts...</td>\n",
       "      <td>We show that a determinant of Stirling cycle...</td>\n",
       "      <td>2007-05-23</td>\n",
       "      <td>[['Callan', 'David', '']]</td>\n",
       "      <td>['Combinatorics']</td>\n",
       "      <td>[ 2.25209355e-01  6.65704012e-02 -5.51288947e-...</td>\n",
       "      <td>[0.225209355, 0.0665704012, -0.000551288947, 0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>704.0005</td>\n",
       "      <td>From dyadic $\\Lambda_{\\alpha}$ to $\\Lambda_{\\a...</td>\n",
       "      <td>In this paper we show how to compute the $\\L...</td>\n",
       "      <td>2013-10-15</td>\n",
       "      <td>[['Abu-Shammala', 'Wael', ''], ['Torchinsky', ...</td>\n",
       "      <td>['Classical Analysis and ODEs', 'Functional An...</td>\n",
       "      <td>[-4.51628491e-03  1.31734580e-01 -2.85967231e-...</td>\n",
       "      <td>[-0.00451628491, 0.13173458, -0.285967231, -0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>704.0006</td>\n",
       "      <td>Bosonic characters of atomic Cooper pairs acro...</td>\n",
       "      <td>We study the two-particle wave function of p...</td>\n",
       "      <td>2015-05-13</td>\n",
       "      <td>[['Pong', 'Y. H.', ''], ['Law', 'C. K.', '']]</td>\n",
       "      <td>['Mesoscale and Nanoscale Physics']</td>\n",
       "      <td>[ 3.04478467e-01 -1.56092525e-01 -2.15403959e-...</td>\n",
       "      <td>[0.304478467, -0.156092525, -0.215403959, -0.0...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>704.0007</td>\n",
       "      <td>Polymer Quantum Mechanics and its Continuum Limit</td>\n",
       "      <td>A rather non-standard quantum representation...</td>\n",
       "      <td>2008-11-26</td>\n",
       "      <td>[['Corichi', 'Alejandro', ''], ['Vukasinac', '...</td>\n",
       "      <td>['General Relativity and Quantum Cosmology']</td>\n",
       "      <td>[ 1.98111072e-01 -4.66411188e-03  4.31902222e-...</td>\n",
       "      <td>[0.198111072, -0.00466411188, 0.0431902222, 0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>704.0008</td>\n",
       "      <td>Numerical solution of shock and ramp compressi...</td>\n",
       "      <td>A general formulation was developed to repre...</td>\n",
       "      <td>2009-02-05</td>\n",
       "      <td>[['Swift', 'Damian C.', '']]</td>\n",
       "      <td>['Materials Science']</td>\n",
       "      <td>[ 1.01819381e-01  1.36552304e-01  1.39253110e-...</td>\n",
       "      <td>[0.101819381, 0.136552304, 0.13925311, 0.10627...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>704.0009</td>\n",
       "      <td>The Spitzer c2d Survey of Large, Nearby, Inste...</td>\n",
       "      <td>We discuss the results from the combined IRA...</td>\n",
       "      <td>2010-03-18</td>\n",
       "      <td>[['Harvey', 'Paul', ''], ['Merin', 'Bruno', ''...</td>\n",
       "      <td>['Astrophysics']</td>\n",
       "      <td>[ 5.14137596e-02 -2.31310248e-01  4.08286601e-...</td>\n",
       "      <td>[0.0514137596, -0.231310248, 0.0408286601, -0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>704.0010</td>\n",
       "      <td>Partial cubes: structures, characterizations, ...</td>\n",
       "      <td>Partial cubes are isometric subgraphs of hyp...</td>\n",
       "      <td>2007-05-23</td>\n",
       "      <td>[['Ovchinnikov', 'Sergei', '']]</td>\n",
       "      <td>['Combinatorics']</td>\n",
       "      <td>[ 5.11773705e-01 -9.93338078e-02  8.86382014e-...</td>\n",
       "      <td>[0.511773705, -0.0993338078, 0.0886382014, -0....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   order        id                                              title  \\\n",
       "0      0  704.0001  Calculation of prompt diphoton production cros...   \n",
       "1      1  704.0002           Sparsity-certifying Graph Decompositions   \n",
       "2      2  704.0003  The evolution of the Earth-Moon system based o...   \n",
       "3      3  704.0004  A determinant of Stirling cycle numbers counts...   \n",
       "4      4  704.0005  From dyadic $\\Lambda_{\\alpha}$ to $\\Lambda_{\\a...   \n",
       "5      5  704.0006  Bosonic characters of atomic Cooper pairs acro...   \n",
       "6      6  704.0007  Polymer Quantum Mechanics and its Continuum Limit   \n",
       "7      7  704.0008  Numerical solution of shock and ramp compressi...   \n",
       "8      8  704.0009  The Spitzer c2d Survey of Large, Nearby, Inste...   \n",
       "9      9  704.0010  Partial cubes: structures, characterizations, ...   \n",
       "\n",
       "                                            abstract update_date  \\\n",
       "0    A fully differential calculation in perturba...  2008-11-26   \n",
       "1    We describe a new algorithm, the $(k,\\ell)$-...  2008-12-13   \n",
       "2    The evolution of Earth-Moon system is descri...  2008-01-13   \n",
       "3    We show that a determinant of Stirling cycle...  2007-05-23   \n",
       "4    In this paper we show how to compute the $\\L...  2013-10-15   \n",
       "5    We study the two-particle wave function of p...  2015-05-13   \n",
       "6    A rather non-standard quantum representation...  2008-11-26   \n",
       "7    A general formulation was developed to repre...  2009-02-05   \n",
       "8    We discuss the results from the combined IRA...  2010-03-18   \n",
       "9    Partial cubes are isometric subgraphs of hyp...  2007-05-23   \n",
       "\n",
       "                                             authors  \\\n",
       "0  [['Balázs', 'C.', ''], ['Berger', 'E. L.', '']...   \n",
       "1  [['Streinu', 'Ileana', ''], ['Theran', 'Louis'...   \n",
       "2                           [['Pan', 'Hongjun', '']]   \n",
       "3                          [['Callan', 'David', '']]   \n",
       "4  [['Abu-Shammala', 'Wael', ''], ['Torchinsky', ...   \n",
       "5      [['Pong', 'Y. H.', ''], ['Law', 'C. K.', '']]   \n",
       "6  [['Corichi', 'Alejandro', ''], ['Vukasinac', '...   \n",
       "7                       [['Swift', 'Damian C.', '']]   \n",
       "8  [['Harvey', 'Paul', ''], ['Merin', 'Bruno', ''...   \n",
       "9                    [['Ovchinnikov', 'Sergei', '']]   \n",
       "\n",
       "                                          Categories  \\\n",
       "0            ['High Energy Physics - Phenomenology']   \n",
       "1        ['Combinatorics', 'Computational Geometry']   \n",
       "2                                ['General Physics']   \n",
       "3                                  ['Combinatorics']   \n",
       "4  ['Classical Analysis and ODEs', 'Functional An...   \n",
       "5                ['Mesoscale and Nanoscale Physics']   \n",
       "6       ['General Relativity and Quantum Cosmology']   \n",
       "7                              ['Materials Science']   \n",
       "8                                   ['Astrophysics']   \n",
       "9                                  ['Combinatorics']   \n",
       "\n",
       "                                          embeddings  \\\n",
       "0  [-1.33643940e-01  7.99820647e-02 -2.59015679e-...   \n",
       "1  [ 1.40859067e-01  2.30508655e-01  1.20633028e-...   \n",
       "2  [ 8.67411792e-02  1.22752758e-02 -6.91849440e-...   \n",
       "3  [ 2.25209355e-01  6.65704012e-02 -5.51288947e-...   \n",
       "4  [-4.51628491e-03  1.31734580e-01 -2.85967231e-...   \n",
       "5  [ 3.04478467e-01 -1.56092525e-01 -2.15403959e-...   \n",
       "6  [ 1.98111072e-01 -4.66411188e-03  4.31902222e-...   \n",
       "7  [ 1.01819381e-01  1.36552304e-01  1.39253110e-...   \n",
       "8  [ 5.14137596e-02 -2.31310248e-01  4.08286601e-...   \n",
       "9  [ 5.11773705e-01 -9.93338078e-02  8.86382014e-...   \n",
       "\n",
       "                                              vector  \n",
       "0  [-0.13364394, 0.0799820647, -0.259015679, -0.0...  \n",
       "1  [0.140859067, 0.230508655, 0.120633028, 0.0760...  \n",
       "2  [0.0867411792, 0.0122752758, -0.069184944, 0.0...  \n",
       "3  [0.225209355, 0.0665704012, -0.000551288947, 0...  \n",
       "4  [-0.00451628491, 0.13173458, -0.285967231, -0....  \n",
       "5  [0.304478467, -0.156092525, -0.215403959, -0.0...  \n",
       "6  [0.198111072, -0.00466411188, 0.0431902222, 0....  \n",
       "7  [0.101819381, 0.136552304, 0.13925311, 0.10627...  \n",
       "8  [0.0514137596, -0.231310248, 0.0408286601, -0....  \n",
       "9  [0.511773705, -0.0993338078, 0.0886382014, -0....  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "528388ad-3e7b-4a24-a2d7-75d8f1702e27",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\sumeet\\anaconda3\\envs\\Optimization-Python\\lib\\site-packages\\elasticsearch\\_sync\\client\\__init__.py:395: SecurityWarning: Connecting to 'https://localhost:9200' using TLS with verify_certs=False is insecure\n",
      "  _transport = transport_class(\n",
      "C:\\Users\\sumeet\\AppData\\Local\\Temp\\ipykernel_23192\\938878863.py:4: DeprecationWarning: The 'http_auth' parameter is deprecated. Use 'basic_auth' or 'bearer_auth' parameters instead\n",
      "  es = Elasticsearch(['https://localhost:9200'], http_auth=('elastic', 'P_YhQNcMjc36xSELpgzJ'), verify_certs=False)\n"
     ]
    }
   ],
   "source": [
    "from elasticsearch import Elasticsearch\n",
    "from elasticsearch import helpers\n",
    "\n",
    "es = Elasticsearch(['https://localhost:9200'], http_auth=('elastic', 'P_YhQNcMjc36xSELpgzJ'), verify_certs=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "97adb188-dad5-44ba-bec7-a45a2a1e0f8c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFBertModel: ['embeddings.position_ids']\n",
      "- This IS expected if you are initializing TFBertModel from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFBertModel from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the weights of TFBertModel were initialized from the PyTorch model.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertModel for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer, TFAutoModel\n",
    "\n",
    "# model_ckpt = \"sentence-transformers/multi-qa-mpnet-base-dot-v1\"\n",
    "model_ckpt = \"sentence-transformers/all-MiniLM-L12-v2\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_ckpt)\n",
    "\n",
    "model = TFAutoModel.from_pretrained(model_ckpt, from_pt=True)\n",
    "\n",
    "\n",
    "def cls_pooling(model_output):\n",
    "    return model_output.last_hidden_state[:, 0]\n",
    "\n",
    "import torch\n",
    "def get_embeddings(text_list):\n",
    "    encoded_input = tokenizer(text_list, padding=True, truncation=True, return_tensors=\"tf\")\n",
    "    encoded_input = {k: v for k, v in encoded_input.items()}\n",
    "    with torch.no_grad():\n",
    "        model_output = model(**encoded_input)\n",
    "    # return cls_pooling(model_output)\n",
    "    # return  mean_pooling(model_output, encoded_input[\"attention_mask\"])\n",
    "    return cls_pooling(model_output)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7482fd9-c39d-44f5-895e-adf1aeea96aa",
   "metadata": {},
   "source": [
    "## Modified below cell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "441fe9b7-66c4-4796-94b2-b940034401a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_index(client):\n",
    "    client.indices.create(\n",
    "        index=\"ssd-search\",\n",
    "        body={\n",
    "                \"settings\": {\n",
    "                    \"number_of_shards\": 1\n",
    "},\n",
    "                \"mappings\": {\n",
    "                    \"dynamic\": \"true\",\n",
    "                    \"_source\": {\n",
    "                        \"enabled\": \"true\"\n",
    "                    },\n",
    "                    \"properties\": {\n",
    "                        \"id\": {\n",
    "                            \"type\": \"keyword\",\n",
    "                            \"index\": \"false\"\n",
    "                        },\n",
    "                        \"abstract\": {\n",
    "                            \"type\": \"text\",\n",
    "                            \"fields\": {\n",
    "                                \"keyword\": {\n",
    "                                    \"type\": \"keyword\",\n",
    "                                    \"index\": \"false\"\n",
    "                                }\n",
    "                            }\n",
    "                        },\n",
    "                        \"title\": {\n",
    "                            \"type\": \"text\",\n",
    "                            \"index\": \"false\"\n",
    "                        },\n",
    "                        \"paper_id\": {\n",
    "                            \"type\": \"text\",\n",
    "                            \"index\": \"false\"\n",
    "                        },\n",
    "                        \"update_date\": {\n",
    "                            \"type\": \"date\",\n",
    "                            \"index\": \"false\"\n",
    "                        },\n",
    "                        \"Authors\": {\n",
    "                            \"type\": \"text\",\n",
    "                            \"index\": \"false\"\n",
    "                        },\n",
    "                        \"categories\": {\n",
    "                            \"type\": \"text\",\n",
    "                            \"index\": \"false\"\n",
    "                        },\n",
    "                        \"embeddings\": {\n",
    "                            \"type\": \"dense_vector\",\n",
    "                            \"dims\": 384\n",
    "                        }\n",
    "                    }\n",
    "}\n",
    "},\n",
    "        ignore=400,\n",
    "    )\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20068720-c66e-4e91-9de9-877b2c418990",
   "metadata": {},
   "source": [
    "## Don't run below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3f06a777-c20c-48c2-933e-7c6a7c2fae93",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_index(client):\n",
    "    client.indices.create(\n",
    "        index=\"ssd-search\",\n",
    "        body={\n",
    "            \"settings\": {\"number_of_shards\": 1},\n",
    "            \"mappings\": {\n",
    "                \"dynamic\": \"true\",\n",
    "                \"_source\": {\n",
    "                    \"enabled\": \"true\"\n",
    "                },\n",
    "                \"properties\": {\n",
    "                    \"id\": {\n",
    "                        \"type\": \"keyword\",\n",
    "                        \"index\":\"false\"\n",
    "                    },\n",
    "                    \"abstract\": {\n",
    "                        \"type\": \"text\",\n",
    "                        \"fields\": {\n",
    "                            \"keyword\": {\n",
    "                                \"type\": \"keyword\", \n",
    "                                \"index\":\"false\"\n",
    "                            }\n",
    "                        }\n",
    "                    },\n",
    "                    \"embeddings\": {\n",
    "                        \"type\": \"dense_vector\",\n",
    "                        \"dims\": 384\n",
    "                    }\n",
    "                }\n",
    "            }\n",
    "},\n",
    "        ignore=400,\n",
    "    )\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c289ef73-9589-40fb-8d05-a44080f2f7b0",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading dataset...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\sumeet\\anaconda3\\envs\\Optimization-Python\\lib\\site-packages\\elasticsearch\\_sync\\client\\__init__.py:395: SecurityWarning: Connecting to 'https://localhost:9200' using TLS with verify_certs=False is insecure\n",
      "  _transport = transport_class(\n",
      "C:\\Users\\sumeet\\AppData\\Local\\Temp\\ipykernel_23192\\1594203483.py:29: DeprecationWarning: The 'http_auth' parameter is deprecated. Use 'basic_auth' or 'bearer_auth' parameters instead\n",
      "  client = Elasticsearch(['https://localhost:9200'], http_auth=('elastic', 'P_YhQNcMjc36xSELpgzJ'), verify_certs=False)\n",
      "C:\\Users\\sumeet\\AppData\\Local\\Temp\\ipykernel_23192\\3079391344.py:2: DeprecationWarning: Passing transport options in the API method is deprecated. Use 'Elasticsearch.options()' instead.\n",
      "  client.indices.create(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating an index...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\sumeet\\AppData\\Local\\Temp\\ipykernel_23192\\3079391344.py:2: DeprecationWarning: The 'body' parameter is deprecated and will be removed in a future version. Instead use individual parameters.\n",
      "  client.indices.create(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Indexing documents...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dab4df15efec4d13a62b977cf470da67",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100001 [00:00<?, ?docs/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Indexed 100000/100001 documents\n"
     ]
    }
   ],
   "source": [
    "def generate_actions(dataset):\n",
    "    def convert(item):\n",
    "        item = item.strip()  # remove spaces at the end\n",
    "        item = item[1:-1]    # remove `[ ]`\n",
    "        item = np.fromstring(item, sep=' ')  # convert string to `numpy.array`\n",
    "        return item\n",
    "\n",
    "    dataset['embeddings'] = dataset['embeddings'].apply(convert)\n",
    "\n",
    "    for index, row in dataset.iterrows():\n",
    "            doc = {\n",
    "                \"id\": index,\n",
    "                \"paper_id\": row.get(\"id\", 0),\n",
    "                \"abstract\": row.get(\"abstract\", 0),\n",
    "                \"embeddings\": row.get(\"embeddings\", 0),\n",
    "                \"title\": row.get(\"title\", 0),\n",
    "                \"update_date\": row.get(\"update_date\", 0),\n",
    "                \"Authors\": row.get(\"authors\", 0),\n",
    "                \"categories\": row.get(\"categories\", 0)\n",
    "            }\n",
    "            yield doc\n",
    "\n",
    "\n",
    "def main():\n",
    "    print(\"Loading dataset...\")\n",
    "    dataset = pd.read_csv(\"D:\\\\arxiv dump\\\\Embeddings/merged_out_1.csv\")\n",
    "    number_of_docs = len(dataset) + 1\n",
    "\n",
    "    client = Elasticsearch(['https://localhost:9200'], http_auth=('elastic', 'P_YhQNcMjc36xSELpgzJ'), verify_certs=False)\n",
    "    print(\"Creating an index...\")\n",
    "    create_index(client)\n",
    "\n",
    "    print(\"Indexing documents...\")\n",
    "    progress = tqdm(unit=\"docs\", total=number_of_docs)\n",
    "    successes = 0\n",
    "    for ok, action in streaming_bulk(\n",
    "        client=client, index=\"ssd-search\", actions=generate_actions(dataset),\n",
    "    ):\n",
    "        progress.update(1)\n",
    "        successes += ok\n",
    "    print(\"Indexed %d/%d documents\" % (successes, number_of_docs))\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "785a73c8-89ca-4bac-ad60-16d6918b00cf",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Do not run below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "4ad70586-ab87-4641-b459-e3fc623af178",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\sumeet\\AppData\\Local\\Temp\\ipykernel_5160\\300644945.py:2: DeprecationWarning: The 'http_auth' parameter is deprecated. Use 'basic_auth' or 'bearer_auth' parameters instead\n",
      "  client = Elasticsearch(['https://localhost:9200'], http_auth=('elastic', 'P_YhQNcMjc36xSELpgzJ'), verify_certs=False)\n"
     ]
    }
   ],
   "source": [
    "action_list = []\n",
    "client = Elasticsearch(['https://localhost:9200'], http_auth=('elastic', 'P_YhQNcMjc36xSELpgzJ'), verify_certs=False)\n",
    "for index, row in dataset.iterrows():\n",
    "\n",
    "            doc = {\n",
    "                \"id\": index,\n",
    "                \"abstract\": row.get(\"abstract\", 0),\n",
    "                \"vector\": row.get(\"vector\", 0),\n",
    "                 \"_index\": \"ssd-search\"\n",
    "            }\n",
    "\n",
    "            action_list.append(doc)\n",
    "helpers.bulk(client, action_list)\n",
    "es.indices.delete(index='ssd-search', ignore=[400, 404])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "85df4823-1229-4b71-8d01-7fdb8512909f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\sumeet\\AppData\\Local\\Temp\\ipykernel_23192\\384970040.py:1: DeprecationWarning: Passing transport options in the API method is deprecated. Use 'Elasticsearch.options()' instead.\n",
      "  es.indices.delete(index='ssd-search', ignore=[400, 404])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "ObjectApiResponse({'acknowledged': True})"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es.indices.delete(index='ssd-search', ignore=[400, 404])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "528f6642-89c8-4f63-b551-701e9e6c1d99",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'question_embedding' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[1;32mIn [9]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[38;5;28mlen\u001b[39m(\u001b[43mquestion_embedding\u001b[49m[\u001b[38;5;241m0\u001b[39m])\n",
      "\u001b[1;31mNameError\u001b[0m: name 'question_embedding' is not defined"
     ]
    }
   ],
   "source": [
    "len(question_embedding[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "936026c2-ec9a-4d40-a7b1-76fc83c9e904",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\sumeet\\AppData\\Local\\Temp\\ipykernel_11400\\2848717253.py:1: DeprecationWarning: The 'http_auth' parameter is deprecated. Use 'basic_auth' or 'bearer_auth' parameters instead\n",
      "  client = Elasticsearch(['https://localhost:9200'], http_auth=('elastic', 'P_YhQNcMjc36xSELpgzJ'), verify_certs=False)\n",
      "C:\\Users\\sumeet\\AppData\\Local\\Temp\\ipykernel_11400\\2848717253.py:19: DeprecationWarning: The 'body' parameter is deprecated and will be removed in a future version. Instead use individual parameters.\n",
      "  ss = client.search(\n",
      "C:\\Users\\sumeet\\anaconda3\\envs\\Optimization-Python\\lib\\site-packages\\urllib3\\connectionpool.py:1045: InsecureRequestWarning: Unverified HTTPS request is being made to host 'localhost'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html#ssl-warnings\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "client = Elasticsearch(['https://localhost:9200'], http_auth=('elastic', 'P_YhQNcMjc36xSELpgzJ'), verify_certs=False)\n",
    "embedding_start = time.time()\n",
    "question = \"Deep learning contextual EMbeddings\"\n",
    "question_embedding = get_embeddings([question]).numpy()\n",
    "\n",
    "embedding_time = time.time() - embedding_start\n",
    "\n",
    "script_query = {\n",
    "    \"script_score\": {\n",
    "        \"query\": {\"match_all\": {}},\n",
    "        \"script\": {\n",
    "            \"source\": \"cosineSimilarity(params.query_vector, 'embeddings' ) + 1.0\",\n",
    "            \"params\": {\"query_vector\": question_embedding[0]}\n",
    "        }\n",
    "    }\n",
    "}\n",
    "\n",
    "\n",
    "ss = client.search(\n",
    "        index=\"ssd-search\",\n",
    "        body={\n",
    "            \"size\": 10,\n",
    "            \"query\": script_query,\n",
    "            \"_source\": {\"includes\": [\"id\",\"title\", \"abstract\", \"update_date\", \"Authors\" ,\"categories\", \"paper_id\"]}\n",
    "        }\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "78a7815e-76e2-4d07-a810-e660307ffec4",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ObjectApiResponse({'took': 5285, 'timed_out': False, '_shards': {'total': 1, 'successful': 1, 'skipped': 0, 'failed': 0}, 'hits': {'total': {'value': 10000, 'relation': 'gte'}, 'max_score': 1.6866598, 'hits': [{'_index': 'ssd-search', '_id': 'zwUHKoQBAlqJc8YlnDc-', '_score': 1.6866598, '_source': {'Authors': \"[['Herdağdelen', 'Amaç', ''], ['Baroni', 'Marco', '']]\", 'id': 8446, 'abstract': '  We introduce a way to represent word pairs instantiating arbitrary semantic\\nrelations that keeps track of the contexts in which the words in the pair occur\\nboth together and independently. The resulting features are of sufficient\\ngenerality to allow us, with the help of a standard supervised machine learning\\nalgorithm, to tackle a variety of unrelated semantic tasks with good results\\nand almost no task-specific tailoring.\\n', 'categories': 0, 'title': 'BagPack: A general framework to represent semantic relations', 'paper_id': 902.223, 'update_date': '2009-02-16'}}, {'_index': 'ssd-search', '_id': 'K6Fwf4QBf6YSOwoa4QhT', '_score': 1.6866598, '_source': {'Authors': \"[['Herdağdelen', 'Amaç', ''], ['Baroni', 'Marco', '']]\", 'id': 8446, 'abstract': '  We introduce a way to represent word pairs instantiating arbitrary semantic\\nrelations that keeps track of the contexts in which the words in the pair occur\\nboth together and independently. The resulting features are of sufficient\\ngenerality to allow us, with the help of a standard supervised machine learning\\nalgorithm, to tackle a variety of unrelated semantic tasks with good results\\nand almost no task-specific tailoring.\\n', 'categories': 0, 'title': 'BagPack: A general framework to represent semantic relations', 'paper_id': 902.223, 'update_date': '2009-02-16'}}, {'_index': 'ssd-search', '_id': 'qhUSsIQBnjDViD7L6Gxd', '_score': 1.6866598, '_source': {'Authors': \"[['Herdağdelen', 'Amaç', ''], ['Baroni', 'Marco', '']]\", 'id': 8446, 'abstract': '  We introduce a way to represent word pairs instantiating arbitrary semantic\\nrelations that keeps track of the contexts in which the words in the pair occur\\nboth together and independently. The resulting features are of sufficient\\ngenerality to allow us, with the help of a standard supervised machine learning\\nalgorithm, to tackle a variety of unrelated semantic tasks with good results\\nand almost no task-specific tailoring.\\n', 'categories': 0, 'title': 'BagPack: A general framework to represent semantic relations', 'paper_id': 902.223, 'update_date': '2009-02-16'}}, {'_index': 'ssd-search', '_id': 'WzewuIQBnWhyGIpaRa1G', '_score': 1.6866598, '_source': {'Authors': \"[['Herdağdelen', 'Amaç', ''], ['Baroni', 'Marco', '']]\", 'id': 8446, 'abstract': '  We introduce a way to represent word pairs instantiating arbitrary semantic\\nrelations that keeps track of the contexts in which the words in the pair occur\\nboth together and independently. The resulting features are of sufficient\\ngenerality to allow us, with the help of a standard supervised machine learning\\nalgorithm, to tackle a variety of unrelated semantic tasks with good results\\nand almost no task-specific tailoring.\\n', 'categories': 0, 'title': 'BagPack: A general framework to represent semantic relations', 'paper_id': 902.223, 'update_date': '2009-02-16'}}, {'_index': 'ssd-search', '_id': 'BQYLKoQBAlqJc8YlvQNd', '_score': 1.6747695, '_source': {'Authors': \"[['Gregor', 'Karol', ''], ['Griffin', 'Gregory', '']]\", 'id': 60468, 'abstract': \"  We apply deep belief networks of restricted Boltzmann machines to bags of\\nwords of sift features obtained from databases of 13 Scenes, 15 Scenes and\\nCaltech 256 and study experimentally their behavior and performance. We find\\nthat the final performance in the supervised phase is reached much faster if\\nthe system is pre-trained. Pre-training the system on a larger dataset keeping\\nthe supervised dataset fixed improves the performance (for the 13 Scenes case).\\nAfter the unsupervised pre-training, neurons arise that form approximate\\nexplicit representations for several categories (meaning they are mostly active\\nfor this category). The last three facts suggest that unsupervised training\\nreally discovers structure in these data. Pre-training can be done on a\\ncompletely different dataset (we use Corel dataset) and we find that the\\nsupervised phase performs just as good (on the 15 Scenes dataset). This leads\\nus to conjecture that one can pre-train the system once (e.g. in a factory) and\\nsubsequently apply it to many supervised problems which then learn much faster.\\nThe best performance is obtained with single hidden layer system suggesting\\nthat the histogram of sift features doesn't have much high level structure. The\\noverall performance is almost equal, but slightly worse then that of the\\nsupport vector machine and the spatial pyramidal matching.\\n\", 'categories': 0, 'title': 'Behavior and performance of the deep belief networks on image\\n  classification', 'paper_id': 912.0717, 'update_date': '2009-12-04'}}, {'_index': 'ssd-search', '_id': 'YaFxf4QBf6YSOwoandMP', '_score': 1.6747695, '_source': {'Authors': \"[['Gregor', 'Karol', ''], ['Griffin', 'Gregory', '']]\", 'id': 60468, 'abstract': \"  We apply deep belief networks of restricted Boltzmann machines to bags of\\nwords of sift features obtained from databases of 13 Scenes, 15 Scenes and\\nCaltech 256 and study experimentally their behavior and performance. We find\\nthat the final performance in the supervised phase is reached much faster if\\nthe system is pre-trained. Pre-training the system on a larger dataset keeping\\nthe supervised dataset fixed improves the performance (for the 13 Scenes case).\\nAfter the unsupervised pre-training, neurons arise that form approximate\\nexplicit representations for several categories (meaning they are mostly active\\nfor this category). The last three facts suggest that unsupervised training\\nreally discovers structure in these data. Pre-training can be done on a\\ncompletely different dataset (we use Corel dataset) and we find that the\\nsupervised phase performs just as good (on the 15 Scenes dataset). This leads\\nus to conjecture that one can pre-train the system once (e.g. in a factory) and\\nsubsequently apply it to many supervised problems which then learn much faster.\\nThe best performance is obtained with single hidden layer system suggesting\\nthat the histogram of sift features doesn't have much high level structure. The\\noverall performance is almost equal, but slightly worse then that of the\\nsupport vector machine and the spatial pyramidal matching.\\n\", 'categories': 0, 'title': 'Behavior and performance of the deep belief networks on image\\n  classification', 'paper_id': 912.0717, 'update_date': '2009-12-04'}}, {'_index': 'ssd-search', '_id': '4BYTsIQBnjDViD7LyDdZ', '_score': 1.6747695, '_source': {'Authors': \"[['Gregor', 'Karol', ''], ['Griffin', 'Gregory', '']]\", 'id': 60468, 'abstract': \"  We apply deep belief networks of restricted Boltzmann machines to bags of\\nwords of sift features obtained from databases of 13 Scenes, 15 Scenes and\\nCaltech 256 and study experimentally their behavior and performance. We find\\nthat the final performance in the supervised phase is reached much faster if\\nthe system is pre-trained. Pre-training the system on a larger dataset keeping\\nthe supervised dataset fixed improves the performance (for the 13 Scenes case).\\nAfter the unsupervised pre-training, neurons arise that form approximate\\nexplicit representations for several categories (meaning they are mostly active\\nfor this category). The last three facts suggest that unsupervised training\\nreally discovers structure in these data. Pre-training can be done on a\\ncompletely different dataset (we use Corel dataset) and we find that the\\nsupervised phase performs just as good (on the 15 Scenes dataset). This leads\\nus to conjecture that one can pre-train the system once (e.g. in a factory) and\\nsubsequently apply it to many supervised problems which then learn much faster.\\nThe best performance is obtained with single hidden layer system suggesting\\nthat the histogram of sift features doesn't have much high level structure. The\\noverall performance is almost equal, but slightly worse then that of the\\nsupport vector machine and the spatial pyramidal matching.\\n\", 'categories': 0, 'title': 'Behavior and performance of the deep belief networks on image\\n  classification', 'paper_id': 912.0717, 'update_date': '2009-12-04'}}, {'_index': 'ssd-search', '_id': 'kTixuIQBnWhyGIpaT3jm', '_score': 1.6747695, '_source': {'Authors': \"[['Gregor', 'Karol', ''], ['Griffin', 'Gregory', '']]\", 'id': 60468, 'abstract': \"  We apply deep belief networks of restricted Boltzmann machines to bags of\\nwords of sift features obtained from databases of 13 Scenes, 15 Scenes and\\nCaltech 256 and study experimentally their behavior and performance. We find\\nthat the final performance in the supervised phase is reached much faster if\\nthe system is pre-trained. Pre-training the system on a larger dataset keeping\\nthe supervised dataset fixed improves the performance (for the 13 Scenes case).\\nAfter the unsupervised pre-training, neurons arise that form approximate\\nexplicit representations for several categories (meaning they are mostly active\\nfor this category). The last three facts suggest that unsupervised training\\nreally discovers structure in these data. Pre-training can be done on a\\ncompletely different dataset (we use Corel dataset) and we find that the\\nsupervised phase performs just as good (on the 15 Scenes dataset). This leads\\nus to conjecture that one can pre-train the system once (e.g. in a factory) and\\nsubsequently apply it to many supervised problems which then learn much faster.\\nThe best performance is obtained with single hidden layer system suggesting\\nthat the histogram of sift features doesn't have much high level structure. The\\noverall performance is almost equal, but slightly worse then that of the\\nsupport vector machine and the spatial pyramidal matching.\\n\", 'categories': 0, 'title': 'Behavior and performance of the deep belief networks on image\\n  classification', 'paper_id': 912.0717, 'update_date': '2009-12-04'}}, {'_index': 'ssd-search', '_id': 'SgUKKoQBAlqJc8Yle8Pi', '_score': 1.6692231, '_source': {'Authors': \"[['Madhavan', 'Jayant', '', 'Google Inc.'], ['Afanasiev', 'Loredana', '', 'Universiteit van\\\\n  Amsterdam'], ['Antova', 'Lyublena', '', 'Cornell University'], ['Halevy', 'Alon', '', 'Google']]\", 'id': 44153, 'abstract': '  Over the past few years, we have built a system that has exposed large\\nvolumes of Deep-Web content to Google.com users. The content that our system\\nexposes contributes to more than 1000 search queries per-second and spans over\\n50 languages and hundreds of domains. The Deep Web has long been acknowledged\\nto be a major source of structured data on the web, and hence accessing\\nDeep-Web content has long been a problem of interest in the data management\\ncommunity. In this paper, we report on where we believe the Deep Web provides\\nvalue and where it does not. We contrast two very different approaches to\\nexposing Deep-Web content -- the surfacing approach that we used, and the\\nvirtual integration approach that has often been pursued in the data management\\nliterature. We emphasize where the values of each of the two approaches lie and\\ncaution against potential pitfalls. We outline important areas of future\\nresearch and, in particular, emphasize the value that can be derived from\\nanalyzing large collections of potentially disparate structured data on the\\nweb.\\n', 'categories': 0, 'title': 'Harnessing the Deep Web: Present and Future', 'paper_id': 909.1785, 'update_date': '2009-09-15'}}, {'_index': 'ssd-search', '_id': 'pqFxf4QBf6YSOwoaY5MI', '_score': 1.6692231, '_source': {'Authors': \"[['Madhavan', 'Jayant', '', 'Google Inc.'], ['Afanasiev', 'Loredana', '', 'Universiteit van\\\\n  Amsterdam'], ['Antova', 'Lyublena', '', 'Cornell University'], ['Halevy', 'Alon', '', 'Google']]\", 'id': 44153, 'abstract': '  Over the past few years, we have built a system that has exposed large\\nvolumes of Deep-Web content to Google.com users. The content that our system\\nexposes contributes to more than 1000 search queries per-second and spans over\\n50 languages and hundreds of domains. The Deep Web has long been acknowledged\\nto be a major source of structured data on the web, and hence accessing\\nDeep-Web content has long been a problem of interest in the data management\\ncommunity. In this paper, we report on where we believe the Deep Web provides\\nvalue and where it does not. We contrast two very different approaches to\\nexposing Deep-Web content -- the surfacing approach that we used, and the\\nvirtual integration approach that has often been pursued in the data management\\nliterature. We emphasize where the values of each of the two approaches lie and\\ncaution against potential pitfalls. We outline important areas of future\\nresearch and, in particular, emphasize the value that can be derived from\\nanalyzing large collections of potentially disparate structured data on the\\nweb.\\n', 'categories': 0, 'title': 'Harnessing the Deep Web: Present and Future', 'paper_id': 909.1785, 'update_date': '2009-09-15'}}]}})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "485a86ab-288f-47ad-ad61-a05122038a96",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'_index': 'ssd-search',\n",
       "  '_id': 'zwUHKoQBAlqJc8YlnDc-',\n",
       "  '_score': 1.6866598,\n",
       "  '_source': {'Authors': \"[['Herdağdelen', 'Amaç', ''], ['Baroni', 'Marco', '']]\",\n",
       "   'id': 8446,\n",
       "   'abstract': '  We introduce a way to represent word pairs instantiating arbitrary semantic\\nrelations that keeps track of the contexts in which the words in the pair occur\\nboth together and independently. The resulting features are of sufficient\\ngenerality to allow us, with the help of a standard supervised machine learning\\nalgorithm, to tackle a variety of unrelated semantic tasks with good results\\nand almost no task-specific tailoring.\\n',\n",
       "   'categories': 0,\n",
       "   'title': 'BagPack: A general framework to represent semantic relations',\n",
       "   'paper_id': 902.223,\n",
       "   'update_date': '2009-02-16'}},\n",
       " {'_index': 'ssd-search',\n",
       "  '_id': 'K6Fwf4QBf6YSOwoa4QhT',\n",
       "  '_score': 1.6866598,\n",
       "  '_source': {'Authors': \"[['Herdağdelen', 'Amaç', ''], ['Baroni', 'Marco', '']]\",\n",
       "   'id': 8446,\n",
       "   'abstract': '  We introduce a way to represent word pairs instantiating arbitrary semantic\\nrelations that keeps track of the contexts in which the words in the pair occur\\nboth together and independently. The resulting features are of sufficient\\ngenerality to allow us, with the help of a standard supervised machine learning\\nalgorithm, to tackle a variety of unrelated semantic tasks with good results\\nand almost no task-specific tailoring.\\n',\n",
       "   'categories': 0,\n",
       "   'title': 'BagPack: A general framework to represent semantic relations',\n",
       "   'paper_id': 902.223,\n",
       "   'update_date': '2009-02-16'}},\n",
       " {'_index': 'ssd-search',\n",
       "  '_id': 'qhUSsIQBnjDViD7L6Gxd',\n",
       "  '_score': 1.6866598,\n",
       "  '_source': {'Authors': \"[['Herdağdelen', 'Amaç', ''], ['Baroni', 'Marco', '']]\",\n",
       "   'id': 8446,\n",
       "   'abstract': '  We introduce a way to represent word pairs instantiating arbitrary semantic\\nrelations that keeps track of the contexts in which the words in the pair occur\\nboth together and independently. The resulting features are of sufficient\\ngenerality to allow us, with the help of a standard supervised machine learning\\nalgorithm, to tackle a variety of unrelated semantic tasks with good results\\nand almost no task-specific tailoring.\\n',\n",
       "   'categories': 0,\n",
       "   'title': 'BagPack: A general framework to represent semantic relations',\n",
       "   'paper_id': 902.223,\n",
       "   'update_date': '2009-02-16'}},\n",
       " {'_index': 'ssd-search',\n",
       "  '_id': 'WzewuIQBnWhyGIpaRa1G',\n",
       "  '_score': 1.6866598,\n",
       "  '_source': {'Authors': \"[['Herdağdelen', 'Amaç', ''], ['Baroni', 'Marco', '']]\",\n",
       "   'id': 8446,\n",
       "   'abstract': '  We introduce a way to represent word pairs instantiating arbitrary semantic\\nrelations that keeps track of the contexts in which the words in the pair occur\\nboth together and independently. The resulting features are of sufficient\\ngenerality to allow us, with the help of a standard supervised machine learning\\nalgorithm, to tackle a variety of unrelated semantic tasks with good results\\nand almost no task-specific tailoring.\\n',\n",
       "   'categories': 0,\n",
       "   'title': 'BagPack: A general framework to represent semantic relations',\n",
       "   'paper_id': 902.223,\n",
       "   'update_date': '2009-02-16'}})"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ss.body['hits']['hits'][0],ss.body['hits']['hits'][1],ss.body['hits']['hits'][2],ss.body['hits']['hits'][3]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "1fc70ffd-bdd9-4fc4-aaf8-9487722bc627",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>abstract</th>\n",
       "      <th>embeddings</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100000</td>\n",
       "      <td>No-scale supersymmetry or gaugino mediation ...</td>\n",
       "      <td>[-4.60246354e-02  4.32418548e-02 -8.72858316e-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100001</td>\n",
       "      <td>Numerous authors have referred to room-tempe...</td>\n",
       "      <td>[ 2.32891068e-01 -3.49794663e-02  1.66781843e-...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                           abstract  \\\n",
       "0      100000    No-scale supersymmetry or gaugino mediation ...   \n",
       "1      100001    Numerous authors have referred to room-tempe...   \n",
       "\n",
       "                                          embeddings  \n",
       "0  [-4.60246354e-02  4.32418548e-02 -8.72858316e-...  \n",
       "1  [ 2.32891068e-01 -3.49794663e-02  1.66781843e-...  "
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client = Elasticsearch(['https://localhost:9200'], http_auth=('elastic', 'P_YhQNcMjc36xSELpgzJ'), verify_certs=False)\n",
    "embedding_start = time.time()\n",
    "question = \"Deep learning contextual EMbeddings\"\n",
    "question_embedding = get_embeddings([question]).numpy()\n",
    "\n",
    "embedding_time = time.time() - embedding_start\n",
    "\n",
    "script_query = {\n",
    "    \"script_score\": {\n",
    "        \"query\": {\"match_all\": {}},\n",
    "        \"script\": {\n",
    "            \"source\": \"cosineSimilarity(params.query_vector, 'embeddings' ) + 1.0\",\n",
    "            \"params\": {\"query_vector\": question_embedding[0]}\n",
    "        }\n",
    "    }\n",
    "}\n",
    "\n",
    "\n",
    "client.search(\n",
    "        index=\"ssd-search\",\n",
    "        body={\n",
    "            \"size\": 10,\n",
    "            \"query\": script_query,\n",
    "            \"_source\": {\"includes\": [\"id\",\"title\", \"abstract\", \"update_date\", \"Authors\" ,\"categories\", \"paper_id\"]}\n",
    "        }\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a7aa7b4d-27de-4cc7-8cce-df23ad25d55c",
   "metadata": {},
   "source": [
    "## Final code ends here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06a0c44b-6f4c-44c1-a9e3-2fae5e5ff914",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7793ead-c889-4117-aaf3-51755072270b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03e65213-a0a8-4558-9dbf-cba7124f5d5b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "8bc2fae3-387b-455e-a278-5b221a0c7720",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gc\n",
    "import os\n",
    "import json\n",
    "from collections import Counter, defaultdict\n",
    "from tqdm.notebook import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import plotly.express as px\n",
    "import re\n",
    "import csv  \n",
    "year_pattern = r'([1-2][0-9]{3})'\n",
    "pd.set_option('display.max_colwidth', None)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "00396d30-ce90-4a95-8dc6-c3c6032eae1e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\sumeet\\AppData\\Local\\Temp\\ipykernel_5160\\143463918.py:1: DtypeWarning: Columns (1) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df = pd.read_csv('D:\\\\arxiv dump\\\\processed.csv')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>id</th>\n",
       "      <th>submitter</th>\n",
       "      <th>title</th>\n",
       "      <th>categories</th>\n",
       "      <th>abstract</th>\n",
       "      <th>update_date</th>\n",
       "      <th>authors_parsed</th>\n",
       "      <th>cat-short</th>\n",
       "      <th>categories_map</th>\n",
       "      <th>new_categories</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>704.0001</td>\n",
       "      <td>Pavel Nadolsky</td>\n",
       "      <td>Calculation of prompt diphoton production cross sections at Tevatron and\\n  LHC energies</td>\n",
       "      <td>hep-ph</td>\n",
       "      <td>A fully differential calculation in perturbative quantum chromodynamics is\\npresented for the production of massive photon pairs at hadron colliders. All\\nnext-to-leading order perturbative contributions from quark-antiquark,\\ngluon-(anti)quark, and gluon-gluon subprocesses are included, as well as\\nall-orders resummation of initial-state gluon radiation valid at\\nnext-to-next-to-leading logarithmic accuracy. The region of phase space is\\nspecified in which the calculation is most reliable. Good agreement is\\ndemonstrated with data from the Fermilab Tevatron, and predictions are made for\\nmore detailed tests with CDF and DO data. Predictions are shown for\\ndistributions of diphoton pairs produced at the energy of the Large Hadron\\nCollider (LHC). Distributions of the diphoton pairs from the decay of a Higgs\\nboson are contrasted with those produced from QCD processes at the LHC, showing\\nthat enhanced sensitivity to the signal can be obtained with judicious\\nselection of events.\\n</td>\n",
       "      <td>2008-11-26</td>\n",
       "      <td>[['Balázs', 'C.', ''], ['Berger', 'E. L.', ''], ['Nadolsky', 'P. M.', ''], ['Yuan', 'C. -P.', '']]</td>\n",
       "      <td>hep-ph</td>\n",
       "      <td>High Energy Physics - Phenomenology</td>\n",
       "      <td>['High Energy Physics - Phenomenology']</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0        id       submitter  \\\n",
       "0           0  704.0001  Pavel Nadolsky   \n",
       "\n",
       "                                                                                      title  \\\n",
       "0  Calculation of prompt diphoton production cross sections at Tevatron and\\n  LHC energies   \n",
       "\n",
       "  categories  \\\n",
       "0     hep-ph   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                abstract  \\\n",
       "0    A fully differential calculation in perturbative quantum chromodynamics is\\npresented for the production of massive photon pairs at hadron colliders. All\\nnext-to-leading order perturbative contributions from quark-antiquark,\\ngluon-(anti)quark, and gluon-gluon subprocesses are included, as well as\\nall-orders resummation of initial-state gluon radiation valid at\\nnext-to-next-to-leading logarithmic accuracy. The region of phase space is\\nspecified in which the calculation is most reliable. Good agreement is\\ndemonstrated with data from the Fermilab Tevatron, and predictions are made for\\nmore detailed tests with CDF and DO data. Predictions are shown for\\ndistributions of diphoton pairs produced at the energy of the Large Hadron\\nCollider (LHC). Distributions of the diphoton pairs from the decay of a Higgs\\nboson are contrasted with those produced from QCD processes at the LHC, showing\\nthat enhanced sensitivity to the signal can be obtained with judicious\\nselection of events.\\n   \n",
       "\n",
       "  update_date  \\\n",
       "0  2008-11-26   \n",
       "\n",
       "                                                                                       authors_parsed  \\\n",
       "0  [['Balázs', 'C.', ''], ['Berger', 'E. L.', ''], ['Nadolsky', 'P. M.', ''], ['Yuan', 'C. -P.', '']]   \n",
       "\n",
       "  cat-short                       categories_map  \\\n",
       "0    hep-ph  High Energy Physics - Phenomenology   \n",
       "\n",
       "                            new_categories  \n",
       "0  ['High Energy Physics - Phenomenology']  "
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('D:\\\\arxiv dump\\\\processed.csv')\n",
    "df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08b73607-b83a-4d8d-9a4d-82e015d3c0f2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:Optimization-Python] *",
   "language": "python",
   "name": "conda-env-Optimization-Python-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}