13_pdf_q_n_a_astradb_notebook.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/luasampaio/CienciaDados/blob/main/13_pdf_q_n_a_astradb_notebook.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Power of Cassandra and ChatGPT for PDF Data Ingestion and Question Answering using AstraDB & Langchain 🦜\n",
        "[**Link to my YouTube Channel**](https://www.youtube.com/BhaveshBhatt8791?sub_confirmation=1)"
      ],
      "metadata": {
        "id": "Q8hkQflCaog4"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Installs"
      ],
      "metadata": {
        "id": "UF0zhpH3bfb5"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YkgQX3rTKzlV"
      },
      "outputs": [],
      "source": [
        "!pip install -q cassandra-driver\n",
        "!pip install -q langchain\n",
        "!pip install -q openai\n",
        "!pip install -q pypdf\n",
        "!pip install -q cassio>=0.1.1\n",
        "!pip install -q tiktoken==0.4.0"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Cassandra Import"
      ],
      "metadata": {
        "id": "Da6eP8eybgvN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import cassandra\n",
        "print (cassandra.__version__)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4r1nYVZ6LSqA",
        "outputId": "303746b5-3951-46ac-bbfe-8dc9f3be0c12"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "3.28.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from cassandra.cluster import Cluster\n",
        "from cassandra.auth import PlainTextAuthProvider\n",
        "import json\n",
        "\n",
        "# This secure connect bundle is autogenerated when you donwload your SCB,\n",
        "# if yours is different update the file name below\n",
        "cloud_config= {\n",
        "  'secure_connect_bundle': 'secure-connect-bhavesh-astra-test.zip'\n",
        "}\n",
        "\n",
        "# This token json file is autogenerated when you donwload your token,\n",
        "# if yours is different update the file name below\n",
        "with open(\"bhavesh_astra_test-token.json\") as f:\n",
        "    secrets = json.load(f)\n",
        "\n",
        "CLIENT_ID = secrets[\"clientId\"]\n",
        "CLIENT_SECRET = secrets[\"secret\"]\n",
        "\n",
        "auth_provider = PlainTextAuthProvider(CLIENT_ID, CLIENT_SECRET)\n",
        "cluster = Cluster(cloud=cloud_config, auth_provider=auth_provider)\n",
        "session = cluster.connect()\n",
        "\n",
        "row = session.execute(\"select release_version from system.local\").one()\n",
        "if row:\n",
        "  print(row[0])\n",
        "else:\n",
        "  print(\"An error occurred.\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PhgsFxAqLZyi",
        "outputId": "bba10c0e-8453-479f-a816-64294b04f762"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:cassandra.cluster:Downgrading core protocol version from 66 to 65 for e312786c-bba2-4fd0-a3c3-17328cc29f6a-us-east1.db.astra.datastax.com:29042:b76400fb-5d89-4041-8fac-032b8afcdffd. To avoid this, it is best practice to explicitly set Cluster(protocol_version) to the version supported by your cluster. http://datastax.github.io/python-driver/api/cassandra/cluster.html#cassandra.cluster.Cluster.protocol_version\n",
            "WARNING:cassandra.cluster:Downgrading core protocol version from 65 to 5 for e312786c-bba2-4fd0-a3c3-17328cc29f6a-us-east1.db.astra.datastax.com:29042:b76400fb-5d89-4041-8fac-032b8afcdffd. To avoid this, it is best practice to explicitly set Cluster(protocol_version) to the version supported by your cluster. http://datastax.github.io/python-driver/api/cassandra/cluster.html#cassandra.cluster.Cluster.protocol_version\n",
            "ERROR:cassandra.connection:Closing connection <AsyncoreConnection(134104449286784) e312786c-bba2-4fd0-a3c3-17328cc29f6a-us-east1.db.astra.datastax.com:29042:b76400fb-5d89-4041-8fac-032b8afcdffd> due to protocol error: Error from server: code=000a [Protocol error] message=\"Beta version of the protocol used (5/v5-beta), but USE_BETA flag is unset\"\n",
            "WARNING:cassandra.cluster:Downgrading core protocol version from 5 to 4 for e312786c-bba2-4fd0-a3c3-17328cc29f6a-us-east1.db.astra.datastax.com:29042:b76400fb-5d89-4041-8fac-032b8afcdffd. To avoid this, it is best practice to explicitly set Cluster(protocol_version) to the version supported by your cluster. http://datastax.github.io/python-driver/api/cassandra/cluster.html#cassandra.cluster.Cluster.protocol_version\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "4.0.7-131836135da7\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Import"
      ],
      "metadata": {
        "id": "2n1p4K1Eag5m"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.llms import OpenAI\n",
        "from langchain.embeddings import OpenAIEmbeddings\n",
        "from langchain.vectorstores.cassandra import Cassandra\n",
        "from langchain.indexes import VectorstoreIndexCreator\n",
        "from langchain.text_splitter import (\n",
        "    CharacterTextSplitter,\n",
        "    RecursiveCharacterTextSplitter,\n",
        ")\n",
        "from langchain.docstore.document import Document\n",
        "from langchain.document_loaders import TextLoader, PyPDFLoader"
      ],
      "metadata": {
        "id": "ADMuR4uFM7ja"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# OS Import"
      ],
      "metadata": {
        "id": "D_w7l_uvamfM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.environ['OPENAI_API_KEY'] = \"Enter your OpenAI Key Here...\""
      ],
      "metadata": {
        "id": "QoOUr-9-Yfrj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Initialization"
      ],
      "metadata": {
        "id": "mEUSrKwVbmsL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "llm = OpenAI(temperature=0)\n",
        "openai_embeddings = OpenAIEmbeddings()"
      ],
      "metadata": {
        "id": "3d0bZ5WPNn5w",
        "outputId": "e95abeee-4513-4b71-dcca-450d41829969",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 159
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "name 'OpenAI' is not defined",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-1-852c2d02cbb7>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mllm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mOpenAI\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtemperature\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mopenai_embeddings\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mOpenAIEmbeddings\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'OpenAI' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "table_name = 'pdf_q_n_a_table_1'\n",
        "keyspace = \"pdf_q_n_a_test\"\n",
        "\n",
        "index_creator = VectorstoreIndexCreator(\n",
        "    vectorstore_cls = Cassandra,\n",
        "    embedding = openai_embeddings,\n",
        "    text_splitter = RecursiveCharacterTextSplitter(\n",
        "        chunk_size = 400,\n",
        "        chunk_overlap = 30,\n",
        "    ),\n",
        "\n",
        "    vectorstore_kwargs={\n",
        "        'session': session,\n",
        "        'keyspace': keyspace,\n",
        "        'table_name': table_name,\n",
        "    },\n",
        ")"
      ],
      "metadata": {
        "id": "G5fK-rJMNsV3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Loading PDF"
      ],
      "metadata": {
        "id": "6B-Hbuskbqu5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "loader = PyPDFLoader(\"Attention Paper.pdf\")\n",
        "pages = loader.load_and_split()"
      ],
      "metadata": {
        "id": "MBGnWIuPN54d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(pages)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9l2NikV3cMq2",
        "outputId": "f37035b4-d9c4-4d92-b6e7-28d4593c9fb7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "12"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pages[1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WjjRWKrIOF-u",
        "outputId": "e1184d63-05c9-4157-dcf5-99e934e45e5a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Document(page_content='Recurrent models typically factor computation along the symbol positions of the input and output\\nsequences. Aligning the positions to steps in computation time, they generate a sequence of hidden\\nstatesht, as a function of the previous hidden state ht−1and the input for position t. This inherently\\nsequential nature precludes parallelization within training examples, which becomes critical at longer\\nsequence lengths, as memory constraints limit batching across examples. Recent work has achieved\\nsigniﬁcant improvements in computational efﬁciency through factorization tricks [ 18] and conditional\\ncomputation [ 26], while also improving model performance in case of the latter. The fundamental\\nconstraint of sequential computation, however, remains.\\nAttention mechanisms have become an integral part of compelling sequence modeling and transduc-\\ntion models in various tasks, allowing modeling of dependencies without regard to their distance in\\nthe input or output sequences [ 2,16]. In all but a few cases [ 22], however, such attention mechanisms\\nare used in conjunction with a recurrent network.\\nIn this work we propose the Transformer, a model architecture eschewing recurrence and instead\\nrelying entirely on an attention mechanism to draw global dependencies between input and output.\\nThe Transformer allows for signiﬁcantly more parallelization and can reach a new state of the art in\\ntranslation quality after being trained for as little as twelve hours on eight P100 GPUs.\\n2 Background\\nThe goal of reducing sequential computation also forms the foundation of the Extended Neural GPU\\n[20], ByteNet [ 15] and ConvS2S [ 8], all of which use convolutional neural networks as basic building\\nblock, computing hidden representations in parallel for all input and output positions. In these models,\\nthe number of operations required to relate signals from two arbitrary input or output positions grows\\nin the distance between positions, linearly for ConvS2S and logarithmically for ByteNet. This makes\\nit more difﬁcult to learn dependencies between distant positions [ 11]. In the Transformer this is\\nreduced to a constant number of operations, albeit at the cost of reduced effective resolution due\\nto averaging attention-weighted positions, an effect we counteract with Multi-Head Attention as\\ndescribed in section 3.2.\\nSelf-attention, sometimes called intra-attention is an attention mechanism relating different positions\\nof a single sequence in order to compute a representation of the sequence. Self-attention has been\\nused successfully in a variety of tasks including reading comprehension, abstractive summarization,\\ntextual entailment and learning task-independent sentence representations [4, 22, 23, 19].\\nEnd-to-end memory networks are based on a recurrent attention mechanism instead of sequence-\\naligned recurrence and have been shown to perform well on simple-language question answering and\\nlanguage modeling tasks [28].\\nTo the best of our knowledge, however, the Transformer is the ﬁrst transduction model relying\\nentirely on self-attention to compute representations of its input and output without using sequence-\\naligned RNNs or convolution. In the following sections, we will describe the Transformer, motivate\\nself-attention and discuss its advantages over models such as [14, 15] and [8].\\n3 Model Architecture\\nMost competitive neural sequence transduction models have an encoder-decoder structure [ 5,2,29].\\nHere, the encoder maps an input sequence of symbol representations (x1,...,x n)to a sequence\\nof continuous representations z= (z1,...,z n). Given z, the decoder then generates an output\\nsequence (y1,...,y m)of symbols one element at a time. At each step the model is auto-regressive\\n[9], consuming the previously generated symbols as additional input when generating the next.\\nThe Transformer follows this overall architecture using stacked self-attention and point-wise, fully', metadata={'source': 'Attention Paper.pdf', 'page': 1})"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Load to Index"
      ],
      "metadata": {
        "id": "Httloa3fcRpH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pdf_index = index_creator.from_loaders([loader])"
      ],
      "metadata": {
        "id": "fLrsEXlkOZmb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "default_query = f'SELECT * FROM {keyspace}.{table_name}'\n",
        "\n",
        "rows = session.execute(default_query)\n",
        "\n",
        "for row_i, row in enumerate(rows):\n",
        "    print(f'\\nRow {row_i}:')\n",
        "    print(f'row_id: {row.row_id}')\n",
        "    print(f'embedding_vector: {str(row.vector)[:64]} ...')\n",
        "    print(f'body_blob: {row.body_blob[:64]} ...')\n",
        "    print(f'metadata_blob: {row.metadata_s}')\n",
        "\n",
        "print('\\n...')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tQrEY9i7Oc9W",
        "outputId": "9313d658-8cb9-4ddc-877b-a5b77bb588d0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Row 0:\n",
            "row_id: 1543e4d885a84b30a713ce3945104a55\n",
            "embedding_vector: [-0.043849579989910126, -0.005807334091514349, -0.00084909476572 ...\n",
            "body_blob: arXiv:1609.08144 , 2016.\n",
            "[32] Jie Zhou, Ying Cao, Xuguang Wang,  ...\n",
            "metadata_blob: {'page': '10.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 1:\n",
            "row_id: cf1d29fa0b504af99e26253116dd3b17\n",
            "embedding_vector: [-0.02232765033841133, 0.006839259527623653, 0.0281220730394125, ...\n",
            "body_blob: traverse in the network. The shorter these paths between any com ...\n",
            "metadata_blob: {'page': '5.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 2:\n",
            "row_id: 1fc0fbbb8c414a779e20c842c75093bf\n",
            "embedding_vector: [-0.023693561553955078, -0.005949937738478184, 0.002372674643993 ...\n",
            "body_blob: connected feed-forward network, which is applied to each positio ...\n",
            "metadata_blob: {'page': '4.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 3:\n",
            "row_id: e7e95b8c02fc4200addcff7a5a6027f4\n",
            "embedding_vector: [-0.01607612706720829, -0.010801807977259159, 0.0183686986565589 ...\n",
            "body_blob: translation quality after being trained for as little as twelve  ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 4:\n",
            "row_id: f5ad7f4c393b4c34a4dc7430802bbaef\n",
            "embedding_vector: [-0.029735980555415154, -0.0010455148294568062, 0.01836469583213 ...\n",
            "body_blob: on recurrent or convolutional layers. On both WMT 2014 English-t ...\n",
            "metadata_blob: {'page': '8.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 5:\n",
            "row_id: cc475a72c3f34086984f722b4339556b\n",
            "embedding_vector: [-0.002292240271344781, -0.013220361433923244, 0.020153829827904 ...\n",
            "body_blob: 5We used values of 2.8, 3.7, 6.0 and 9.5 TFLOPS for K80, K40, M4 ...\n",
            "metadata_blob: {'page': '7.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 6:\n",
            "row_id: fb72c42f3cb24d4389f6d84a51022877\n",
            "embedding_vector: [-0.025479335337877274, 0.0019815010018646717, 0.017256274819374 ...\n",
            "body_blob: because it may allow the model to extrapolate to sequence length ...\n",
            "metadata_blob: {'page': '5.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 7:\n",
            "row_id: 8aae95681a8a436ab22b847dc7ccbc09\n",
            "embedding_vector: [-0.0074414219707250595, 0.010410835966467857, 0.003921057097613 ...\n",
            "body_blob: the input sequence centered around the respective output positio ...\n",
            "metadata_blob: {'page': '6.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 8:\n",
            "row_id: df447be1f53b49c49e5c6f21bf8a1d7c\n",
            "embedding_vector: [-0.0074414219707250595, 0.010410835966467857, 0.003921057097613 ...\n",
            "body_blob: the input sequence centered around the respective output positio ...\n",
            "metadata_blob: {'page': '6.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 9:\n",
            "row_id: 07d60982cc184ef2a098e865d5b48675\n",
            "embedding_vector: [-0.03434796631336212, 0.025906283408403397, 0.03955136612057686 ...\n",
            "body_blob: Scaled Dot-Product Attention\n",
            " Multi-Head Attention\n",
            "Figure 2: (le ...\n",
            "metadata_blob: {'page': '3.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 10:\n",
            "row_id: 90dfbf10598f492cad1b277de2cda2e7\n",
            "embedding_vector: [-0.024865008890628815, -0.007470231968909502, 0.015986565500497 ...\n",
            "body_blob: Advances in Neural Information Processing Systems 28 , pages 244 ...\n",
            "metadata_blob: {'page': '10.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 11:\n",
            "row_id: 706a174b84bd4482aad2dc1dd75ecbec\n",
            "embedding_vector: [-0.0255027674138546, 0.001944096526131034, 0.017267359420657158 ...\n",
            "body_blob: because it may allow the model to extrapolate to sequence length ...\n",
            "metadata_blob: {'page': '5.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 12:\n",
            "row_id: b4d13787662943d2b19964643e84558c\n",
            "embedding_vector: [-0.024877432733774185, -0.0074632298201322556, 0.01595910824835 ...\n",
            "body_blob: Advances in Neural Information Processing Systems 28 , pages 244 ...\n",
            "metadata_blob: {'page': '10.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 13:\n",
            "row_id: 51743f07e9274b3685bedfca35bc47cd\n",
            "embedding_vector: [-0.024278121069073677, 0.02531438134610653, 0.01009345427155494 ...\n",
            "body_blob: different layer types.\n",
            "As noted in Table 1, a self-attention lay ...\n",
            "metadata_blob: {'page': '5.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 14:\n",
            "row_id: 39d6600f6d6e438a9fe236b5c8361b4c\n",
            "embedding_vector: [-0.03371512144804001, 0.007825237698853016, 0.00436701020225882 ...\n",
            "body_blob: itself. To facilitate these residual connections, all sub-layers ...\n",
            "metadata_blob: {'page': '2.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 15:\n",
            "row_id: 36ff1a5ed89d478ca2ae727b601c5cbd\n",
            "embedding_vector: [-0.008008867502212524, -0.006723493803292513, 0.000782612594775 ...\n",
            "body_blob: lrate =d−0.5\n",
            "model·min(step_num−0.5,step _num·warmup _steps−1.5) ...\n",
            "metadata_blob: {'page': '6.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 16:\n",
            "row_id: fe3fa448cc6a424288a5471f9d19c455\n",
            "embedding_vector: [-0.044530078768730164, 0.025041494518518448, 0.0343319438397884 ...\n",
            "body_blob: depicted in Figure 2.\n",
            "Multi-head attention allows the model to j ...\n",
            "metadata_blob: {'page': '3.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 17:\n",
            "row_id: 9d95befbfd28451f827151b7345d3143\n",
            "embedding_vector: [-0.019569559022784233, -0.0009797124657779932, 0.02072651870548 ...\n",
            "body_blob: machine translation. CoRR , abs/1406.1078, 2014.\n",
            "[6]Francois Cho ...\n",
            "metadata_blob: {'page': '9.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 18:\n",
            "row_id: c91344e22d3d440a9c9dcb5259d639f8\n",
            "embedding_vector: [-0.03578288108110428, 0.021066805347800255, 0.03428859263658523 ...\n",
            "body_blob: described in section 3.2.\n",
            "Self-attention, sometimes called intra ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 19:\n",
            "row_id: 59cc2e33f2274952ac3ff5c64c22391b\n",
            "embedding_vector: [-0.004167643375694752, -0.007041879463940859, 0.001422576140612 ...\n",
            "body_blob: Figure 1: The Transformer - model architecture.\n",
            "wise fully conne ...\n",
            "metadata_blob: {'page': '2.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 20:\n",
            "row_id: 6a5d06db8ca948a08a671be64f495e41\n",
            "embedding_vector: [-0.007048842962831259, -0.005564702674746513, 0.031117580831050 ...\n",
            "body_blob: 31st Conference on Neural Information Processing Systems (NIPS 2 ...\n",
            "metadata_blob: {'page': '0.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 21:\n",
            "row_id: bdc023ee148b4062b0cab5add65b0027\n",
            "embedding_vector: [-0.023575125262141228, 0.002641733968630433, 0.0049215378239750 ...\n",
            "body_blob: matrix multiplication code.\n",
            "While for small values of dkthe two  ...\n",
            "metadata_blob: {'page': '3.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 22:\n",
            "row_id: 5abdfd87c6b148aaab15915056243b91\n",
            "embedding_vector: [-0.004167643375694752, -0.007041879463940859, 0.001422576140612 ...\n",
            "body_blob: Figure 1: The Transformer - model architecture.\n",
            "wise fully conne ...\n",
            "metadata_blob: {'page': '2.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 23:\n",
            "row_id: c28eb9f432b446cda20bbc8cb402506e\n",
            "embedding_vector: [-0.013538183644413948, -0.013891412876546383, 0.026234051212668 ...\n",
            "body_blob: Table 2: The Transformer achieves better BLEU scores than previo ...\n",
            "metadata_blob: {'page': '7.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 24:\n",
            "row_id: cfb33d413d864bcc9a3dd87f31972c57\n",
            "embedding_vector: [-0.023693561553955078, -0.005949937738478184, 0.002372674643993 ...\n",
            "body_blob: connected feed-forward network, which is applied to each positio ...\n",
            "metadata_blob: {'page': '4.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 25:\n",
            "row_id: 988fed2dd3d8458687af3de02bf94dfc\n",
            "embedding_vector: [-0.010905753821134567, -0.014614815823733807, 0.011147949844598 ...\n",
            "body_blob: Rethinking the inception architecture for computer vision. CoRR  ...\n",
            "metadata_blob: {'page': '10.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 26:\n",
            "row_id: 17fe528b10d6408894b786820485f6b8\n",
            "embedding_vector: [-0.0063390471041202545, 0.026244904845952988, 0.023051079362630 ...\n",
            "body_blob: Table 1: Maximum path lengths, per-layer complexity and minimum  ...\n",
            "metadata_blob: {'page': '5.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 27:\n",
            "row_id: 9c815b79e65e4925bfdce4614648230f\n",
            "embedding_vector: [-0.015332161448895931, 0.010403966531157494, 0.0137455910444259 ...\n",
            "body_blob: Recurrent O(n·d2) O(n) O(n)\n",
            "Convolutional O(k·n·d2)O(1) O(logk(n ...\n",
            "metadata_blob: {'page': '5.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 28:\n",
            "row_id: 2aa2344d26ac427f99dd3f4d2a46a1b1\n",
            "embedding_vector: [-0.01605009287595749, -0.009503709152340889, 0.0232698563486337 ...\n",
            "body_blob: BLEU, establishing a new state-of-the-art BLEU score of 28.4. Th ...\n",
            "metadata_blob: {'page': '7.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 29:\n",
            "row_id: 6f3c5eb4e30e4e18b19c7fdb43cafc52\n",
            "embedding_vector: [0.011458257213234901, 0.024805542081594467, 0.00792150292545557 ...\n",
            "body_blob: orO(logk(n))in the case of dilated convolutions [ 15], increasin ...\n",
            "metadata_blob: {'page': '6.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 30:\n",
            "row_id: 75512d04391e46ddad932e028cfe4dfe\n",
            "embedding_vector: [-0.005453823134303093, 0.002798903500661254, 0.0063247494399547 ...\n",
            "body_blob: with subword units. arXiv preprint arXiv:1508.07909 , 2015.\n",
            "[26] ...\n",
            "metadata_blob: {'page': '10.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 31:\n",
            "row_id: bb8fe3eed43f46ff8a70cdeadb4ecf96\n",
            "embedding_vector: [-0.013139872811734676, 0.005170072428882122, 0.0159745104610919 ...\n",
            "body_blob: For the base models, we used a single model obtained by averagin ...\n",
            "metadata_blob: {'page': '7.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 32:\n",
            "row_id: 713ef972c2be4e5aae3b9d4f5e2966fa\n",
            "embedding_vector: [-0.004545195959508419, -0.0021413483191281557, 0.01022669114172 ...\n",
            "body_blob: inference to input length + 50, but terminate early when possibl ...\n",
            "metadata_blob: {'page': '7.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 33:\n",
            "row_id: 5545d383e0214d06b7b4d80e18a6dc28\n",
            "embedding_vector: [-0.01342056319117546, -0.012849325314164162, 0.0033022484276443 ...\n",
            "body_blob: single-precision ﬂoating-point capacity of each GPU5.\n",
            "6.2 Model  ...\n",
            "metadata_blob: {'page': '7.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 34:\n",
            "row_id: 7eebc14328224f4fbd99c4643b40fdaa\n",
            "embedding_vector: [-0.008308163844048977, -0.006507070269435644, 0.021692438051104 ...\n",
            "body_blob: sentence pairs. Sentences were encoded using byte-pair encoding  ...\n",
            "metadata_blob: {'page': '6.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 35:\n",
            "row_id: db76db62f1a141dea6db2cb97efb3af4\n",
            "embedding_vector: [-0.02588088996708393, -0.005006260238587856, 0.0061367056332528 ...\n",
            "body_blob: sequential nature precludes parallelization within training exam ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 36:\n",
            "row_id: 38fb53e8e9bd43a1908c6aec0d56786a\n",
            "embedding_vector: [-0.0477626658976078, -0.01527959294617176, 0.03136777877807617, ...\n",
            "body_blob: machine translation architectures. CoRR , abs/1703.03906, 2017.\n",
            " ...\n",
            "metadata_blob: {'page': '9.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 37:\n",
            "row_id: 4712dbbb53e74d7d9d0036006bf011fb\n",
            "embedding_vector: [-0.03199830278754234, 0.0008491338230669498, -0.003291095606982 ...\n",
            "body_blob: tokens in the sequence. To this end, we add \"positional encoding ...\n",
            "metadata_blob: {'page': '4.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 38:\n",
            "row_id: 46625c68dc8144af905be15ce7198900\n",
            "embedding_vector: [-0.023476174101233482, 0.01488665584474802, -0.0047332104295492 ...\n",
            "body_blob: of1√dk. Additive attention computes the compatibility function u ...\n",
            "metadata_blob: {'page': '3.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 39:\n",
            "row_id: 2bda672ab33542ecacb0242fe99cf598\n",
            "embedding_vector: [-0.037647638469934464, -0.030410893261432648, 0.010537472553551 ...\n",
            "body_blob: sequence (y1,...,y m)of symbols one element at a time. At each s ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 40:\n",
            "row_id: f4f575b72b27449b9afea3a67552ad4c\n",
            "embedding_vector: [-0.03228265792131424, -0.009443367831408978, 0.0188039597123861 ...\n",
            "body_blob: ensembles, by over 2 BLEU. On the WMT 2014 English-to-French tra ...\n",
            "metadata_blob: {'page': '0.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 41:\n",
            "row_id: 5a7cf4f21c14433db3e3dc2be59914fb\n",
            "embedding_vector: [-0.010459700599312782, -0.011344649828970432, 0.020466946065425 ...\n",
            "body_blob: heads clearly learn to perform different tasks, many appear to e ...\n",
            "metadata_blob: {'page': '6.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 42:\n",
            "row_id: f637472a5184425d9546e4edadca64b0\n",
            "embedding_vector: [-0.031971633434295654, -0.0010852740379050374, 0.01276219636201 ...\n",
            "body_blob: [23] Romain Paulus, Caiming Xiong, and Richard Socher. A deep re ...\n",
            "metadata_blob: {'page': '10.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 43:\n",
            "row_id: 80c6edb11a2e4315a84493637d814dd2\n",
            "embedding_vector: [-0.01692274957895279, 0.02297365665435791, 0.030254540964961052 ...\n",
            "body_blob: lengthnis smaller than the representation dimensionality d, whic ...\n",
            "metadata_blob: {'page': '5.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 44:\n",
            "row_id: ed2dfb4aa1314d398931d39ee3e902c5\n",
            "embedding_vector: [-0.022851573303341866, -0.007329107727855444, 0.019816491752862 ...\n",
            "body_blob: References\n",
            "[1]Jimmy Lei Ba, Jamie Ryan Kiros, and Geoffrey E Hin ...\n",
            "metadata_blob: {'page': '9.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 45:\n",
            "row_id: a85edac9ecdb422eb9ef7c6ea0285cb0\n",
            "embedding_vector: [-0.010951397009193897, -0.014606478624045849, 0.011034467257559 ...\n",
            "body_blob: Rethinking the inception architecture for computer vision. CoRR  ...\n",
            "metadata_blob: {'page': '10.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 46:\n",
            "row_id: eb69c4b5975b4f07b514e1cb503a2203\n",
            "embedding_vector: [-0.04791904613375664, -0.015722131356596947, 0.0314163863658905 ...\n",
            "body_blob: machine translation architectures. CoRR , abs/1703.03906, 2017.\n",
            " ...\n",
            "metadata_blob: {'page': '9.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 47:\n",
            "row_id: e5b4dc747f1d419b99ecaf2c626cea3c\n",
            "embedding_vector: [-0.0238182470202446, -0.004377948120236397, 0.02713016979396343 ...\n",
            "body_blob: [15] Nal Kalchbrenner, Lasse Espeholt, Karen Simonyan, Aaron van ...\n",
            "metadata_blob: {'page': '9.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 48:\n",
            "row_id: 2a528d9e7eac41f6962257ac0b479a13\n",
            "embedding_vector: [-0.022184738889336586, 0.0034397009294480085, 0.012138945050537 ...\n",
            "body_blob: respectively.\n",
            "3.1 Encoder and Decoder Stacks\n",
            "Encoder: The encode ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 49:\n",
            "row_id: 6ef33c2fe60c42d799f7fa5ef2a2e47b\n",
            "embedding_vector: [-0.020810583606362343, -0.007724390830844641, 0.021744301542639 ...\n",
            "body_blob: plan to extend the Transformer to problems involving input and o ...\n",
            "metadata_blob: {'page': '8.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 50:\n",
            "row_id: 7bec41243d154b058987e183daa8b566\n",
            "embedding_vector: [-0.016666973009705544, 0.022860782220959663, 0.0312787294387817 ...\n",
            "body_blob: lengthnis smaller than the representation dimensionality d, whic ...\n",
            "metadata_blob: {'page': '5.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 51:\n",
            "row_id: 3e50a7e973944436ad7586c71602d4de\n",
            "embedding_vector: [-0.03578288108110428, 0.021066805347800255, 0.03428859263658523 ...\n",
            "body_blob: described in section 3.2.\n",
            "Self-attention, sometimes called intra ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 52:\n",
            "row_id: 6759d00359d54090ad62a1941e58a8d7\n",
            "embedding_vector: [-0.03310201317071915, -0.022889409214258194, 0.0110427895560860 ...\n",
            "body_blob: language modeling tasks [28].\n",
            "To the best of our knowledge, howe ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 53:\n",
            "row_id: c78cd89f6cb848cda6921db25034a2a0\n",
            "embedding_vector: [-0.01660635508596897, -0.009747208096086979, 0.0101558966562151 ...\n",
            "body_blob: MoE [26] 26.03 40.56 2.0·10191.2·1020\n",
            "Deep-Att + PosUnk Ensemble ...\n",
            "metadata_blob: {'page': '7.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 54:\n",
            "row_id: a4023ddddad645758dea9611e2a99fef\n",
            "embedding_vector: [-0.03434796631336212, 0.025906283408403397, 0.03955136612057686 ...\n",
            "body_blob: Scaled Dot-Product Attention\n",
            " Multi-Head Attention\n",
            "Figure 2: (le ...\n",
            "metadata_blob: {'page': '3.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 55:\n",
            "row_id: 44d8da9dc01b4fb49b2c53669cacb8e9\n",
            "embedding_vector: [-0.03726242855191231, 8.219048322644085e-05, 0.0264128744602203 ...\n",
            "body_blob: textual entailment and learning task-independent sentence repres ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 56:\n",
            "row_id: d45c8f473ab84c43829ccda0f7e90b5d\n",
            "embedding_vector: [-0.05037495121359825, 0.011250224895775318, 0.05509760603308678 ...\n",
            "body_blob: [21] Minh-Thang Luong, Hieu Pham, and Christopher D Manning. Eff ...\n",
            "metadata_blob: {'page': '10.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 57:\n",
            "row_id: ec04ee6f7a3047d786d1c4f1accf5878\n",
            "embedding_vector: [-0.022313375025987625, 0.0068215178325772285, 0.028093602508306 ...\n",
            "body_blob: traverse in the network. The shorter these paths between any com ...\n",
            "metadata_blob: {'page': '5.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 58:\n",
            "row_id: bab9003441ff4587a756428b7e9d2422\n",
            "embedding_vector: [-0.026029463857412338, -0.00977668259292841, 0.0189140941947698 ...\n",
            "body_blob: are used in conjunction with a recurrent network.\n",
            "In this work w ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 59:\n",
            "row_id: 4ca4b892689347b59ea4abb67ac0836f\n",
            "embedding_vector: [-0.00794966984540224, 0.03249814361333847, 0.019450461491942406 ...\n",
            "body_blob: convolution is equal to the combination of a self-attention laye ...\n",
            "metadata_blob: {'page': '6.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 60:\n",
            "row_id: f7921764acd943639235be3583054e67\n",
            "embedding_vector: [-0.03544716164469719, 0.013497081585228443, 0.00724047748371958 ...\n",
            "body_blob: function than dot product may be beneﬁcial. We further observe i ...\n",
            "metadata_blob: {'page': '8.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 61:\n",
            "row_id: 76950d67090e44ab850bbc0351324848\n",
            "embedding_vector: [-0.013599230907857418, -0.013809808529913425, 0.026233881711959 ...\n",
            "body_blob: Table 2: The Transformer achieves better BLEU scores than previo ...\n",
            "metadata_blob: {'page': '7.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 62:\n",
            "row_id: 20e579d34ddd47dda5cedce11e6f1228\n",
            "embedding_vector: [-0.019996190443634987, -0.0036130130756646395, 0.00268958788365 ...\n",
            "body_blob: tensorflow/tensor2tensor .\n",
            "Acknowledgements We are grateful to N ...\n",
            "metadata_blob: {'page': '8.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 63:\n",
            "row_id: b186b2c31f734e3da8171f5cce8817bd\n",
            "embedding_vector: [-0.02350560389459133, 0.009333519265055656, 0.01232080720365047 ...\n",
            "body_blob: chose this function because we hypothesized it would allow the m ...\n",
            "metadata_blob: {'page': '5.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 64:\n",
            "row_id: 75c77c63a3704e1196f3dcf8e3c91283\n",
            "embedding_vector: [-0.02261974848806858, -0.005508508533239365, 0.0126137873157858 ...\n",
            "body_blob: attention over the output of the encoder stack. Similar to the e ...\n",
            "metadata_blob: {'page': '2.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 65:\n",
            "row_id: c43ec6b6e5c647e095cdf0811c251b19\n",
            "embedding_vector: [-0.008308163844048977, -0.006507070269435644, 0.021692438051104 ...\n",
            "body_blob: sentence pairs. Sentences were encoded using byte-pair encoding  ...\n",
            "metadata_blob: {'page': '6.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 66:\n",
            "row_id: 5604b14c54b7421b931eeff612b37b74\n",
            "embedding_vector: [-0.012731826864182949, 0.020214004442095757, 0.0301854871213436 ...\n",
            "body_blob: 0.0 4.67 25.3\n",
            "0.2 5.47 25.7\n",
            "(E) positional embedding instead of  ...\n",
            "metadata_blob: {'page': '8.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 67:\n",
            "row_id: 2555edc109ba4c4bb28909613a03637f\n",
            "embedding_vector: [-0.047096069902181625, -0.0038798328023403883, 0.02606851048767 ...\n",
            "body_blob: nov. Dropout: a simple way to prevent neural networks from overﬁ ...\n",
            "metadata_blob: {'page': '10.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 68:\n",
            "row_id: aac401de0592439981e7a7cb99527e48\n",
            "embedding_vector: [-0.012171432375907898, 0.014311314560472965, 0.0415982194244861 ...\n",
            "body_blob: checkpoint averaging. We present these results in Table 3.\n",
            "In Ta ...\n",
            "metadata_blob: {'page': '7.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 69:\n",
            "row_id: 95674899fd3a46029a63151a70e02ad0\n",
            "embedding_vector: [-0.019996190443634987, -0.0036130130756646395, 0.00268958788365 ...\n",
            "body_blob: tensorflow/tensor2tensor .\n",
            "Acknowledgements We are grateful to N ...\n",
            "metadata_blob: {'page': '8.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 70:\n",
            "row_id: d0dc133025a2405390775b213bb53376\n",
            "embedding_vector: [-0.030264100059866905, -0.0054894802160561085, 0.00666139181703 ...\n",
            "body_blob: Residual Dropout We apply dropout [ 27] to the output of each su ...\n",
            "metadata_blob: {'page': '6.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 71:\n",
            "row_id: 098e8ab6e67b4f48993ceaffe7c2dd4b\n",
            "embedding_vector: [0.0016399690648540854, -0.009022930637001991, -0.01187936495989 ...\n",
            "body_blob: trained the base models for a total of 100,000 steps or 12 hours ...\n",
            "metadata_blob: {'page': '6.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 72:\n",
            "row_id: 2041f98e3b634f0abf356eb75f83780d\n",
            "embedding_vector: [-0.02215864323079586, 0.0034331229981034994, 0.0121059250086545 ...\n",
            "body_blob: respectively.\n",
            "3.1 Encoder and Decoder Stacks\n",
            "Encoder: The encode ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 73:\n",
            "row_id: a95415c22de045e3a14ee1e053d094f3\n",
            "embedding_vector: [-0.031853727996349335, 0.0009013250237330794, -0.00375034101307 ...\n",
            "body_blob: tokens in the sequence. To this end, we add \"positional encoding ...\n",
            "metadata_blob: {'page': '4.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 74:\n",
            "row_id: e67f4d47ad8e4eee88170c51b1b62ffd\n",
            "embedding_vector: [-0.01363587286323309, 0.02386108599603176, 0.04044068232178688, ...\n",
            "body_blob: Attention Is All You Need\n",
            "Ashish Vaswani∗\n",
            "Google Brain\n",
            "avaswani@ ...\n",
            "metadata_blob: {'page': '0.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 75:\n",
            "row_id: 0e9057361faa4729861edafa939460b1\n",
            "embedding_vector: [-0.022851573303341866, -0.007329107727855444, 0.019816491752862 ...\n",
            "body_blob: References\n",
            "[1]Jimmy Lei Ba, Jamie Ryan Kiros, and Geoffrey E Hin ...\n",
            "metadata_blob: {'page': '9.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 76:\n",
            "row_id: 0a4fd7680a96436aa2c5587ecc48cc74\n",
            "embedding_vector: [-0.02270355634391308, 0.0062330313958227634, 0.0180096309632062 ...\n",
            "body_blob: (x1,...,x n)to another sequence of equal length (z1,...,z n), wi ...\n",
            "metadata_blob: {'page': '5.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 77:\n",
            "row_id: aa4a84c4964d47e89071266a4927be63\n",
            "embedding_vector: [-0.04613142088055611, 0.014356251806020737, 0.03511128947138786 ...\n",
            "body_blob: encoder. Each position in the encoder can attend to all position ...\n",
            "metadata_blob: {'page': '4.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 78:\n",
            "row_id: fcbd8cc1907b402590624019f25e4ecb\n",
            "embedding_vector: [-0.004676527343690395, -0.002141300356015563, 0.023404324427247 ...\n",
            "body_blob: Table 3: Variations on the Transformer architecture. Unlisted va ...\n",
            "metadata_blob: {'page': '8.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 79:\n",
            "row_id: dce51766c74b4c1e96191e57c3c013ac\n",
            "embedding_vector: [-0.006276054307818413, 0.026257315650582314, 0.0229647308588027 ...\n",
            "body_blob: Table 1: Maximum path lengths, per-layer complexity and minimum  ...\n",
            "metadata_blob: {'page': '5.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 80:\n",
            "row_id: 68de6ae6e24143fab4df85dd2ee99ec1\n",
            "embedding_vector: [-0.02641398087143898, -0.013999003916978836, 0.0064308769069612 ...\n",
            "body_blob: in particular, have been ﬁrmly established as state of the art a ...\n",
            "metadata_blob: {'page': '0.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 81:\n",
            "row_id: 3d0121b19f1c4c66a603368e1b909226\n",
            "embedding_vector: [-0.019136866554617882, -0.02117994800209999, 0.0229778606444597 ...\n",
            "body_blob: architectures [31, 21, 13].\n",
            "∗Equal contribution. Listing order i ...\n",
            "metadata_blob: {'page': '0.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 82:\n",
            "row_id: 734e5ec493c74a33b8aba9d5afa85870\n",
            "embedding_vector: [-0.018170135095715523, -0.015682224184274673, 0.004154670983552 ...\n",
            "body_blob: batch contained a set of sentence pairs containing approximately ...\n",
            "metadata_blob: {'page': '6.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 83:\n",
            "row_id: f0e8141cbd8a45999e9e08aeb9f94f8d\n",
            "embedding_vector: [-0.023476174101233482, 0.01488665584474802, -0.0047332104295492 ...\n",
            "body_blob: of1√dk. Additive attention computes the compatibility function u ...\n",
            "metadata_blob: {'page': '3.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 84:\n",
            "row_id: 9959f53fb6214d7587f598a43fc5590a\n",
            "embedding_vector: [-0.02641398087143898, -0.013999003916978836, 0.0064308769069612 ...\n",
            "body_blob: in particular, have been ﬁrmly established as state of the art a ...\n",
            "metadata_blob: {'page': '0.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 85:\n",
            "row_id: 09954451a03149e78bd4d77ff91582cf\n",
            "embedding_vector: [-0.019445935264229774, -0.0007021260680630803, 0.02058981545269 ...\n",
            "body_blob: machine translation. CoRR , abs/1406.1078, 2014.\n",
            "[6]Francois Cho ...\n",
            "metadata_blob: {'page': '9.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 86:\n",
            "row_id: a01d34c2ce114d119d67abf19c23df5f\n",
            "embedding_vector: [-0.02271842025220394, -0.022832797840237617, -0.011602263897657 ...\n",
            "body_blob: Recurrent models typically factor computation along the symbol p ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 87:\n",
            "row_id: 3aa3ec92b6034bc0a595c3dc61b58a65\n",
            "embedding_vector: [-0.01658528670668602, 0.005932947620749474, -0.0095871053636074 ...\n",
            "body_blob: linear transformation, similar to [ 24]. In the embedding layers ...\n",
            "metadata_blob: {'page': '4.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 88:\n",
            "row_id: 9ad3d8b754954e699e65bac8ef6fbf91\n",
            "embedding_vector: [-0.0240273829549551, 0.0013280215207487345, 0.029253039509058,  ...\n",
            "body_blob: into a matrix Q. The keys and values are also packed together in ...\n",
            "metadata_blob: {'page': '3.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 89:\n",
            "row_id: 09eeaa85911c441dbc4c5a8310820416\n",
            "embedding_vector: [0.011142938397824764, -0.004568043164908886, 0.0175283756107091 ...\n",
            "body_blob: steps (dev) (dev)×106\n",
            "base 6 512 2048 8 64 64 0.1 0.1 100K 4.92  ...\n",
            "metadata_blob: {'page': '8.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 90:\n",
            "row_id: b162d199550a4fb3bd71eb1eca7ec61e\n",
            "embedding_vector: [-0.006293328944593668, 0.007911421358585358, -0.001225391984917 ...\n",
            "body_blob: learned and ﬁxed [8].\n",
            "In this work, we use sine and cosine funct ...\n",
            "metadata_blob: {'page': '5.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 91:\n",
            "row_id: 275b696114ef45aa8bd90589218247d8\n",
            "embedding_vector: [-0.03218371048569679, -0.001701378496363759, 0.0064970864914357 ...\n",
            "body_blob: Illia Polosukhin∗‡\n",
            "illia.polosukhin@gmail.com\n",
            "Abstract\n",
            "The domin ...\n",
            "metadata_blob: {'page': '0.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 92:\n",
            "row_id: 8628621ea7c346689ba0cee4d45bd8f0\n",
            "embedding_vector: [-0.03547557815909386, 0.01342806126922369, 0.00749698793515563, ...\n",
            "body_blob: function than dot product may be beneﬁcial. We further observe i ...\n",
            "metadata_blob: {'page': '8.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 93:\n",
            "row_id: b42e645130eb4e3d938a06273fc49e3a\n",
            "embedding_vector: [-0.026029463857412338, -0.00977668259292841, 0.0189140941947698 ...\n",
            "body_blob: are used in conjunction with a recurrent network.\n",
            "In this work w ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 94:\n",
            "row_id: f0904c54c7894b8fa1ad0c033e474bc9\n",
            "embedding_vector: [-0.018258364871144295, -0.023662956431508064, 0.021932914853096 ...\n",
            "body_blob: block, computing hidden representations in parallel for all inpu ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 95:\n",
            "row_id: c05caf9b92634585a97f3e4080cbb8ab\n",
            "embedding_vector: [-0.05037495121359825, 0.011250224895775318, 0.05509760603308678 ...\n",
            "body_blob: [21] Minh-Thang Luong, Hieu Pham, and Christopher D Manning. Eff ...\n",
            "metadata_blob: {'page': '10.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 96:\n",
            "row_id: c37e706280384a689b5241f01cdf4579\n",
            "embedding_vector: [-0.007997782900929451, -0.006832028739154339, 0.000832366466056 ...\n",
            "body_blob: lrate =d−0.5\n",
            "model·min(step_num−0.5,step _num·warmup _steps−1.5) ...\n",
            "metadata_blob: {'page': '6.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 97:\n",
            "row_id: 4c0f54956e954af6962d135e3f91a8c9\n",
            "embedding_vector: [-0.01793324388563633, 0.006438305135816336, 0.01737772673368454 ...\n",
            "body_blob: attention and the parameter-free position representation and bec ...\n",
            "metadata_blob: {'page': '0.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 98:\n",
            "row_id: 9a5cd7fe98854887b4ff86125162672b\n",
            "embedding_vector: [-0.03228265792131424, -0.009443367831408978, 0.0188039597123861 ...\n",
            "body_blob: ensembles, by over 2 BLEU. On the WMT 2014 English-to-French tra ...\n",
            "metadata_blob: {'page': '0.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 99:\n",
            "row_id: ecf6a32cfba844abaf3ced19968205aa\n",
            "embedding_vector: [-0.028885526582598686, 0.0028913195710629225, 0.013287618756294 ...\n",
            "body_blob: based solely on attention mechanisms, dispensing with recurrence ...\n",
            "metadata_blob: {'page': '0.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 100:\n",
            "row_id: 14128f383d3b489887b38b8aeca49bfa\n",
            "embedding_vector: [-0.005491903517395258, -0.003464805195108056, -0.01101106218993 ...\n",
            "body_blob: from layer to layer. Another way of describing this is as two co ...\n",
            "metadata_blob: {'page': '4.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 101:\n",
            "row_id: a9dc547e02fb4ca697e7da3ece4d4178\n",
            "embedding_vector: [-0.044702015817165375, 0.024886345490813255, 0.0344272255897522 ...\n",
            "body_blob: depicted in Figure 2.\n",
            "Multi-head attention allows the model to j ...\n",
            "metadata_blob: {'page': '3.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 102:\n",
            "row_id: 588ef99ab97c46ea917f7927a2c39b82\n",
            "embedding_vector: [-0.01321055181324482, 0.005219285376369953, 0.01593365706503391 ...\n",
            "body_blob: For the base models, we used a single model obtained by averagin ...\n",
            "metadata_blob: {'page': '7.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 103:\n",
            "row_id: 84a75fd9425d4c1ba0720558c08c07ac\n",
            "embedding_vector: [-0.04098382964730263, -0.010711060836911201, 0.0055436240509152 ...\n",
            "body_blob: 7 Conclusion\n",
            "In this work, we presented the Transformer, the ﬁrs ...\n",
            "metadata_blob: {'page': '8.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 104:\n",
            "row_id: f9552e01d07d40a88f0b12a074304cdb\n",
            "embedding_vector: [-0.012171432375907898, 0.014311314560472965, 0.0415982194244861 ...\n",
            "body_blob: checkpoint averaging. We present these results in Table 3.\n",
            "In Ta ...\n",
            "metadata_blob: {'page': '7.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 105:\n",
            "row_id: 25b4568298b74d269f3246cdd8a7d220\n",
            "embedding_vector: [-0.011360409669578075, -0.009139370173215866, -0.00506286602467 ...\n",
            "body_blob: i=1qiki, has mean 0and variance dk.\n",
            "4 ...\n",
            "metadata_blob: {'page': '3.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 106:\n",
            "row_id: a9cc95175dd6456d8e68012a36dc5e70\n",
            "embedding_vector: [0.0016524465754628181, -0.00888865627348423, -0.011681424453854 ...\n",
            "body_blob: trained the base models for a total of 100,000 steps or 12 hours ...\n",
            "metadata_blob: {'page': '6.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 107:\n",
            "row_id: 1e404d207fec49ed8c796aeb22017cdd\n",
            "embedding_vector: [-0.015448170714080334, -0.010011181235313416, 0.016899859532713 ...\n",
            "body_blob: age recognition. In Proceedings of the IEEE Conference on Comput ...\n",
            "metadata_blob: {'page': '9.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 108:\n",
            "row_id: c89a4bcebdbe4dfeb3fee96a1235d9fc\n",
            "embedding_vector: [-0.02261974848806858, -0.005508508533239365, 0.0126137873157858 ...\n",
            "body_blob: attention over the output of the encoder stack. Similar to the e ...\n",
            "metadata_blob: {'page': '2.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 109:\n",
            "row_id: 43c09944f99a4d67b2524825eea6136e\n",
            "embedding_vector: [-0.01342056319117546, -0.012849325314164162, 0.0033022484276443 ...\n",
            "body_blob: single-precision ﬂoating-point capacity of each GPU5.\n",
            "6.2 Model  ...\n",
            "metadata_blob: {'page': '7.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 110:\n",
            "row_id: 1ec7d1fa48b04163b6b132e81757a413\n",
            "embedding_vector: [-0.03371512144804001, 0.007825237698853016, 0.00436701020225882 ...\n",
            "body_blob: itself. To facilitate these residual connections, all sub-layers ...\n",
            "metadata_blob: {'page': '2.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 111:\n",
            "row_id: d0c34f4596db49f8975ba36dbdfe4f7e\n",
            "embedding_vector: [-0.028048940002918243, -0.004964107181876898, -0.00888679362833 ...\n",
            "body_blob: [8]Jonas Gehring, Michael Auli, David Grangier, Denis Yarats, an ...\n",
            "metadata_blob: {'page': '9.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 112:\n",
            "row_id: be4d8ec10003409facdba26c4f3922db\n",
            "embedding_vector: [-0.015332161448895931, 0.010403966531157494, 0.0137455910444259 ...\n",
            "body_blob: Recurrent O(n·d2) O(n) O(n)\n",
            "Convolutional O(k·n·d2)O(1) O(logk(n ...\n",
            "metadata_blob: {'page': '5.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 113:\n",
            "row_id: 109fe8b610c84e74b0ba978a00e8d3f3\n",
            "embedding_vector: [-0.01599283143877983, -0.009523509070277214, 0.0232118181884288 ...\n",
            "body_blob: BLEU, establishing a new state-of-the-art BLEU score of 28.4. Th ...\n",
            "metadata_blob: {'page': '7.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 114:\n",
            "row_id: efc64d6ce2424ba584a1cfa8f92405fb\n",
            "embedding_vector: [-0.024207478389143944, 0.025405067950487137, 0.0101055121049284 ...\n",
            "body_blob: different layer types.\n",
            "As noted in Table 1, a self-attention lay ...\n",
            "metadata_blob: {'page': '5.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 115:\n",
            "row_id: fad6493dcf6d46ea8c619244175806d4\n",
            "embedding_vector: [-0.022442281246185303, 0.009031649678945541, 0.0144643234089016 ...\n",
            "body_blob: it more difﬁcult to learn dependencies between distant positions ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 116:\n",
            "row_id: 1795d6c44ea240b3b2550ba7b6260c13\n",
            "embedding_vector: [-0.018258364871144295, -0.023662956431508064, 0.021932914853096 ...\n",
            "body_blob: block, computing hidden representations in parallel for all inpu ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 117:\n",
            "row_id: d1e290c2174e4bb8aabd6d5fcbf1506f\n",
            "embedding_vector: [-0.01363587286323309, 0.02386108599603176, 0.04044068232178688, ...\n",
            "body_blob: Attention Is All You Need\n",
            "Ashish Vaswani∗\n",
            "Google Brain\n",
            "avaswani@ ...\n",
            "metadata_blob: {'page': '0.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 118:\n",
            "row_id: c1441ff2fb1145e584356d3039f302b4\n",
            "embedding_vector: [-0.03471637889742851, -0.0107490886002779, 0.032037559896707535 ...\n",
            "body_blob: constraint of sequential computation, however, remains.\n",
            "Attentio ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 119:\n",
            "row_id: ef16b7e224014c209270c6d101c2cb1b\n",
            "embedding_vector: [-0.02630132995545864, 0.018571238964796066, 0.03064865618944168 ...\n",
            "body_blob: is similar to that of single-head attention with full dimensiona ...\n",
            "metadata_blob: {'page': '4.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 120:\n",
            "row_id: ef6f385c50a44623b1a0607a82e714d1\n",
            "embedding_vector: [-0.03218371048569679, -0.001701378496363759, 0.0064970864914357 ...\n",
            "body_blob: Illia Polosukhin∗‡\n",
            "illia.polosukhin@gmail.com\n",
            "Abstract\n",
            "The domin ...\n",
            "metadata_blob: {'page': '0.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 121:\n",
            "row_id: 9849192ba3524914add0c026aa7ae151\n",
            "embedding_vector: [-0.01658528670668602, 0.005932947620749474, -0.0095871053636074 ...\n",
            "body_blob: linear transformation, similar to [ 24]. In the embedding layers ...\n",
            "metadata_blob: {'page': '4.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 122:\n",
            "row_id: 90840cbd7c95423f9ceb4af169610ebb\n",
            "embedding_vector: [-0.037647638469934464, -0.030410893261432648, 0.010537472553551 ...\n",
            "body_blob: sequence (y1,...,y m)of symbols one element at a time. At each s ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 123:\n",
            "row_id: 9b8c174861ff4b9eb0dd76a77811113d\n",
            "embedding_vector: [-0.030240120366215706, 0.001599768758751452, 0.0070728436112403 ...\n",
            "body_blob: linear projections to dk,dkanddvdimensions, respectively. On eac ...\n",
            "metadata_blob: {'page': '3.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 124:\n",
            "row_id: 414b8690fcc54b35bc6119f5627cdea3\n",
            "embedding_vector: [-0.02588088996708393, -0.005006260238587856, 0.0061367056332528 ...\n",
            "body_blob: sequential nature precludes parallelization within training exam ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 125:\n",
            "row_id: 1305e422b1704c0bb516e10687989304\n",
            "embedding_vector: [-0.023575125262141228, 0.002641733968630433, 0.0049215378239750 ...\n",
            "body_blob: matrix multiplication code.\n",
            "While for small values of dkthe two  ...\n",
            "metadata_blob: {'page': '3.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 126:\n",
            "row_id: 70e3da0fb7f042fdacca260e04055d60\n",
            "embedding_vector: [-0.004676527343690395, -0.002141300356015563, 0.023404324427247 ...\n",
            "body_blob: Table 3: Variations on the Transformer architecture. Unlisted va ...\n",
            "metadata_blob: {'page': '8.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 127:\n",
            "row_id: 70a19ca4da674c89986092648678b5d4\n",
            "embedding_vector: [-0.005553919821977615, -0.003512939438223839, -0.01105332281440 ...\n",
            "body_blob: from layer to layer. Another way of describing this is as two co ...\n",
            "metadata_blob: {'page': '4.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 128:\n",
            "row_id: cb35b1fae6654d32972af421549cd6fa\n",
            "embedding_vector: [-0.012731826864182949, 0.020214004442095757, 0.0301854871213436 ...\n",
            "body_blob: 0.0 4.67 25.3\n",
            "0.2 5.47 25.7\n",
            "(E) positional embedding instead of  ...\n",
            "metadata_blob: {'page': '8.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 129:\n",
            "row_id: a67e8bbe614b413e8f9183410bb97e54\n",
            "embedding_vector: [-0.047096069902181625, -0.0038798328023403883, 0.02606851048767 ...\n",
            "body_blob: nov. Dropout: a simple way to prevent neural networks from overﬁ ...\n",
            "metadata_blob: {'page': '10.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 130:\n",
            "row_id: 99f99256fa0741e486051e9db23b70e5\n",
            "embedding_vector: [-0.01655850186944008, -0.009785188362002373, 0.0100577566772699 ...\n",
            "body_blob: MoE [26] 26.03 40.56 2.0·10191.2·1020\n",
            "Deep-Att + PosUnk Ensemble ...\n",
            "metadata_blob: {'page': '7.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 131:\n",
            "row_id: 28414ec3f78641ffa43705e5b9c33bba\n",
            "embedding_vector: [-0.01793324388563633, 0.006438305135816336, 0.01737772673368454 ...\n",
            "body_blob: attention and the parameter-free position representation and bec ...\n",
            "metadata_blob: {'page': '0.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 132:\n",
            "row_id: bd8a861aaa9b41d3a341bd8309260042\n",
            "embedding_vector: [-0.0240273829549551, 0.0013280215207487345, 0.029253039509058,  ...\n",
            "body_blob: into a matrix Q. The keys and values are also packed together in ...\n",
            "metadata_blob: {'page': '3.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 133:\n",
            "row_id: 9251ebbb7cf547acb34abddd3d5241a0\n",
            "embedding_vector: [-0.020146569237113, 0.007318378426134586, -5.323173900251277e-0 ...\n",
            "body_blob: [17] Diederik Kingma and Jimmy Ba. Adam: A method for stochastic ...\n",
            "metadata_blob: {'page': '9.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 134:\n",
            "row_id: bf66efdd77544e94ac02e4eedeb6155d\n",
            "embedding_vector: [-0.014692949131131172, -0.010069439187645912, -0.00161373638547 ...\n",
            "body_blob: inside of scaled dot-product attention by masking out (setting t ...\n",
            "metadata_blob: {'page': '4.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 135:\n",
            "row_id: bdd5d114764a4ca6b80ec459d362aea7\n",
            "embedding_vector: [-0.020146569237113, 0.007318378426134586, -5.323173900251277e-0 ...\n",
            "body_blob: [17] Diederik Kingma and Jimmy Ba. Adam: A method for stochastic ...\n",
            "metadata_blob: {'page': '9.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 136:\n",
            "row_id: 8814b15b50a14332962544973c0976b9\n",
            "embedding_vector: [-0.0045154583640396595, -0.001946504577063024, 0.01010311394929 ...\n",
            "body_blob: inference to input length + 50, but terminate early when possibl ...\n",
            "metadata_blob: {'page': '7.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 137:\n",
            "row_id: 7175a18a7fc844efa603ed4aabf160cc\n",
            "embedding_vector: [-0.028885526582598686, 0.0028913195710629225, 0.013287618756294 ...\n",
            "body_blob: based solely on attention mechanisms, dispensing with recurrence ...\n",
            "metadata_blob: {'page': '0.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 138:\n",
            "row_id: 7d2324d19f994062a2f59774d145b396\n",
            "embedding_vector: [-0.022142091765999794, -0.0040940530598163605, -0.0187495779246 ...\n",
            "body_blob: tokens and output tokens to vectors of dimension dmodel. We also ...\n",
            "metadata_blob: {'page': '4.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 139:\n",
            "row_id: 6bf9161ad8ca4ff48efff4fafaab21e1\n",
            "embedding_vector: [-0.002369735622778535, -0.01309717446565628, 0.0200447291135787 ...\n",
            "body_blob: 5We used values of 2.8, 3.7, 6.0 and 9.5 TFLOPS for K80, K40, M4 ...\n",
            "metadata_blob: {'page': '7.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 140:\n",
            "row_id: 968d1cf2e3c34c578e18a19e874bdca7\n",
            "embedding_vector: [-0.028048940002918243, -0.004964107181876898, -0.00888679362833 ...\n",
            "body_blob: [8]Jonas Gehring, Michael Auli, David Grangier, Denis Yarats, an ...\n",
            "metadata_blob: {'page': '9.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 141:\n",
            "row_id: 0b04fb51b817413fb12be12f0455c57c\n",
            "embedding_vector: [-0.02010841853916645, 0.012344060465693474, 0.01482414640486240 ...\n",
            "body_blob: efﬁcient inference and visualizations. Lukasz and Aidan spent co ...\n",
            "metadata_blob: {'page': '0.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 142:\n",
            "row_id: 40960bee334a4802b097670927062107\n",
            "embedding_vector: [-0.02010841853916645, 0.012344060465693474, 0.01482414640486240 ...\n",
            "body_blob: efﬁcient inference and visualizations. Lukasz and Aidan spent co ...\n",
            "metadata_blob: {'page': '0.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 143:\n",
            "row_id: f4fb87cd468540c3bd5e6236821eb4bd\n",
            "embedding_vector: [-0.022679852321743965, 0.0062690009362995625, 0.018068654462695 ...\n",
            "body_blob: (x1,...,x n)to another sequence of equal length (z1,...,z n), wi ...\n",
            "metadata_blob: {'page': '5.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 144:\n",
            "row_id: d5a76b9d1c1a4d6b8bc9d34b9a9dc694\n",
            "embedding_vector: [-0.007041687145829201, -0.005557667929679155, 0.031088721007108 ...\n",
            "body_blob: 31st Conference on Neural Information Processing Systems (NIPS 2 ...\n",
            "metadata_blob: {'page': '0.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 145:\n",
            "row_id: aba072e8f46d48e6b8dbe91584012510\n",
            "embedding_vector: [-0.04692409187555313, 0.0007153992191888392, 0.0348338857293129 ...\n",
            "body_blob: self-attention and discuss its advantages over models such as [1 ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 146:\n",
            "row_id: 365a925fcf9d425ab040d98fcbec7c84\n",
            "embedding_vector: [-0.03556596115231514, 0.014283355325460434, 0.01332028023898601 ...\n",
            "body_blob: MultiHead( Q,K,V ) = Concat(head 1,...,head h)WO\n",
            "where head i= A ...\n",
            "metadata_blob: {'page': '4.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 147:\n",
            "row_id: 37533e21a5f6495e959e13beb8ff646e\n",
            "embedding_vector: [-0.02582927606999874, -0.028244374319911003, 0.0120004480704665 ...\n",
            "body_blob: hurts perplexity, as the model learns to be more unsure, but imp ...\n",
            "metadata_blob: {'page': '7.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 148:\n",
            "row_id: 4bd38db2726c4fea83bac54526b55dca\n",
            "embedding_vector: [-0.022145401686429977, -0.004045794252306223, -0.01868256367743 ...\n",
            "body_blob: tokens and output tokens to vectors of dimension dmodel. We also ...\n",
            "metadata_blob: {'page': '4.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 149:\n",
            "row_id: ea550d11245d467d91737a034eb92449\n",
            "embedding_vector: [-0.025380566716194153, 0.005738393869251013, 0.0440971925854682 ...\n",
            "body_blob: arXiv:1703.03130 , 2017.\n",
            "[20] Samy Bengio Łukasz Kaiser. Can act ...\n",
            "metadata_blob: {'page': '9.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 150:\n",
            "row_id: cbb2e207992646c88efa7b2c3487f704\n",
            "embedding_vector: [-0.030820587649941444, -0.002478353213518858, 0.021797275170683 ...\n",
            "body_blob: predictions for position ican depend only on the known outputs a ...\n",
            "metadata_blob: {'page': '2.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 151:\n",
            "row_id: 264175f38d424744affe785be25254dd\n",
            "embedding_vector: [-0.031971633434295654, -0.0010852740379050374, 0.01276219636201 ...\n",
            "body_blob: [23] Romain Paulus, Caiming Xiong, and Richard Socher. A deep re ...\n",
            "metadata_blob: {'page': '10.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 152:\n",
            "row_id: 33a64816187b465cbe7dad54bec9203e\n",
            "embedding_vector: [-0.03556596115231514, 0.014283355325460434, 0.01332028023898601 ...\n",
            "body_blob: MultiHead( Q,K,V ) = Concat(head 1,...,head h)WO\n",
            "where head i= A ...\n",
            "metadata_blob: {'page': '4.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 153:\n",
            "row_id: 1214a2a8f7464e87818112173e0fd457\n",
            "embedding_vector: [-0.033869385719299316, 0.015471032820641994, 0.0222011357545852 ...\n",
            "body_blob: extremely small gradients4. To counteract this effect, we scale  ...\n",
            "metadata_blob: {'page': '3.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 154:\n",
            "row_id: 9f8193b5744f43c9b8a89e7dee3b1ba2\n",
            "embedding_vector: [-0.014692949131131172, -0.010069439187645912, -0.00161373638547 ...\n",
            "body_blob: inside of scaled dot-product attention by masking out (setting t ...\n",
            "metadata_blob: {'page': '4.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 155:\n",
            "row_id: 6be22291efb04fb880e4d3c902d73d8a\n",
            "embedding_vector: [-0.011360409669578075, -0.009139370173215866, -0.00506286602467 ...\n",
            "body_blob: i=1qiki, has mean 0and variance dk.\n",
            "4 ...\n",
            "metadata_blob: {'page': '3.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 156:\n",
            "row_id: 9a7d5aca37f74655a6668e501acf2278\n",
            "embedding_vector: [-0.046062733978033066, 0.014469826593995094, 0.0352035276591777 ...\n",
            "body_blob: encoder. Each position in the encoder can attend to all position ...\n",
            "metadata_blob: {'page': '4.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 157:\n",
            "row_id: 19b627fc92274db783c27ecc39574aa2\n",
            "embedding_vector: [-0.018170135095715523, -0.015682224184274673, 0.004154670983552 ...\n",
            "body_blob: batch contained a set of sentence pairs containing approximately ...\n",
            "metadata_blob: {'page': '6.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 158:\n",
            "row_id: f287cfad36034c9796b10bda76711752\n",
            "embedding_vector: [-0.015394619666039944, -0.0011894599301740527, 0.01076666358858 ...\n",
            "body_blob: 9(8):1735–1780, 1997.\n",
            "[13] Rafal Jozefowicz, Oriol Vinyals, Mike ...\n",
            "metadata_blob: {'page': '9.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 159:\n",
            "row_id: db16994863b742acbc20b9b8b90f3fb2\n",
            "embedding_vector: [-0.033869385719299316, 0.015471032820641994, 0.0222011357545852 ...\n",
            "body_blob: extremely small gradients4. To counteract this effect, we scale  ...\n",
            "metadata_blob: {'page': '3.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 160:\n",
            "row_id: b1d7844a187e48b492acf714e1853f3a\n",
            "embedding_vector: [-0.03310201317071915, -0.022889409214258194, 0.0110427895560860 ...\n",
            "body_blob: language modeling tasks [28].\n",
            "To the best of our knowledge, howe ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 161:\n",
            "row_id: fe3218dbb676474bbbc9cfdda398d18e\n",
            "embedding_vector: [-0.02557532675564289, -0.004798027686774731, 0.0144790038466453 ...\n",
            "body_blob: be parallelized, as measured by the minimum number of sequential ...\n",
            "metadata_blob: {'page': '5.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 162:\n",
            "row_id: 355ed9ace1ba4183b4fb500e2d520674\n",
            "embedding_vector: [-0.02630132995545864, 0.018571238964796066, 0.03064865618944168 ...\n",
            "body_blob: is similar to that of single-head attention with full dimensiona ...\n",
            "metadata_blob: {'page': '4.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 163:\n",
            "row_id: 6d0476266b3b4c469c4770152ca450da\n",
            "embedding_vector: [-0.03726242855191231, 8.219048322644085e-05, 0.0264128744602203 ...\n",
            "body_blob: textual entailment and learning task-independent sentence repres ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 164:\n",
            "row_id: 06847263c16b43ab8d800e0fb6871b75\n",
            "embedding_vector: [-0.00794966984540224, 0.03249814361333847, 0.019450461491942406 ...\n",
            "body_blob: convolution is equal to the combination of a self-attention laye ...\n",
            "metadata_blob: {'page': '6.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 165:\n",
            "row_id: 281332d1d8364801a2dee19d50d985d0\n",
            "embedding_vector: [-0.05291248857975006, 0.005001457408070564, 0.02222869917750358 ...\n",
            "body_blob: position in the decoder to attend over all positions in the inpu ...\n",
            "metadata_blob: {'page': '4.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 166:\n",
            "row_id: cadfa70ce22e475da244ed5a291f72b6\n",
            "embedding_vector: [-0.01537340972572565, -0.0012266928097233176, 0.010825671255588 ...\n",
            "body_blob: 9(8):1735–1780, 1997.\n",
            "[13] Rafal Jozefowicz, Oriol Vinyals, Mike ...\n",
            "metadata_blob: {'page': '9.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 167:\n",
            "row_id: 55451e1b1bc24484870c49a8dd2d2aa8\n",
            "embedding_vector: [-0.030820587649941444, -0.002478353213518858, 0.021797275170683 ...\n",
            "body_blob: predictions for position ican depend only on the known outputs a ...\n",
            "metadata_blob: {'page': '2.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 168:\n",
            "row_id: 9fbe9ff06a2e43fab2d227d9ceb2c01b\n",
            "embedding_vector: [-0.022442281246185303, 0.009031649678945541, 0.0144643234089016 ...\n",
            "body_blob: it more difﬁcult to learn dependencies between distant positions ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 169:\n",
            "row_id: 26d05b1036b8481a82b3e58cad2b8876\n",
            "embedding_vector: [-0.030240120366215706, 0.001599768758751452, 0.0070728436112403 ...\n",
            "body_blob: linear projections to dk,dkanddvdimensions, respectively. On eac ...\n",
            "metadata_blob: {'page': '3.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 170:\n",
            "row_id: a295fda4e4754448a8b9e8c7960e0cff\n",
            "embedding_vector: [-0.025797853246331215, -0.028185280039906502, 0.012012167833745 ...\n",
            "body_blob: hurts perplexity, as the model learns to be more unsure, but imp ...\n",
            "metadata_blob: {'page': '7.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 171:\n",
            "row_id: 2ba330047b9c42c7b634c442a84810f2\n",
            "embedding_vector: [-0.03471637889742851, -0.0107490886002779, 0.032037559896707535 ...\n",
            "body_blob: constraint of sequential computation, however, remains.\n",
            "Attentio ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 172:\n",
            "row_id: 0521a02637e44bf2bdf0f14da9d1a09e\n",
            "embedding_vector: [-0.043849579989910126, -0.005807334091514349, -0.00084909476572 ...\n",
            "body_blob: arXiv:1609.08144 , 2016.\n",
            "[32] Jie Zhou, Ying Cao, Xuguang Wang,  ...\n",
            "metadata_blob: {'page': '10.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 173:\n",
            "row_id: 6fd90b473abf4f48949e971cbb30ca50\n",
            "embedding_vector: [-0.05291248857975006, 0.005001457408070564, 0.02222869917750358 ...\n",
            "body_blob: position in the decoder to attend over all positions in the inpu ...\n",
            "metadata_blob: {'page': '4.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 174:\n",
            "row_id: a67370c54b3e4b33bb2b65babdc6f921\n",
            "embedding_vector: [-0.010459700599312782, -0.011344649828970432, 0.020466946065425 ...\n",
            "body_blob: heads clearly learn to perform different tasks, many appear to e ...\n",
            "metadata_blob: {'page': '6.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 175:\n",
            "row_id: 91fa8ef1077042d884acba96d7e9bd7c\n",
            "embedding_vector: [-0.025421299040317535, 0.005779437720775604, 0.0441787615418434 ...\n",
            "body_blob: arXiv:1703.03130 , 2017.\n",
            "[20] Samy Bengio Łukasz Kaiser. Can act ...\n",
            "metadata_blob: {'page': '9.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 176:\n",
            "row_id: 1a7ea3ef8ca24e0e99a76dc547bf14a4\n",
            "embedding_vector: [-0.03027210757136345, -0.005484077613800764, 0.0066323061473667 ...\n",
            "body_blob: Residual Dropout We apply dropout [ 27] to the output of each su ...\n",
            "metadata_blob: {'page': '6.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 177:\n",
            "row_id: f94911b8833b4e1cbe715994cdc87acc\n",
            "embedding_vector: [0.011163578368723392, -0.004680142272263765, 0.0174715965986251 ...\n",
            "body_blob: steps (dev) (dev)×106\n",
            "base 6 512 2048 8 64 64 0.1 0.1 100K 4.92  ...\n",
            "metadata_blob: {'page': '8.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 178:\n",
            "row_id: cd0a420785c641e9a1a515dd23a7ac01\n",
            "embedding_vector: [-0.023727815598249435, -0.00439288979396224, 0.0270544048398733 ...\n",
            "body_blob: [15] Nal Kalchbrenner, Lasse Espeholt, Karen Simonyan, Aaron van ...\n",
            "metadata_blob: {'page': '9.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 179:\n",
            "row_id: 4daef8c074924393af2874f31ae91c9b\n",
            "embedding_vector: [-0.01607612706720829, -0.010801807977259159, 0.0183686986565589 ...\n",
            "body_blob: translation quality after being trained for as little as twelve  ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 180:\n",
            "row_id: 66b256c233a843e59d0304954f08472c\n",
            "embedding_vector: [-0.006293286569416523, 0.007931635715067387, -0.001240584766492 ...\n",
            "body_blob: learned and ﬁxed [8].\n",
            "In this work, we use sine and cosine funct ...\n",
            "metadata_blob: {'page': '5.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 181:\n",
            "row_id: 4691f7ef100a4c59a214a185e2566e8e\n",
            "embedding_vector: [-0.025825563818216324, -0.013635127805173397, 0.016744652763009 ...\n",
            "body_blob: the competitive models.\n",
            "On the WMT 2014 English-to-French transl ...\n",
            "metadata_blob: {'page': '7.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 182:\n",
            "row_id: fc27b57e97614b29b4001f8eec799991\n",
            "embedding_vector: [-0.020689306780695915, -0.007788582239300013, 0.021949641406536 ...\n",
            "body_blob: plan to extend the Transformer to problems involving input and o ...\n",
            "metadata_blob: {'page': '8.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 183:\n",
            "row_id: 3144af36f99347aeae6699732a68a599\n",
            "embedding_vector: [-0.018252432346343994, 0.005742142908275127, 0.0249718893319368 ...\n",
            "body_blob: of the values, where the weight assigned to each value is comput ...\n",
            "metadata_blob: {'page': '2.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 184:\n",
            "row_id: ee2728737e994903b2316b1037dcbd81\n",
            "embedding_vector: [-0.018252432346343994, 0.005742142908275127, 0.0249718893319368 ...\n",
            "body_blob: of the values, where the weight assigned to each value is comput ...\n",
            "metadata_blob: {'page': '2.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 185:\n",
            "row_id: 5a943834c3994b3eaf3c3be0838af54f\n",
            "embedding_vector: [-0.02549763023853302, -0.004523316398262978, 0.0145680522546172 ...\n",
            "body_blob: be parallelized, as measured by the minimum number of sequential ...\n",
            "metadata_blob: {'page': '5.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 186:\n",
            "row_id: 0ba55ce53b9949de8f2fdc974cbcf926\n",
            "embedding_vector: [-0.02271842025220394, -0.022832797840237617, -0.011602263897657 ...\n",
            "body_blob: Recurrent models typically factor computation along the symbol p ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 187:\n",
            "row_id: e16c295733144441a457ee3478c65c43\n",
            "embedding_vector: [-0.04692409187555313, 0.0007153992191888392, 0.0348338857293129 ...\n",
            "body_blob: self-attention and discuss its advantages over models such as [1 ...\n",
            "metadata_blob: {'page': '1.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 188:\n",
            "row_id: 1757e14ddac845eeace913a42b9bbd04\n",
            "embedding_vector: [0.010954023338854313, 0.024167314171791077, 0.00846198294311761 ...\n",
            "body_blob: orO(logk(n))in the case of dilated convolutions [ 15], increasin ...\n",
            "metadata_blob: {'page': '6.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 189:\n",
            "row_id: 4772e3acfe4549418da1134e28655f7d\n",
            "embedding_vector: [-0.029735980555415154, -0.0010455148294568062, 0.01836469583213 ...\n",
            "body_blob: on recurrent or convolutional layers. On both WMT 2014 English-t ...\n",
            "metadata_blob: {'page': '8.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 190:\n",
            "row_id: c593f7be8f3d4b2a88f44fb2c37f19b7\n",
            "embedding_vector: [-0.019136866554617882, -0.02117994800209999, 0.0229778606444597 ...\n",
            "body_blob: architectures [31, 21, 13].\n",
            "∗Equal contribution. Listing order i ...\n",
            "metadata_blob: {'page': '0.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 191:\n",
            "row_id: a8360b355cad472d9c851f476f4af61c\n",
            "embedding_vector: [-0.0155562162399292, -0.009941737167537212, 0.01685713231563568 ...\n",
            "body_blob: age recognition. In Proceedings of the IEEE Conference on Comput ...\n",
            "metadata_blob: {'page': '9.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 192:\n",
            "row_id: a375ccfdef124f669a977ec3003be0d5\n",
            "embedding_vector: [-0.04098382964730263, -0.010711060836911201, 0.0055436240509152 ...\n",
            "body_blob: 7 Conclusion\n",
            "In this work, we presented the Transformer, the ﬁrs ...\n",
            "metadata_blob: {'page': '8.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 193:\n",
            "row_id: 745daef5c28842928e84eec253348e1e\n",
            "embedding_vector: [-0.025436652824282646, -0.013509352691471577, 0.017402580007910 ...\n",
            "body_blob: the competitive models.\n",
            "On the WMT 2014 English-to-French transl ...\n",
            "metadata_blob: {'page': '7.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 194:\n",
            "row_id: 5c28bde60c834be284456f3b2152af6e\n",
            "embedding_vector: [-0.023947717621922493, 0.009092824533581734, 0.0113414367660880 ...\n",
            "body_blob: chose this function because we hypothesized it would allow the m ...\n",
            "metadata_blob: {'page': '5.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "Row 195:\n",
            "row_id: 79be4e591a2848519685dde237bbcd33\n",
            "embedding_vector: [-0.0054652513936161995, 0.0028098979964852333, 0.00638900557532 ...\n",
            "body_blob: with subword units. arXiv preprint arXiv:1508.07909 , 2015.\n",
            "[26] ...\n",
            "metadata_blob: {'page': '10.0', 'source': 'Attention Paper.pdf'}\n",
            "\n",
            "...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Asking Questions to the PDF"
      ],
      "metadata": {
        "id": "6a75vM3tfzih"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "query_1 = \"What is multi-head attention?\"\n",
        "pdf_index.query_with_sources(query_1, llm=llm)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FKXHMgbVOdC3",
        "outputId": "eadab9cf-1f0f-4cf8-dc40-4ebe25667b3a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'question': 'What is multi-head attention?',\n",
              " 'answer': ' Multi-head attention is a technique used in natural language processing where multiple attention layers are employed in parallel, each with a reduced dimension.\\n',\n",
              " 'sources': 'Attention Paper.pdf'}"
            ]
          },
          "metadata": {},
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "query_2 = \"What are positional encodings?\"\n",
        "pdf_index.query_with_sources(query_2, llm=llm)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BdW--gMhOdFk",
        "outputId": "622a97b4-8ec0-44fa-e45d-9db98783de26"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'question': 'What are positional encodings?',\n",
              " 'answer': ' Positional encodings are additional information added to the input embeddings of a sequence to make use of the order of the sequence.\\n',\n",
              " 'sources': 'Attention Paper.pdf'}"
            ]
          },
          "metadata": {},
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "3aeyU4KmOZrC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "eWoQcwyIOX9K"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}