enc-dec-mt.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install torch\n",
    "# !pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu117\n",
    "# !pip install torch==1.8.0+cpu torchvision==0.9.0+cpu torchaudio==0.8.0 -f https://download.pytorch.org/whl/torch_stable.html\n",
    "# !pip install torchtext==0.9.1\n",
    "# !pip install spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.8.1+cpu\n",
      "False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\siddp\\OneDrive\\Desktop\\project-venv\\enc-torch\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "print(torch.cuda.is_available())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9.0+cpu\n",
      "1.8.1+cpu\n",
      "0.8.0\n",
      "3.5.3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\siddp\\OneDrive\\Desktop\\project-venv\\enc-torch\\lib\\site-packages\\torchaudio\\extension\\extension.py:13: UserWarning: torchaudio C++ extension is not available.\n",
      "  warnings.warn('torchaudio C++ extension is not available.')\n",
      "c:\\Users\\siddp\\OneDrive\\Desktop\\project-venv\\enc-torch\\lib\\site-packages\\torchaudio\\backend\\utils.py:89: UserWarning: No audio backend is available.\n",
      "  warnings.warn('No audio backend is available.')\n"
     ]
    }
   ],
   "source": [
    "import torchvision\n",
    "import torchaudio\n",
    "import spacy\n",
    "\n",
    "print(torchvision.__version__)\n",
    "print(torch.__version__)\n",
    "print(torchaudio.__version__)\n",
    "print(spacy.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torchtext.datasets import Multi30k\n",
    "from torchtext.data import Field, BucketIterator\n",
    "import numpy as np\n",
    "import spacy\n",
    "import random\n",
    "from torch.utils.tensorboard import SummaryWriter  # to print to tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import spacy\n",
    "from torchtext.data import bleu_score\n",
    "import sys\n",
    "\n",
    "\n",
    "def translate_sentence(model, sentence, german, english, device, max_length=50):\n",
    "    # print(sentence)\n",
    "\n",
    "    # sys.exit()\n",
    "\n",
    "    # Load german tokenizer\n",
    "    spacy_ger = spacy.load(\"de_core_news_sm\")\n",
    "\n",
    "    # Create tokens using spacy and everything in lower case (which is what our vocab is)\n",
    "    if type(sentence) == str:\n",
    "        tokens = [token.text.lower() for token in spacy_ger(sentence)]\n",
    "    else:\n",
    "        tokens = [token.lower() for token in sentence]\n",
    "\n",
    "    # print(tokens)\n",
    "\n",
    "    # sys.exit()\n",
    "    # Add <SOS> and <EOS> in beginning and end respectively\n",
    "    tokens.insert(0, german.init_token)\n",
    "    tokens.append(german.eos_token)\n",
    "\n",
    "    # Go through each german token and convert to an index\n",
    "    text_to_indices = [german.vocab.stoi[token] for token in tokens]\n",
    "\n",
    "    # Convert to Tensor\n",
    "    sentence_tensor = torch.LongTensor(text_to_indices).unsqueeze(1).to(device)\n",
    "\n",
    "    # Build encoder hidden, cell state\n",
    "    with torch.no_grad():\n",
    "        hidden, cell = model.encoder(sentence_tensor)\n",
    "\n",
    "    outputs = [english.vocab.stoi[\"<sos>\"]]\n",
    "\n",
    "    for _ in range(max_length):\n",
    "        previous_word = torch.LongTensor([outputs[-1]]).to(device)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            output, hidden, cell = model.decoder(previous_word, hidden, cell)\n",
    "            best_guess = output.argmax(1).item()\n",
    "\n",
    "        outputs.append(best_guess)\n",
    "\n",
    "        # Model predicts it's the end of the sentence\n",
    "        if output.argmax(1).item() == english.vocab.stoi[\"<eos>\"]:\n",
    "            break\n",
    "\n",
    "    translated_sentence = [english.vocab.itos[idx] for idx in outputs]\n",
    "\n",
    "    # remove start token\n",
    "    return translated_sentence[1:]\n",
    "\n",
    "\n",
    "def bleu(data, model, german, english, device):\n",
    "    targets = []\n",
    "    outputs = []\n",
    "\n",
    "    for example in data:\n",
    "        src = vars(example)[\"src\"]\n",
    "        trg = vars(example)[\"trg\"]\n",
    "\n",
    "        prediction = translate_sentence(model, src, german, english, device)\n",
    "        prediction = prediction[:-1]  # remove <eos> token\n",
    "\n",
    "        targets.append([trg])\n",
    "        outputs.append(prediction)\n",
    "\n",
    "    return bleu_score(outputs, targets)\n",
    "\n",
    "\n",
    "def save_checkpoint(state, filename=\"my_checkpoint.pth.tar\"):\n",
    "    print(\"=> Saving checkpoint\")\n",
    "    torch.save(state, filename)\n",
    "\n",
    "\n",
    "def load_checkpoint(checkpoint, model, optimizer):\n",
    "    print(\"=> Loading checkpoint\")\n",
    "    model.load_state_dict(checkpoint[\"state_dict\"])\n",
    "    optimizer.load_state_dict(checkpoint[\"optimizer\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !python -m spacy download de_core_news_sm\n",
    "# !python -m spacy download en_core_web_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Hello', 'is', 'this', 'even', 'working', '!']\n"
     ]
    }
   ],
   "source": [
    "spacy_ger = spacy.load('de_core_news_sm')\n",
    "spacy_eng = spacy.load('en_core_web_sm')\n",
    "\n",
    "def tokenizer_ger(text):\n",
    "    return [token.text for token in spacy_ger.tokenizer(text)]\n",
    "\n",
    "def tokenizer_eng(text):\n",
    "    return [token.text for token in spacy_eng.tokenizer(text)]    \n",
    "\n",
    "print(tokenizer_eng(\"Hello is this even working!\"))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "german = Field(tokenize=tokenizer_ger, lower=True, \n",
    "            init_token='<start>', eos_token='<end>')\n",
    "\n",
    "english = Field(tokenize=tokenizer_eng, lower=True, \n",
    "            init_token='<start>', eos_token='<end>')            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, validation_data, test_data = Multi30k.splits(exts=('.de', '.en'),\n",
    "                                                    fields=(german, english))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "german.build_vocab(train_data, max_size=7000, min_freq=2)\n",
    "english.build_vocab(train_data, max_size=7000, min_freq=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self, input_size, embedding_size, hidden_size, num_layers, dropout_):\n",
    "        \"\"\"\n",
    "        input_size = size of the vocabulary\n",
    "        embedding_size = representation of word with d size vectors\n",
    "        \"\"\"\n",
    "        super(Encoder, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "        self.dropout = nn.Dropout(dropout_)\n",
    "        self.embedding = nn.Embedding(input_size, embedding_size)\n",
    "        self.rnn = nn.LSTM(embedding_size, hidden_size, num_layers, dropout=dropout_)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        x = vector representation of the word that is tokenized and mapped to the index in vocabulary\n",
    "        \"\"\" \n",
    "        # shape of x - (seq_length, N), we have sequence of words in N batches. So in encoder we can feed in the whole sentence.\n",
    "        embedding = self.dropout(self.embedding(x))  \n",
    "        # shape - (seq_length, N, embedding_size) \n",
    "        outputs, (hidden, cell) = self.rnn(embedding)\n",
    "        return hidden, cell\n",
    "\n",
    "class Decoder(nn.Module):\n",
    "    def __init__(self, input_size, embedding_size, hidden_size, output_size, num_layers, dropout_):\n",
    "        \"\"\"\n",
    "        input_size = size of the vocabulary\n",
    "        embedding_size = representation of word with d size vectors\n",
    "        output_size = same size as input size, vector representation of word translated into the other language. It is the prob distribution over the vocuabulary - a size(vocab) lenght vector having probs of each word from the vocabulary.\n",
    "        \"\"\"\n",
    "        super(Decoder, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "        self.dropout = nn.Dropout(dropout_)\n",
    "        self.embedding = nn.Embedding(input_size, embedding_size)   \n",
    "        self.rnn = nn.LSTM(embedding_size, hidden_size, num_layers, dropout=dropout_)\n",
    "        # hidden size of the decoder and encoder are the same.\n",
    "        self.fc = nn.Linear(hidden_size, output_size)\n",
    "\n",
    "    def forward(self, x, hidden, cell):\n",
    "        # shape of x - (1, N), here in decoder we feed in a single word from the sentence.  \n",
    "        x = x.unsqueeze(0) # reshaped from (N) to (1,N)\n",
    "        embedding = self.dropout(self.embedding(x))  \n",
    "        # shape - (1, N, embedding_size)\n",
    "        outputs, (hidden, cell) = self.rnn(embedding, (hidden, cell))\n",
    "        # shape - (1, N, hidden_size)\n",
    "        predictions = self.fc(outputs)\n",
    "        # shape  - (1, N, len(vocab))\n",
    "        predictions.squeeze(0) # changing the shape from (1, N, len(vocab)) to (N, len(vocab))\n",
    "\n",
    "        return predictions, hidden, cell\n",
    "\n",
    "class Seq2Seq(nn.Module):\n",
    "    def __init__(self, Encoder, Decoder):\n",
    "        super(Seq2Seq, self).__init__()\n",
    "        self.encoder = Encoder\n",
    "        self.decoder = Decoder\n",
    "\n",
    "    def forward(self, source, target, teacher_force_ratio=0.5):\n",
    "        \"\"\"\n",
    "        teacher_force_ratio = While decoding, predictions of words happen word-by-word, so if the predictions is wrong, then the wrong word gets fed into the decoder for next prediction. This ratio foretells ratio of how many predicted and actual words to feed for next prediction while decoding.\n",
    "        \"\"\"\n",
    "        batch_size = source.shape[1]\n",
    "        target_len = target.shape[0]\n",
    "        target_vocab_size = len(english.vocab)\n",
    "\n",
    "        outputs = torch.zeros(target_len, batch_size, target_vocab_size).to(device)\n",
    "\n",
    "        hidden, cell = self.encoder(source)\n",
    "\n",
    "        x = target[0] # grabbing start token\n",
    "        for t in range(1, target_len):\n",
    "            output, hidden, cell = self.decoder(x, hidden, cell)\n",
    "            outputs[t] = output\n",
    "            best_guess = output.argmax(1)\n",
    "            x = target[t] if random.random() < teacher_force_ratio else best_guess\n",
    "        \n",
    "        return outputs    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch [0/20]\n",
      "=> Saving checkpoint\n",
      "=> Saving checkpoint\n",
      "torch.Size([46, 64]) torch.Size([42, 64])\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "input must have 3 dimensions, got 4",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[20], line 64\u001b[0m\n\u001b[0;32m     61\u001b[0m target \u001b[38;5;241m=\u001b[39m batch\u001b[38;5;241m.\u001b[39mtrg\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[0;32m     62\u001b[0m \u001b[38;5;28mprint\u001b[39m(inp_data\u001b[38;5;241m.\u001b[39mshape, target\u001b[38;5;241m.\u001b[39mshape)\n\u001b[1;32m---> 64\u001b[0m output \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43minp_data\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtarget\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# shape (trg_len, batch_size, output_dim)\u001b[39;00m\n\u001b[0;32m     65\u001b[0m output \u001b[38;5;241m=\u001b[39m output[\u001b[38;5;241m1\u001b[39m:]\u001b[38;5;241m.\u001b[39mreshape(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m, output\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m2\u001b[39m])\n\u001b[0;32m     66\u001b[0m target \u001b[38;5;241m=\u001b[39m target[\u001b[38;5;241m1\u001b[39m:]\u001b[38;5;241m.\u001b[39mreshape(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m) \n",
      "File \u001b[1;32mc:\\Users\\siddp\\OneDrive\\Desktop\\project-venv\\enc-torch\\lib\\site-packages\\torch\\nn\\modules\\module.py:889\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/module.py?line=886'>887</a>\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_slow_forward(\u001b[39m*\u001b[39m\u001b[39minput\u001b[39m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/module.py?line=887'>888</a>\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m--> <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/module.py?line=888'>889</a>\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mforward(\u001b[39m*\u001b[39m\u001b[39minput\u001b[39m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/module.py?line=889'>890</a>\u001b[0m \u001b[39mfor\u001b[39;00m hook \u001b[39min\u001b[39;00m itertools\u001b[39m.\u001b[39mchain(\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/module.py?line=890'>891</a>\u001b[0m         _global_forward_hooks\u001b[39m.\u001b[39mvalues(),\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/module.py?line=891'>892</a>\u001b[0m         \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks\u001b[39m.\u001b[39mvalues()):\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/module.py?line=892'>893</a>\u001b[0m     hook_result \u001b[39m=\u001b[39m hook(\u001b[39mself\u001b[39m, \u001b[39minput\u001b[39m, result)\n",
      "Cell \u001b[1;32mIn[11], line 73\u001b[0m, in \u001b[0;36mSeq2Seq.forward\u001b[1;34m(self, source, target, teacher_force_ratio)\u001b[0m\n\u001b[0;32m     71\u001b[0m x \u001b[38;5;241m=\u001b[39m target[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;66;03m# grabbing start token\u001b[39;00m\n\u001b[0;32m     72\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m t \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;241m1\u001b[39m, target_len):\n\u001b[1;32m---> 73\u001b[0m     output, hidden, cell \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecoder\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhidden\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcell\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     74\u001b[0m     outputs[t] \u001b[38;5;241m=\u001b[39m output\n\u001b[0;32m     75\u001b[0m     best_guess \u001b[38;5;241m=\u001b[39m output\u001b[38;5;241m.\u001b[39margmax(\u001b[38;5;241m1\u001b[39m)\n",
      "File \u001b[1;32mc:\\Users\\siddp\\OneDrive\\Desktop\\project-venv\\enc-torch\\lib\\site-packages\\torch\\nn\\modules\\module.py:889\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/module.py?line=886'>887</a>\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_slow_forward(\u001b[39m*\u001b[39m\u001b[39minput\u001b[39m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/module.py?line=887'>888</a>\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m--> <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/module.py?line=888'>889</a>\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mforward(\u001b[39m*\u001b[39m\u001b[39minput\u001b[39m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/module.py?line=889'>890</a>\u001b[0m \u001b[39mfor\u001b[39;00m hook \u001b[39min\u001b[39;00m itertools\u001b[39m.\u001b[39mchain(\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/module.py?line=890'>891</a>\u001b[0m         _global_forward_hooks\u001b[39m.\u001b[39mvalues(),\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/module.py?line=891'>892</a>\u001b[0m         \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks\u001b[39m.\u001b[39mvalues()):\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/module.py?line=892'>893</a>\u001b[0m     hook_result \u001b[39m=\u001b[39m hook(\u001b[39mself\u001b[39m, \u001b[39minput\u001b[39m, result)\n",
      "Cell \u001b[1;32mIn[11], line 45\u001b[0m, in \u001b[0;36mDecoder.forward\u001b[1;34m(self, x, hidden, cell)\u001b[0m\n\u001b[0;32m     43\u001b[0m embedding \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdropout(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39membedding(x))  \n\u001b[0;32m     44\u001b[0m \u001b[38;5;66;03m# shape - (1, N, embedding_size)\u001b[39;00m\n\u001b[1;32m---> 45\u001b[0m outputs, (hidden, cell) \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrnn\u001b[49m\u001b[43m(\u001b[49m\u001b[43membedding\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m(\u001b[49m\u001b[43mhidden\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcell\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     46\u001b[0m \u001b[38;5;66;03m# shape - (1, N, hidden_size)\u001b[39;00m\n\u001b[0;32m     47\u001b[0m predictions \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfc(outputs)\n",
      "File \u001b[1;32mc:\\Users\\siddp\\OneDrive\\Desktop\\project-venv\\enc-torch\\lib\\site-packages\\torch\\nn\\modules\\module.py:889\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/module.py?line=886'>887</a>\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_slow_forward(\u001b[39m*\u001b[39m\u001b[39minput\u001b[39m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/module.py?line=887'>888</a>\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m--> <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/module.py?line=888'>889</a>\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mforward(\u001b[39m*\u001b[39m\u001b[39minput\u001b[39m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/module.py?line=889'>890</a>\u001b[0m \u001b[39mfor\u001b[39;00m hook \u001b[39min\u001b[39;00m itertools\u001b[39m.\u001b[39mchain(\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/module.py?line=890'>891</a>\u001b[0m         _global_forward_hooks\u001b[39m.\u001b[39mvalues(),\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/module.py?line=891'>892</a>\u001b[0m         \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks\u001b[39m.\u001b[39mvalues()):\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/module.py?line=892'>893</a>\u001b[0m     hook_result \u001b[39m=\u001b[39m hook(\u001b[39mself\u001b[39m, \u001b[39minput\u001b[39m, result)\n",
      "File \u001b[1;32mc:\\Users\\siddp\\OneDrive\\Desktop\\project-venv\\enc-torch\\lib\\site-packages\\torch\\nn\\modules\\rnn.py:659\u001b[0m, in \u001b[0;36mLSTM.forward\u001b[1;34m(self, input, hx)\u001b[0m\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=653'>654</a>\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=654'>655</a>\u001b[0m     \u001b[39m# Each batch of the hidden state should match the input sequence that\u001b[39;00m\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=655'>656</a>\u001b[0m     \u001b[39m# the user believes he/she is passing in.\u001b[39;00m\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=656'>657</a>\u001b[0m     hx \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpermute_hidden(hx, sorted_indices)\n\u001b[1;32m--> <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=658'>659</a>\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mcheck_forward_args(\u001b[39minput\u001b[39;49m, hx, batch_sizes)\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=659'>660</a>\u001b[0m \u001b[39mif\u001b[39;00m batch_sizes \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=660'>661</a>\u001b[0m     result \u001b[39m=\u001b[39m _VF\u001b[39m.\u001b[39mlstm(\u001b[39minput\u001b[39m, hx, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_flat_weights, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbias, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mnum_layers,\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=661'>662</a>\u001b[0m                       \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdropout, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtraining, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbidirectional, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbatch_first)\n",
      "File \u001b[1;32mc:\\Users\\siddp\\OneDrive\\Desktop\\project-venv\\enc-torch\\lib\\site-packages\\torch\\nn\\modules\\rnn.py:605\u001b[0m, in \u001b[0;36mLSTM.check_forward_args\u001b[1;34m(self, input, hidden, batch_sizes)\u001b[0m\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=603'>604</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mcheck_forward_args\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39minput\u001b[39m: Tensor, hidden: Tuple[Tensor, Tensor], batch_sizes: Optional[Tensor]):  \u001b[39m# type: ignore\u001b[39;00m\n\u001b[1;32m--> <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=604'>605</a>\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mcheck_input(\u001b[39minput\u001b[39;49m, batch_sizes)\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=605'>606</a>\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcheck_hidden_size(hidden[\u001b[39m0\u001b[39m], \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mget_expected_hidden_size(\u001b[39minput\u001b[39m, batch_sizes),\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=606'>607</a>\u001b[0m                            \u001b[39m'\u001b[39m\u001b[39mExpected hidden[0] size \u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m, got \u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m'\u001b[39m)\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=607'>608</a>\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcheck_hidden_size(hidden[\u001b[39m1\u001b[39m], \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mget_expected_cell_size(\u001b[39minput\u001b[39m, batch_sizes),\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=608'>609</a>\u001b[0m                            \u001b[39m'\u001b[39m\u001b[39mExpected hidden[1] size \u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m, got \u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m'\u001b[39m)\n",
      "File \u001b[1;32mc:\\Users\\siddp\\OneDrive\\Desktop\\project-venv\\enc-torch\\lib\\site-packages\\torch\\nn\\modules\\rnn.py:198\u001b[0m, in \u001b[0;36mRNNBase.check_input\u001b[1;34m(self, input, batch_sizes)\u001b[0m\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=195'>196</a>\u001b[0m expected_input_dim \u001b[39m=\u001b[39m \u001b[39m2\u001b[39m \u001b[39mif\u001b[39;00m batch_sizes \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39melse\u001b[39;00m \u001b[39m3\u001b[39m\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=196'>197</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39minput\u001b[39m\u001b[39m.\u001b[39mdim() \u001b[39m!=\u001b[39m expected_input_dim:\n\u001b[1;32m--> <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=197'>198</a>\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mRuntimeError\u001b[39;00m(\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=198'>199</a>\u001b[0m         \u001b[39m'\u001b[39m\u001b[39minput must have \u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m dimensions, got \u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m.\u001b[39mformat(\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=199'>200</a>\u001b[0m             expected_input_dim, \u001b[39minput\u001b[39m\u001b[39m.\u001b[39mdim()))\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=200'>201</a>\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39minput_size \u001b[39m!=\u001b[39m \u001b[39minput\u001b[39m\u001b[39m.\u001b[39msize(\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m):\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=201'>202</a>\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mRuntimeError\u001b[39;00m(\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=202'>203</a>\u001b[0m         \u001b[39m'\u001b[39m\u001b[39minput.size(-1) must be equal to input_size. Expected \u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m, got \u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m.\u001b[39mformat(\n\u001b[0;32m    <a href='file:///c%3A/Users/siddp/OneDrive/Desktop/project-venv/enc-torch/lib/site-packages/torch/nn/modules/rnn.py?line=203'>204</a>\u001b[0m             \u001b[39mself\u001b[39m\u001b[39m.\u001b[39minput_size, \u001b[39minput\u001b[39m\u001b[39m.\u001b[39msize(\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m)))\n",
      "\u001b[1;31mRuntimeError\u001b[0m: input must have 3 dimensions, got 4"
     ]
    }
   ],
   "source": [
    "# Training the model now\n",
    "\n",
    "num_epoch = 20\n",
    "learning_rate = 0.001\n",
    "batch_size = 64\n",
    "\n",
    "load_model = False\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "input_size_encoder = len(german.vocab)\n",
    "input_size_decoder = len(english.vocab)\n",
    "output_size = len(english.vocab)\n",
    "encoder_embedding_size = 300\n",
    "decoder_embedding_size = 300\n",
    "hidden_size = 1024\n",
    "num_layers = 2\n",
    "enc_dropout = 0.5\n",
    "dec_dropout = 0.5\n",
    "\n",
    "# Tensorboard\n",
    "writer = SummaryWriter(f'runs/loss_plot')\n",
    "step = 0\n",
    "\n",
    "train_iterator, valid_iterator, test_iterator = BucketIterator.splits(\n",
    "    (train_data, validation_data, test_data),\n",
    "    batch_size=batch_size,\n",
    "    sort_within_batch = True,\n",
    "    sort_key = lambda x: len(x.src), # this helps in sorting sentences with similar sizes to same batches.\n",
    "    device = device,\n",
    ")\n",
    "\n",
    "encoder_net = Encoder(input_size_encoder, encoder_embedding_size, hidden_size, num_layers, enc_dropout).to(device)\n",
    "decoder_net = Decoder(input_size_decoder, decoder_embedding_size, hidden_size, output_size, num_layers, dec_dropout).to(device)\n",
    "\n",
    "model = Seq2Seq(encoder_net, decoder_net).to(device)\n",
    "optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "pad_idx = english.vocab.stoi['<pad>']\n",
    "criterion = nn.CrossEntropyLoss(ignore_index=pad_idx)\n",
    "\n",
    "if load_model:\n",
    "    load_checkpoint(torch.load('my_checkpoint.pth.ptar'), model, optimizer)\n",
    "\n",
    "for epoch in range(num_epoch):\n",
    "    print(f'epoch [{epoch}/{num_epoch}]')\n",
    "    checkpoint = {'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict()}\n",
    "    save_checkpoint(checkpoint)\n",
    "\n",
    "    save_checkpoint(checkpoint)\n",
    "\n",
    "    # sentence = \"ein boot mit mehreren männern darauf wird von einem großen pferdegespann ans ufer gezogen.\"\n",
    "    # model.eval()\n",
    "\n",
    "    # translated_sentence = translate_sentence(\n",
    "    #     model, sentence, german, english, device, max_length=50\n",
    "    # )\n",
    "\n",
    "    # print(f\"Translated example sentence: \\n {translated_sentence}\")\n",
    "\n",
    "    for batch_idx, batch in enumerate(train_iterator):\n",
    "        inp_data = batch.src.to(device)\n",
    "        target = batch.trg.to(device)\n",
    "        print(inp_data.shape, target.shape)\n",
    "\n",
    "        output = model(inp_data, target) # shape (trg_len, batch_size, output_dim)\n",
    "        output = output[1:].reshape(-1, output.shape[2])\n",
    "        target = target[1:].reshape(-1) \n",
    "        optimizer.zero_grad()\n",
    "        loss = criterion(output, target)\n",
    "        loss.backward()\n",
    "\n",
    "        torch.nn.utils.clip_grad_norm(model.parameters(), max_norm=1)\n",
    "        optimizer.step()\n",
    "\n",
    "        writer.add_scalar(\"Training loss\", loss, global_step=step)\n",
    "        step += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "d607cf0d3893fb2aef6d2d8c0b138e238ffdff59e216acb6a1dfb919e2a1bb02"
  },
  "kernelspec": {
   "display_name": "Python 3.9.12 ('enc-torch')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}