EgyptianTranslation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "EgyptianTranslation.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "1AotXG91-849vtqfbLNc4z0grfHvK6u5a",
      "authorship_tag": "ABX9TyODQJXl0HyvTGJO4oToqpYb"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "60ShPLbx1GiY",
        "outputId": "d995b4b6-8212-431c-f93d-5f14713d3c98"
      },
      "source": [
        "#!git clone https://github.com/fayrose/EgyptianTranslation.git\n",
        "%cd /content/drive/MyDrive"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/MyDrive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UrdeqEco2G03",
        "outputId": "15a80f0d-883d-4e02-dbf2-a01a658df6b3"
      },
      "source": [
        "!pip install OpenNMT-py"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting OpenNMT-py\n",
            "  Downloading OpenNMT_py-2.2.0-py3-none-any.whl (216 kB)\n",
            "\u001b[?25l\r\u001b[K     |█▌                              | 10 kB 25.4 MB/s eta 0:00:01\r\u001b[K     |███                             | 20 kB 29.5 MB/s eta 0:00:01\r\u001b[K     |████▌                           | 30 kB 24.0 MB/s eta 0:00:01\r\u001b[K     |██████                          | 40 kB 19.2 MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 51 kB 14.9 MB/s eta 0:00:01\r\u001b[K     |█████████                       | 61 kB 10.9 MB/s eta 0:00:01\r\u001b[K     |██████████▋                     | 71 kB 12.1 MB/s eta 0:00:01\r\u001b[K     |████████████                    | 81 kB 13.4 MB/s eta 0:00:01\r\u001b[K     |█████████████▋                  | 92 kB 14.3 MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 102 kB 12.9 MB/s eta 0:00:01\r\u001b[K     |████████████████▋               | 112 kB 12.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████▏             | 122 kB 12.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████▋            | 133 kB 12.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▏          | 143 kB 12.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 153 kB 12.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████▏       | 163 kB 12.9 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▊      | 174 kB 12.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▏    | 184 kB 12.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▊   | 194 kB 12.9 MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 204 kB 12.9 MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▊| 215 kB 12.9 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 216 kB 12.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from OpenNMT-py) (3.13)\n",
            "Requirement already satisfied: flask in /usr/local/lib/python3.7/dist-packages (from OpenNMT-py) (1.1.4)\n",
            "Collecting pyonmttok<2,>=1.23\n",
            "  Downloading pyonmttok-1.28.0-cp37-cp37m-manylinux2010_x86_64.whl (15.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 15.8 MB 135 kB/s \n",
            "\u001b[?25hCollecting configargparse\n",
            "  Downloading ConfigArgParse-1.5.2-py3-none-any.whl (20 kB)\n",
            "Requirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from OpenNMT-py) (1.9.0+cu102)\n",
            "Requirement already satisfied: tensorboard>=2.3 in /usr/local/lib/python3.7/dist-packages (from OpenNMT-py) (2.6.0)\n",
            "Collecting waitress\n",
            "  Downloading waitress-2.0.0-py3-none-any.whl (56 kB)\n",
            "\u001b[K     |████████████████████████████████| 56 kB 4.1 MB/s \n",
            "\u001b[?25hCollecting torchtext==0.5.0\n",
            "  Downloading torchtext-0.5.0-py3-none-any.whl (73 kB)\n",
            "\u001b[K     |████████████████████████████████| 73 kB 1.6 MB/s \n",
            "\u001b[?25hCollecting sentencepiece\n",
            "  Downloading sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2 MB 41.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from torchtext==0.5.0->OpenNMT-py) (2.23.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from torchtext==0.5.0->OpenNMT-py) (1.15.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from torchtext==0.5.0->OpenNMT-py) (4.62.2)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torchtext==0.5.0->OpenNMT-py) (1.19.5)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.3->OpenNMT-py) (0.6.1)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.3->OpenNMT-py) (0.4.6)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.3->OpenNMT-py) (1.8.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.3->OpenNMT-py) (1.0.1)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.3->OpenNMT-py) (0.37.0)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.3->OpenNMT-py) (57.4.0)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.3->OpenNMT-py) (0.12.0)\n",
            "Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.3->OpenNMT-py) (1.40.0)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.3->OpenNMT-py) (1.35.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.3->OpenNMT-py) (3.3.4)\n",
            "Requirement already satisfied: protobuf>=3.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.3->OpenNMT-py) (3.17.3)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard>=2.3->OpenNMT-py) (4.7.2)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard>=2.3->OpenNMT-py) (4.2.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard>=2.3->OpenNMT-py) (0.2.8)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.3->OpenNMT-py) (1.3.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard>=2.3->OpenNMT-py) (4.8.1)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard>=2.3->OpenNMT-py) (0.4.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext==0.5.0->OpenNMT-py) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext==0.5.0->OpenNMT-py) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext==0.5.0->OpenNMT-py) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->torchtext==0.5.0->OpenNMT-py) (2021.5.30)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.3->OpenNMT-py) (3.1.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch>=1.6.0->OpenNMT-py) (3.7.4.3)\n",
            "Requirement already satisfied: click<8.0,>=5.1 in /usr/local/lib/python3.7/dist-packages (from flask->OpenNMT-py) (7.1.2)\n",
            "Requirement already satisfied: itsdangerous<2.0,>=0.24 in /usr/local/lib/python3.7/dist-packages (from flask->OpenNMT-py) (1.1.0)\n",
            "Requirement already satisfied: Jinja2<3.0,>=2.10.1 in /usr/local/lib/python3.7/dist-packages (from flask->OpenNMT-py) (2.11.3)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from Jinja2<3.0,>=2.10.1->flask->OpenNMT-py) (2.0.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->markdown>=2.6.8->tensorboard>=2.3->OpenNMT-py) (3.5.0)\n",
            "Installing collected packages: sentencepiece, waitress, torchtext, pyonmttok, configargparse, OpenNMT-py\n",
            "  Attempting uninstall: torchtext\n",
            "    Found existing installation: torchtext 0.10.0\n",
            "    Uninstalling torchtext-0.10.0:\n",
            "      Successfully uninstalled torchtext-0.10.0\n",
            "Successfully installed OpenNMT-py-2.2.0 configargparse-1.5.2 pyonmttok-1.28.0 sentencepiece-0.1.96 torchtext-0.5.0 waitress-2.0.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4ofRY0VT2V37"
      },
      "source": [
        "string = \"\"\"\n",
        "# egy_eng.yaml\n",
        "\n",
        "## Where the samples will be written\n",
        "save_data: EgyptianTranslation/onmt-files/run/example\n",
        "## Where the vocab(s) will be written\n",
        "src_vocab: EgyptianTranslation/onmt-files/run/example.vocab.egy\n",
        "tgt_vocab: EgyptianTranslation/onmt-files/run/example.vocab.eng\n",
        "# Prevent overwriting existing files in the folder\n",
        "overwrite: True\n",
        "\n",
        "# Corpus opts:\n",
        "data:\n",
        "    corpus_1:\n",
        "        path_src: EgyptianTranslation/compiled_corpora/aligned_train.egy.csv\n",
        "        path_tgt: EgyptianTranslation/compiled_corpora/aligned_train.eng.csv\n",
        "    valid:\n",
        "        path_src: EgyptianTranslation/compiled_corpora/aligned_val.egy.csv\n",
        "        path_tgt: EgyptianTranslation/compiled_corpora/aligned_val.eng.csv\n",
        "\"\"\"\n",
        "with open(\"EgyptianTranslation/egy_eng.yaml\", 'w') as yaml_file:\n",
        "  yaml_file.write(string)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZCW19tnM3G31",
        "outputId": "979c1110-b8d3-4c4d-a312-29f5d19607f7"
      },
      "source": [
        "!onmt_build_vocab -config EgyptianTranslation/egy_eng.yaml -n_sample -1"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Corpus corpus_1's weight should be given. We default it to 1 for you.\n",
            "[2021-09-20 03:14:24,754 INFO] Counter vocab from -1 samples.\n",
            "[2021-09-20 03:14:24,754 INFO] n_sample=-1: Build vocab on full datasets.\n",
            "[2021-09-20 03:14:24,764 INFO] corpus_1's transforms: TransformPipe()\n",
            "[2021-09-20 03:14:25,867 INFO] Counters src:3183\n",
            "[2021-09-20 03:14:25,868 INFO] Counters tgt:6356\n",
            "[2021-09-20 03:14:26,234 WARNING] path EgyptianTranslation/onmt-files/run/example.vocab.egy exists, may overwrite...\n",
            "[2021-09-20 03:14:26,721 WARNING] path EgyptianTranslation/onmt-files/run/example.vocab.eng exists, may overwrite...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fFzWt6Nt3Vyy"
      },
      "source": [
        "string = \"\"\"\n",
        "# Vocabulary files that were just created\n",
        "src_vocab: EgyptianTranslation/onmt-files/run/example.vocab.egy\n",
        "tgt_vocab: EgyptianTranslation/onmt-files/run/example.vocab.eng\n",
        "\n",
        "# Train on a single GPU\n",
        "world_size: 1\n",
        "\n",
        "# Where to save the checkpoints\n",
        "save_model: EgyptianTranslation/onmt-files/run/model\n",
        "save_checkpoint_steps: 1000\n",
        "train_steps: 3000\n",
        "valid_steps: 1500\n",
        "\n",
        "# Optimization\n",
        "model_dtype: \"fp32\"\n",
        "optim: \"adam\"\n",
        "learning_rate: 2\n",
        "warmup_steps: 8000\n",
        "decay_method: \"noam\"\n",
        "adam_beta2: 0.998\n",
        "max_grad_norm: 0\n",
        "param_init: 0\n",
        "param_init_glorot: true\n",
        "label_smoothing: 0.1\n",
        "normalization: \"tokens\"\n",
        "\n",
        "# Model\n",
        "encoder_type: transformer\n",
        "decoder_type: transformer\n",
        "position_encoding: true\n",
        "heads: 8\n",
        "rnn_size: 512\n",
        "word_vec_size: 512\n",
        "max_generator_batches: 2\n",
        "transformer_ff: 2048\n",
        "dropout_steps: [0]\n",
        "dropout: [0.1]\n",
        "accum_count: 4\n",
        "attention_dropout: [0.1]\n",
        "enc_layers: 6\n",
        "dec_layers: 6\n",
        "gpu_ranks: 0\n",
        "\"\"\"\n",
        "\n",
        "with open(\"EgyptianTranslation/egy_eng.yaml\", 'a') as yaml_file:\n",
        "  yaml_file.write(string)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H1l3ulnX_uTw",
        "outputId": "0a580b0c-9d46-48f8-d6d7-addec2634529"
      },
      "source": [
        "!onmt_train --config EgyptianTranslation/egy_eng.yaml"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[2021-09-20 03:14:39,956 INFO] Missing transforms field for corpus_1 data, set to default: [].\n",
            "[2021-09-20 03:14:39,957 WARNING] Corpus corpus_1's weight should be given. We default it to 1 for you.\n",
            "[2021-09-20 03:14:39,959 INFO] Missing transforms field for valid data, set to default: [].\n",
            "[2021-09-20 03:14:39,960 INFO] Parsed 2 corpora from -data.\n",
            "[2021-09-20 03:14:39,961 INFO] Get special vocabs from Transforms: {'src': set(), 'tgt': set()}.\n",
            "[2021-09-20 03:14:39,961 INFO] Loading vocab from text file...\n",
            "[2021-09-20 03:14:39,961 INFO] Loading src vocabulary from EgyptianTranslation/onmt-files/run/example.vocab.egy\n",
            "[2021-09-20 03:14:39,972 INFO] Loaded src vocab has 3183 tokens.\n",
            "[2021-09-20 03:14:39,974 INFO] Loading tgt vocabulary from EgyptianTranslation/onmt-files/run/example.vocab.eng\n",
            "[2021-09-20 03:14:39,992 INFO] Loaded tgt vocab has 6356 tokens.\n",
            "[2021-09-20 03:14:39,995 INFO] Building fields with vocab in counters...\n",
            "[2021-09-20 03:14:40,003 INFO]  * tgt vocab size: 6360.\n",
            "[2021-09-20 03:14:40,007 INFO]  * src vocab size: 3185.\n",
            "[2021-09-20 03:14:40,007 INFO]  * src vocab size = 3185\n",
            "[2021-09-20 03:14:40,007 INFO]  * tgt vocab size = 6360\n",
            "[2021-09-20 03:14:40,009 INFO] Building model...\n",
            "[2021-09-20 03:14:50,053 INFO] NMTModel(\n",
            "  (encoder): TransformerEncoder(\n",
            "    (embeddings): Embeddings(\n",
            "      (make_embedding): Sequential(\n",
            "        (emb_luts): Elementwise(\n",
            "          (0): Embedding(3185, 512, padding_idx=1)\n",
            "        )\n",
            "        (pe): PositionalEncoding(\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (transformer): ModuleList(\n",
            "      (0): TransformerEncoderLayer(\n",
            "        (self_attn): MultiHeadedAttention(\n",
            "          (linear_keys): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_values): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_query): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (softmax): Softmax(dim=-1)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "          (final_linear): Linear(in_features=512, out_features=512, bias=True)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "          (dropout_1): Dropout(p=0.1, inplace=False)\n",
            "          (dropout_2): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (dropout): Dropout(p=0.1, inplace=False)\n",
            "      )\n",
            "      (1): TransformerEncoderLayer(\n",
            "        (self_attn): MultiHeadedAttention(\n",
            "          (linear_keys): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_values): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_query): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (softmax): Softmax(dim=-1)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "          (final_linear): Linear(in_features=512, out_features=512, bias=True)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "          (dropout_1): Dropout(p=0.1, inplace=False)\n",
            "          (dropout_2): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (dropout): Dropout(p=0.1, inplace=False)\n",
            "      )\n",
            "      (2): TransformerEncoderLayer(\n",
            "        (self_attn): MultiHeadedAttention(\n",
            "          (linear_keys): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_values): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_query): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (softmax): Softmax(dim=-1)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "          (final_linear): Linear(in_features=512, out_features=512, bias=True)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "          (dropout_1): Dropout(p=0.1, inplace=False)\n",
            "          (dropout_2): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (dropout): Dropout(p=0.1, inplace=False)\n",
            "      )\n",
            "      (3): TransformerEncoderLayer(\n",
            "        (self_attn): MultiHeadedAttention(\n",
            "          (linear_keys): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_values): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_query): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (softmax): Softmax(dim=-1)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "          (final_linear): Linear(in_features=512, out_features=512, bias=True)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "          (dropout_1): Dropout(p=0.1, inplace=False)\n",
            "          (dropout_2): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (dropout): Dropout(p=0.1, inplace=False)\n",
            "      )\n",
            "      (4): TransformerEncoderLayer(\n",
            "        (self_attn): MultiHeadedAttention(\n",
            "          (linear_keys): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_values): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_query): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (softmax): Softmax(dim=-1)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "          (final_linear): Linear(in_features=512, out_features=512, bias=True)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "          (dropout_1): Dropout(p=0.1, inplace=False)\n",
            "          (dropout_2): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (dropout): Dropout(p=0.1, inplace=False)\n",
            "      )\n",
            "      (5): TransformerEncoderLayer(\n",
            "        (self_attn): MultiHeadedAttention(\n",
            "          (linear_keys): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_values): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_query): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (softmax): Softmax(dim=-1)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "          (final_linear): Linear(in_features=512, out_features=512, bias=True)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "          (dropout_1): Dropout(p=0.1, inplace=False)\n",
            "          (dropout_2): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (dropout): Dropout(p=0.1, inplace=False)\n",
            "      )\n",
            "    )\n",
            "    (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "  )\n",
            "  (decoder): TransformerDecoder(\n",
            "    (embeddings): Embeddings(\n",
            "      (make_embedding): Sequential(\n",
            "        (emb_luts): Elementwise(\n",
            "          (0): Embedding(6360, 512, padding_idx=1)\n",
            "        )\n",
            "        (pe): PositionalEncoding(\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "    (transformer_layers): ModuleList(\n",
            "      (0): TransformerDecoderLayer(\n",
            "        (self_attn): MultiHeadedAttention(\n",
            "          (linear_keys): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_values): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_query): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (softmax): Softmax(dim=-1)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "          (final_linear): Linear(in_features=512, out_features=512, bias=True)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "          (dropout_1): Dropout(p=0.1, inplace=False)\n",
            "          (dropout_2): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (layer_norm_1): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (drop): Dropout(p=0.1, inplace=False)\n",
            "        (context_attn): MultiHeadedAttention(\n",
            "          (linear_keys): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_values): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_query): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (softmax): Softmax(dim=-1)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "          (final_linear): Linear(in_features=512, out_features=512, bias=True)\n",
            "        )\n",
            "        (layer_norm_2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "      )\n",
            "      (1): TransformerDecoderLayer(\n",
            "        (self_attn): MultiHeadedAttention(\n",
            "          (linear_keys): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_values): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_query): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (softmax): Softmax(dim=-1)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "          (final_linear): Linear(in_features=512, out_features=512, bias=True)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "          (dropout_1): Dropout(p=0.1, inplace=False)\n",
            "          (dropout_2): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (layer_norm_1): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (drop): Dropout(p=0.1, inplace=False)\n",
            "        (context_attn): MultiHeadedAttention(\n",
            "          (linear_keys): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_values): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_query): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (softmax): Softmax(dim=-1)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "          (final_linear): Linear(in_features=512, out_features=512, bias=True)\n",
            "        )\n",
            "        (layer_norm_2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "      )\n",
            "      (2): TransformerDecoderLayer(\n",
            "        (self_attn): MultiHeadedAttention(\n",
            "          (linear_keys): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_values): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_query): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (softmax): Softmax(dim=-1)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "          (final_linear): Linear(in_features=512, out_features=512, bias=True)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "          (dropout_1): Dropout(p=0.1, inplace=False)\n",
            "          (dropout_2): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (layer_norm_1): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (drop): Dropout(p=0.1, inplace=False)\n",
            "        (context_attn): MultiHeadedAttention(\n",
            "          (linear_keys): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_values): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_query): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (softmax): Softmax(dim=-1)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "          (final_linear): Linear(in_features=512, out_features=512, bias=True)\n",
            "        )\n",
            "        (layer_norm_2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "      )\n",
            "      (3): TransformerDecoderLayer(\n",
            "        (self_attn): MultiHeadedAttention(\n",
            "          (linear_keys): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_values): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_query): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (softmax): Softmax(dim=-1)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "          (final_linear): Linear(in_features=512, out_features=512, bias=True)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "          (dropout_1): Dropout(p=0.1, inplace=False)\n",
            "          (dropout_2): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (layer_norm_1): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (drop): Dropout(p=0.1, inplace=False)\n",
            "        (context_attn): MultiHeadedAttention(\n",
            "          (linear_keys): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_values): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_query): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (softmax): Softmax(dim=-1)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "          (final_linear): Linear(in_features=512, out_features=512, bias=True)\n",
            "        )\n",
            "        (layer_norm_2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "      )\n",
            "      (4): TransformerDecoderLayer(\n",
            "        (self_attn): MultiHeadedAttention(\n",
            "          (linear_keys): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_values): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_query): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (softmax): Softmax(dim=-1)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "          (final_linear): Linear(in_features=512, out_features=512, bias=True)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "          (dropout_1): Dropout(p=0.1, inplace=False)\n",
            "          (dropout_2): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (layer_norm_1): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (drop): Dropout(p=0.1, inplace=False)\n",
            "        (context_attn): MultiHeadedAttention(\n",
            "          (linear_keys): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_values): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_query): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (softmax): Softmax(dim=-1)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "          (final_linear): Linear(in_features=512, out_features=512, bias=True)\n",
            "        )\n",
            "        (layer_norm_2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "      )\n",
            "      (5): TransformerDecoderLayer(\n",
            "        (self_attn): MultiHeadedAttention(\n",
            "          (linear_keys): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_values): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_query): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (softmax): Softmax(dim=-1)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "          (final_linear): Linear(in_features=512, out_features=512, bias=True)\n",
            "        )\n",
            "        (feed_forward): PositionwiseFeedForward(\n",
            "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
            "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
            "          (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "          (dropout_1): Dropout(p=0.1, inplace=False)\n",
            "          (dropout_2): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (layer_norm_1): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (drop): Dropout(p=0.1, inplace=False)\n",
            "        (context_attn): MultiHeadedAttention(\n",
            "          (linear_keys): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_values): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (linear_query): Linear(in_features=512, out_features=512, bias=True)\n",
            "          (softmax): Softmax(dim=-1)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "          (final_linear): Linear(in_features=512, out_features=512, bias=True)\n",
            "        )\n",
            "        (layer_norm_2): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (generator): Sequential(\n",
            "    (0): Linear(in_features=512, out_features=6360, bias=True)\n",
            "    (1): Cast()\n",
            "    (2): LogSoftmax(dim=-1)\n",
            "  )\n",
            ")\n",
            "[2021-09-20 03:14:50,097 INFO] encoder: 20546048\n",
            "[2021-09-20 03:14:50,097 INFO] decoder: 31744216\n",
            "[2021-09-20 03:14:50,098 INFO] * number of parameters: 52290264\n",
            "[2021-09-20 03:14:50,102 INFO] Starting training on GPU: [0]\n",
            "[2021-09-20 03:14:50,102 INFO] Start training loop and validate every 1500 steps...\n",
            "[2021-09-20 03:14:50,102 INFO] corpus_1's transforms: TransformPipe()\n",
            "[2021-09-20 03:14:50,102 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 1\n",
            "[2021-09-20 03:15:20,120 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 2\n",
            "[2021-09-20 03:15:27,817 INFO] Step 50/ 3000; acc:   6.65; ppl: 1508.55; xent: 7.32; lr: 0.00001; 2192/2956 tok/s;     38 sec\n",
            "[2021-09-20 03:15:50,218 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 3\n",
            "[2021-09-20 03:16:05,037 INFO] Step 100/ 3000; acc:  11.56; ppl: 748.68; xent: 6.62; lr: 0.00001; 2208/2975 tok/s;     75 sec\n",
            "[2021-09-20 03:16:27,082 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 4\n",
            "[2021-09-20 03:16:42,489 INFO] Step 150/ 3000; acc:  11.47; ppl: 513.05; xent: 6.24; lr: 0.00002; 2199/2980 tok/s;    112 sec\n",
            "[2021-09-20 03:16:57,117 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 5\n",
            "[2021-09-20 03:17:19,881 INFO] Step 200/ 3000; acc:  11.64; ppl: 357.15; xent: 5.88; lr: 0.00002; 2202/2981 tok/s;    150 sec\n",
            "[2021-09-20 03:17:33,955 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 6\n",
            "[2021-09-20 03:17:58,638 INFO] Step 250/ 3000; acc:  13.24; ppl: 247.85; xent: 5.51; lr: 0.00003; 2241/3017 tok/s;    189 sec\n",
            "[2021-09-20 03:18:03,853 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 7\n",
            "[2021-09-20 03:18:37,255 INFO] Step 300/ 3000; acc:  15.09; ppl: 177.57; xent: 5.18; lr: 0.00004; 2242/3018 tok/s;    227 sec\n",
            "[2021-09-20 03:18:40,734 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 8\n",
            "[2021-09-20 03:19:10,505 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 9\n",
            "[2021-09-20 03:19:15,597 INFO] Step 350/ 3000; acc:  17.62; ppl: 138.14; xent: 4.93; lr: 0.00004; 2224/3017 tok/s;    265 sec\n",
            "[2021-09-20 03:19:47,381 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 10\n",
            "[2021-09-20 03:19:53,412 INFO] Step 400/ 3000; acc:  19.72; ppl: 117.60; xent: 4.77; lr: 0.00005; 2196/2967 tok/s;    303 sec\n",
            "[2021-09-20 03:20:17,299 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 11\n",
            "[2021-09-20 03:20:30,658 INFO] Step 450/ 3000; acc:  21.87; ppl: 101.06; xent: 4.62; lr: 0.00006; 2228/3001 tok/s;    341 sec\n",
            "[2021-09-20 03:20:54,231 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 12\n",
            "[2021-09-20 03:21:08,235 INFO] Step 500/ 3000; acc:  23.85; ppl: 84.82; xent: 4.44; lr: 0.00006; 2198/2958 tok/s;    378 sec\n",
            "[2021-09-20 03:21:24,200 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 13\n",
            "[2021-09-20 03:21:45,653 INFO] Step 550/ 3000; acc:  25.97; ppl: 70.61; xent: 4.26; lr: 0.00007; 2202/2999 tok/s;    416 sec\n",
            "[2021-09-20 03:22:01,135 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 14\n",
            "[2021-09-20 03:22:23,877 INFO] Step 600/ 3000; acc:  28.62; ppl: 57.66; xent: 4.05; lr: 0.00007; 2215/2983 tok/s;    454 sec\n",
            "[2021-09-20 03:22:31,077 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 15\n",
            "[2021-09-20 03:23:02,742 INFO] Step 650/ 3000; acc:  31.25; ppl: 46.93; xent: 3.85; lr: 0.00008; 2250/3026 tok/s;    493 sec\n",
            "[2021-09-20 03:23:07,870 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 16\n",
            "[2021-09-20 03:23:37,824 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 17\n",
            "[2021-09-20 03:23:41,286 INFO] Step 700/ 3000; acc:  33.62; ppl: 38.67; xent: 3.66; lr: 0.00009; 2237/3016 tok/s;    531 sec\n",
            "[2021-09-20 03:24:14,662 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 18\n",
            "[2021-09-20 03:24:19,055 INFO] Step 750/ 3000; acc:  35.69; ppl: 32.69; xent: 3.49; lr: 0.00009; 2196/2979 tok/s;    569 sec\n",
            "[2021-09-20 03:24:44,832 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 19\n",
            "[2021-09-20 03:24:56,683 INFO] Step 800/ 3000; acc:  37.93; ppl: 27.65; xent: 3.32; lr: 0.00010; 2192/2959 tok/s;    607 sec\n",
            "[2021-09-20 03:25:21,916 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 20\n",
            "[2021-09-20 03:25:34,532 INFO] Step 850/ 3000; acc:  39.77; ppl: 23.69; xent: 3.17; lr: 0.00011; 2205/2961 tok/s;    644 sec\n",
            "[2021-09-20 03:25:51,821 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 21\n",
            "[2021-09-20 03:26:11,783 INFO] Step 900/ 3000; acc:  42.25; ppl: 19.96; xent: 2.99; lr: 0.00011; 2225/3016 tok/s;    682 sec\n",
            "[2021-09-20 03:26:28,505 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 22\n",
            "[2021-09-20 03:26:49,348 INFO] Step 950/ 3000; acc:  44.81; ppl: 16.64; xent: 2.81; lr: 0.00012; 2228/3014 tok/s;    719 sec\n",
            "[2021-09-20 03:26:58,282 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 23\n",
            "[2021-09-20 03:27:27,725 INFO] Step 1000/ 3000; acc:  47.09; ppl: 14.15; xent: 2.65; lr: 0.00012; 2244/3020 tok/s;    758 sec\n",
            "[2021-09-20 03:27:27,757 INFO] Saving checkpoint EgyptianTranslation/onmt-files/run/model_step_1000.pt\n",
            "[2021-09-20 03:27:47,561 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 24\n",
            "[2021-09-20 03:28:17,589 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 25\n",
            "[2021-09-20 03:28:19,235 INFO] Step 1050/ 3000; acc:  49.78; ppl: 11.90; xent: 2.48; lr: 0.00013; 1698/2282 tok/s;    809 sec\n",
            "[2021-09-20 03:28:54,498 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 26\n",
            "[2021-09-20 03:28:57,447 INFO] Step 1100/ 3000; acc:  52.39; ppl: 10.12; xent: 2.31; lr: 0.00014; 2193/2982 tok/s;    847 sec\n",
            "[2021-09-20 03:29:24,453 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 27\n",
            "[2021-09-20 03:29:34,711 INFO] Step 1150/ 3000; acc:  54.42; ppl:  8.92; xent: 2.19; lr: 0.00014; 2207/2983 tok/s;    885 sec\n",
            "[2021-09-20 03:30:01,278 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 28\n",
            "[2021-09-20 03:30:12,095 INFO] Step 1200/ 3000; acc:  56.84; ppl:  7.69; xent: 2.04; lr: 0.00015; 2216/2983 tok/s;    922 sec\n",
            "[2021-09-20 03:30:31,056 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 29\n",
            "[2021-09-20 03:30:49,473 INFO] Step 1250/ 3000; acc:  58.91; ppl:  6.89; xent: 1.93; lr: 0.00015; 2226/3010 tok/s;    959 sec\n",
            "[2021-09-20 03:31:07,976 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 30\n",
            "[2021-09-20 03:31:27,350 INFO] Step 1300/ 3000; acc:  61.57; ppl:  5.89; xent: 1.77; lr: 0.00016; 2214/2996 tok/s;    997 sec\n",
            "[2021-09-20 03:31:38,000 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 31\n",
            "[2021-09-20 03:32:05,885 INFO] Step 1350/ 3000; acc:  64.12; ppl:  5.15; xent: 1.64; lr: 0.00017; 2225/2996 tok/s;   1036 sec\n",
            "[2021-09-20 03:32:14,866 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 32\n",
            "[2021-09-20 03:32:44,957 INFO] Step 1400/ 3000; acc:  66.78; ppl:  4.50; xent: 1.50; lr: 0.00017; 2224/3004 tok/s;   1075 sec\n",
            "[2021-09-20 03:32:44,964 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 33\n",
            "[2021-09-20 03:33:21,971 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 34\n",
            "[2021-09-20 03:33:23,313 INFO] Step 1450/ 3000; acc:  69.27; ppl:  3.97; xent: 1.38; lr: 0.00018; 2213/2980 tok/s;   1113 sec\n",
            "[2021-09-20 03:33:52,175 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 35\n",
            "[2021-09-20 03:34:01,149 INFO] Step 1500/ 3000; acc:  70.96; ppl:  3.67; xent: 1.30; lr: 0.00019; 2178/2951 tok/s;   1151 sec\n",
            "[2021-09-20 03:34:01,150 INFO] valid's transforms: TransformPipe()\n",
            "[2021-09-20 03:34:03,586 INFO] Validation perplexity: 54.2323\n",
            "[2021-09-20 03:34:03,586 INFO] Validation accuracy: 38.8448\n",
            "[2021-09-20 03:34:31,496 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 36\n",
            "[2021-09-20 03:34:41,122 INFO] Step 1550/ 3000; acc:  73.53; ppl:  3.26; xent: 1.18; lr: 0.00019; 2060/2777 tok/s;   1191 sec\n",
            "[2021-09-20 03:35:01,602 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 37\n",
            "[2021-09-20 03:35:18,493 INFO] Step 1600/ 3000; acc:  75.24; ppl:  3.02; xent: 1.11; lr: 0.00020; 2208/2990 tok/s;   1228 sec\n",
            "[2021-09-20 03:35:38,615 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 38\n",
            "[2021-09-20 03:35:56,535 INFO] Step 1650/ 3000; acc:  77.63; ppl:  2.73; xent: 1.01; lr: 0.00020; 2212/2986 tok/s;   1266 sec\n",
            "[2021-09-20 03:36:08,601 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 39\n",
            "[2021-09-20 03:36:35,017 INFO] Step 1700/ 3000; acc:  80.33; ppl:  2.45; xent: 0.90; lr: 0.00021; 2238/3006 tok/s;   1305 sec\n",
            "[2021-09-20 03:36:45,479 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 40\n",
            "[2021-09-20 03:37:13,628 INFO] Step 1750/ 3000; acc:  82.28; ppl:  2.26; xent: 0.82; lr: 0.00022; 2228/3021 tok/s;   1344 sec\n",
            "[2021-09-20 03:37:15,420 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 41\n",
            "[2021-09-20 03:37:45,302 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 42\n",
            "[2021-09-20 03:37:52,350 INFO] Step 1800/ 3000; acc:  84.33; ppl:  2.09; xent: 0.74; lr: 0.00022; 2218/2995 tok/s;   1382 sec\n",
            "[2021-09-20 03:38:22,136 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 43\n",
            "[2021-09-20 03:38:29,751 INFO] Step 1850/ 3000; acc:  85.45; ppl:  2.01; xent: 0.70; lr: 0.00023; 2210/2981 tok/s;   1420 sec\n",
            "[2021-09-20 03:38:52,171 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 44\n",
            "[2021-09-20 03:39:06,936 INFO] Step 1900/ 3000; acc:  87.54; ppl:  1.87; xent: 0.62; lr: 0.00023; 2210/2978 tok/s;   1457 sec\n",
            "[2021-09-20 03:39:28,924 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 45\n",
            "[2021-09-20 03:39:44,315 INFO] Step 1950/ 3000; acc:  89.07; ppl:  1.78; xent: 0.57; lr: 0.00024; 2204/2986 tok/s;   1494 sec\n",
            "[2021-09-20 03:39:58,936 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 46\n",
            "[2021-09-20 03:40:21,616 INFO] Step 2000/ 3000; acc:  90.44; ppl:  1.69; xent: 0.52; lr: 0.00025; 2208/2988 tok/s;   1532 sec\n",
            "[2021-09-20 03:40:21,648 INFO] Saving checkpoint EgyptianTranslation/onmt-files/run/model_step_2000.pt\n",
            "[2021-09-20 03:40:47,907 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 47\n",
            "[2021-09-20 03:41:12,611 INFO] Step 2050/ 3000; acc:  91.50; ppl:  1.61; xent: 0.48; lr: 0.00025; 1703/2293 tok/s;   1583 sec\n",
            "[2021-09-20 03:41:17,817 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 48\n",
            "[2021-09-20 03:41:51,142 INFO] Step 2100/ 3000; acc:  92.85; ppl:  1.54; xent: 0.43; lr: 0.00026; 2247/3024 tok/s;   1621 sec\n",
            "[2021-09-20 03:41:54,635 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 49\n",
            "[2021-09-20 03:42:24,528 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 50\n",
            "[2021-09-20 03:42:29,637 INFO] Step 2150/ 3000; acc:  93.69; ppl:  1.49; xent: 0.40; lr: 0.00027; 2215/3005 tok/s;   1660 sec\n",
            "[2021-09-20 03:43:01,387 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 51\n",
            "[2021-09-20 03:43:07,443 INFO] Step 2200/ 3000; acc:  94.37; ppl:  1.45; xent: 0.37; lr: 0.00027; 2196/2968 tok/s;   1697 sec\n",
            "[2021-09-20 03:43:31,376 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 52\n",
            "[2021-09-20 03:43:44,737 INFO] Step 2250/ 3000; acc:  95.14; ppl:  1.41; xent: 0.34; lr: 0.00028; 2225/2997 tok/s;   1735 sec\n",
            "[2021-09-20 03:44:08,352 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 53\n",
            "[2021-09-20 03:44:22,302 INFO] Step 2300/ 3000; acc:  95.64; ppl:  1.38; xent: 0.32; lr: 0.00028; 2199/2958 tok/s;   1772 sec\n",
            "[2021-09-20 03:44:38,276 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 54\n",
            "[2021-09-20 03:44:59,681 INFO] Step 2350/ 3000; acc:  96.02; ppl:  1.35; xent: 0.30; lr: 0.00029; 2205/3002 tok/s;   1810 sec\n",
            "[2021-09-20 03:45:15,231 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 55\n",
            "[2021-09-20 03:45:38,018 INFO] Step 2400/ 3000; acc:  96.22; ppl:  1.34; xent: 0.29; lr: 0.00030; 2208/2974 tok/s;   1848 sec\n",
            "[2021-09-20 03:45:45,242 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 56\n",
            "[2021-09-20 03:46:16,866 INFO] Step 2450/ 3000; acc:  96.46; ppl:  1.32; xent: 0.28; lr: 0.00030; 2251/3027 tok/s;   1887 sec\n",
            "[2021-09-20 03:46:22,002 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 57\n",
            "[2021-09-20 03:46:51,992 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 58\n",
            "[2021-09-20 03:46:55,442 INFO] Step 2500/ 3000; acc:  96.77; ppl:  1.30; xent: 0.26; lr: 0.00031; 2235/3013 tok/s;   1925 sec\n",
            "[2021-09-20 03:47:28,910 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 59\n",
            "[2021-09-20 03:47:33,315 INFO] Step 2550/ 3000; acc:  97.08; ppl:  1.28; xent: 0.25; lr: 0.00032; 2190/2971 tok/s;   1963 sec\n",
            "[2021-09-20 03:47:58,997 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 60\n",
            "[2021-09-20 03:48:10,802 INFO] Step 2600/ 3000; acc:  97.46; ppl:  1.26; xent: 0.23; lr: 0.00032; 2200/2970 tok/s;   2001 sec\n",
            "[2021-09-20 03:48:36,016 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 61\n",
            "[2021-09-20 03:48:48,624 INFO] Step 2650/ 3000; acc:  97.69; ppl:  1.25; xent: 0.22; lr: 0.00033; 2207/2963 tok/s;   2039 sec\n",
            "[2021-09-20 03:49:06,010 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 62\n",
            "[2021-09-20 03:49:26,084 INFO] Step 2700/ 3000; acc:  97.79; ppl:  1.24; xent: 0.22; lr: 0.00033; 2213/2999 tok/s;   2076 sec\n",
            "[2021-09-20 03:49:42,899 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 63\n",
            "[2021-09-20 03:50:03,884 INFO] Step 2750/ 3000; acc:  97.85; ppl:  1.23; xent: 0.21; lr: 0.00034; 2214/2996 tok/s;   2114 sec\n",
            "[2021-09-20 03:50:12,877 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 64\n",
            "[2021-09-20 03:50:42,487 INFO] Step 2800/ 3000; acc:  97.93; ppl:  1.22; xent: 0.20; lr: 0.00035; 2231/3002 tok/s;   2152 sec\n",
            "[2021-09-20 03:50:49,858 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 65\n",
            "[2021-09-20 03:51:19,897 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 66\n",
            "[2021-09-20 03:51:21,567 INFO] Step 2850/ 3000; acc:  97.94; ppl:  1.22; xent: 0.20; lr: 0.00035; 2238/3007 tok/s;   2191 sec\n",
            "[2021-09-20 03:51:56,970 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 67\n",
            "[2021-09-20 03:51:59,918 INFO] Step 2900/ 3000; acc:  98.03; ppl:  1.21; xent: 0.19; lr: 0.00036; 2185/2971 tok/s;   2230 sec\n",
            "[2021-09-20 03:52:26,960 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 68\n",
            "[2021-09-20 03:52:37,229 INFO] Step 2950/ 3000; acc:  98.05; ppl:  1.21; xent: 0.19; lr: 0.00036; 2204/2980 tok/s;   2267 sec\n",
            "[2021-09-20 03:53:03,917 INFO] Weighted corpora loaded so far:\n",
            "\t\t\t* corpus_1: 69\n",
            "[2021-09-20 03:53:14,767 INFO] Step 3000/ 3000; acc:  98.07; ppl:  1.21; xent: 0.19; lr: 0.00037; 2207/2971 tok/s;   2305 sec\n",
            "[2021-09-20 03:53:16,274 INFO] Validation perplexity: 66.8696\n",
            "[2021-09-20 03:53:16,274 INFO] Validation accuracy: 41.9458\n",
            "[2021-09-20 03:53:16,308 INFO] Saving checkpoint EgyptianTranslation/onmt-files/run/model_step_3000.pt\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HPZv8-MfJwUf",
        "outputId": "8daeccb1-3159-4f90-c688-af4201053cf9"
      },
      "source": [
        "!onmt_translate -model EgyptianTranslation/onmt-files/run/model_step_3000.pt -src /content/drive/MyDrive/EgyptianTranslation/compiled_corpora/aligned_test.egy.csv -output EgyptianTranslation/onmt-files/run/pred_3000.txt"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[2021-09-20 04:10:02,001 INFO] Translating shard 0.\n",
            "/usr/local/lib/python3.7/dist-packages/torch/_tensor.py:575: UserWarning: floor_divide is deprecated, and will be removed in a future version of pytorch. It currently rounds toward 0 (like the 'trunc' function NOT 'floor'). This results in incorrect rounding for negative values.\n",
            "To keep the current behavior, use torch.div(a, b, rounding_mode='trunc'), or for actual floor division, use torch.div(a, b, rounding_mode='floor'). (Triggered internally at  /pytorch/aten/src/ATen/native/BinaryOps.cpp:467.)\n",
            "  return torch.floor_divide(self, other)\n",
            "[2021-09-20 04:10:44,674 INFO] PRED AVG SCORE: -0.9037, PRED PPL: 2.4687\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kFNGcImY8V86",
        "outputId": "721e0477-14c9-451d-a594-e8b527eb6178"
      },
      "source": [
        "ref_path = \"EgyptianTranslation/compiled_corpora/aligned_test.eng.csv\"\n",
        "with open(ref_path, 'r') as ref_file:\n",
        "  ref = [list(map(lambda x: x.strip('\\n'), ref_file.readlines()))]\n",
        "\n",
        "print(len(ref[0]))\n",
        "ref[0][-10:]"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "660\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['and the fields were flooded three palms high with the liquid',\n",
              " 'by the might of the majesty of this god',\n",
              " 'then this goddess came in the early morning',\n",
              " 'and found that these fields were flooded',\n",
              " 'and her gaze was pleased by it',\n",
              " 'then she drank and it pleased her heart',\n",
              " 'after she returned drunk without having perceived mankind',\n",
              " 'the majesty of re said to this goddess',\n",
              " 'welcome in peace o gracious one',\n",
              " 'thus beautiful women came into being in momemphis']"
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8uTi6z-j-St7",
        "outputId": "067b9dc8-6ec8-4568-a396-332356232991"
      },
      "source": [
        "pred_path = \"EgyptianTranslation/onmt-files/run/pred_3000.txt\"\n",
        "with open(pred_path, 'r') as pred_file:\n",
        "  pred = list(map(lambda x: x.strip('\\n'), pred_file.readlines()))\n",
        "\n",
        "print(len(pred))\n",
        "pred[-10:]"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "660\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['then there was very many fields after the water',\n",
              " 'as the might of the majesty of this god',\n",
              " 'then this official who is in the morning',\n",
              " 'she found a fan',\n",
              " 'and it is good out there',\n",
              " 'then there was done on account of the goodness',\n",
              " 'when she came out without a plummet',\n",
              " 'then the majesty of Re said to this city',\n",
              " 'when it is born in peace',\n",
              " 'how good it is a master of tent']"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AbAfOQUi9jUm",
        "outputId": "eb60d959-dcd4-448f-a2db-b39ec63ded61"
      },
      "source": [
        "!pip install sacrebleu"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting sacrebleu\n",
            "  Downloading sacrebleu-2.0.0-py3-none-any.whl (90 kB)\n",
            "\u001b[?25l\r\u001b[K     |███▋                            | 10 kB 21.9 MB/s eta 0:00:01\r\u001b[K     |███████▏                        | 20 kB 26.0 MB/s eta 0:00:01\r\u001b[K     |██████████▉                     | 30 kB 23.5 MB/s eta 0:00:01\r\u001b[K     |██████████████▍                 | 40 kB 18.7 MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 51 kB 14.7 MB/s eta 0:00:01\r\u001b[K     |█████████████████████▋          | 61 kB 10.5 MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▎      | 71 kB 11.8 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▉   | 81 kB 13.1 MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 90 kB 5.6 MB/s \n",
            "\u001b[?25hCollecting colorama\n",
            "  Downloading colorama-0.4.4-py2.py3-none-any.whl (16 kB)\n",
            "Requirement already satisfied: tabulate>=0.8.9 in /usr/local/lib/python3.7/dist-packages (from sacrebleu) (0.8.9)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from sacrebleu) (1.19.5)\n",
            "Collecting portalocker\n",
            "  Downloading portalocker-2.3.2-py2.py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from sacrebleu) (2019.12.20)\n",
            "Installing collected packages: portalocker, colorama, sacrebleu\n",
            "Successfully installed colorama-0.4.4 portalocker-2.3.2 sacrebleu-2.0.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fjpvrFF4RhzJ",
        "outputId": "47da1582-0f6e-4364-b152-31666e1fd610"
      },
      "source": [
        "from nltk.translate import bleu_score\n",
        "from nltk.translate.bleu_score import SmoothingFunction\n",
        "\n",
        "split_ref = [[item] for item in ref[0]]\n",
        "split_pred = [item for item in pred]\n",
        "bleu_score.corpus_bleu(split_ref, split_pred, smoothing_function=SmoothingFunction().method5) * 100"
      ],
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "41.68753829829091"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iaqJeY1pWkVA",
        "outputId": "fd0b6332-a329-4989-89f4-572d92a82216"
      },
      "source": [
        "from sacrebleu.metrics import BLEU\n",
        "\n",
        "bleu = BLEU()\n",
        "split_ref = [[item] for item in ref[0]]\n",
        "split_pred = [item for item in pred]\n",
        "bleu.corpus_score(split_pred, split_ref)"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BLEU = 33.57 80.0/44.4/25.0/14.3 (BP = 1.000 ratio = 1.000 hyp_len = 10 ref_len = 10)"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    }
  ]
}