Bert_Classification_Final.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Bert_Classification_Final",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0b43eabb2c834a7bb49cc322ff32e533": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_177bff930e1b4ce38aa8725993df775c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3d31141e2eed4ecbb8667b451e08c40e",
              "IPY_MODEL_5bc89353df6243749681b5688346017f"
            ]
          }
        },
        "177bff930e1b4ce38aa8725993df775c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3d31141e2eed4ecbb8667b451e08c40e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e25a1ec2f93a46daac511bfd8cfe2d79",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 570,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 570,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_22ebc82125604d09bda094f37e25cc28"
          }
        },
        "5bc89353df6243749681b5688346017f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_483958c444934f2396feb477b0b35a0b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 570/570 [00:00&lt;00:00, 3.32kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3f1cbbc4ef6f461eaf88b33221495955"
          }
        },
        "e25a1ec2f93a46daac511bfd8cfe2d79": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "22ebc82125604d09bda094f37e25cc28": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "483958c444934f2396feb477b0b35a0b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3f1cbbc4ef6f461eaf88b33221495955": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "77449725d6d946199f6fbea8ff4cb815": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4c273703093248c5bb3d5c3408beafd6",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e048648e998040f9ad886b91bb042469",
              "IPY_MODEL_7a516f8546044a0b9aaa26fe99ff4d8b"
            ]
          }
        },
        "4c273703093248c5bb3d5c3408beafd6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e048648e998040f9ad886b91bb042469": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6ff540b1f29e4ff0a01adabe32f93c1f",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1fa526b40e384824b5ea28fc7e099c4f"
          }
        },
        "7a516f8546044a0b9aaa26fe99ff4d8b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_729aea8d5c8c48279fc501cea8707c98",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [00:09&lt;00:00, 47.7MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9663551ac23e4a448f562392e5e30083"
          }
        },
        "6ff540b1f29e4ff0a01adabe32f93c1f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1fa526b40e384824b5ea28fc7e099c4f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "729aea8d5c8c48279fc501cea8707c98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9663551ac23e4a448f562392e5e30083": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "143ce0cf7b4446feb213b83115fc848f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f18caf57fcd74d2f839e5a3eea9535e4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_383a2610eeda4be09481c2977cf9e9df",
              "IPY_MODEL_e923480016b54fe380589a99fad35f79"
            ]
          }
        },
        "f18caf57fcd74d2f839e5a3eea9535e4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "383a2610eeda4be09481c2977cf9e9df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_13973da2bae742308772316b0e4c582f",
            "_dom_classes": [],
            "description": "[Train] : 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 8438,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 8438,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2090e5ffe3c04b3b9fc9814ede5ff4ca"
          }
        },
        "e923480016b54fe380589a99fad35f79": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_38f9201a6f524b1e9977623c4d3add82",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 8438/8438 [57:24&lt;00:00,  2.45it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d4cf8ebc039940c4a8e93fcdd372c57a"
          }
        },
        "13973da2bae742308772316b0e4c582f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2090e5ffe3c04b3b9fc9814ede5ff4ca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "38f9201a6f524b1e9977623c4d3add82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d4cf8ebc039940c4a8e93fcdd372c57a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f14000b4bd7a437caf8e46afae9d9f72": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_661672cf41b746e7ab955cd9015a2af5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_528ce850b46b45dfb7e935f97c72a554",
              "IPY_MODEL_6670c242755e4fc5ae2832f887a3552e"
            ]
          }
        },
        "661672cf41b746e7ab955cd9015a2af5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "528ce850b46b45dfb7e935f97c72a554": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0f49774b0be3452f99708aabd76e13eb",
            "_dom_classes": [],
            "description": "[Validation]: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 938,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 938,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_587f32ae32a347c0b649e366f2528c46"
          }
        },
        "6670c242755e4fc5ae2832f887a3552e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_630b6b18964b4daf865fe8de0f6857d5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 938/938 [02:13&lt;00:00,  7.00it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_48b7164ad5504679adfbfe47208c1d7c"
          }
        },
        "0f49774b0be3452f99708aabd76e13eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "587f32ae32a347c0b649e366f2528c46": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "630b6b18964b4daf865fe8de0f6857d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "48b7164ad5504679adfbfe47208c1d7c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "17635bc0412645d09e6033bcce308eb8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_12ae66e6d3f94ce0a0988edccaa70b38",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9b6fd2a792b44f83bece2151c8c450e9",
              "IPY_MODEL_5cb625885ec44ee9a4dfb8bd00aadabb"
            ]
          }
        },
        "12ae66e6d3f94ce0a0988edccaa70b38": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9b6fd2a792b44f83bece2151c8c450e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4b40f829f58f4fbdb9cae38ea338f01b",
            "_dom_classes": [],
            "description": "[Train] : 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 8438,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 8438,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_803d1005562e4c7bace6da998af99649"
          }
        },
        "5cb625885ec44ee9a4dfb8bd00aadabb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2a31ed05aa1b48cdb166f7ca2ebba757",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 8438/8438 [57:34&lt;00:00,  2.44it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9b635f93eb8d4bac9d4275677a7acb46"
          }
        },
        "4b40f829f58f4fbdb9cae38ea338f01b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "803d1005562e4c7bace6da998af99649": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2a31ed05aa1b48cdb166f7ca2ebba757": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9b635f93eb8d4bac9d4275677a7acb46": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4480a94f37d146539313abdc2816719e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_fb9bf85f33d741bbb462d76fe28b3e4d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4e4aafa6a68a4e8a8747cfdd5fe11348",
              "IPY_MODEL_1fcff08013354cc9aafc6ea0a70deddd"
            ]
          }
        },
        "fb9bf85f33d741bbb462d76fe28b3e4d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4e4aafa6a68a4e8a8747cfdd5fe11348": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e1faad19bb20464196ab6483c89f5039",
            "_dom_classes": [],
            "description": "[Validation]: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 938,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 938,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_286d2c9b5b9e43458eadd2050b2c1463"
          }
        },
        "1fcff08013354cc9aafc6ea0a70deddd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d2713b6db2b74c0fa54c9514a35387e2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 938/938 [02:14&lt;00:00,  6.99it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_003dd7de268c4074841561672e16af18"
          }
        },
        "e1faad19bb20464196ab6483c89f5039": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "286d2c9b5b9e43458eadd2050b2c1463": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d2713b6db2b74c0fa54c9514a35387e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "003dd7de268c4074841561672e16af18": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c965ba0807504066974ed399d6774715": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_567023686bcb441790f1d45a0e428916",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4126a982d8fb44c9a2705c4d7cd7536a",
              "IPY_MODEL_769d77c247724175b5ff2f146388df54"
            ]
          }
        },
        "567023686bcb441790f1d45a0e428916": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4126a982d8fb44c9a2705c4d7cd7536a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5b78adeb66c04802bafa68af5ffc480e",
            "_dom_classes": [],
            "description": "[Train] : 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 8438,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 8438,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_670a50033bd04d37ba1830a3c0a82fed"
          }
        },
        "769d77c247724175b5ff2f146388df54": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fad3d09959fb4ad6858458392421199f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 8438/8438 [57:31&lt;00:00,  2.44it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7616f3c593c547b7b0f7ef9e37d5422a"
          }
        },
        "5b78adeb66c04802bafa68af5ffc480e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "670a50033bd04d37ba1830a3c0a82fed": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fad3d09959fb4ad6858458392421199f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7616f3c593c547b7b0f7ef9e37d5422a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "822f2fe4446d44b3850f7e2a24770d0d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_72bf367661e34c4c9f0253720836e29c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_620ff7f268404e5db55cb5500e5abc63",
              "IPY_MODEL_0309ff80fd6d427eb61e5b035afa0ab7"
            ]
          }
        },
        "72bf367661e34c4c9f0253720836e29c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "620ff7f268404e5db55cb5500e5abc63": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cdcc1852b61344439816947b547471ff",
            "_dom_classes": [],
            "description": "[Validation]: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 938,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 938,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a56fff1aa30249f1b1fabbf3ca3394ad"
          }
        },
        "0309ff80fd6d427eb61e5b035afa0ab7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a9e3d0ae627843498ac9ab9ae34e5c95",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 938/938 [02:13&lt;00:00,  7.00it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cc30abd7b0f84c6c9e87562df90bb501"
          }
        },
        "cdcc1852b61344439816947b547471ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a56fff1aa30249f1b1fabbf3ca3394ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a9e3d0ae627843498ac9ab9ae34e5c95": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cc30abd7b0f84c6c9e87562df90bb501": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X93q5IjhSvZA",
        "outputId": "20f6282d-593c-4414-b599-09c8d6a1d17b"
      },
      "source": [
        "from google.colab import drive \n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "rer9crY0SxW8",
        "outputId": "6475930a-f59d-4f11-e767-2cd16c477d3c"
      },
      "source": [
        "import time\n",
        "PROJECT_TIME = time.asctime()\n",
        "PROJECT_TIME"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Wed Jul 14 14:13:05 2021'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vf9E0FLM8GYW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "53a9188c-b0da-4cb4-88a2-a95b2724288a"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fd/1a/41c644c963249fd7f3836d926afa1e3f1cc234a1c40d80c5f03ad8f6f1b2/transformers-4.8.2-py3-none-any.whl (2.5MB)\n",
            "\r\u001b[K     |▏                               | 10kB 24.1MB/s eta 0:00:01\r\u001b[K     |▎                               | 20kB 30.7MB/s eta 0:00:01\r\u001b[K     |▍                               | 30kB 32.9MB/s eta 0:00:01\r\u001b[K     |▌                               | 40kB 35.8MB/s eta 0:00:01\r\u001b[K     |▋                               | 51kB 27.5MB/s eta 0:00:01\r\u001b[K     |▉                               | 61kB 29.8MB/s eta 0:00:01\r\u001b[K     |█                               | 71kB 28.0MB/s eta 0:00:01\r\u001b[K     |█                               | 81kB 28.1MB/s eta 0:00:01\r\u001b[K     |█▏                              | 92kB 28.2MB/s eta 0:00:01\r\u001b[K     |█▎                              | 102kB 29.4MB/s eta 0:00:01\r\u001b[K     |█▍                              | 112kB 29.4MB/s eta 0:00:01\r\u001b[K     |█▋                              | 122kB 29.4MB/s eta 0:00:01\r\u001b[K     |█▊                              | 133kB 29.4MB/s eta 0:00:01\r\u001b[K     |█▉                              | 143kB 29.4MB/s eta 0:00:01\r\u001b[K     |██                              | 153kB 29.4MB/s eta 0:00:01\r\u001b[K     |██                              | 163kB 29.4MB/s eta 0:00:01\r\u001b[K     |██▎                             | 174kB 29.4MB/s eta 0:00:01\r\u001b[K     |██▍                             | 184kB 29.4MB/s eta 0:00:01\r\u001b[K     |██▌                             | 194kB 29.4MB/s eta 0:00:01\r\u001b[K     |██▋                             | 204kB 29.4MB/s eta 0:00:01\r\u001b[K     |██▊                             | 215kB 29.4MB/s eta 0:00:01\r\u001b[K     |██▉                             | 225kB 29.4MB/s eta 0:00:01\r\u001b[K     |███                             | 235kB 29.4MB/s eta 0:00:01\r\u001b[K     |███▏                            | 245kB 29.4MB/s eta 0:00:01\r\u001b[K     |███▎                            | 256kB 29.4MB/s eta 0:00:01\r\u001b[K     |███▍                            | 266kB 29.4MB/s eta 0:00:01\r\u001b[K     |███▌                            | 276kB 29.4MB/s eta 0:00:01\r\u001b[K     |███▊                            | 286kB 29.4MB/s eta 0:00:01\r\u001b[K     |███▉                            | 296kB 29.4MB/s eta 0:00:01\r\u001b[K     |████                            | 307kB 29.4MB/s eta 0:00:01\r\u001b[K     |████                            | 317kB 29.4MB/s eta 0:00:01\r\u001b[K     |████▏                           | 327kB 29.4MB/s eta 0:00:01\r\u001b[K     |████▎                           | 337kB 29.4MB/s eta 0:00:01\r\u001b[K     |████▌                           | 348kB 29.4MB/s eta 0:00:01\r\u001b[K     |████▋                           | 358kB 29.4MB/s eta 0:00:01\r\u001b[K     |████▊                           | 368kB 29.4MB/s eta 0:00:01\r\u001b[K     |████▉                           | 378kB 29.4MB/s eta 0:00:01\r\u001b[K     |█████                           | 389kB 29.4MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 399kB 29.4MB/s eta 0:00:01\r\u001b[K     |█████▎                          | 409kB 29.4MB/s eta 0:00:01\r\u001b[K     |█████▍                          | 419kB 29.4MB/s eta 0:00:01\r\u001b[K     |█████▌                          | 430kB 29.4MB/s eta 0:00:01\r\u001b[K     |█████▋                          | 440kB 29.4MB/s eta 0:00:01\r\u001b[K     |█████▊                          | 450kB 29.4MB/s eta 0:00:01\r\u001b[K     |██████                          | 460kB 29.4MB/s eta 0:00:01\r\u001b[K     |██████                          | 471kB 29.4MB/s eta 0:00:01\r\u001b[K     |██████▏                         | 481kB 29.4MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 491kB 29.4MB/s eta 0:00:01\r\u001b[K     |██████▍                         | 501kB 29.4MB/s eta 0:00:01\r\u001b[K     |██████▌                         | 512kB 29.4MB/s eta 0:00:01\r\u001b[K     |██████▊                         | 522kB 29.4MB/s eta 0:00:01\r\u001b[K     |██████▉                         | 532kB 29.4MB/s eta 0:00:01\r\u001b[K     |███████                         | 542kB 29.4MB/s eta 0:00:01\r\u001b[K     |███████                         | 552kB 29.4MB/s eta 0:00:01\r\u001b[K     |███████▏                        | 563kB 29.4MB/s eta 0:00:01\r\u001b[K     |███████▍                        | 573kB 29.4MB/s eta 0:00:01\r\u001b[K     |███████▌                        | 583kB 29.4MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 593kB 29.4MB/s eta 0:00:01\r\u001b[K     |███████▊                        | 604kB 29.4MB/s eta 0:00:01\r\u001b[K     |███████▉                        | 614kB 29.4MB/s eta 0:00:01\r\u001b[K     |████████                        | 624kB 29.4MB/s eta 0:00:01\r\u001b[K     |████████▏                       | 634kB 29.4MB/s eta 0:00:01\r\u001b[K     |████████▎                       | 645kB 29.4MB/s eta 0:00:01\r\u001b[K     |████████▍                       | 655kB 29.4MB/s eta 0:00:01\r\u001b[K     |████████▌                       | 665kB 29.4MB/s eta 0:00:01\r\u001b[K     |████████▋                       | 675kB 29.4MB/s eta 0:00:01\r\u001b[K     |████████▉                       | 686kB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████                       | 696kB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████                       | 706kB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 716kB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████▎                      | 727kB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████▍                      | 737kB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████▋                      | 747kB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████▊                      | 757kB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████▉                      | 768kB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████                      | 778kB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████                      | 788kB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████▎                     | 798kB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 808kB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████▌                     | 819kB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████▋                     | 829kB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████▊                     | 839kB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████▉                     | 849kB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████                     | 860kB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████▏                    | 870kB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████▎                    | 880kB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████▍                    | 890kB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████▌                    | 901kB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████▊                    | 911kB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████▉                    | 921kB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████                    | 931kB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████                    | 942kB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████▏                   | 952kB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████▎                   | 962kB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████▌                   | 972kB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████▋                   | 983kB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████▊                   | 993kB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████▉                   | 1.0MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 1.0MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 1.0MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████▎                  | 1.0MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████▍                  | 1.0MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████▌                  | 1.1MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████▋                  | 1.1MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████▊                  | 1.1MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 1.1MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 1.1MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████▏                 | 1.1MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████▎                 | 1.1MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████▍                 | 1.1MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 1.1MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████▊                 | 1.1MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████▉                 | 1.2MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 1.2MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 1.2MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 1.2MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 1.2MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████▌                | 1.2MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████▋                | 1.2MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 1.2MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████▉                | 1.2MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████                | 1.2MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████▏               | 1.3MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████▎               | 1.3MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████▍               | 1.3MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████▌               | 1.3MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████▋               | 1.3MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 1.3MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 1.3MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 1.3MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████▏              | 1.3MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████▎              | 1.4MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 1.4MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████▋              | 1.4MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████▊              | 1.4MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████▉              | 1.4MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 1.4MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 1.4MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 1.4MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████▍             | 1.4MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████▌             | 1.4MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████▋             | 1.5MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████▊             | 1.5MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████▉             | 1.5MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 1.5MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████▏            | 1.5MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████▎            | 1.5MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████▍            | 1.5MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████▌            | 1.5MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████▋            | 1.5MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████▉            | 1.5MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 1.6MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 1.6MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████▏           | 1.6MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████▎           | 1.6MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████▌           | 1.6MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████▋           | 1.6MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████▊           | 1.6MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████▉           | 1.6MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 1.6MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 1.6MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████▎          | 1.7MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████▍          | 1.7MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████▌          | 1.7MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████▋          | 1.7MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████▊          | 1.7MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 1.7MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 1.7MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████▏         | 1.7MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████▎         | 1.7MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████▍         | 1.8MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████▌         | 1.8MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 1.8MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████▉         | 1.8MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 1.8MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 1.8MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████▏        | 1.8MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 1.8MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████▌        | 1.8MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████▋        | 1.8MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████▊        | 1.9MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 1.9MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 1.9MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████▏       | 1.9MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 1.9MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████▍       | 1.9MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████▌       | 1.9MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████▋       | 1.9MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████▉       | 1.9MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 1.9MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 2.0MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▏      | 2.0MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▎      | 2.0MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▍      | 2.0MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 2.0MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▊      | 2.0MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▉      | 2.0MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 2.0MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 2.0MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▏     | 2.0MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▍     | 2.1MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▌     | 2.1MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▋     | 2.1MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▊     | 2.1MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 2.1MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 2.1MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▏    | 2.1MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▎    | 2.1MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▍    | 2.1MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▌    | 2.2MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▋    | 2.2MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▉    | 2.2MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 2.2MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 2.2MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▏   | 2.2MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▎   | 2.2MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▌   | 2.2MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▋   | 2.2MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▊   | 2.2MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▉   | 2.3MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 2.3MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 2.3MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▎  | 2.3MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▍  | 2.3MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▌  | 2.3MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 2.3MB 29.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▊  | 2.3MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 2.3MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 2.3MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▏ | 2.4MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 2.4MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▍ | 2.4MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▌ | 2.4MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▊ | 2.4MB 29.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 2.4MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 2.4MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 2.4MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▏| 2.4MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▍| 2.4MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▌| 2.5MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▋| 2.5MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▊| 2.5MB 29.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▉| 2.5MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 2.5MB 29.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 2.5MB 29.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/e2/df3543e8ffdab68f5acc73f613de9c2b155ac47f162e725dcac87c521c11/tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 49.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Collecting huggingface-hub==0.0.12\n",
            "  Downloading https://files.pythonhosted.org/packages/2f/ee/97e253668fda9b17e968b3f97b2f8e53aa0127e8807d24a547687423fe0b/huggingface_hub-0.0.12-py3-none-any.whl\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (4.6.0)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from transformers) (3.13)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
            "\u001b[K     |████████████████████████████████| 901kB 44.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.5.30)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from huggingface-hub==0.0.12->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Installing collected packages: tokenizers, huggingface-hub, sacremoses, transformers\n",
            "Successfully installed huggingface-hub-0.0.12 sacremoses-0.0.45 tokenizers-0.10.3 transformers-4.8.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uCA2ELd-8R44"
      },
      "source": [
        "import torch\n",
        "\n",
        "from transformers import BertTokenizer\n",
        "from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
        "from transformers import get_linear_schedule_with_warmup\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "from tqdm.notebook import tqdm\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import random\n",
        "import time\n",
        "import datetime"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HTysMTAT8c5Y",
        "outputId": "d0c2e0f3-26d7-4d66-b331-aefa2f367f1f"
      },
      "source": [
        "if torch.cuda.is_available():    \n",
        "    device = torch.device(\"cuda\")\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "else:\n",
        "    device = torch.device(\"cpu\")\n",
        "    print('No GPU available, using the CPU instead.')"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla T4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RqEhbhdqSGa5",
        "outputId": "299fd852-76f5-455b-8e40-7a04f156d184"
      },
      "source": [
        "!git clone https://github.com/perfume-reconmendation/topic_modeling.git"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'topic_modeling'...\n",
            "remote: Enumerating objects: 20, done.\u001b[K\n",
            "remote: Counting objects: 100% (20/20), done.\u001b[K\n",
            "remote: Compressing objects: 100% (14/14), done.\u001b[K\n",
            "remote: Total 20 (delta 7), reused 16 (delta 5), pack-reused 0\u001b[K\n",
            "Unpacking objects: 100% (20/20), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 391
        },
        "id": "nsRld6JLUBwT",
        "outputId": "f20b0e4b-dc4c-4c07-fe98-2dae147d5fbb"
      },
      "source": [
        "df = pd.read_csv('/content/topic_modeling/data/final_data.csv')\n",
        "df = df.dropna().reset_index().drop(['index'], axis=1)\n",
        "df.info()\n",
        "df.head()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 75000 entries, 0 to 74999\n",
            "Data columns (total 4 columns):\n",
            " #   Column   Non-Null Count  Dtype \n",
            "---  ------   --------------  ----- \n",
            " 0   gender   75000 non-null  object\n",
            " 1   name     75000 non-null  object\n",
            " 2   accords  75000 non-null  object\n",
            " 3   review   75000 non-null  object\n",
            "dtypes: object(4)\n",
            "memory usage: 2.3+ MB\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>gender</th>\n",
              "      <th>name</th>\n",
              "      <th>accords</th>\n",
              "      <th>review</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>female</td>\n",
              "      <td>Alien Mugler for women</td>\n",
              "      <td>['white floral', 'amber', 'woody']</td>\n",
              "      <td>Got a sample of this today, and my 9 year old ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>female</td>\n",
              "      <td>Alien Mugler for women</td>\n",
              "      <td>['white floral', 'amber', 'woody']</td>\n",
              "      <td>First impressions. Test on blotter.\\nI have he...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>female</td>\n",
              "      <td>Alien Mugler for women</td>\n",
              "      <td>['white floral', 'amber', 'woody']</td>\n",
              "      <td>This perfume reminds me of my best friend. Act...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>female</td>\n",
              "      <td>Alien Mugler for women</td>\n",
              "      <td>['white floral', 'amber', 'woody']</td>\n",
              "      <td>Imagine tripping over your own feet and fallin...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>female</td>\n",
              "      <td>Alien Mugler for women</td>\n",
              "      <td>['white floral', 'amber', 'woody']</td>\n",
              "      <td>Gorgeous Gorgeous Blend ..\\nLove the scent...\\...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   gender  ...                                             review\n",
              "0  female  ...  Got a sample of this today, and my 9 year old ...\n",
              "1  female  ...  First impressions. Test on blotter.\\nI have he...\n",
              "2  female  ...  This perfume reminds me of my best friend. Act...\n",
              "3  female  ...  Imagine tripping over your own feet and fallin...\n",
              "4  female  ...  Gorgeous Gorgeous Blend ..\\nLove the scent...\\...\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ULlwQuuHVOxP",
        "outputId": "1d5b3595-0e3b-4f32-9eb0-e5170f80a6d7"
      },
      "source": [
        "len(df['name'].unique())"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "89"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        },
        "id": "A13O8Od4UpCu",
        "outputId": "cddf0f71-3a41-4e13-f2ed-d7219d52ca1c"
      },
      "source": [
        "df_label = pd.read_csv('/content/topic_modeling/data/labeled_data.csv', usecols=['name', 'label'])\n",
        "df_label = df_label.dropna().reset_index().drop(['index'], axis=1)\n",
        "df_label.info()\n",
        "df_label.head()"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 89 entries, 0 to 88\n",
            "Data columns (total 2 columns):\n",
            " #   Column  Non-Null Count  Dtype  \n",
            "---  ------  --------------  -----  \n",
            " 0   name    89 non-null     object \n",
            " 1   label   89 non-null     float64\n",
            "dtypes: float64(1), object(1)\n",
            "memory usage: 1.5+ KB\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>name</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Alien Mugler for women</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Coco Mademoiselle Chanel for women</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Black Orchid Tom Ford for women</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Black Opium Yves Saint Laurent for women</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Hypnotic Poison Christian Dior for women</td>\n",
              "      <td>2.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                       name  label\n",
              "0                    Alien Mugler for women    2.0\n",
              "1        Coco Mademoiselle Chanel for women    2.0\n",
              "2           Black Orchid Tom Ford for women    2.0\n",
              "3  Black Opium Yves Saint Laurent for women    2.0\n",
              "4  Hypnotic Poison Christian Dior for women    2.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "q4J9leU7tBcY",
        "outputId": "66589d18-67f9-468c-e956-7b2a51da22ac"
      },
      "source": [
        "df_label.label = df_label.label.astype(int)\n",
        "df_label"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>name</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Alien Mugler for women</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Coco Mademoiselle Chanel for women</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Black Orchid Tom Ford for women</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Black Opium Yves Saint Laurent for women</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Hypnotic Poison Christian Dior for women</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84</th>\n",
              "      <td>Spicebomb Extreme Viktor&amp;Rolf for men</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85</th>\n",
              "      <td>Cool Water Davidoff for men</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>86</th>\n",
              "      <td>Herod Parfums de Marly for men</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>87</th>\n",
              "      <td>L’Homme Ideal Eau de Parfum Guerlain for men</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>88</th>\n",
              "      <td>Jazz Club Maison Martin Margiela for men</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>89 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                            name  label\n",
              "0                         Alien Mugler for women      2\n",
              "1             Coco Mademoiselle Chanel for women      2\n",
              "2                Black Orchid Tom Ford for women      2\n",
              "3       Black Opium Yves Saint Laurent for women      2\n",
              "4       Hypnotic Poison Christian Dior for women      2\n",
              "..                                           ...    ...\n",
              "84         Spicebomb Extreme Viktor&Rolf for men      1\n",
              "85                   Cool Water Davidoff for men      0\n",
              "86                Herod Parfums de Marly for men      1\n",
              "87  L’Homme Ideal Eau de Parfum Guerlain for men      1\n",
              "88      Jazz Club Maison Martin Margiela for men      1\n",
              "\n",
              "[89 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kWS3wNe48Q8T",
        "outputId": "12a01d32-4180-47fe-fb56-0fa9c7f79541"
      },
      "source": [
        "len(df_label['label'].unique())"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "id": "H2oYSo56-k0C",
        "outputId": "e256125e-178d-42f5-83fe-78dc5b28cb9c"
      },
      "source": [
        "train_df = pd.merge(df, df_label, how='inner', on='name')\n",
        "train_df"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>gender</th>\n",
              "      <th>name</th>\n",
              "      <th>accords</th>\n",
              "      <th>review</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>female</td>\n",
              "      <td>Alien Mugler for women</td>\n",
              "      <td>['white floral', 'amber', 'woody']</td>\n",
              "      <td>Got a sample of this today, and my 9 year old ...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>female</td>\n",
              "      <td>Alien Mugler for women</td>\n",
              "      <td>['white floral', 'amber', 'woody']</td>\n",
              "      <td>First impressions. Test on blotter.\\nI have he...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>female</td>\n",
              "      <td>Alien Mugler for women</td>\n",
              "      <td>['white floral', 'amber', 'woody']</td>\n",
              "      <td>This perfume reminds me of my best friend. Act...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>female</td>\n",
              "      <td>Alien Mugler for women</td>\n",
              "      <td>['white floral', 'amber', 'woody']</td>\n",
              "      <td>Imagine tripping over your own feet and fallin...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>female</td>\n",
              "      <td>Alien Mugler for women</td>\n",
              "      <td>['white floral', 'amber', 'woody']</td>\n",
              "      <td>Gorgeous Gorgeous Blend ..\\nLove the scent...\\...</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>74995</th>\n",
              "      <td>male</td>\n",
              "      <td>Jazz Club Maison Martin Margiela for men</td>\n",
              "      <td>['tobacco', 'rum', 'sweet', 'vanilla', 'woody'...</td>\n",
              "      <td>I've been wanting to try this on for a while a...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>74996</th>\n",
              "      <td>male</td>\n",
              "      <td>Jazz Club Maison Martin Margiela for men</td>\n",
              "      <td>['tobacco', 'rum', 'sweet', 'vanilla', 'woody'...</td>\n",
              "      <td>Goes on sweet. Sweet and boozy with too much d...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>74997</th>\n",
              "      <td>male</td>\n",
              "      <td>Jazz Club Maison Martin Margiela for men</td>\n",
              "      <td>['tobacco', 'rum', 'sweet', 'vanilla', 'woody'...</td>\n",
              "      <td>Yesterday I walked in to a store looking for a...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>74998</th>\n",
              "      <td>male</td>\n",
              "      <td>Jazz Club Maison Martin Margiela for men</td>\n",
              "      <td>['tobacco', 'rum', 'sweet', 'vanilla', 'woody'...</td>\n",
              "      <td>Tried this today  and now looking at notes I c...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>74999</th>\n",
              "      <td>male</td>\n",
              "      <td>Jazz Club Maison Martin Margiela for men</td>\n",
              "      <td>['tobacco', 'rum', 'sweet', 'vanilla', 'woody'...</td>\n",
              "      <td>Oh Lawd!  i am so disppointed in this fragranc...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>75000 rows × 5 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       gender  ... label\n",
              "0      female  ...     2\n",
              "1      female  ...     2\n",
              "2      female  ...     2\n",
              "3      female  ...     2\n",
              "4      female  ...     2\n",
              "...       ...  ...   ...\n",
              "74995    male  ...     1\n",
              "74996    male  ...     1\n",
              "74997    male  ...     1\n",
              "74998    male  ...     1\n",
              "74999    male  ...     1\n",
              "\n",
              "[75000 rows x 5 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QAU593msY3Um"
      },
      "source": [
        "# 셔플\n",
        "shuffle = train_df.sample(frac=1)\n",
        "\n",
        "# data\n",
        "train = shuffle.reset_index(drop=True)\n",
        "\n",
        "# 토큰 사전\n",
        "document_bert = [\"[CLS] \" + str(s) + \" [SEP]\" for s in train.review]\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\n",
        "tokenized_texts = [tokenizer.tokenize(s) for s in document_bert]"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "id": "mPfqvF6SZ04N",
        "outputId": "f17d5be6-1404-4c74-edff-4b2ed02b0513"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "\n",
        "arr  = [len(t) for t in tokenized_texts]\n",
        "v_df = pd.DataFrame(arr, columns=['length'])\n",
        "sns.histplot(v_df['length'])\n",
        "plt.show()"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZYAAAEGCAYAAABGnrPVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAY0klEQVR4nO3de5CldX3n8fe3b+gCuwxhMkWA2kEz2Sxmd0d2uERJikjJzSToxgW8jgYdVsDVTeIuxKrAmqLWNd7W3QlycQTCTbyVE2WDIxl1WRUYcOS6hAkXmamRGQF1wHL6nO7v/nF+B860fTnd/ZxLd79fVaf6Ob/nOed852lOf3h+v+f5PZGZSJJUlYFeFyBJWlwMFklSpQwWSVKlDBZJUqUMFklSpYZ6XUAnHHLIIbly5cpelyFJC8rdd9/948xcPt/3WZTBsnLlSrZs2dLrMiRpQYmIJ6p4H7vCJEmVMlgkSZUyWCRJlTJYJEmVMlgkSZUyWCRJlTJYJEmVMlgkSZUyWKaRmYyOjuI9aySpfQbLFDKT559/nrPWb6ZWq/W6HElaMAyWKdRqNd5y2TeJGOx1KZK0oBgs0xgYXJRTqUlSRxkskqRKGSySpEoZLJKkShkskqRKGSySpEoZLJKkShkskqRKGSySpEoZLDNwvjBJmh2DZQY5Vuetl9/ufGGS1CaDpQ1O7SJJ7TNYJEmVMlgkSZUyWCbRHLCXJM2ewTKJ5r1YcrxxJphnhklS+wyWKbQO2HtmmCS1z2Bpk2eGSVJ7DBZJUqUMFklSpToWLBFxRERsjogHI+KBiHhfab8kInZExNbyOL3lNRdFxLaIeDgiTmlpP7W0bYuICztVsyRp/jo5cFAH/jQz74mIA4G7I2JTWfeJzPxo68YRcRRwNvAK4NeAb0TEb5TV64HXAtuBuyJiY2Y+2MHaJUlz1LFgycydwM6yvCciHgIOm+YlZwA3ZeZe4LGI2AYcW9Zty8xHASLiprKtwSJJfagrYywRsRJ4JXBHabogIu6NiA0Rsay0HQY82fKy7aVtqvaJn7EuIrZExJbdu3dX/C+QJLWr48ESEQcAXwTen5k/Ay4DXg6spnFE87EqPiczr8jMNZm5Zvny5VW8pSRpDjp6cUZEDNMIlesz80sAmflUy/orga+WpzuAI1pefnhpY5p2SVKf6eRZYQF8BngoMz/e0n5oy2ZvAO4vyxuBsyNiv4g4ElgF3AncBayKiCMjYoTGAP/GTtUtSZqfTh6xvBp4G3BfRGwtbX8OvCkiVgMJPA6cC5CZD0TEzTQG5evA+Zk5BhARFwC3AoPAhsx8oIN1S5LmoZNnhd0OxCSrbpnmNZcCl07Sfst0r5Mk9Q+vvJckVcpgkSRVymCZxni9RuZ4r8uQpAXFYJEkVcpgkSRVymCZQf0XP2d8fKzXZUjSgmGwTGHi+Ir3vZek9hgsbfK+95LUHoNlFrzvvSTNzGCRJFXKYJEkVcpgkSRVymCRJFXKYJEkVcpgkSRVymCRJFXKYJEkVcpgaYPT50tS+wwWSVKlDJZJjI6OkuNONilJc2GwSJIqZbBIkiplsMzCeL3G6Ohor8uQpL5msEiSKmWwSJIqZbBIkiplsEiSKtWxYImIIyJic0Q8GBEPRMT7SvvBEbEpIh4pP5eV9oiIT0XEtoi4NyKObnmvtWX7RyJibadqno5X30tSezp5xFIH/jQzjwKOB86PiKOAC4HbMnMVcFt5DnAasKo81gGXQSOIgIuB44BjgYubYSRJ6j8dC5bM3JmZ95TlPcBDwGHAGcA1ZbNrgNeX5TOAa7Phe8BBEXEocAqwKTOfycxngU3AqZ2qW5I0P10ZY4mIlcArgTuAFZm5s6z6EbCiLB8GPNnysu2lbap2SVIf6niwRMQBwBeB92fmz1rXZWYClUzKFRHrImJLRGzZvXt3FW8pSZqDjgZLRAzTCJXrM/NLpfmp0sVF+bmrtO8Ajmh5+eGlbar2fWTmFZm5JjPXLF++fF51j46OOlAvSXPUybPCAvgM8FBmfrxl1UageWbXWuArLe1vL2eHHQ/8tHSZ3QqcHBHLyqD9yaVNktSHhjr43q8G3gbcFxFbS9ufAx8Gbo6Ic4AngDPLuluA04FtwM+BdwJk5jMR8ZfAXWW7D2XmMx2sW5I0Dx0Llsy8HYgpVp80yfYJnD/Fe20ANlRXnSSpU7zyfhac3ViSZmawSJIqZbBIkiplsEiSKmWwSJIqZbBIkiplsMxSZpYr8yuZiUaSFh2DZZZqtRpnrd9MrVbrdSmS1JcMljkYGOzkhAWStLAZLJKkShkskqRKGSySpEoZLJKkShkskqRKGSySpEoZLJKkShkskqRKGSySpEoZLHPgfGGSNLW2giUiXt1O22L3QqCM1Xnr5bc7X5gkTaLdI5b/2WbbopZjdc677h5yPJ0vTJKmMO1fx4j4beBVwPKI+JOWVf8UGOxkYf1qYMhAkaTpzPRXcgQ4oGx3YEv7z4A3dqooSdLCNW2wZOa3gG9FxNWZ+USXapIkLWDt9uvsFxFXACtbX5OZr+lEUZKkhavdYPk88GngKmCsc+VIkha6doOlnpmXdbSSBWK8XiOGo9dlSFLfavd047+NiPMi4tCIOLj56GhlkqQFqd1gWQt8APgOcHd5bJnuBRGxISJ2RcT9LW2XRMSOiNhaHqe3rLsoIrZFxMMRcUpL+6mlbVtEXDibf5wkqfva6grLzCPn8N5XA/8LuHZC+ycy86OtDRFxFHA28Arg14BvRMRvlNXrgdcC24G7ImJjZj44h3okSV3QVrBExNsna8/MiaHRuu7bEbGyzTrOAG7KzL3AYxGxDTi2rNuWmY+WOm4q2xosktSn2u0KO6bl8TvAJcAfzvEzL4iIe0tX2bLSdhjwZMs220vbVO2/JCLWRcSWiNiye/fuOZYmSZqvtoIlM9/b8ng3cDSNK/Jn6zLg5cBqYCfwsTm8x1Q1XpGZazJzzfLly6t6W0nSLM114qvngVmPu2TmU83liLgS+Gp5ugM4omXTw0sb07RLkvpQu2Msfws0bz4yCPxL4ObZflhEHJqZO8vTNwDNM8Y2AjdExMdpDN6vAu4EAlgVEUfSCJSzgTfP9nMlSd3T7hFL61lcdeCJzNw+3Qsi4kbgROCQiNgOXAycGBGraYTU48C5AJn5QETcTGNQvg6cn5lj5X0uAG6lEWgbMvOBNmvuqPF6jdHRUUZGRnpdiiT1lXZPN/5WRKygMXgP8Egbr3nTJM2fmWb7S4FLJ2m/BbilnTolSb3X7h0kz6TRNfXvgTOBOyJiUU6b37xL5HTG6zUyx7tUkSQtLO12hX0QOCYzdwFExHLgG8AXOlVYr9RqNc658tvE4DARzgkmSbPV7nUsA81QKZ6exWsXHG87LElz1+5f0L+LiFuBG8vzs3DcQ5I0iZnuef/rwIrM/EBE/DvghLLqu8D1nS5OkrTwzHTE8kngIoDM/BLwJYCI+Fdl3R90tDpJ0oIz0zjJisy8b2JjaVvZkYokSQvaTMFy0DTrXlplIZKkxWGmYNkSEe+e2BgR76Jxs68lq3nlvSRpXzONsbwf+HJEvIUXg2QNMEJjri9JkvYxbbCU2YhfFRG/B/xWaf5aZv59xytbAEZHR50vTJImaHeusM3A5g7XIklaBBbt1fOSpN4wWCRJlTJYJEmVMljmqDm9fmbOvLEkLSEGyxzlWJ3zrruHWq3W61Ikqa8YLPMwMOT0+pI0kcEiSaqUwSJJqpTBIkmqlMEiSaqUwSJJqpTBIkmqlMEiSaqUwSJJqpTBIkmqVMeCJSI2RMSuiLi/pe3giNgUEY+Un8tKe0TEpyJiW0TcGxFHt7xmbdn+kYhY26l658o5wyRpX508YrkaOHVC24XAbZm5CritPAc4DVhVHuuAy6ARRMDFwHHAscDFzTDqF7VajbPWb3bOMEkqOhYsmflt4JkJzWcA15Tla4DXt7Rfmw3fAw6KiEOBU4BNmflMZj4LbOKXw6rnBgadM0ySmro9xrIiM3eW5R8BK8ryYcCTLdttL21Ttf+SiFgXEVsiYsvu3burrXoKzW4wSdKLejZ4n41BicoGJjLzisxck5lrli9fXtXbTv+ZY3XeffUdjI+NO84iSUW3g+Wp0sVF+bmrtO8AjmjZ7vDSNlV73xgYHCLH6rz18tsdZ5Ekuh8sG4HmmV1rga+0tL+9nB12PPDT0mV2K3ByRCwrg/Ynl7a+MV6vkTnuOIskFR37axgRNwInAodExHYaZ3d9GLg5Is4BngDOLJvfApwObAN+DrwTIDOfiYi/BO4q230oMyeeECBJ6iMdC5bMfNMUq06aZNsEzp/ifTYAGyosTZLUQV55L0mqlMEiSaqUwSJJqpTBIkmqlMEiSaqUwVKR8XrN6V0kiQ6ebryUjNcbV9wbLJLkEYskqWIGiySpUgaLJKlSBssEo6Oj5LjT30vSXBkskqRKGSwVGh0d9cwwSUuewSJJqpTBIkmqlMFSMbvDJC11BktFMrNxRll6Rpmkpc1gqUiO1Tnvunuo1Wq9LkWSespgqdDAkFOvSZLBIkmqlMEiSaqUwVIhB/AlyWCpVI7VeffVdziAL2lJM1gqNjA45LUskpY0g6VidodJWuoMlorZHSZpqTNYOmBg0OtZJC1dPQmWiHg8Iu6LiK0RsaW0HRwRmyLikfJzWWmPiPhURGyLiHsj4uhe1CxJak8vj1h+LzNXZ+aa8vxC4LbMXAXcVp4DnAasKo91wGVdr3SWxus1B+8lLVn91BV2BnBNWb4GeH1L+7XZ8D3goIg4tBcFzoZnhklaqnoVLAl8PSLujoh1pW1FZu4syz8CVpTlw4AnW167vbTtIyLWRcSWiNiye/fuTtUtSZpBr0aZT8jMHRHxq8CmiPh/rSszMyNiVufrZuYVwBUAa9as8VxfSeqRnhyxZOaO8nMX8GXgWOCpZhdX+bmrbL4DOKLl5YeXtr7ltSySlrKuB0tE7B8RBzaXgZOB+4GNwNqy2VrgK2V5I/D2cnbY8cBPW7rM+pL3ZpG0lPWiK2wF8OWIaH7+DZn5dxFxF3BzRJwDPAGcWba/BTgd2Ab8HHhn90uevYGhxtQuzz33HCMjI4yMjPS6JEnqiq4HS2Y+CvybSdqfBk6apD2B87tQWqWa3WHDw8O9LkWSuqqfTjdeVJzaRdJSZbB0kFO7SFqKDBZJUqUMlg7z1GNJS43B0mG1Wo23Xn67Yy2SlgyDpYMyk+eee44cH3PeMElLhsHSQTlW573X3WU3mKQlxWDpsOaZYc2LJT1ykbTYGSySpEoZLF3gmWGSlhKDpQuaV+E3b/5lwEhazAyWLomBQZ5//nlPPZa06BksXZJjdd5z7Z3EgLtc0uLmX7kucu4wSUuBwdJlDuRLWuwMli4ar9cYr9d48/rNPPvss17TImlRMli6bLxec5xF0qLmX7ges2tM0mJjsPRAM0zGx8d5/vnnOWv9Zk9BlrRoGCw9kGN13vXZ77Fr1y7evH4zEYO9LkmSKmOw9EgA5274vxDR61IkqVJeWNFDrde1ZCZ79+4FYGRkhDBwJC1QBkuPjddrjEXw1FNPcf4N32dweD9ufu9rGBkZ6XVpkjQndoX1geZgPpmMj9XZs2cPv/jFL9i7d69ni0lacDxi6QPNO00OjrwExuq848rbiRggBoe4/j/8Dvvvv79dY5IWDI9Y+kTreMt4bRQiCHA2ZEkLjkcsfWq8XiOjTmby9NNPs2zZMgAiwsF9SX3NYOlzOVbnnCtvJwaC8XqNwZGXcuN7T3phcN+QkdRvFkxXWEScGhEPR8S2iLiw1/V008DQEAODjUcMDvLss8/yRx/9Gm/4yEZ++MMfsmfPHjLzl6aHaZ7C7EkAkrppQRyxROPS9PXAa4HtwF0RsTEzH+xtZd3XvGHYwOAQMT7Ge665k4HhYda/+ZUAXHDjD7hi7THsv//+1Go13nnVd4jBIa479wSGh4dnfP+IYHh4mHq9ztDQEPV6neHh4b45KspMarVaX9UkaV8LIliAY4FtmfkoQETcBJwBdCRYxsfqMD7Q6H4aqxPJtMvN18y03XxfM3F5fKxODAyRY3XWXfVtYmCIweFh3vHX3yAGhl64qD8GhjjzY1998TPLusmWh1/6T7jyHcfxrs98h79+2xrO+5stXHXOqzjggANe6H4bHR1ldHR00mttZrr+pnmrgLlepzM6OspbLvsm17/nRK/1kSbRD9+LWAhdJBHxRuDUzHxXef424LjMvKBlm3XAuvL0XwAPz/HjDgF+PI9yO8365qef6+vn2sD65quf62vW9s8zc/l832yhHLHMKDOvAK6Y7/tExJbMXFNBSR1hffPTz/X1c21gffPVz/VVXdtCGbzfARzR8vzw0iZJ6jMLJVjuAlZFxJERMQKcDWzscU2SpEksiK6wzKxHxAXArcAgsCEzH+jQx827O63DrG9++rm+fq4NrG+++rm+SmtbEIP3kqSFY6F0hUmSFgiDRZJUKYOlRb9MGxMRj0fEfRGxNSK2lLaDI2JTRDxSfi4r7RERnyo13xsRR1dcy4aI2BUR97e0zbqWiFhbtn8kItZ2uL5LImJH2X9bI+L0lnUXlfoejohTWto78ruPiCMiYnNEPBgRD0TE+0p7z/fhNLX1xf6LiJdExJ0R8YNS338t7UdGxB3lsz5XTughIvYrz7eV9StnqrtD9V0dEY+17L/Vpb0X34/BiPh+RHy1PO/OvmvOMbXUHzROCvhH4GXACPAD4Kge1fI4cMiEto8AF5blC4H/XpZPB/43EMDxwB0V1/K7wNHA/XOtBTgYeLT8XFaWl3WwvkuAP5tk26PK73U/4Mjy+x7s5O8eOBQ4uiwfCPxDqaPn+3Ca2vpi/5V9cEBZHgbuKPvkZuDs0v5p4D1l+Tzg02X5bOBz09XdwfquBt44yfa9+H78CXAD8NXyvCv7ziOWF70wbUxmjgLNaWP6xRnANWX5GuD1Le3XZsP3gIMi4tCqPjQzvw08M89aTgE2ZeYzmfkssAk4tYP1TeUM4KbM3JuZjwHbaPzeO/a7z8ydmXlPWd4DPAQcRh/sw2lqm0pX91/ZB8+Vp8PlkcBrgC+U9on7rrlPvwCcFBExTd2dqm8qXf1+RMThwOuAq8rzoEv7zmB50WHAky3PtzP9l6yTEvh6RNwdjalqAFZk5s6y/CNgRVnuRd2zraUXNV5Quhs2NLuZel1f6V54JY3/s+2rfTihNuiT/Ve6crYCu2j8wf1H4CeZWZ/ks16oo6z/KfAr3awvM5v779Ky/z4REftNrG9CHZ2q75PAfwbGy/NfoUv7zmDpTydk5tHAacD5EfG7rSuzcYzaF+eJ91MtLS4DXg6sBnYCH+ttORARBwBfBN6fmT9rXdfrfThJbX2z/zJzLDNX05ht41jgN3tVy2Qm1hcRvwVcRKPOY2h0b/2XbtcVEb8P7MrMu7v92WCwtOqbaWMyc0f5uQv4Mo0v1FPNLq7yc1fZvBd1z7aWrtaYmU+VL/w4cCUvHrr3pL6IGKbxh/v6zPxSae6LfThZbf22/0pNPwE2A79NowupeXF362e9UEdZ/8+Ap7tc36mlizEzcy/wWXqz/14N/GFEPE6ja/I1wP+gW/uuigGixfCgMQvBozQGqJoDkK/oQR37Awe2LH+HRn/rX7HvYO9HyvLr2HdA8M4O1LSSfQfHZ1ULjf9re4zGwOSysnxwB+s7tGX5P9HoIwZ4BfsORD5KY+C5Y7/7si+uBT45ob3n+3Ca2vpi/wHLgYPK8kuB/wP8PvB59h2APq8sn8++A9A3T1d3B+s7tGX/fhL4cI+/Hyfy4uB9V/ZdpX+AFvqDxlkb/0CjH/eDParhZeUX+QPggWYdNPo7bwMeAb7R/A+v/Ee6vtR8H7Cm4npupNEdUqPRv3rOXGoB/pjGwN824J0dru9vyuffS2NOudY/lB8s9T0MnNbp3z1wAo1urnuBreVxej/sw2lq64v9B/xr4PuljvuBv2j5jtxZ9sPngf1K+0vK821l/ctmqrtD9f192X/3A9fx4pljXf9+lPc+kReDpSv7zildJEmVcoxFklQpg0WSVCmDRZJUKYNFklQpg0WSVCmDRZqDiHhu5q1m/Z6rJ8wkfElE/FnVnyN1msEi9Y/VNK4HkRY0g0Wap4j4QETcVSYdbN6TY2VEPBQRV5Z7dXw9Il5a1h1Ttt0aEX8VEfeX+2J8CDirtJ9V3v6oiPhmRDwaEf+xR/9EaVYMFmkeIuJkYBWN+aBWA/+2ZdLQVcD6zHwF8BPgj0r7Z4FzszF54RhANqab/wsa98FYnZmfK9v+Jo1p1Y8FLi5ze0l9zWCR5ufk8vg+cA+NIFhV1j2WmVvL8t3Ayog4iMZccN8t7TfM8P5fy8a9MH5MY6LKFTNsL/Xc0MybSJpGAP8tMy/fp7Fxf5O9LU1jNCYqnK2J7+F3Vn3PIxZpfm4F/rjc04SIOCwifnWqjbMxvfqeiDiuNJ3dsnoPjVsESwuawSLNQ2Z+nUZ31ncj4j4at3WdKRzOAa4sdx7cn8bd+qBxP4+jJgzeSwuOsxtLXRYRB2S5V3pEXEhjWvr39bgsqTL210rd97qIuIjG9+8J4B29LUeqlkcskqRKOcYiSaqUwSJJqpTBIkmqlMEiSaqUwSJJqtT/B4K7BP05lv+rAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6RYDY52Lbye4"
      },
      "source": [
        "# 패딩\n",
        "MAX_LEN = 256\n",
        "input_ids = [tokenizer.convert_tokens_to_ids(x) for x in tokenized_texts]\n",
        "input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype='long', truncating='post', padding='post')\n",
        "\n",
        "# 성능을 위한 마스킹\n",
        "attention_masks = []\n",
        "for seq in input_ids:\n",
        "    seq_mask = [float(i>0) for i in seq]\n",
        "    attention_masks.append(seq_mask)"
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bff1IHdSFSpY"
      },
      "source": [
        "# Validation split\n",
        "train_inputs, validation_inputs, train_labels, validation_labels = \\\n",
        "    train_test_split(input_ids, train['label'].values, random_state=1, test_size=0.1)\n",
        "train_masks, validation_masks, _, _ = \\\n",
        "    train_test_split(attention_masks, input_ids, random_state=1, test_size=0.1)"
      ],
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a3MXTRvi5ZLY"
      },
      "source": [
        "# To Tensor\n",
        "train_inputs = torch.tensor(train_inputs)\n",
        "train_labels = torch.tensor(train_labels)\n",
        "train_masks = torch.tensor(train_masks)\n",
        "\n",
        "validation_inputs = torch.tensor(validation_inputs)\n",
        "validation_labels = torch.tensor(validation_labels)\n",
        "validation_masks = torch.tensor(validation_masks)"
      ],
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ooC-_hz9Gho9"
      },
      "source": [
        "# To Loader\n",
        "BATCH_SIZE = 8\n",
        "\n",
        "train_data = TensorDataset(train_inputs, train_masks, train_labels)\n",
        "train_sampler = RandomSampler(train_data)\n",
        "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=BATCH_SIZE)\n",
        "\n",
        "validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)\n",
        "validation_sampler = SequentialSampler(validation_data)\n",
        "validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=BATCH_SIZE)"
      ],
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1iZygYgpdIjZ",
        "outputId": "6e55b04f-9a17-4d9c-9bef-b5478c5e0cc8"
      },
      "source": [
        "train_inputs.shape, train_labels.shape, train_masks.shape"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(torch.Size([67500, 256]), torch.Size([67500]), torch.Size([67500, 256]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u2TjKvhgdhMb",
        "outputId": "18680c72-5415-4e9c-e42e-5656cce4c63d"
      },
      "source": [
        "validation_inputs.shape, validation_labels.shape, validation_masks.shape"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(torch.Size([7500, 256]), torch.Size([7500]), torch.Size([7500, 256]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "0b43eabb2c834a7bb49cc322ff32e533",
            "177bff930e1b4ce38aa8725993df775c",
            "3d31141e2eed4ecbb8667b451e08c40e",
            "5bc89353df6243749681b5688346017f",
            "e25a1ec2f93a46daac511bfd8cfe2d79",
            "22ebc82125604d09bda094f37e25cc28",
            "483958c444934f2396feb477b0b35a0b",
            "3f1cbbc4ef6f461eaf88b33221495955",
            "77449725d6d946199f6fbea8ff4cb815",
            "4c273703093248c5bb3d5c3408beafd6",
            "e048648e998040f9ad886b91bb042469",
            "7a516f8546044a0b9aaa26fe99ff4d8b",
            "6ff540b1f29e4ff0a01adabe32f93c1f",
            "1fa526b40e384824b5ea28fc7e099c4f",
            "729aea8d5c8c48279fc501cea8707c98",
            "9663551ac23e4a448f562392e5e30083"
          ]
        },
        "id": "v6OxE5e47Nv0",
        "outputId": "330f0ce5-8c67-4e86-a564-ebc561b92bd3"
      },
      "source": [
        "model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\", num_labels=4)\n",
        "model.cuda()"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0b43eabb2c834a7bb49cc322ff32e533",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=570.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "77449725d6d946199f6fbea8ff4cb815",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForSequenceClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=4, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QIkXD-U67ca4"
      },
      "source": [
        "optimizer = AdamW(model.parameters(), lr=2e-5, eps=1e-8)\n",
        "epochs = 3\n",
        "total_steps = len(train_dataloader) * epochs\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps = 0, num_training_steps = total_steps)"
      ],
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "68IDXhiE71Do"
      },
      "source": [
        "# 정확도 계산 함수\n",
        "def flat_accuracy(preds, labels):\n",
        "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
        "\n",
        "# 시간 표시 함수\n",
        "def format_time(elapsed):\n",
        "    elapsed_rounded = int(round((elapsed)))\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))"
      ],
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tXDMDin5lC7T",
        "outputId": "c858c6c6-3604-4f45-d370-815aa80f3e7e"
      },
      "source": [
        "import gc\n",
        "gc.collect()"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "454"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SICP5bR7f0mU",
        "outputId": "9aa2d896-2e0c-4bee-8ce2-2a3bb9fba959"
      },
      "source": [
        "model.save_pretrained(\"/content/drive/MyDrive/SAI/p/lunab_model_21_07_14\")\n",
        "tokenizer.save_pretrained(\"/content/drive/MyDrive/SAI/p/lunab_tokenizer_21_07_14\")"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('/content/drive/MyDrive/SAI/p/lunab_tokenizer_21_07_14/tokenizer_config.json',\n",
              " '/content/drive/MyDrive/SAI/p/lunab_tokenizer_21_07_14/special_tokens_map.json',\n",
              " '/content/drive/MyDrive/SAI/p/lunab_tokenizer_21_07_14/vocab.txt',\n",
              " '/content/drive/MyDrive/SAI/p/lunab_tokenizer_21_07_14/added_tokens.json')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "143ce0cf7b4446feb213b83115fc848f",
            "f18caf57fcd74d2f839e5a3eea9535e4",
            "383a2610eeda4be09481c2977cf9e9df",
            "e923480016b54fe380589a99fad35f79",
            "13973da2bae742308772316b0e4c582f",
            "2090e5ffe3c04b3b9fc9814ede5ff4ca",
            "38f9201a6f524b1e9977623c4d3add82",
            "d4cf8ebc039940c4a8e93fcdd372c57a",
            "f14000b4bd7a437caf8e46afae9d9f72",
            "661672cf41b746e7ab955cd9015a2af5",
            "528ce850b46b45dfb7e935f97c72a554",
            "6670c242755e4fc5ae2832f887a3552e",
            "0f49774b0be3452f99708aabd76e13eb",
            "587f32ae32a347c0b649e366f2528c46",
            "630b6b18964b4daf865fe8de0f6857d5",
            "48b7164ad5504679adfbfe47208c1d7c",
            "17635bc0412645d09e6033bcce308eb8",
            "12ae66e6d3f94ce0a0988edccaa70b38",
            "9b6fd2a792b44f83bece2151c8c450e9",
            "5cb625885ec44ee9a4dfb8bd00aadabb",
            "4b40f829f58f4fbdb9cae38ea338f01b",
            "803d1005562e4c7bace6da998af99649",
            "2a31ed05aa1b48cdb166f7ca2ebba757",
            "9b635f93eb8d4bac9d4275677a7acb46",
            "4480a94f37d146539313abdc2816719e",
            "fb9bf85f33d741bbb462d76fe28b3e4d",
            "4e4aafa6a68a4e8a8747cfdd5fe11348",
            "1fcff08013354cc9aafc6ea0a70deddd",
            "e1faad19bb20464196ab6483c89f5039",
            "286d2c9b5b9e43458eadd2050b2c1463",
            "d2713b6db2b74c0fa54c9514a35387e2",
            "003dd7de268c4074841561672e16af18",
            "c965ba0807504066974ed399d6774715",
            "567023686bcb441790f1d45a0e428916",
            "4126a982d8fb44c9a2705c4d7cd7536a",
            "769d77c247724175b5ff2f146388df54",
            "5b78adeb66c04802bafa68af5ffc480e",
            "670a50033bd04d37ba1830a3c0a82fed",
            "fad3d09959fb4ad6858458392421199f",
            "7616f3c593c547b7b0f7ef9e37d5422a",
            "822f2fe4446d44b3850f7e2a24770d0d",
            "72bf367661e34c4c9f0253720836e29c",
            "620ff7f268404e5db55cb5500e5abc63",
            "0309ff80fd6d427eb61e5b035afa0ab7",
            "cdcc1852b61344439816947b547471ff",
            "a56fff1aa30249f1b1fabbf3ca3394ad",
            "a9e3d0ae627843498ac9ab9ae34e5c95",
            "cc30abd7b0f84c6c9e87562df90bb501"
          ]
        },
        "id": "hseDsGAT77yq",
        "outputId": "b59b26cd-68e0-4b37-ed22-6c5cd239b319"
      },
      "source": [
        "# 재현을 위해 랜덤시드 고정\n",
        "seed_val = 1\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "# cuda delete cache\n",
        "torch.cuda.empty_cache()\n",
        "\n",
        "# 그래디언트 초기화\n",
        "model.zero_grad()\n",
        "\n",
        "# 에폭만큼 반복\n",
        "for epoch_i in range(0, epochs):\n",
        "    \n",
        "    # ========================================\n",
        "    #               Training\n",
        "    # ========================================\n",
        "    \n",
        "    print(\"\")\n",
        "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "    print('Training...')\n",
        "\n",
        "    # 시작 시간 설정\n",
        "    t0 = time.time()\n",
        "\n",
        "    # 로스 초기화\n",
        "    total_loss = 0\n",
        "\n",
        "    # 훈련모드로 변경\n",
        "    model.train()\n",
        "        \n",
        "    # 데이터로더에서 배치만큼 반복하여 가져옴\n",
        "    for step, batch in enumerate(tqdm(train_dataloader, desc='[Train] ')):\n",
        "        # 경과 정보 표시\n",
        "        if step % 500 == 0 and not step == 0:\n",
        "            elapsed = format_time(time.time() - t0)\n",
        "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
        "\n",
        "        # 배치를 GPU에 넣음\n",
        "        batch = tuple(t.to(device) for t in batch)\n",
        "\n",
        "        # 배치에서 데이터 추출\n",
        "        b_input_ids, b_input_mask, b_labels = batch\n",
        "\n",
        "        # Forward 수행                \n",
        "        outputs = model(b_input_ids, \n",
        "                        token_type_ids=None, \n",
        "                        attention_mask=b_input_mask, \n",
        "                        labels=b_labels)\n",
        "        \n",
        "        # 로스 구함\n",
        "        loss = outputs[0]\n",
        "\n",
        "        # 총 로스 계산\n",
        "        total_loss += loss.item()\n",
        "\n",
        "        # Backward 수행으로 그래디언트 계산\n",
        "        loss.backward()\n",
        "\n",
        "        # 그래디언트 클리핑\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "        # 그래디언트를 통해 가중치 파라미터 업데이트\n",
        "        optimizer.step()\n",
        "\n",
        "        # 스케줄러로 학습률 감소\n",
        "        scheduler.step()\n",
        "\n",
        "        # 그래디언트 초기화\n",
        "        model.zero_grad()\n",
        "\n",
        "    # 평균 로스 계산\n",
        "    avg_train_loss = total_loss / len(train_dataloader)            \n",
        "\n",
        "    print(\"\")\n",
        "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "    print(\"  Training epcoh took: {:}\".format(format_time(time.time() - t0)))\n",
        "        \n",
        "    # ========================================\n",
        "    #               Validation\n",
        "    # ========================================\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"Running Validation...\")\n",
        "\n",
        "    #시작 시간 설정\n",
        "    t0 = time.time()\n",
        "\n",
        "    # 평가모드로 변경\n",
        "    model.eval()\n",
        "\n",
        "    # 변수 초기화\n",
        "    eval_loss, eval_accuracy = 0, 0\n",
        "    nb_eval_steps, nb_eval_examples = 0, 0\n",
        "\n",
        "    # 데이터로더에서 배치만큼 반복하여 가져옴\n",
        "    for batch in tqdm(validation_dataloader, desc=\"[Validation]\"):\n",
        "        # 배치를 GPU에 넣음\n",
        "        batch = tuple(t.to(device) for t in batch)\n",
        "        \n",
        "        # 배치에서 데이터 추출\n",
        "        b_input_ids, b_input_mask, b_labels = batch\n",
        "        \n",
        "        # 그래디언트 계산 안함\n",
        "        with torch.no_grad():     \n",
        "            # Forward 수행\n",
        "            outputs = model(b_input_ids, \n",
        "                            token_type_ids=None, \n",
        "                            attention_mask=b_input_mask)\n",
        "        \n",
        "        # 로스 구함\n",
        "        logits = outputs[0]\n",
        "\n",
        "        # CPU로 데이터 이동\n",
        "        logits = logits.detach().cpu().numpy()\n",
        "        label_ids = b_labels.to('cpu').numpy()\n",
        "        \n",
        "        # 출력 로짓과 라벨을 비교하여 정확도 계산\n",
        "        tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n",
        "        eval_accuracy += tmp_eval_accuracy\n",
        "        nb_eval_steps += 1\n",
        "\n",
        "    print(\"  Accuracy: {0:.2f}\".format(eval_accuracy/nb_eval_steps))\n",
        "    print(\"  Validation took: {:}\".format(format_time(time.time() - t0)))\n",
        "    model.save_pretrained(\"/content/drive/MyDrive/SAI/p/lunab_model_21_07_14_\"+str(epoch_i))\n",
        "\n",
        "print(\"\")\n",
        "print(\"Training complete!\")"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "======== Epoch 1 / 3 ========\n",
            "Training...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "143ce0cf7b4446feb213b83115fc848f",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='[Train] ', max=8438.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "  Batch   500  of  8,438.    Elapsed: 0:03:15.\n",
            "  Batch 1,000  of  8,438.    Elapsed: 0:06:40.\n",
            "  Batch 1,500  of  8,438.    Elapsed: 0:10:05.\n",
            "  Batch 2,000  of  8,438.    Elapsed: 0:13:30.\n",
            "  Batch 2,500  of  8,438.    Elapsed: 0:16:54.\n",
            "  Batch 3,000  of  8,438.    Elapsed: 0:20:19.\n",
            "  Batch 3,500  of  8,438.    Elapsed: 0:23:44.\n",
            "  Batch 4,000  of  8,438.    Elapsed: 0:27:08.\n",
            "  Batch 4,500  of  8,438.    Elapsed: 0:30:33.\n",
            "  Batch 5,000  of  8,438.    Elapsed: 0:33:57.\n",
            "  Batch 5,500  of  8,438.    Elapsed: 0:37:22.\n",
            "  Batch 6,000  of  8,438.    Elapsed: 0:40:47.\n",
            "  Batch 6,500  of  8,438.    Elapsed: 0:44:11.\n",
            "  Batch 7,000  of  8,438.    Elapsed: 0:47:36.\n",
            "  Batch 7,500  of  8,438.    Elapsed: 0:51:00.\n",
            "  Batch 8,000  of  8,438.    Elapsed: 0:54:25.\n",
            "\n",
            "\n",
            "  Average training loss: 0.61\n",
            "  Training epcoh took: 0:57:24\n",
            "\n",
            "Running Validation...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f14000b4bd7a437caf8e46afae9d9f72",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='[Validation]', max=938.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Accuracy: 0.81\n",
            "  Validation took: 0:02:14\n",
            "\n",
            "======== Epoch 2 / 3 ========\n",
            "Training...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "17635bc0412645d09e6033bcce308eb8",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='[Train] ', max=8438.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "  Batch   500  of  8,438.    Elapsed: 0:03:25.\n",
            "  Batch 1,000  of  8,438.    Elapsed: 0:06:50.\n",
            "  Batch 1,500  of  8,438.    Elapsed: 0:10:14.\n",
            "  Batch 2,000  of  8,438.    Elapsed: 0:13:38.\n",
            "  Batch 2,500  of  8,438.    Elapsed: 0:17:03.\n",
            "  Batch 3,000  of  8,438.    Elapsed: 0:20:28.\n",
            "  Batch 3,500  of  8,438.    Elapsed: 0:23:53.\n",
            "  Batch 4,000  of  8,438.    Elapsed: 0:27:17.\n",
            "  Batch 4,500  of  8,438.    Elapsed: 0:30:42.\n",
            "  Batch 5,000  of  8,438.    Elapsed: 0:34:07.\n",
            "  Batch 5,500  of  8,438.    Elapsed: 0:37:32.\n",
            "  Batch 6,000  of  8,438.    Elapsed: 0:40:57.\n",
            "  Batch 6,500  of  8,438.    Elapsed: 0:44:21.\n",
            "  Batch 7,000  of  8,438.    Elapsed: 0:47:46.\n",
            "  Batch 7,500  of  8,438.    Elapsed: 0:51:11.\n",
            "  Batch 8,000  of  8,438.    Elapsed: 0:54:36.\n",
            "\n",
            "\n",
            "  Average training loss: 0.41\n",
            "  Training epcoh took: 0:57:35\n",
            "\n",
            "Running Validation...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4480a94f37d146539313abdc2816719e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='[Validation]', max=938.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Accuracy: 0.83\n",
            "  Validation took: 0:02:14\n",
            "\n",
            "======== Epoch 3 / 3 ========\n",
            "Training...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c965ba0807504066974ed399d6774715",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='[Train] ', max=8438.0, style=ProgressStyle(description_wi…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "  Batch   500  of  8,438.    Elapsed: 0:03:25.\n",
            "  Batch 1,000  of  8,438.    Elapsed: 0:06:49.\n",
            "  Batch 1,500  of  8,438.    Elapsed: 0:10:14.\n",
            "  Batch 2,000  of  8,438.    Elapsed: 0:13:38.\n",
            "  Batch 2,500  of  8,438.    Elapsed: 0:17:02.\n",
            "  Batch 3,000  of  8,438.    Elapsed: 0:20:27.\n",
            "  Batch 3,500  of  8,438.    Elapsed: 0:23:51.\n",
            "  Batch 4,000  of  8,438.    Elapsed: 0:27:16.\n",
            "  Batch 4,500  of  8,438.    Elapsed: 0:30:41.\n",
            "  Batch 5,000  of  8,438.    Elapsed: 0:34:05.\n",
            "  Batch 5,500  of  8,438.    Elapsed: 0:37:30.\n",
            "  Batch 6,000  of  8,438.    Elapsed: 0:40:54.\n",
            "  Batch 6,500  of  8,438.    Elapsed: 0:44:19.\n",
            "  Batch 7,000  of  8,438.    Elapsed: 0:47:43.\n",
            "  Batch 7,500  of  8,438.    Elapsed: 0:51:08.\n",
            "  Batch 8,000  of  8,438.    Elapsed: 0:54:33.\n",
            "\n",
            "\n",
            "  Average training loss: 0.28\n",
            "  Training epcoh took: 0:57:32\n",
            "\n",
            "Running Validation...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "822f2fe4446d44b3850f7e2a24770d0d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='[Validation]', max=938.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  Accuracy: 0.82\n",
            "  Validation took: 0:02:14\n",
            "\n",
            "Training complete!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZNNTl5USw9fm"
      },
      "source": [
        "def evaluate(e_model, e_tokenizer, text):\n",
        "    e_model.cuda()\n",
        "    e_model.eval()\n",
        "\n",
        "    m = {\n",
        "        0: 'A type',\n",
        "        1: 'B type',\n",
        "        2: 'C type',\n",
        "        3: 'D type'\n",
        "    }\n",
        "\n",
        "    sentence = text\n",
        "    sentences = [\"[CLS] \" + sentence + \" [SEP]\"]\n",
        "\n",
        "    tokenized_texts = [e_tokenizer.tokenize(sent) for sent in sentences]\n",
        "\n",
        "    input_ids = [e_tokenizer.convert_tokens_to_ids(x) for x in tokenized_texts]\n",
        "    input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
        "\n",
        "    attention_masks = []\n",
        "    for seq in input_ids:\n",
        "        seq_mask = [float(i>0) for i in seq]\n",
        "        attention_masks.append(seq_mask)\n",
        "\n",
        "    test_inputs = torch.tensor(input_ids).to(device)\n",
        "    test_masks = torch.tensor(attention_masks).to(device)\n",
        "\n",
        "    outputs = e_model(test_inputs, token_type_ids=None, attention_mask=test_masks)\n",
        "    \n",
        "    # 로스 구함\n",
        "    logits = outputs[0]\n",
        "\n",
        "    # CPU로 데이터 이동\n",
        "    logits = logits.detach().cpu().numpy()\n",
        "    print('this text predicted :', m[logits.argmax(1)[0]])"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-CnZbGDVDff1"
      },
      "source": [
        "evaluate(model, tokenizer, \"i feel the wind while looking at the night sky in summer.\")\n",
        "evaluate(model, tokenizer, \"it's like having a strong spice in your mouth.\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hMMTf1gLIT4j",
        "outputId": "56ad29c1-f172-4b2e-adbd-927fc5113a08"
      },
      "source": [
        "from google.colab import drive \n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3cdCJBfsJqTD",
        "outputId": "27ea8f77-24bc-4359-b8ed-60da69e00c49"
      },
      "source": [
        "from transformers import AutoModelForSequenceClassification\n",
        "from transformers import AutoTokenizer\n",
        "\n",
        "model.save_pretrained(\"/content/drive/MyDrive/SAI/p/lunab_model_21_06_30\")\n",
        "tokenizer.save_pretrained(\"/content/drive/MyDrive/SAI/p/lunab_tokenizer_21_06_30\")\n",
        "\n",
        "print('Done...')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Done...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YSGWEsYtOCki",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "65817028-e7df-44f4-bb97-d7a2bf31f7e4"
      },
      "source": [
        "load_model = AutoModelForSequenceClassification.from_pretrained(\"/content/drive/MyDrive/SAI/p/lunab_model_21_06_30\")\n",
        "load_tokenizer = AutoTokenizer.from_pretrained(\"/content/drive/MyDrive/SAI/p/lunab_tokenizer_21_06_30\")\n",
        "\n",
        "print('Load...')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Load...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XknhXFsQZycu",
        "outputId": "d48c616d-7dd2-4023-f25a-ef7a56e1acb1"
      },
      "source": [
        "evaluate(load_model, load_tokenizer, \"i feel the wind while looking at the night sky in summer.\")\n",
        "evaluate(load_model, load_tokenizer, \"it's like having a strong spice in your mouth.\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "this text predicted : A type\n",
            "this text predicted : C type\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zklf6m0zakPJ"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}