freeAdv_FinalProject.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "freeAdv_FinalProject.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "6edba301490942349a6f5fd8f559bccc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_47bbe763779a45a39434d9560c010225",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b73555ea4f0c449a877382f7fa9bf44e",
              "IPY_MODEL_88f127c849fc466194420a55333fecdd"
            ]
          }
        },
        "47bbe763779a45a39434d9560c010225": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b73555ea4f0c449a877382f7fa9bf44e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3337212847254535a2c38b3926d0c294",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 9912422,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 9912422,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d2726fc7f97546228e7e04df4a2bab64"
          }
        },
        "88f127c849fc466194420a55333fecdd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_48fd252b0767483c8b3c1ef92cad5034",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 9913344/? [00:38&lt;00:00, 254449.64it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ade1b8ace264429d84fa7425ff03ef18"
          }
        },
        "3337212847254535a2c38b3926d0c294": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d2726fc7f97546228e7e04df4a2bab64": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "48fd252b0767483c8b3c1ef92cad5034": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ade1b8ace264429d84fa7425ff03ef18": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "26c74e069471497eb6cff4048f151def": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_655a5df3bc764f9e8531f955e1bb2684",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b9de2b930f9c44b79a6ec8703793b871",
              "IPY_MODEL_2bbff307c7834bb4be5d203812be8a68"
            ]
          }
        },
        "655a5df3bc764f9e8531f955e1bb2684": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b9de2b930f9c44b79a6ec8703793b871": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_8206bff93d444650889d379ca3c17396",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 28881,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 28881,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_529ced219dda42cfa458b757e5871903"
          }
        },
        "2bbff307c7834bb4be5d203812be8a68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1f53a8d341f44b409abbd59f363f65e7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 29696/? [00:37&lt;00:00, 799.92it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cf495abac84a4fffbc631235a66c5dda"
          }
        },
        "8206bff93d444650889d379ca3c17396": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "529ced219dda42cfa458b757e5871903": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1f53a8d341f44b409abbd59f363f65e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cf495abac84a4fffbc631235a66c5dda": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "29591bcb11ca44869dd9f1b6d4538889": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4d708f8c51934a36834896da3a40544d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ca410f3cdcd9456f88f774f3192d0450",
              "IPY_MODEL_d953fda683284b9e9bc13e1afc7f63d4"
            ]
          }
        },
        "4d708f8c51934a36834896da3a40544d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ca410f3cdcd9456f88f774f3192d0450": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5ea0d151ffb74e878e5d9742bad53daa",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1648877,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1648877,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1340eecfcc214cce8d10ba324fc6aeb7"
          }
        },
        "d953fda683284b9e9bc13e1afc7f63d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7d5adbad95c24519a32f19f982a20686",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1649664/? [00:04&lt;00:00, 402796.97it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_afee9e6a438e4c00bf737cc826830ca3"
          }
        },
        "5ea0d151ffb74e878e5d9742bad53daa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1340eecfcc214cce8d10ba324fc6aeb7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7d5adbad95c24519a32f19f982a20686": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "afee9e6a438e4c00bf737cc826830ca3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "060f24b5c7fc48dc8d785e7faf7ff470": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2770eb3bfda44994a52598a0ef015ea5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c8e9744149294f919546c342eb3340c2",
              "IPY_MODEL_2517f826e7314014ad4191ea9ca1c6ff"
            ]
          }
        },
        "2770eb3bfda44994a52598a0ef015ea5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c8e9744149294f919546c342eb3340c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9509a62b0a4243c192f743ffcdba0209",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 4542,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 4542,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0646b35e7d9842a7a3711c7235830522"
          }
        },
        "2517f826e7314014ad4191ea9ca1c6ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a89a6c8d8fc34c71ba4926917d5b6a28",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 5120/? [00:02&lt;00:00, 1853.62it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_834bfa206dcb4c46948248440d4d5bbb"
          }
        },
        "9509a62b0a4243c192f743ffcdba0209": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0646b35e7d9842a7a3711c7235830522": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a89a6c8d8fc34c71ba4926917d5b6a28": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "834bfa206dcb4c46948248440d4d5bbb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ez8vbJ-O7ogr"
      },
      "source": [
        "# Game Theory and Machine Learning Final Project"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cSekwt0h7yj7"
      },
      "source": [
        "In this notebook, I tried to first produce the result of the \"Adversarial Training for Free\" paper for the MNIST dataset, by changing their own released code for imageNet dataset. Then I did few experiments for possible further improvements."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F4tuJBuyGzUb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "243b2c8e-810f-4f83-f716-9af769f4140d"
      },
      "source": [
        "!pwd"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Lh_EUIokFGPT",
        "outputId": "0ce476d6-36c5-410f-88ab-2a071f34c609"
      },
      "source": [
        "!git clone https://github.com/saba96/freeAdv.git"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "fatal: destination path 'freeAdv' already exists and is not an empty directory.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "71PzMTuaGNMi"
      },
      "source": [
        "!mv ./freeAdv/* ."
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iPEj6lHx8b31"
      },
      "source": [
        "## Adversarial Traning for Free for MNIST\n",
        "\n",
        "The following code is used to train models for different hyper-parameters. For training the saved models in this environment, I just set the hypyer-parameters few times and re-runned following cells.\n",
        "\n",
        "First, let's import what we will need."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BOxquwccs3uT"
      },
      "source": [
        "import argparse\n",
        "import logging\n",
        "import time\n",
        "\n",
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torchvision\n",
        "from torchvision import datasets, transforms\n",
        "from torch.utils.data import DataLoader, Dataset\n",
        "\n",
        "from torch.autograd import Variable"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y5TgBNqF7lHV"
      },
      "source": [
        "Now, we can prepare our data and define the architecture we want to use for training our model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OCA_PGedtM2Y",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 860,
          "referenced_widgets": [
            "6edba301490942349a6f5fd8f559bccc",
            "47bbe763779a45a39434d9560c010225",
            "b73555ea4f0c449a877382f7fa9bf44e",
            "88f127c849fc466194420a55333fecdd",
            "3337212847254535a2c38b3926d0c294",
            "d2726fc7f97546228e7e04df4a2bab64",
            "48fd252b0767483c8b3c1ef92cad5034",
            "ade1b8ace264429d84fa7425ff03ef18",
            "26c74e069471497eb6cff4048f151def",
            "655a5df3bc764f9e8531f955e1bb2684",
            "b9de2b930f9c44b79a6ec8703793b871",
            "2bbff307c7834bb4be5d203812be8a68",
            "8206bff93d444650889d379ca3c17396",
            "529ced219dda42cfa458b757e5871903",
            "1f53a8d341f44b409abbd59f363f65e7",
            "cf495abac84a4fffbc631235a66c5dda",
            "29591bcb11ca44869dd9f1b6d4538889",
            "4d708f8c51934a36834896da3a40544d",
            "ca410f3cdcd9456f88f774f3192d0450",
            "d953fda683284b9e9bc13e1afc7f63d4",
            "5ea0d151ffb74e878e5d9742bad53daa",
            "1340eecfcc214cce8d10ba324fc6aeb7",
            "7d5adbad95c24519a32f19f982a20686",
            "afee9e6a438e4c00bf737cc826830ca3",
            "060f24b5c7fc48dc8d785e7faf7ff470",
            "2770eb3bfda44994a52598a0ef015ea5",
            "c8e9744149294f919546c342eb3340c2",
            "2517f826e7314014ad4191ea9ca1c6ff",
            "9509a62b0a4243c192f743ffcdba0209",
            "0646b35e7d9842a7a3711c7235830522",
            "a89a6c8d8fc34c71ba4926917d5b6a28",
            "834bfa206dcb4c46948248440d4d5bbb"
          ]
        },
        "outputId": "e82e63f2-2b06-4ca1-f8ff-c54a69afeb20"
      },
      "source": [
        "# Compose is used for chaining transfotmation, but here we only have one tranformation\n",
        "\"\"\"ToTensor: Converts a PIL Image or numpy.ndarray (H x W x C) in the range [0, 255] to \n",
        "a torch.FloatTensor of shape (C x H x W) in the range [0.0, 1.0]\"\"\"\n",
        "T = torchvision.transforms.Compose([torchvision.transforms.ToTensor()])\n",
        "#!mkdir data\n",
        "mnist_data = torchvision.datasets.MNIST('./data', train=True, transform=T, download=True)\n",
        "class Flatten(nn.Module):\n",
        "    def forward(self, x):\n",
        "        return x.view(x.size(0), -1)\n",
        "\n",
        "def mnist_net():\n",
        "    model = nn.Sequential(\n",
        "        nn.Conv2d(1, 16, 4, stride=2, padding=1),\n",
        "        nn.ReLU(),\n",
        "        nn.Conv2d(16, 32, 4, stride=2, padding=1),\n",
        "        nn.ReLU(),\n",
        "        Flatten(),\n",
        "        nn.Linear(32*7*7,100),\n",
        "        nn.ReLU(),\n",
        "        nn.Linear(100, 10)\n",
        "    )\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n",
            "Failed to download (trying next):\n",
            "HTTP Error 503: Service Unavailable\n",
            "\n",
            "Downloading https://ossci-datasets.s3.amazonaws.com/mnist/train-images-idx3-ubyte.gz\n",
            "Downloading https://ossci-datasets.s3.amazonaws.com/mnist/train-images-idx3-ubyte.gz to ./data/MNIST/raw/train-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6edba301490942349a6f5fd8f559bccc",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=9912422.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extracting ./data/MNIST/raw/train-images-idx3-ubyte.gz to ./data/MNIST/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n",
            "Failed to download (trying next):\n",
            "HTTP Error 503: Service Unavailable\n",
            "\n",
            "Downloading https://ossci-datasets.s3.amazonaws.com/mnist/train-labels-idx1-ubyte.gz\n",
            "Downloading https://ossci-datasets.s3.amazonaws.com/mnist/train-labels-idx1-ubyte.gz to ./data/MNIST/raw/train-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "26c74e069471497eb6cff4048f151def",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=28881.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extracting ./data/MNIST/raw/train-labels-idx1-ubyte.gz to ./data/MNIST/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n",
            "Failed to download (trying next):\n",
            "HTTP Error 503: Service Unavailable\n",
            "\n",
            "Downloading https://ossci-datasets.s3.amazonaws.com/mnist/t10k-images-idx3-ubyte.gz\n",
            "Downloading https://ossci-datasets.s3.amazonaws.com/mnist/t10k-images-idx3-ubyte.gz to ./data/MNIST/raw/t10k-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "29591bcb11ca44869dd9f1b6d4538889",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=1648877.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extracting ./data/MNIST/raw/t10k-images-idx3-ubyte.gz to ./data/MNIST/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n",
            "Failed to download (trying next):\n",
            "HTTP Error 503: Service Unavailable\n",
            "\n",
            "Downloading https://ossci-datasets.s3.amazonaws.com/mnist/t10k-labels-idx1-ubyte.gz\n",
            "Downloading https://ossci-datasets.s3.amazonaws.com/mnist/t10k-labels-idx1-ubyte.gz to ./data/MNIST/raw/t10k-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "060f24b5c7fc48dc8d785e7faf7ff470",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=4542.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Extracting ./data/MNIST/raw/t10k-labels-idx1-ubyte.gz to ./data/MNIST/raw\n",
            "\n",
            "Processing...\n",
            "Done!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torchvision/datasets/mnist.py:502: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at  /pytorch/torch/csrc/utils/tensor_numpy.cpp:143.)\n",
            "  return torch.from_numpy(parsed.astype(m[2], copy=False)).view(*s)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w2jzs9UatjAd"
      },
      "source": [
        "model = mnist_net().cuda()\n",
        "model.train()\n",
        "\n",
        "lr=.0001\n",
        "opt = torch.optim.Adam(model.parameters(), lr=.0001)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5aUP-O4gtmZ0"
      },
      "source": [
        "batch_size=100\n",
        "total_epochs=1000\n",
        "replays= 40\n",
        "epochs = total_epochs / replays\n",
        "attack='fgsm'\n",
        "epsilon=0.3\n",
        "alpha=0.01\n",
        "attack_iters=40\n",
        "lr_max=5e-3\n",
        "lr_type='flat'\n",
        "fname= '/content/mnist_model'\n",
        "seed=0\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F6uClKI6tpaM"
      },
      "source": [
        "# It automatically converts NumPy arrays and Python numerical values into PyTorch Tensors.\n",
        "# It outputs a dictionary with the same set of keys but batched Tensors as values \n",
        "train_loader = torch.utils.data.DataLoader(mnist_data, batch_size, shuffle=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-dqAxOIctvS2",
        "outputId": "c9d989c0-e2a3-48d3-e941-11a1bcebeb66"
      },
      "source": [
        "criterion = nn.CrossEntropyLoss()\n",
        "logger = logging.getLogger(__name__)\n",
        "logging.basicConfig(\n",
        "    format='[%(asctime)s] - %(message)s',\n",
        "    datefmt='%Y/%m/%d %H:%M:%S',\n",
        "    level=logging.DEBUG)\n",
        "\n",
        "logger.info('Epoch \\t Time \\t LR \\t \\t Train Loss \\t Train Acc')\n",
        "global_noise_data = torch.zeros([batch_size, 1, 28, 28]).cuda()\n",
        "for epoch in range(int(epochs)):\n",
        "    start_time = time.time()\n",
        "    train_loss = 0\n",
        "    train_acc = 0\n",
        "    train_n = 0\n",
        "    # switch to the train mode\n",
        "    model.train()\n",
        "    for i, (X, y) in enumerate(train_loader):\n",
        "        input, target = X.cuda(), y.cuda()\n",
        "        for j in range(replays):\n",
        "\n",
        "            # Ascend on the global noise\n",
        "            noise_batch = Variable(global_noise_data[0:input.size(0)], requires_grad=True).cuda()\n",
        "            in1 = input + noise_batch\n",
        "            in1.clamp_(0, 1.0)\n",
        "            output = model(in1)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # compute gradient\n",
        "            opt.zero_grad()\n",
        "            loss.backward()\n",
        "            \n",
        "            # Update the noise for the next iteration\n",
        "            pert = alpha * torch.sign(noise_batch.grad)\n",
        "            global_noise_data[0:input.size(0)] += pert.data\n",
        "            global_noise_data.clamp_(-epsilon, epsilon)\n",
        "          \n",
        "            #finally do SGD step\n",
        "            opt.step()\n",
        "        train_loss += loss.item() * target.size(0)\n",
        "        train_acc += (output.max(1)[1] == target).sum().item()\n",
        "        train_n += target.size(0)\n",
        "\n",
        "    train_time = time.time()\n",
        "    logger.info('%d \\t %.1f \\t %.4f \\t %.4f \\t %.4f',\n",
        "        epoch, train_time - start_time, lr, train_loss/train_n, train_acc/train_n)\n",
        "    model_name= '/content/mnist_model'+ str(total_epochs) + '_' + str(replays)\n",
        "    torch.save(model.state_dict(),  model_name)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[2021/04/19 14:18:47] - Epoch \t Time \t LR \t \t Train Loss \t Train Acc\n",
            "[2021/04/19 14:23:09] - 0 \t 261.6 \t 0.0001 \t 0.0409 \t 0.9930\n",
            "[2021/04/19 14:27:31] - 1 \t 261.9 \t 0.0001 \t 0.0048 \t 1.0000\n",
            "[2021/04/19 14:31:52] - 2 \t 261.0 \t 0.0001 \t 0.0026 \t 1.0000\n",
            "[2021/04/19 14:36:13] - 3 \t 260.8 \t 0.0001 \t 0.0017 \t 1.0000\n",
            "[2021/04/19 14:40:33] - 4 \t 260.7 \t 0.0001 \t 0.0013 \t 1.0000\n",
            "[2021/04/19 14:44:54] - 5 \t 260.7 \t 0.0001 \t 0.0010 \t 1.0000\n",
            "[2021/04/19 14:49:15] - 6 \t 260.9 \t 0.0001 \t 0.0009 \t 1.0000\n",
            "[2021/04/19 14:53:36] - 7 \t 260.7 \t 0.0001 \t 0.0007 \t 1.0000\n",
            "[2021/04/19 14:57:57] - 8 \t 261.0 \t 0.0001 \t 0.0006 \t 1.0000\n",
            "[2021/04/19 15:02:17] - 9 \t 260.1 \t 0.0001 \t 0.0004 \t 1.0000\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yav1LmZW-5sC"
      },
      "source": [
        "### Testing saved models\n",
        "\n",
        "I trained 3 models, using 1000 total epochs with three different number of replays(10, 40, 100). Then I test them under the L-Infinity PGD attacks. I wrote my benchmark using advertorch. First we need to install advertorch."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SGKFPvb2AbfQ",
        "outputId": "0318b801-399e-4ecb-afd3-39702ba1be2b"
      },
      "source": [
        "!pip install advertorch"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting advertorch\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/2f/b1/84602596294c32f49396bac9c36f1f72b00577bbcb26ebbe776e64791cac/advertorch-0.2.3.tar.gz (5.7MB)\n",
            "\u001b[K     |████████████████████████████████| 5.7MB 6.2MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: advertorch\n",
            "  Building wheel for advertorch (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for advertorch: filename=advertorch-0.2.3-cp37-none-any.whl size=5696220 sha256=3437b2697d3aecd9fa1e1cacebc5fa74416b19e1e9f014abbd41831a979b8ad4\n",
            "  Stored in directory: /root/.cache/pip/wheels/9b/53/6e/6b2509701b0da68443fa3d4499733f5455d6d583afa8c46676\n",
            "Successfully built advertorch\n",
            "Installing collected packages: advertorch\n",
            "Successfully installed advertorch-0.2.3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_wlE6C56gDJu"
      },
      "source": [
        "Now, let's import what we will need."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "60L48UPhBV2Z"
      },
      "source": [
        "import pandas as pd\n",
        "# importing advertorch\n",
        "from advertorch.attacks import LinfPGDAttack\n",
        "from advertorch.attacks.utils import attack_whole_dataset\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C2MN6O29gIgg"
      },
      "source": [
        "This bench mark is written using [advertorch](https://github.com/BorealisAI/advertorch). We first attack 3 models that we trained in the previous section(using 1000 total epochs with three different number of replays(10, 40, 100)). Then we visualize the result to understand it easier. Then we pick the best replays number to train our model with 100k total epochs in the next section. Please note that **m** refers to number of replays we used in the training mode for free adversarial training. Also, PGD-K here refers to L-Infinity PGD attack using K steps."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "Mgle0jwjAa_D",
        "outputId": "c4d46186-b88a-4736-b492-8de6c8a5b6a3"
      },
      "source": [
        "pgd_report = []\n",
        "\n",
        "# MNIST Test dataset and dataloader declaration\n",
        "loader = torch.utils.data.DataLoader(\n",
        "    datasets.MNIST('./data', train=False, download=False, transform=transforms.Compose([\n",
        "            transforms.ToTensor(),\n",
        "            ])),\n",
        "        batch_size=200, shuffle=True)\n",
        "\n",
        "n_replays_list = [10, 40, 100]\n",
        "attack_n_itrs_list = [10, 40, 100]\n",
        "epsilons = [.1, .2, .3]\n",
        "total_epochs = 1000\n",
        "for attack_n_itrs in attack_n_itrs_list:\n",
        "    fig, axis = plt.subplots()\n",
        "    title = 'Accuracy on PGD-' + str(attack_n_itrs)\n",
        "    axis.set_title(title, fontsize=12)\n",
        "    for n_replay in n_replays_list:\n",
        "        acc_list = []\n",
        "        for epsilon in epsilons: \n",
        "            # Define what device we are using\n",
        "            device = torch.device('cuda')\n",
        "            # Load model and put it in evaluation mode\n",
        "            path = '/content/mnist_model-' + str(total_epochs) + '_'+ str(n_replay)\n",
        "            model = mnist_net().cuda()\n",
        "            model.load_state_dict(torch.load(path))\n",
        "            model.eval()\n",
        "\n",
        "            adversary = LinfPGDAttack(\n",
        "                model, loss_fn=nn.CrossEntropyLoss(reduction=\"sum\"), eps=epsilon,\n",
        "                nb_iter=attack_n_itrs, eps_iter=0.01, rand_init=True, clip_min=0.0, clip_max=1.0,\n",
        "                targeted=False)\n",
        "\n",
        "            _, label, pred, advpred = attack_whole_dataset(\n",
        "                adversary, loader, device=\"cuda\")\n",
        "            \n",
        "            clean_acc = 100. * (label == pred).sum().item() / len(label)\n",
        "            adv_acc = 100. * (label == advpred).sum().item() / len(label)\n",
        "            acc_list.append(adv_acc)\n",
        "            pgd_report.append([total_epochs, n_replay, 'flat', 'LinfPGD', attack_n_itrs, epsilon, adv_acc, clean_acc])\n",
        "\n",
        "            print(\"Accuracy: {:.2f}%, Robust Accuracy: {:.2f}%\".format(\n",
        "                100. * (label == pred).sum().item() / len(label),\n",
        "                100. * (label == advpred).sum().item() / len(label)))\n",
        "        plt.plot(range(len(epsilons)), acc_list, marker='p', label='m=' + str(n_replay))\n",
        "    \n",
        "    axis.set_xlabel('epsilon')\n",
        "    axis.set_ylabel('Accuracy %')\n",
        "    axis.set_ylim(bottom=0)\n",
        "    axis.set_ylim(top=100)\n",
        "    plt.xticks(range(len(epsilons)),epsilons)\n",
        "    axis.legend()\n",
        "    plt.savefig('LinfPGD-' + str(attack_n_itrs))\n",
        "    plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy: 99.02%, Robust Accuracy: 95.35%\n",
            "Accuracy: 99.02%, Robust Accuracy: 93.66%\n",
            "Accuracy: 99.02%, Robust Accuracy: 91.40%\n",
            "Accuracy: 99.10%, Robust Accuracy: 97.54%\n",
            "Accuracy: 99.10%, Robust Accuracy: 97.27%\n",
            "Accuracy: 99.10%, Robust Accuracy: 97.11%\n",
            "Accuracy: 98.79%, Robust Accuracy: 96.97%\n",
            "Accuracy: 98.79%, Robust Accuracy: 96.48%\n",
            "Accuracy: 98.79%, Robust Accuracy: 95.99%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZxcZZ3v8c+3ujvpJCQEMAkhCRBJEAQ1SNhcmGjMFRECXheCIkEZuS4zLC5DHPXCOI4kDhdUFsc4CFFBEBQSUBgBiQooY4KM7CRAAg0hCVlIyNrL7/5xTp2u7q7urnR3dXW6v+/Xq19V5znL81SlUt/znOUpRQRmZmYAuUo3wMzM+g6HgpmZZRwKZmaWcSiYmVnGoWBmZhmHgpmZZRwKZmaWcShYr5K0WNIGSYMr3Za+TtLFkuolvS5po6QHJR1XMH+spB9Jejld5jlJ10k6JJ1/oKRI570uabWkOyTN6KTewyX9l6RXJbW5kUnS3pJulbRF0kpJH+/5V2+V4lCwXiPpQODdQAAze7nu6t6srwfdFBF7AKOA+4FfKbEP8CAwlOQ9HQ68Hfg90PpLf2S6jbcBdwO3SjqrgzrrgV8AZ7cz/ypgJzAG+ATwA0mHdeG1WR/kULDedCbwZ+A6YHbhDEkTJP1K0lpJ6yRdWTDvM5KelLRZ0hOS3p6Wh6RJBctdJ+lb6fNpkuokXSjpFeBaSXule8pr097KHZLGF6y/t6Rr0z3vDZJuS8sfk3RywXI16V70EcVeZNre5ZLWS1okab+CeSHps5KWpXv/V0lSZ29cRNQDC4B9gX2AC4BNwCcj4tlIbIyIayPiina28UpEfA+4GJgnqej//4h4OiKuAR4v8tqGAR8GvhERr0fE/cAi4JOdvQbbPTgUrDedCVyf/r1f0hgASVXAHcBK4EBgHHBjOu+jJF9iZwIjSHoY60qsb19gb+AA4BySz/u16fT+wDbgyoLlf0qy530YMBq4PC3/CXBGwXInAqsi4q+tK5T0XuAS4GPA2PQ13dhqsZOAo4C3psu9v7MXkh5uOwt4MSJeBd4H3BoRTZ2tW8SvSF7fm7qw7sFAQ0Q8U1D2PyTvmfUDDgXrFZLeRfJl/IuIWAo8C+SPRR8N7Ad8JSK2RMT2dA8U4O+B70TEX9K94eURsbLEapuAiyJiR0Rsi4h1EfHLiNgaEZuBfwP+Lm3fWOADwGcjYkNE1EfE79Pt/Aw4UdKIdPqTJAFSzCeAH0fEwxGxA/gqcFx66CxvbrpX/wJwHzClg9fwMUkbgReBI4EPpeVvAF7JLyRpZtrz2Czpt528Ly+nj3t3slwxe5D0UAq9RnL4yvoBh4L1ltnAb9O9XIAbaD6ENAFYGRENRdabQBIgXbE2IrbnJyQNlfTD9OToJuAPwMi0pzIBWB8RG1pvJCJeBh4APixpJEl4XN9OnfuR9A7y675O0rMZV7DMKwXPt5J80bbnFxExMiJGR8R700Al3ebYgnoWRcRIksNKgzrYHgVtWS/pEwUnou/sZD2A10l6bIVGAJtLWNd2A7vryTfbjUgaQnKYpCo9vg8wmOQL+W0ke8H7S6ouEgwvAge1s+mtJId78vYF6gqmW1858yWSQybHRMQrkqYAfwWU1rO3pJERsbFIXQtIei3VwJ8i4qV22vQySY8IyI7B7wO0t3xX3QucKulfunAI6UPAGuDpiHiS9gOumGeAakmTI2JZWvY2ipx/sN2TewrWG04FGoE3kxwqmQIcCvyR5FzBfwOrgLmShkmqlfTOdN3/BL4s6cj0qptJkvJfuo8AH5dUJekE0kNBHRhOch5ho6S9gYvyMyJiFXAncHV6QrpG0vEF695GcnXPeSTnGNrzc+BTkqak5wG+DTwUESs6aduuugzYC/ippIPS92Y4HRyKkjRG0j+QvO6vthcm6bZqSXsc6b/HYICI2EJyTuKb6b/VO4FTaP9wmu1mHArWG2YD10bEC+kVMK9ExCskJ3k/QbKnfjIwCXiBZG//NICIuJnk2P8NJIcobqP5WPh56Xob0+3c1kk7vgsMAV4luQrqrlbzP0lyOeZTJHvS5+dnRMQ24JfARJIvxaIi4h7gG+myq0h6ObM6adcuSw/DHQtsJ7lUdTNJSA4HPtdq8Y2StgCPkpwk/2hE/LiDzR9AEp75vf9twNMF8z9P8j6uIQnBz0WEewr9hPwjO2alkfR/gYMj4oxOFzbbTfmcglkJ0sNNZ+Pr8a2fK9vhI0k/lrRG0mMFZXtLuju9ceduSXul5ZL0/fSGn7/lb04y6wskfYbkRPSdEfGHSrfHrJzKeU7hOuCEVmVzgHsjYjLJ1RNz0vIPAJPTv3OAH5SxXWa7JCJ+FBHDIuKzlW6LWbmVLRTSPar1rYpPIbm0j/Tx1ILyn6Q3J/2Z5FLFsZiZWa/q7XMKY9JL/yC5gWdM+nwcSfc8ry4tW0Urks4h6U0wbNiwIw855JDytdbMrB9aunTpqxExqti8ip1ojohQkWF5S1hvPjAfYOrUqbFkyZIeb5uZWX8mqd2hYnr7PoXV+cNC6eOatPwlkmEG8sbT83eAmplZJ3o7FBbRPN7NbGBhQfmZ6VVIxwKvFRxm6llNjfDA92HeRHjwimTarKf482W7uXJekvpz4E/Am5SMa382MBeYIWkZydC/c9PFfwM8BywHfkRyx2TPW/csjfOP59oll/OuUUO47i+X0fjDd8PqJ6BhBzTshMZ6aGxI/jM3NYFv7rNSrXsW5v8dLL4Etq2H+74N86cl5WY9pcw7Hrv1Hc27ek5h5f+bxJeGV/FCTTXbcjmGNDVxQH0Dl655lQMaig3Q2ZpA6uJjJ+tD97dR9vVLbWc51y/yOrq8zR5uyx8upbF+Gz8ZsQfXjBzB32/cxCc3vU5VzVB4zz8X1Efbdree1/p3d4rO62QbJW2/r9VNkXnleG19qe5d2P7GF+DXX4QNK6FhG9QMhX0mwUevg33aGzeyLUlLI2Jq0XkDKRSOv+5tvEYjTYX/GBFUIQ6rHUUOyCHy/xz55zkp61LlSJ63mN9qOl/Wen5zOSiEFOQChMiJ9DmttgkKyCl9pLl7lwvIKbLybN2IVtP5+ZFtT0SL8vx6SZ355SJpW7Sdr7SOHAGRrtNiuXy78q+xcDrS+pPlk21FwXrRXEdQsH47y6X/jhAdPNLJ/E4eS7CyupovjX5DN3Y6zDrXCPxkxPDmHY/NW6gasjf8U+k90o5CYUANczFpjwn8ZcvKloUSe1TVMmzvSQRBRNBEE03RREMEQdAUTUl5NNFE8jxf3pQONJl/Xrh84TKF00FaR1NTc52t1s3Xky+viFY7L5VVsOdURE5JBEsiRy6ZltLATZ7nlCNH8fJsOn3Mpb9UmX+e364AKXnWsg7x6NpHaYjGbM9uWy7HU4NqOHXCeI4be0y2XHMdyXJJe5Vtt00dkNavFu1obm8yD6Xbyi8nJTsAaV3pO4SU37lJ6myxrfR50q6C9qbr5Z/ny1W4nEQu2i6Xbaug7uZtKtvRSt7v5nbmd4QKX2PyKSjYiVIu2cGQUKTlFNaZvndB27rTeS2CP9tJjpbPS5pXZLkW093f/so/fZcv1WzJdjyu3mtPfr3HMC4ddGDzeO3dNKBC4d0Hnsbjj85la675VMrQpibOPuxcPjW1bw9p0zqU8mHRJqwKQq3NchQJqHbCrk1QtReCBQHWXgi22+ZWbStaXqQNhW0maP/1thPi7bW5u69rSM1QNte/3vIfTmJwdS3rd25qWUeJ70nRf6+0vMPXTvNrso4V7iy02LlIy1Hny6hF2BfsROTDsr0dlXbWbb1Mvs4HhjdR31TTYsfjmUE1fLJmIz01/sqACoUr7tqTnfsNRWQ/xsWWGMrlvxnOseM2UVOVY1BVjppqpY/pdFWOqlxld5clUaUqqqiqaDusfbc/ezvf+vO32NqwNSsbWj2Urx37dU4+6OSKtKm9QOlub7fUnZP2esGlhlqbNrTTtpLaX6SdXW1/hzsaJQR6i52egmUamhpa7AS13lmoztVQHy1PLDdJTN67527iHVChcPDoUfz5mYuLzvvA9/7Y4bo5waDqXHNwpOGRn87Pq6lKygZn0+k6BcvWZNtRwfzmQGpTVrDd5uWSskEFy1bnkr0Sq4xpE6ZxyUOXtCirUhXTJkyrTIOgxV6n7f7a2/E49eCP9FgdAyoUTjtqAo/WvcaWnc1JW1ud44xjD2DqgXuxo6GJ+sagvrGJ+sYmdqbTyWNalj7WNwQ789PZ/GTZzfUNrG9sWda8XlrWWJ6u/aB8gBQE2KDq5lBpGTZpWTo9qFXQFS5fU6WC4MtvQ22Cb1BVVZttFNY/qCpHrsK9rnIZPmg4f5z1ANfc/xxXL36Wz087iLPf9caK9zKt/+iNHY8BdfXRpu31vGvu79i0vflKkBG11dw/572MqK0pRxPbFREtAygNkPo0QHYWBE0+oLJgKQylFoEVWfDszIKtZcjtbOxgu+myOwvKy/HxqMop6+UMatGjKtLLSsNnUOueV0HPqU3wtO7RpSE5uKCXVqz+wlDsSq/r+Ve38IXrH+b5V7ewrb6RITVVvHHUMK78+NuZ+IZhPf9GmnWRL0m1Lmtsag6KLGzSQGobNpH1mpoDq1XwNQQ7Gxvb9sAKelCFwbejRU8sWWZHi+mkrKdJFARLc48oC6zqtmX3L1vLzlZtEVBbU8UX3nMQtTVVDK7OMbi6isE1hY+5lvPy0+m8QVU5Hxa0HuVLUq3LqnKiKldFbU3fPcGd73UVHsrLh9TOwumCkGodLDsLemnN24iCwGo/+LbsaKC+MajO5djZ2PIkYADb6hu59LfPdPn1SbQNjOpcFiy1NUXmVecYXFNFbfpYWJbfVm1HwVSTo7a6ipoqn6caaBwKttuTlJzPqM7B4Mq149a/1vH1Wx9rcc5q2KAq/vXUwznprfuxo6GR7fVN7GhoZEdDEzvqm9je0MiOgrLt9em8hiZ25J/XF5lXuK36JjZtayiY37Ke7hwMyAdS0Z5MicE0uFUw1ZYYTA6kynAomPWQ6YeO4aKFj7coq8qJ9715THJivjrH8NrebVO+F9Vu+NS3DZEdrYJpe6uywvDZUd/Ea9vq0+203Nb2+u5dTJETLQKj3fApEkytA6l1j6lNr6jVvL58JV9jU5T1YgafUzCzsohIDum1Hz5te0pFw6eEnlLrsOrJQKrdhWAa3CqYaksJppqW2+ookHrqYgafUzCzXicp/YKsggr0kAoDqUXvpkhPqUUwtQqk7UXmbatvZOO2na2CrecCqdj5ncE1OZ54eRMNTc078tvqG3ly1SY+8oMHWfqNGd192wCHgpn1Q30lkLbXtw2Mwp5S60BqfU5pe6t5QwdVtbikHqAp4OAxw3us/Q4FM7MeVBhIPX3/U3sXM3zsqPE9VofvfTcz201MP3RMm5PKVTkx/dAxPVaHewpmZruJEbU1/O3i95e1DvcUzMws41AwM7OMQ8HMzDIOBTMzyzgUzMws41AwM7OMQ8HMzDIOBTMzyzgUzMws41AwM7OMQ8HMzDIOBTMzyzgUzMws41AwM7OMQ8HMzDIOBTMzyzgUzMws41AwM7NMRUJB0gWSHpf0mKSfS6qVNFHSQ5KWS7pJ0qBKtM3MbCDr9VCQNA44F5gaEYcDVcAsYB5weURMAjYAZ/d228zMBrpKHT6qBoZIqgaGAquA9wK3pPMXAKdWqG1mZgNWr4dCRLwEXAq8QBIGrwFLgY0R0ZAuVgeMK7a+pHMkLZG0ZO3atb3RZDOzAaMSh4/2Ak4BJgL7AcOAE0pdPyLmR8TUiJg6atSoMrXSzGxgqsTho/cBz0fE2oioB34FvBMYmR5OAhgPvFSBtpmZDWiVCIUXgGMlDZUkYDrwBHAf8JF0mdnAwgq0zcxsQKvEOYWHSE4oPww8mrZhPnAh8EVJy4F9gGt6u21mZgNddeeL9LyIuAi4qFXxc8DRFWiOmZmlfEezmZllHApmZpZxKJiZWcahYGZmGYeCmZllHApmZpZxKJiZWcahYGZmGYeCmZllHApmZpZxKJiZWcahYGZmGYeCmZllHApmZpZxKJiZWcahYGZmGYeCmZllHApmZpZxKJiZWcahYGZmGYeCmZllHApmZpZxKJiZWcahYGZmGYeCmZllHApmZpZxKJiZWcahYGZmGYeCmZllHApmZpZxKJiZWcahYGZmmZJDQdIkST+T9EtJx5WzUWZmVhnV7c2QVBsR2wuK/hX4p/T57cCUcjbMzMx6X0c9hdslnVkwXQ8cCBwANHanUkkjJd0i6SlJT0o6TtLeku6WtCx93Ks7dZiZ2a7rKBROAEZIukvS8cCXgfcDHwI+0c16vwfcFRGHAG8DngTmAPdGxGTg3nTazMx6kSKi4wWkPYFvAOOAr0fEs92qMNneI8Abo6BySU8D0yJilaSxwOKIeFNH25o6dWosWbKkO80xMxtwJC2NiKnF5nV0TuEY4CvATuDbwDbg3yS9BPxrRGzsYnsmAmuBayW9DVgKnAeMiYhV6TKvAGPaadc5wDkA+++/fxebYGZmxXR0+OiHwLnAxcAPI+LZiJgFLAJu6kad1cDbgR9ExBHAFlodKkp7EEW7MBExPyKmRsTUUaNGdaMZZmbWWkeh0EDzieWd+cKI+H1EvL8bddYBdRHxUDp9C0lIrE4PG5E+rulGHWZm1gUdhcLHgQ8D7wXO7GC5XRIRrwAvSsqfL5gOPEHSA5mdls0GFvZUnWZmVpp2zylExDPAl8pU7z8C10saBDwHfIokoH4h6WxgJfCxMtVtZmbtaDcUyikiHgGKnfme3tttMTOzZh77yMzMMp2GgqSTJTk8zMwGgFK+7E8Dlkn6jqRDyt0gMzOrnE5DISLOAI4AngWuk/QnSedIGl721pmZWa8q6bBQRGwiuZ/gRmAsyfhHD0v6xzK2zczMelkp5xRmSroVWAzUAEdHxAdIBrIr1yWrZmZWAaVckvph4PKI+ENhYURsTe8pMDOzfqKUULgYyA9Uh6QhJIPXrYiIe8vVMDMz632lnFO4GWgqmG5My8zMrJ8pJRSqI6JwQLydwKDyNcnMzCqllFBYK2lmfkLSKcCr5WuSmZlVSinnFD5LMnjdlYCAF+nBUVPNzKzv6DQU0p/fPFbSHun062VvlZmZVURJo6RK+iBwGFArCYCI+GYZ22VmZhVQys1r/0Ey/tE/khw++ijJr7GZmVk/U8qJ5ndExJnAhoj4F+A44ODyNsvMzCqhlFDYnj5ulbQfUE8y/pGZmfUzpZxTuF3SSODfgYeBAH5U1laZmVlFdBgK6Y/r3BsRG4FfSroDqI2I13qldWZm1qs6PHwUEU3AVQXTOxwIZmb9VynnFO6V9GHlr0U1M7N+q5RQ+D8kA+DtkLRJ0mZJm8rcLjMzq4BS7mj2z26amQ0QnYaCpOOLlbf+0R0zM9v9lXJJ6lcKntcCRwNLgfeWpUVmZlYxpRw+OrlwWtIE4Ltla5GZmVVMKSeaW6sDDu3phpiZWeWVck7hCpK7mCEJkSkkdzabmVk/U8o5hSUFzxuAn0fEA2Vqj5mZVVApoXALsD0iGgEkVUkaGhFby9s0MzPrbSXd0QwMKZgeAtxTnuaYmVkllRIKtYU/wZk+H1q+JpmZWaWUEgpbJL09PyHpSGBb+ZpkZmaVUso5hfOBmyW9TPJznPuS/DynmZn1M6XcvPYXSYcAb0qLno6I+vI2y8zMKqHTw0eSvgAMi4jHIuIxYA9Jn+9uxelVTH9Nf7gHSRMlPSRpuaSbJA3qbh1mZrZrSjmn8Jn0l9cAiIgNwGd6oO7zgCcLpucBl0fEJGADcHYP1GFmZruglFCoKvyBHUlVQLf24iWNBz4I/Gc6LZIB9m5JF1kAnNqdOszMbNeVEgp3ATdJmi5pOvDztKw7vgv8E9CUTu8DbIyIhnS6DhhXbEVJ50haImnJ2rVru9kMMzMrVEooXAj8Dvhc+ncvLYfT3iWSTgLWRMTSrqwfEfMjYmpETB01alRXm2FmZkWUcvVRE/Af6R+S3g1cAXyhi3W+E5gp6USS32cYAXwPGCmpOu0tjAde6uL2zcysi0oaOlvSEZK+I2kF8E3gqa5WGBFfjYjxEXEgMAv4XUR8ArgP+Ei62GxgYVfrMDOzrmm3pyDpYOD09O9V4CZAEfGeMrXlQuBGSd8C/gpcU6Z6zMysHR0dPnoK+CNwUkQsB5B0QU9WHhGLgcXp8+dIfurTzMwqpKPDR/8bWAXcJ+lH6ZVH6mB5MzPbzbUbChFxW0TMAg4hOd5/PjBa0g8k/a/eaqCZmfWeTk80R8SWiLghIk4muSroryTH/83MrJ8p6eqjvIjYkN4nML1cDTIzs8rZpVAwM7P+zaFgZmYZh4KZmWUcCmZmlnEomJlZxqFgZmYZh4KZmWUcCmZmlnEomJlZxqFgZmYZh4KZmWUcCmZmlnEomJlZxqFgZmYZh4KZmWUcCmZmlnEomJlZxqFgZmYZh4KZmWUcCmZmlnEomJlZxqFgZmYZh4KZmWUcCmZmlqmudAN6Wn19PXV1dWzfvr3STemTamtrGT9+PDU1NZVuipn1Qf0uFOrq6hg+fDgHHnggkirdnD4lIli3bh11dXVMnDix0s0xsz6o3x0+2r59O/vss48DoQhJ7LPPPu5FmVm7+l0oAA6EDvi9MbOO9MtQMDOzrhnwodDYFMz/w7NM+eZvmf+HZ2lsiko3KfPpT3+a0aNHc/jhh7coX79+PTNmzGDy5MnMmDGDDRs2VKiFZtbf9HooSJog6T5JT0h6XNJ5afneku6WtCx93KvcbXn+1S2cfMX9XH73MjZurefyu5cx88r7ef7VLeWuuiRnnXUWd911V5vyuXPnMn36dJYtW8b06dOZO3duBVpnZv1RJa4+agC+FBEPSxoOLJV0N3AWcG9EzJU0B5gDXNidiv7l9sd54uVN7c5funIDDQU9g231jTz+8iZmXPZ7jjygeCa9eb8RXHTyYR3Wu2LFCk444QSOPfZYHnzwQY466ig+9alPcdFFF7FmzRquv/56jj766E7bf/zxx7NixYo25QsXLmTx4sUAzJ49m2nTpjFv3rxOt2dm1pleD4WIWAWsSp9vlvQkMA44BZiWLrYAWEw3Q6EzQwZVsXl7Q9Hy7lq+fDk333wzP/7xjznqqKO44YYbuP/++1m0aBHf/va3Oe+887jgggvarDd06FAefPDBDre9evVqxo4dC8C+++7L6tWru91eMzOo8H0Kkg4EjgAeAsakgQHwCjCmnXXOAc4B2H///Tvcfmd79Lf+tY6v3/oYW3Y2ZmXDBlXxzVMO40NHjC/pNbRn4sSJvOUtbwHgsMMOY/r06UjiLW95CytWrOA973kPjzzySLfqgORqIl9RZGY9pWInmiXtAfwSOD8iWhzjiYgAip7xjYj5ETE1IqaOGjWqW22YfugYqnItv1CrcmL6oUXzaJcMHjw4e57L5bLpXC5HQ0MD9913H1OmTGnz9453vKPTbY8ZM4ZVq5L8XLVqFaNHj+52e83MoEI9BUk1JIFwfUT8Ki1eLWlsRKySNBZYU+52jKit4W8Xv7/c1RTVnZ7CzJkzWbBgAXPmzGHBggWccsopPdw6MxuoKnH1kYBrgCcj4rKCWYuA2enz2cDC3m5bX3P66adz3HHH8fTTTzN+/HiuueYaAObMmcPdd9/N5MmTueeee5gzZ06FW2pm/YWSIzW9WKH0LuCPwKNAU1r8zyTnFX4B7A+sBD4WEes72tbUqVNjyZIlLcqefPJJDj300J5udr/i98hsYJO0NCKmFptXiauP7gfaOzM6vTfbYmZmLQ34O5rNzKyZQ8HMzDIOBTMzyzgUzMws41AwM7OMQ6GpER74PsybCA9ekUz3IY2NjRxxxBGcdNJJWdnzzz/PMcccw6RJkzjttNPYuXNnBVtoZv3JwA6Fdc/C/L+DxZfAtvVw37dh/rSkvI/43ve+1+aeggsvvJALLriA5cuXs9dee2U3tZmZdVdFB8QruzvnwCuPtj//xYegqb55un4rvPI3uOoYmHBM8XX2fQt8oOPfL+ipobPr6ur49a9/zde+9jUuuyy5+Tsi+N3vfscNN9wAJENnX3zxxXzuc5/rdHtmZp3p36HQmZohsKO+SPnQbm+6J4bOPv/88/nOd77D5s2bs/nr1q1j5MiRVFcn/3Tjx4/npZde6nZ7zcygv4dCJ3v0/M9N8Osvws7Xm8sG7QEn/ju87bRuVd3dobPvuOMORo8ezZFHHpn9oI6ZWbn171DozJtOgDu/0rIsV5WUd1MpQ2d31FN44IEHWLRoEb/5zW/Yvn07mzZt4owzzuCnP/0pGzdupKGhgerqaurq6hg3bly322tmBgM9FGr3hDkvVKTqznoKl1xyCZdccgkAixcv5tJLL+VnP/tZtu4tt9zCrFmzPHS2mfWogX310W5q3rx5XHbZZUyaNIl169Zx9tlnV7pJZtZP9PrQ2T3JQ2d3jd8js4Gto6Gz3VMwM7OMQ8HMzDIOBTMzyzgUzMws41AwM7OMQ8HMzDIDPhQamxq59rFredeN7+K6x6+jsQ8Nnf3pT3+a0aNHc/jhh7coX79+PTNmzGDy5MnMmDGDDRs2AMlgeeeeey6TJk3irW99Kw8//HAlmm1mu7EBHQorN63ktDtO4wf/8wNe2/EaVz9yNbN+PYuVm1ZWumkAnHXWWdx1111tyufOncv06dNZtmwZ06dPZ+7cZIynO++8k2XLlrFs2TLmz5/vkVPNbJf162Eu5v33PJ5a/1S78x9Z8wgN0ZBNb2vYxlPrn+LU205lyugpRdc5ZO9DuPDoCzust6eGzj7++ONZsWJFm/KFCxdmg+TNnj2badOmMW/ePBYuXMiZZ56JJI499lg2btzIqlWrGDt2bKd1mZlBPw+FzgypHsLm+s1Fy7urJ4bObs/q1auzL/p9992X1atXA/DSSy8xYcKEbLn8sNoOBTMrVb8Ohc726G9/9na+9edvsbVha1Y2tHooXz3mq5x80Mndqru7Q2eXShKSur0dMzMY4OcUpk2YRpWqWpRVqYppE6Z1e9ulDJ09ZcqUNn/veMc7Ot32mDFjWLVqFQCrVq1i9OjRAIwbN44XX3wxW+wHACgAAAU8SURBVM7DapvZrurXPYXODB80nAc/3vGhmnLpTk9h5syZLFiwgDlz5rQYOnvmzJlceeWVzJo1i4ceeog999zTh47MbJcM6J5CX3f66adz3HHH8fTTTzN+/HiuueYaAObMmcPdd9/N5MmTueeee5gzZw4AJ554Im984xuZNGkSn/nMZ7j66qsr2Xwz2w156OwByO+R2cDmobPNzKwkDgUzM8v0y1DYnQ+JlZvfGzPrSL8LhdraWtatW+cvvyIignXr1lFbW1vppphZH9XvLkkdP348dXV1rF27ttJN6ZNqa2sZP358pZthZn1UvwuFmpoaJk6cWOlmmJntlvrU4SNJJ0h6WtJySXMq3R4zs4Gmz4SCpCrgKuADwJuB0yW9ubKtMjMbWPpMKABHA8sj4rmI2AncCJxS4TaZmQ0ofemcwjjgxYLpOuCY1gtJOgc4J518XdLTXazvDcCrXVzXrDP+fFm5deczdkB7M/pSKJQkIuYD87u7HUlL2rvN26y7/PmycivXZ6wvHT56CZhQMD0+LTMzs17Sl0LhL8BkSRMlDQJmAYsq3CYzswGlzxw+iogGSf8A/BdQBfw4Ih4vY5XdPgRl1gF/vqzcyvIZ262HzjYzs57Vlw4fmZlZhTkUzMws0+9DobOhMyQdL+lhSQ2SPlKJNtruq4TP1xclPSHpb5LuldTu9eFmrZXw+fqspEclPSLp/p4YBaJfn1NIh854BphBcjPcX4DTI+KJgmUOBEYAXwYWRcQtvd9S2x2V+Pl6D/BQRGyV9DlgWkScVpEG226lxM/XiIjYlD6fCXw+Ik7oTr39vafQ6dAZEbEiIv4GNFWigbZbK+XzdV9EbE0n/0xy/41ZKUr5fG0qmBwGdHsvv7+HQrGhM8ZVqC3W/+zq5+ts4M6ytsj6k5I+X5K+IOlZ4DvAud2ttL+HglmfIOkMYCrw75Vui/UvEXFVRBwEXAh8vbvb6++h4KEzrJxK+nxJeh/wNWBmROzopbbZ7m9Xv79uBE7tbqX9PRQ8dIaVU6efL0lHAD8kCYQ1FWij7b5K+XxNLpj8ILCsu5X261CIiAYgP3TGk8AvIuJxSd9Mz9Qj6ShJdcBHgR9KKufQGtaPlPL5IjlctAdwc3rZoHdKrCQlfr7+QdLjkh4BvgjM7m69/fqSVDMz2zX9uqdgZma7xqFgZmYZh4KZmWUcCmZmlnEomJlZxqFgVgaSZuZHtZR0saQvV7pNZqXoMz/HadafRMQifKOk7YbcUzArQtIZkv47veHsh5KqJL0u6fL0ZqF7JY1Klz234DcTbkzLzpJ0ZZHtTpH053TZWyXtlZYvljQvrfMZSe/u3VdslnAomLUi6VDgNOCdETEFaAQ+QTI08ZKIOAz4PXBRusoc4IiIeCvw2U42/xPgwnTZRwu2AVAdEUcD57cqN+s1Pnxk1tZ04EjgL5IAhgBrSH5z46Z0mZ8Bv0qf/w24XtJtwG3tbVTSnsDIiPh9WrQAuLlgkfz2lgIHdvtVmHWBewpmbQlYEBFT0r83RcTFRZbLjxHzQeAq4O0kQdLVna38CKqNeIfNKsShYNbWvcBHJI0GkLR3+tvKOSD/O94fB+6XlAMmRMR9JOPZ70kyAF4bEfEasKHgfMEnSQ5DmfUZ3hsxayUinpD0deC36Zd+PfAFYAtwdDpvDcl5hyrgZ+mhIQHfj4iN6WGnYmYD/yFpKPAc8KnyvhqzXeNRUs1KJOn1iCjaCzDrL3z4yMzMMu4pmJlZxj0FMzPLOBTMzCzjUDAzs4xDwczMMg4FMzPL/H94XZUGtRl0fgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Accuracy: 99.02%, Robust Accuracy: 93.20%\n",
            "Accuracy: 99.02%, Robust Accuracy: 63.50%\n",
            "Accuracy: 99.02%, Robust Accuracy: 18.84%\n",
            "Accuracy: 99.10%, Robust Accuracy: 96.99%\n",
            "Accuracy: 99.10%, Robust Accuracy: 92.92%\n",
            "Accuracy: 99.10%, Robust Accuracy: 86.79%\n",
            "Accuracy: 98.79%, Robust Accuracy: 96.40%\n",
            "Accuracy: 98.79%, Robust Accuracy: 90.51%\n",
            "Accuracy: 98.79%, Robust Accuracy: 81.43%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3hUddbA8e+Z9E4CBAIBAtKkKEhHBVZErIhtwQLBtbyWXQtbxHeLrusq7vqirm1l1wL2hmJXQIpIUUGkIy2BQGgppNf5vX/cmzBJJsmkTjI5n+fJM7l37tx7Jgxz7q/cc8UYg1JKKQXg8HYASimlWg5NCkoppcppUlBKKVVOk4JSSqlymhSUUkqV06SglFKqnCYFpZRS5TQpqGYlIitEJENEgrwdS0snIg+KSLGI5IhIpoisEZExLs/Hich/ROSwvc0+EXlFRPrbzyeIiLGfyxGRoyLyiYhMqkMMy+x9+LusSxCR5SKSJyI7ReT8xn3nyps0KahmIyIJwLmAAaY087H9a9+qRXrbGBMOdARWA4vE0h5YA4Ri/U0jgLOAlUDlL/129j7OBJYAH4jIrNoOLCLXAwFunnoT+BFoD/wReE9EOtbjvakWSJOCak4zgXXAK0Ci6xMi0k1EFonIcRFJE5FnXJ67RUR2iEi2iGwXkbPs9UZEerts94qIPGz/PkFEUkTkPhE5ArwsItH2mfJxu7XyiYjEu7w+RkRets+8M0TkQ3v9VhG5zGW7ABE5ISJD3b1JO949IpIuIh+JSBeX54yI3CYiu+2z/2dFRGr7wxljioEFQGesL+N7gSxghjFmr7FkGmNeNsY8Xc0+jhhjngIeBB4TkWr//4tIFPAA8IdK6/tiJZ8HjDH5xpj3gS3AVbW9B9U6aFJQzWkm8Lr9M1lEOgGIiB/wCZAMJABdgbfs567B+hKbCURitTDSPDxeZyAG6AHcivV5f9le7g7kA8+4bP8q1pn3QCAWeMJevxC4wWW7i4FUY8yPlQ8oIucBjwK/BOLs9/RWpc0uBUYAZ9jbTa7tjdjdbbOAg8aYE8D5wAfGGGdtr3VjEdb761fDNo8AzwNHKq0fCOwzxmS7rPvJXq98gCYF1SxE5BysL+N3jDEbgL3AdfbTI4EuwO+NMbnGmAJjzGr7uZuBfxhjvrfPhvcYY5I9PKwT64y20D6rTTPGvG+MybO/1P4OjLfjiwMuAm4zxmQYY4qNMSvt/bwGXCwikfbyDKwE4s71wEvGmI3GmELgfmCM3XVWZq59Vn8AWA4MqeE9/FJEMoGDwDDgCnt9B1y+sEVkit3yyBaRr2r5uxy2H2PcPSkiw4GzAXctjnDgZKV1J7G6r5QP0KSgmksi8JV9lgvwBqe6kLoBycaYEjev64aVQOrjuDGmoGxBREJF5AURSRaRLGAV0M5uqXQD0o0xGZV3Yow5DHwLXCUi7bCSx+vVHLMLVuug7LU5WC2bri7buJ5952F90VbnHWNMO2NMrDHmPDuhYu8zzuU4Hxlj2mF1KwXWsD9cYkkXketdBqI/t7uUngPurubfIwerxeYqEsh2s61qhVrr4JtqRUQkBKubxM/u3wcIwvpCPhPrLLi7iPi7+SI6CJxWza7zsLp7ynQGUlyWK5cA/i1Wl8koY8wRERmCNWAq9nFiRKSdMSbTzbEWYLVa/IG1xphD1cR0GKtFBICIhGGNAVS3fX0tA6aKyF/r0YV0BXAM2GWM2YFLgrOT3nDgbXuow89+KsXuytsG9BKRCJcupDOxkrzyAdpSUM1hKlAKDMDqKhkCnA58gzVW8B2QCswVkTARCRaRs+3X/hf4nYgMs2fd9BaRsi/dTcB1IuInIhdidwXVIAJrHCFTRGKwBlIBMMakAp8Dz9kD0gEiMs7ltR9iDbDejTXGUJ03gRtFZIg9DvAIsN4Yk1RLbHU1D4gGXhWR0+y/TQQ1dEWJSCcR+TXW+76/mmRyEqu1U/bvdLG9fhjW+/gZ6+/+gP3vdAXW2Mj7jfXGlHdpUlDNIRF42RhzwJ4Bc8QYcwRrkPd6rDP1y4DewAGss/1pAMaYd7H6/t/A6qL4kFN94Xfbr8u09/NhLXE8CYQAJ7BmQX1R6fkZQDGwE+tM+p6yJ4wx+VhffD2xBmrdMsYsBf5sb5uK1cqZXktcdWZ3w40GCrCmqmZjfVlHALdX2jxTRHKxZgldDFxjjHmpmv2aSv9Gx+2njhpjiuzfp2O1JjKAucDVxpjjbnanWiHRm+wo5RkR+QvQ1xhzQ60bK9VK6ZiCUh6wu5tuwmpNKOWzmqz7SEReEpFjIrLVZV2MiCyxL9xZIiLR9noRkX/ZF/xsLrs4SamWQERuwRqI/twYs8rb8SjVlJpyTOEV4MJK6+YAy4wxfbBmT8yx118E9LF/bsW6aEapFsEY8x9jTJgx5jZvx6JUU2uypGCfUaVXWn051tQ+7MepLusX2oNc67CmKsahlFKqWTX3mEIne+ofWBfwdLJ/74rVPC+TYq9LpRIRuRWrNUFYWNiw/v37N120SinlgzZs2HDCGOO2iKHXBpqNMUZE6jz1yRgzH5gPMHz4cPPDDz80emxKKeXLRKTaUjHNfZ3C0bJuIfvxmL3+EFaZgTLxNP4VoEoppWrR3EnhI07Vu0kEFrusn2nPQhoNnHTpZlJKKdVMmnJK6pvAWqCfWHXtb8K6+nGSiOzGKv071978M2AfsAf4D3BHU8WFsxS+/Rc81hPWPG0tK6WUAlr5Fc11HlNI20vpuzNZWHiYFyNCuDk7nxlBXfG7ZgG0r67mmlJK+RYR2WCMGe7uuTZ1RXPyK5P5bYQfByJDyXc4eC4ylE+Lj/P4/LPpMfAaCIqE4Cj7MdJ6DIo49XvZc/61VSZWSqnWqU0lhRntwzhJKU777of5Dge7AgOY3rk98/ctIbogm5jCXEJraz35B1eTOKJcEkilx8rr/IOh9rswKqVUs2pTSaF3eDe+z604E8uIkCPCddEOrOKbMQT7BRETGEV0QBgx/qFEO4JoL4HE4Ee0gZhSJzGlxcQUFRJdlE9wYQ7kHIWCLCjMhiIP7jfiCKg9cVR5jKrYcgkM08SilGpUbSopXDFoJtvWPkSe49QXaYjTcOPgmxnY+SzSC9JJL0gnoyCD9IJ00grSSCvIYHf+YdLz0ylyFrndb2hIKNHR0bQP7k10cDQxQdHEBIQR7RdCjCOQGPEnxjiIdhpiSooJLM6Hwiw7ibg8ZiRVXK5yj5hKxM99K6VCl1dkzd1igRHg0ArqSilLm0oKE3pdxKMbn4TiU2fy/kGR3HDGTUQE1nyLWWMMeSV5VRKH609GQQZH846yI30H6QXplDjd3c0QwgPCiQmOsRJIdAwxwfHEBMecWhccQ0xgO2L8g2lnHAQU57kkkJMuiSO7anLJSqm4bGqbXSVWgnDb5eVmPMXdNkGR4NemPkpK+ay2NfuoGRljyCnOKU8WaQVpbpNJ2XJGQQal1XyBRwZGlicN18QRHRxN++D2FZbbBbXD3+FfFgRUaZWcdNNKya7huSwodd9CqiAgrIbxlIjqk4prt5gO4CvVLHT2kYtSp+HF1ft4bsVe7phwGjed0ws/R+P3y4sIEYERRARG0COyR63bO42T7KJs0grSKiSKystJWUlsPLaRzMJMnG7upigIUUFRFVsdlRNK+KmWSVRQFA6ppfuouMBKDoXZ1SeOssey3wsyIfPAqeWS/Nr/aJUH8GsaT6n8XNmyf5COsyjVAG2qpbD/RC53vr6R/SdyyS8uJSTAj14dw3jmurPo2SGsCSNtfKXOUk4Wnay25VF5ObPQ3b3owSEO2gW1c9sScbccGRiJ1OdLt7S4auKo8njSZZvsqtsU5dR+HHcD+DVOM3YzaywgtP6JxVkKa5+F1U/AubNh9B3g8KvfvpRqIjW1FNpUUhj2tyVk5BXhdHnLDoHo0EA2/HlSE0TYcpQ4S8gszKwxcbi2TrKrmUHlL/60C25XIWFU7sJyXQ4PCK9fEnHHWVqpVZJdS7dYNcmlvgP4tXWL5Z+EpX+BzGSr2y4gFNr3hmte0YsjVYui3Ue2Pp3CWbev4i0enAY6hAdS6jRN0o3UUvg7/OkQ0oEOIR082r64tJiMwgy3XVhlM7MyCjI4dOIQGQUZ5BS7P4sPcAS4TRzVtURC/UOrTyIOPwiJtn7qy+m0WhxVEkct3WJZKXCsLgP4QHEeHNkMz46CAZdDeCyEdYCwWAjraP2E248BIfV/T0o1ojaVFKaN6MaWlJPkFp36Dy3ArqM5jPvHcmaM6cG04d2IDtMBzwC/AGJDY4kNjfVo+8LSwiqJw11rJDkrmfSCdPKrGWMI8guqfizEzXKIfx2/TB0O66w+OBKi6vbScsZYX/iVu76+/CMc31l1+4BgOLQBco9X3wUWGGEnjI5Vk0dZ4ij7CYnWcRPVZNpU91FWQTHnzP2arIJTU0Ujgv15aMpA3v7hIOv2pRPk72DqkK7MHNuDgV3q+62hapNfkk9GQYZHM7PSC9IpLC10u58Q/xCPZmWVLQf5BTXdm/rpbfh0dsUv/sBwuGQenDnNWi7Kg7wTkHPcShK5x6zH8mWXn7w0cDOZAIe/nSBcWx0d7GTS0V7X4VQS0VldqhIdU/DQziNZLFybzAcbD5FfXMqIhGhmjkngwkGdCfDTC7y8xRhDfkl+lYThrlur7KfYWex2X2EBYe5bHkHRxITEEBMUQ0yIvRwcQ4BfgOeBFpyk9MnBLAx08mK7SG7OzGJGkQO/e7ZYYw915SyFvHSX5HECco5VXC5PKMegpMD9foKjqm91lLdM7N+DIrQV0gZoUqijk3nFvLvhIAvXJnMgPY9OkUFcP6oH147sTseIJjzTVI3CGENucW61XVjurhEpMe4vNIwIjKiSOKKDomkf0r7K8snCk9y36j4OZB8gvySfEP8QekT24PHxj3s0LbmBb9pqoeQer5Q87J8clySSewzyM9zvxy+oUvKIrb5bK7S9XrTYSmlSqCen07Di52MsWJPMyp+PE+AnXDI4jsSxCQzp1q7xZtUorzLGkFWUVWviKOvmqu4aEXcEIdQ/lGcmPkN8RDwdQzri1xKmqJYWuyQJd8njWMUE47blJRAa46bVUc1gemDrmvbtyzQpNIJ9x3NYuDaZ9zakkFNYwhnxUSSOSeCSM+IIDmgB/8lVs3EaJycLT1bpwnp568sczj1c42sDHAF0Ce9C1/Cu5T/xEfHEh8fTNbwrUUFRLe9kwxhrdlZNyaN8TOSENejuTkCYh4PpsdZgutbkajKaFBpRTmEJH2xMYcHaZPYcy6F9WCDTR3bjhtE9iIvSaYVt2cd7P+bhdQ+TV5JXvi7UP5Tbz7ydvtF9SclJISUnhUPZhziUY/1UvqgwLCCsPEF0jbCTRng88RHxdAnvUvfZVt5QXGAPplfqsnLXrZV7wv30XvGrOFheudVReTA9ILj532crpkmhCRhjWLM3jVfWJLFsx1FEhAsGdCJxbAKjesa0vLM91eSyi7KZ/N5ksl0KLkYERPDl1V9WW3AxpyiHQzmHKiQL198LSisOHrcPbl8lWZS1ODqHdT5V96q1cDqt8Q3XmVg1jYkU57rfT1Ckm8RRzWB6cFSbH0zXpNDEDqbn8dr6ZN7+/iCZecX07xzBzDEJTB3ahdDAVvafVLUYxhjSCtKsRJGdUt66OJRtJY4juUcqFFH0Ez86h3W2WhoRFbunuoZ3pX1w+9Z/slKUW33iqNytlZeO26vXHQGeD6aHdYC6zEBrJTQpNJOC4lI+2nSYV9YksT01i8hgf345vBszxyTQvX2ot8NTPqbEWcKR3CPlySIl2+6eshNHWkFahe1D/EPoEtalQuuia0TX8u6q8MBwL72TJlJaAvnp1SSP41W7taq5FoaQ6IrJw7XVUWUwPbxVtEI0KTQzYww/JGewYE0SX2w9QqkxnNcvlsSxCZzTuwMOHy6noVqOvOI8DuccPtUl5dLKOJRziNxKXTHtgtpVSRZlrY4uYV3qds1Ga2OMVe7Ek5lYucetKsDu+IdUaoW4Dqa7joPEWjO36jMTrRGKLmpS8KIjJwt447sDvLH+ACdyCunVIYyZY3pw1bB4IoJ9+D+ZatGMMZwsPFmeMCp0T9k/rjeJEoTY0NjyVoZrF1V8eDwdQzvWXoLdl5QUWYPpVVod1Vyd7u6GW+KwrvWocTC9Un2stL3wbqL1WJxX76KLmhRagMKSUj7fcoQFa5P48UAmYYF+XDUsnpljEugd62PNdtXqlTpLOZ5/vEKyKPs9JSeF43nHMS799YGOwPKptu66p+pdct0XGGMPptc2E8tOKNXd4z0w3EoEla+REQeExMAf9nockiaFFmZzSiavrEnik59SKSp1ck7vDiSOTeC8/rE+XalV+Y6i0iIO5xyuOmvKTh5ZRVkVto8IiKgw+O16fUaX8C4E++uU0nLF+RWn7Lomjy3vWo+VJZwLsz7x+BCaFFqotJxC3vr+IK+tSyb1ZAHx0SHMGN2DaSO60S5Ui5ip1iu7KLvCGEbl7qnKBQ47hHSo0MpwvVajU2in1jfVtql4UnTRA5oUWriSUidLth/llTVJrN+fTnCAXal1TAIDukR6OzylGpUxhhP5J6q9PuNI3pEKZUT8xZ/OYZ0rzJRy7aKKCW5D1wUVnIQnB1uPZYKjoI5FFzUptCI7j2SxYE0yH/yYQkGxk5EJMcwc24PJA7VSq2obip3F5VNty1sYLokjvaDijbJC/EOqDH67dk+FBuh08Mo0KbRClSu1do4M5vpR3ZmulVpVG5dXnFehK6ry9RmuZUYAooOiK16T4TJrKi4szren2lZDk0IrVuo0rNh1jFfWJPHN7hME+jm45Iw4Zo7pwdDuDbgtpVI+yBhDZmGm22SRkpNCak5qhTLpDnHQKbRT1esz7O6pDiEdfHKqrSYFH7H3eA6vulRqPTM+isSxVqXWIH+t1KpUbUqdpRzLO1ZhppRr+ZBj+ccqbF821dbt9RkR1lTb1kiTgo/JKSxh0cYUFqxJYu/xXNqHBXLtyO5cP7q7VmpVqgEKSgo4nHu4QiVb1+szsitdQxARGHFqppTrALidOJr09q8NoEnBRxlj+HaPXal151EcIkwe2InEMQmM1EqtSjW6rKKsU6VCXEqGlLU0ipxFFbbvGNKx4sV8LgPgsaGxXrvhkiaFNuBgeh6vrUvmre8PcjLfqtSaODaBqUO6EhKoXUtKNTWncZ6aaptdtXvqaN7RilNtHf7EhcVVe31GdFC02xO7UmcpC7cv5MWtL3Lz4JuZcfqMOicXTQptSH5RKR/9dIhX1iSzw67UOm1EN2aM1kqtSnlTcak11fZgzsGK02ztxJFRWPG+2aH+oRVmSnUN70qgXyCvbn+VI7lHKCgtqPd9wDUptEHGGL5PymDBWqtSq9MYJvaPZeaYBM7t00G7lpRqYXKLc6tUsnX9Pb8k3+3rHOIgKiiKVdNWeXysmpKCV64dF5F7gZux7oCxBbgRiAPeAtoDG4AZxpiianeiaiQijOwZw8ieMRw5WcDr65N587sDLN3xHb06hpE4JoErz+qqlVqVaiHCAsLoG92XvtF9qzxnjCG9IJ3bl97OjvQdFZ5zGid92vVptDiafQKuiHQF7gKGG2MGAX7AdOAx4AljTG8gA7ipuWPzVZ2jgvntBf34ds55PDHtTCKDA3jgo22MfmQZDyzeyp5jObXvRCnlNSJC+5D2zBgwg1D/it3Aof6hTO09tdGO5a2rMvyBEBHxB0KBVOA84D37+QVA471LBUCQvx9XDI3nwzvPZvGdZzN5YGfe/O4g589byYwX17N0+1FKna23O1EpXzeh2wT8pOKgsp/4MaHbhEY7hlfGFETkbuDvQD7wFXA3sM5uJSAi3YDP7ZZE5dfeCtwK0L1792HJycnNFrcvOpFTyFvfHeC1dQc4klVAtxirUusvh2ulVqV8VYsaaBaRaOB9YBqQCbyL1UJ40JOk4EoHmhtPsUul1u9cKrUmjk3g9LjWedWmUsq9ljbQfD6w3xhzHEBEFgFnA+1ExN8YUwLEA4e8EFubFeDn4OLBcVw8OI7th7N4dV0SH/x4iLe+P8jIhBgSxyZwwcBOWqlVKR/njf/hB4DRIhIq1rzIicB2YDlwtb1NIrDYC7EpYECXSB698gzW3T+RP158OqlZ+dz5xkbOfWw5Ty/bzYmcwtp3opRqlbw1pvBXrO6jEuBHrOmpXbGmpMbY624wxtT47aPdR82j1GlYvvMYC9aeqtR66RlxzBybwJBu7bwdnlKqjlrUmEJj0qTQ/PYcy+G1dS6VWru1I3FMD63UqlQroklBNbrsgmIWbTzEgrVJ7DueS4dwu1LrqB50jtKbsCvVkmlSUE3GGMPqPSdYsCaJZTuP4RDhwoGdSRybwIgE9wW9lFLe1dJmHykfIiKc26cj5/bpyMH0PF5dl8zb3x/k0y2pnB4XSeKYHlyulVqVajW0paAaXX5RKYs3HeKVNUnsPJJNVEiAXam1B91itFKrUt6m3UfKK8orta5J4ottZZVaO5E4tgfn9NZKrUp5i3YfKa9wrdSaejKfN9Yf4I31B1i64yindQxj5pgErhoWT3iQfgyVaim0paCaVWFJKZ9uTmXB2mR+OphJeJA/Vw+LZ8aYHpzWMdzb4SnVJmj3kWqRNh3MZOGaJD7ZnEpRqZNz+3Rg1tgEJvSLxc+hXUtKNRVNCqpFO55tV2pdn8zRrEK6x4SWV2qNCtWbACnV2DQpqFahuNTJV9uOsmBNEt8lWZVarxjalZljtFKrUo1Jk4JqdbYfzmLh2iQ+3HSIgmInI3vGMGtsAhcM6IS/VmpVqkE0KahWKzOviHd+OMjCtcmkZOQTFxXM9aO6M31kdzqEB3k7PKVaJU0KqtUrdRq+3nmMhWtdKrWeGUfimATO1EqtStWJXqegWj0/hzBpQCcmDejEnmM5LFybxPsbUli08RBDurUjcWwPLh6slVqVaihtKahWK7ugmPc3pLBwbTL7TliVWq8b2Z3rtFKrUjXS7iPl05zOU5Vav951DD8RJg/qTOIYrdSqlDvafaR8msMhjOvbkXF9O3IgLY9X1yVZlVo3W5VaZ43twZQztVKrUp7QloLySXlFJSzedJgFdqXWdqEBTBvejRu0UqtS2n2k2i5jDN/tT2fB2iS+3Ha0vFLrrLEJnN27vXYtqTZJu49UmyUijOrVnlG92nM406rU+uZ3pyq1Jo5N4MqztFKrUmW0paDanILiskqtSWxOOVleqXXmmB700kqtqg3Q7iOlqvHjgQwWrk3mk82HKS41jOvbkcQxPfhFv1gcWqlV+ShNCkrV4nh2IW9+d4DXXSq1zhzTg2uGaaVW5Xs0KSjloeJSJ19uO8KCNUl8n5RBSIAfU4d2JXFsD/p31kqtyjdoUlCqHrYdPsnCNcl8uOkQhSVORtmVWidppVbVymlSUKoBMnKtSq2vrjtVqfWG0T2YPqIb7bVSq2qFNCko1QjKKrUuWJPE6j2nKrXOGpvAGfFaqVW1HnqdglKNoGKl1mwWrk2uUKl11tgEJg/szKvrknhuxV7umHAaN53TS+83rVoVbSko1QBZLpVa95/Ixc8hCFDiNIQE+NGrYxjPXHcWPTuEeTtUpcrV1FLQ0TKlGiAyOIAbz+7JstnjiQj2p9RpKHFaJ1r5xaXsSM3i6ufXeDlKpTynSUGpRuBwCAO7VJ2y6jTovR1Uq6JJQalGMm1EN8IqlecWYNvhLO5ftIWT+cXeCUypOvA4KYhIbxF5TUTeF5ExTRmUUq3RxNM7VRlUDg/2J3FMD97+/gCT5q3ki62pXopOKc9UO9AsIsHGmAKX5TeBP9iLHxtjhjRDfDXSgWbVWmxJOcmcRZvZdjiLCwZ04qHLB2m3kvKa+g40fywiM12Wi4EEoAdQ2sCA2onIeyKyU0R2iMgYEYkRkSUistt+jG7IMZRqSQbHR7H4zrO5/6L+rPz5OJPmreS1dck4na139p/yTTUlhQuBSBH5QkTGAb8DJgNXANc38LhPAV8YY/oDZwI7gDnAMmNMH2CZvayUz/D3c/A/40/jq3vHcUa3KP704VamzV/LnmPZ3g5NqXK1XqcgIlHAn4GuwJ+MMXsbdEBrf5uAXsbl4CKyC5hgjEkVkThghTGmX0370u4j1VoZY3hvQwoPf7qD/KJS7vxFb26fcBqB/jr3QzW9epW5EJFRwO+BIuARIB/4O3AI+JsxJrOewQwB5gPbsVoJG4C7gUPGmHb2NgJklC1Xev2twK0A3bt3H5acnFyfMJRqEU7kFPLQx9v56KfD9IkNZ+5VgxnWI8bbYSkfV9+ksAm4GAgHXjbGnG2vHw/8rzFmcj2DGQ6sA842xqwXkaeALOA3rklARDKMMTWOK2hLQfmK5TuP8ccPtpCaVcCM0T34/eR+RATrfRxU06jvQHMJpwaWi8pWGmNW1jch2FKAFGPMenv5PeAs4KjdbYT9eKwBx1CqVflF/1i+mj2exDEJvLoumUnzVrFk+1Fvh6XaoJqSwnXAVcB5wMwatqsTY8wR4KCIlI0XTMTqSvoISLTXJQKLG+uYSrUG4UH+PDhlIItuH0u70ABuWfgDd76+kWPZBbW/WKlG4pWCePa4wn+BQGAfcCNWgnoH6A4kA780xqTXtB/tPlK+qrjUyfxV+3hq2W6C/R388ZLT+eXwbljDbUo1jN5PQalWat/xHO5ftIX1+9MZ3SuGR688QyuuqgbTKqlKtVK9Oobz5i2jefTKwWw7nMXkJ1fx7PI9FJc6vR2a8lG1JgURuUxENHko5SUOh3DtyO4smz2eif1j+eeXu7js6dVsOlivWeFK1ciTL/tpwG4R+YeI9G/qgJRS7sVGBvP8DcOYP2MYmXnFXPnctzz08XZyC0u8HZryIbUmBWPMDcBQYC/wioisFZFbRSSiyaNTSlVxwcDOLJk9jutH9eClb/dzwROrWL5LZ3CrxuFRt5AxJgvreoK3gDis+kcbReQ3TRibUqoaEcEB/G3qIN67bQwhgX7c+PL33P3Wj6TlFHo7NNXKeTKmMEVEPgBWAAHASGPMRVglKn7btOEppWoyPCGGT+86h3vO78NnW1KZOG8l721IoTXPKlTe5Q2G5BEAABqHSURBVElL4SrgCWPMYGPMP40xxwCMMXnATU0anVKqVkH+ftxzfl8+u+tcTusYzu/e/YkZL37HgbQ8b4emWiFPqqT2BFLLbrgjIiFAJ2NMUtOHVzO9TkGpipxOw+vrk3nsi12UOJ3MntSXX53dE38/nUCoTmnodQrvAq6TokvtdUqpFsbhEGaMSWDJ7HGc26cjj3y2k6nPfcvWQye9HZpqJTxJCv7GGNeCeEVY5SmUUi1UXFQI82cM4/nrz+JoViGXP/stj35m3btBqZp4khSOi8iUsgURuRw40XQhKaUag4hw0eA4lt47nl8Oj+eFVfuY/OQqVu/W/76qep4khduA/xWRAyJyELgP+J+mDUsp1ViiQgN49MozeOvW0fg5hBteXM9v3/mJjNyi2l+s2hyPC+KJSDiAMSanSSOqAx1oVqpuCopLefrr3bywch9RIQH85bIBTDmzi1ZfbWMaXCVVRC4BBgLBZeuMMQ81WoT1pElBqfrZkZrFnEVb+OlgJhP6deThqYOIjw71dliqmTRo9pGI/Bur/tFvAAGuwbobm1KqlTo9LpJFt4/lgcsG8N3+dC54YhUvrd5PqVMvemvrPBlTGGuMmQlkGGP+CowB+jZtWEqppubnEG48uydf3TuOUT1jeOiT7Vz5/Bp2pGZ5OzTlRZ4khbJ7AeaJSBegGKv+kVLKB8RHh/LSrBE8NX0IKel5XPb0av755U4KinX6alvkSVL4WETaAf8ENgJJwBtNGZRSqnmJCJcP6crS2eOZOrQrzy7fy0VPfcPavWneDk01sxqTgn1znWXGmExjzPtYYwn9jTF/aZbolFLNKjoskMevOZPXbhpFqdNw7X/WMef9zZzMK/Z2aKqZ1JgUjDFO4FmX5UJjjF4vr5SPO6dPB768Zxz/M64X725IYeK8lXy2JVWrr7YBnnQfLRORq0QnMivVpoQE+nH/xaez+M6z6RwVxB2vb+SWhRtIPZnv7dBUE/KkSmo2EAaUYA06C2CMMZFNH17N9DoFpZpHSamTl79N4v+W7MLf4eC+C/tx/ageOBx6rtgaNeg6BWNMhDHGYYwJNMZE2steTwhKqebj7+fglnG9+Oqe8Qzt3o4/L97GNS+sZffRbG+HphqZJy2Fce7WG2NWNUlEdaAtBaWanzGGRRsP8bdPt5NbWMIdE3pzxy9OI8jfz9uhKQ/V1FLw9+D1v3f5PRgYCWwAzmuE2JRSrYyIcNWweCb068jfPtnOU8t28+mWVOZeOZjhCTHeDk81kCfdR5e5/EwCBgEZTR+aUqolax8exJPTh/LKjSPILyrl6n+v5U8fbiGrQKevtmb1uUdfCnB6YweilGqdJvSL5at7x3HTOT15Y/0BJs1byVfbjng7LFVPtXYficjTQNnAgwMYgnVls1JKARAW5M+fL7XKcN/3/mZufXUDFw3qzF+nDCQ2Mrj2HagWw5OB5kSXxRIgyRjzbZNG5SEdaFaq5SkudfKfb/bx5NLdBPk7uP+i05k+optOX21BGnQ/BREJAwqMMaX2sh8QZIzJa/RI60iTglIt1/4Tudy/aDPr9qUzsmcMj145mNM6hns7LEUDr1MAlgEhLsshwNLGCEwp5bt6dgjjzVtG89hVg9mZmsVFT33DM1/vpqjE6e3QVA08SQrBrrfgtH/XWzQppWolIkwb0Z2lvx3PpAGdePyrn7ns6dX8eEAnMLZUniSFXBE5q2xBRIYBWvxEKeWx2Ihgnr3uLP47czhZBcVc+fwaHvxoGzmFJd4OTVXiycVr9wDvishhrLpHnbFuz6mUUnVy/oBOjOoVw+Nf7mLB2iS+2naEh68YxHn9O3k7NGWrdaAZQEQCgH724i5jTIu4OkUHmpVqvTYkZzDn/c3sPpbDZWd24S+XDqBjRJC3w2oTGjTQLCJ3AmHGmK3GmK1AuIjc0QhB+YnIjyLyib3cU0TWi8geEXlbRAIbegylVMs1rEc0n951LrMn9eXLrUc4f95K3vnhoN6zwcs8GVO4xRiTWbZgjMkAbmmEY98N7HBZfgx4whjTG6uMxk2NcAylVAsW6O/grol9+Ozuc+nbKZw/vLeZG15cT3JarrdDa7M8SQp+rjfYsa9TaNBZvIjEA5cA/7WXBavA3nv2JguAqQ05hlKq9egdG87bt47h71cMYvPBk1zwxCr+vXIvJaU6fbW5eZIUvgDeFpGJIjIReNNe1xBPAn8Ayv7F2wOZxpiyqQgpQFd3LxSRW0XkBxH54fjx4w0MQynVUjgcwvWjerBk9ngm9OvI3M93MuWZb9mSoncAbk6eJIX7gK+B2+2fZVQsp10nInIpcMwYs6E+rzfGzDfGDDfGDO/YsWN9w1BKtVCdo4J5YcZw/n3DWZzIKeTyZ1fz8CfbySvS6avNwZPS2U5jzL+NMVcbY64GtgNPN+CYZwNTRCQJeAur2+gpoJ2IlE2RjQcONeAYSqlW7sJBcSyZPZ7pI7vz39X7ueCJVaz6WXsHmppHpbNFZKiI/MP+In8I2FnfAxpj7jfGxBtjEoDpwNfGmOuB5cDV9maJwOL6HkMp5RuiQgJ45IrBvH3raAL9Hcx86Ttmv72J9Nwib4fms6pNCiLSV0QeEJGdWC2Dg1jXNfzCGNOQlkJ17gNmi8gerDGGF5vgGEqpVmhUr/Z8dte53HVebz7efJjz563kwx8P6fTVJlDtxWsi4gS+AW4yxuyx1+0zxvRqxvhqpBevKdX27DqSzZxFm/nxQCbj+nbk71MH0S1Gy7HVRX0vXrsSSAWWi8h/7JlHWhBdKeVV/TpH8N5tY/nrlIFsSErngidW8d9v9un01UZSbVIwxnxojJkO9Mfq778HiBWR50XkguYKUCmlKvNzCIljE1gyezxjT2vPw5/u4Mrn17D9cJa3Q2v1PJl9lGuMecMYcxnWrKAfsfr/lVLKq7q0C+G/icN5+tqhHM7M57JnVvPYFzspKC71dmitlkcF8VoqHVNQSpXJzCvikc928M4PKSS0D+WRKwYztncHb4fVIjX0zmtKKdXitQsN5B9Xn8kbN4/CANf9dz1/eO8nMvN0+mpdaFJQSvmUsb078OU947h9wmm8v/EQ589bycc/Hdbpqx7SpKCU8jnBAX7cd2F/Pvr12XRpF8Jv3vyRmxb8wKFMvWlkbTQpKKV81sAuUSy6fSx/uuR01u5N44J5K3nl2/2UOrXVUB1NCkopn+bv5+Dmc3vx1b3jGJYQw4Mfb+fqf69h15Fsb4fWImlSUEq1Cd1iQllw4wienDaE5LQ8Ln36G+Z9tUunr1aiSUEp1WaICFOHdmXp7PFcdkYX/vX1Hi7+1zd8tz/d26G1GJoUlFJtTkxYIPOmDWHhr0ZSVOLkly+s5f5FWziZX+zt0LxOk4JSqs0a17cjX907jlvO7cnb3x9g0ryVfLE11dtheZUmBaVUmxYa6M8fLxnAh3eeTYfwIG57bSP/8+oPHM0q8HZoXqFJQSmlgDPi27H412cz56L+rNh1nPP/byWvrUvG2camr2pSUEopW4Cfg9vGn8aX94xjcHwUf/pwK9Pmr2XPsRxvh9ZsNCkopVQlCR3CeP3mUfzz6jP4+WgOFz/1DU8t3U1Rie/fs0GTglJKuSEiXDO8G0tnj2fyoM48sfRnLn36GzYkZ3g7tCalSUEppWrQMSKIp68dykuzhpNTUMLV/17DXxZvJbvAN6evalJQSikPnNe/E1/NHk/imAReXZfMBU+sYun2o94Oq9FpUlBKKQ+FB/nz4JSBvH/7WCKDA7h54Q/c+cZGjmX7zvRVTQpKKVVHZ3WP5uPfnMPvLujLku1HOf//VvLO9wd94p4NmhSUUqoeAv0d/Pq8Pnx+97n0j4vkD+9v5rr/rGf/iVxvh9YgmhSUUqoBTusYzlu3jObRKwez9fBJJj+5imeX76G4tHVOX9WkoJRSDeRwCNeO7M6y2eOZ2D+Wf365i8ueXs1PBzO9HVqdaVJQSqlGEhsZzPM3DOOFGcPIyCviiue+5aGPt5NbWOLt0DymSUEppRrZ5IGdWTJ7PNeN6s5L3+7ngidWsWLXMW+H5RFNCkop1QQigwN4eOpg3r1tDMEBDma9/D13v/UjaTmF3g6tRtKap1ANHz7c/PDDDxXWFRcXk5KSQkGB78wbbkzBwcHEx8cTEBDg7VCUajMKS0p5bvlenluxh/Agf/50yQCuPKsrIuKVeERkgzFmuNvnfC0p7N+/n4iICNq3b++1P3hLZYwhLS2N7Oxsevbs6e1wlGpzfj6azZz3N7PxQCbn9unA36cOpnv70GaPo6ak4HPdRwUFBZoQqiEitG/fXltRSnlJ304RvHfbWP52+UB+PJDJBU+uZP6qvZS0oOmrPpcUAE0INdC/jVLe5XAIM8YksGT2OM7p3ZFHPtvJ1Oe+Zeuhk94ODfDRpKCUUi1dXFQI/5k5jOeuP4ujWYVc/uy3PPrZDvKLSr0aV5tPCqVOw/xVexny0FfMX7WX0hZ0671f/epXxMbGMmjQoArr09PTmTRpEn369GHSpElkZPh2fXelfJWIcPHgOJbeO55rhsXzwqp9TH5yFat3n/BaTM2eFESkm4gsF5HtIrJNRO6218eIyBIR2W0/Rjd1LPtP5HLZ06t5YsluMvOKeWLJbqY8s7rF1C6ZNWsWX3zxRZX1c+fOZeLEiezevZuJEycyd+5cL0SnlGosUaEBzL3qDN68ZTR+DuGGF9fz23d+IiO3qNljafbZRyISB8QZYzaKSASwAZgKzALSjTFzRWQOEG2Mua+mfbmbfbRjxw5OP/10AP768Ta2H86q9vUbkjMocdMy8HcIw3q4z0kDukTywGUDawqLpKQkLrzwQkaPHs2aNWsYMWIEN954Iw888ADHjh3j9ddfZ+TIkTXuw3Vfl156KVu3bi1f169fP1asWEFcXBypqalMmDCBXbt2ebQ/qPg3Ukq1LAXFpTz99W5eWLmPqJAA/nLZAKac2aVRxwNb1OwjY0yqMWaj/Xs2sAPoClwOLLA3W4CVKJpUSKBfndbXxZ49e/jtb3/Lzp072blzJ2+88QarV6/m8ccf55FHHmH58uUMGTKkys/YsWNr3ffRo0eJi4sDoHPnzhw96ns3+lCqrQoO8OP3k/vz8W/OIT46hLvf2sSvXvmeQ5n5zXJ8/2Y5SjVEJAEYCqwHOhljUu2njgCdqnnNrcCtAN27d69x/7Wd0X/wYwp/+mAruS4DO2GBfjx0+UCuGBrv0XuoTs+ePRk8eDAAAwcOZOLEiYgIgwcPJikpiV/84hds2rSpQccAq09SZxQp5XtOj4tk0R1ns2BNEo9/tYtJ81byuwv6ccPoHryyZj/PrdjLHRNO46ZzeuHnaLzvAK8lBREJB94H7jHGZLl+sRljjIi47dcyxswH5oPVfdSQGCae3okHFm+rsM7PIUw83W0+qpOgoKDy3x0OR/myw+GgpKSE5cuXc++991Z5XWhoKGvWrKlx3506dSI1NbW8+yg2NrbB8SqlWh4/h/Crc3pywcBO/OnDrTz0yXb+8eVOMFBQ4uSJJbtZvOkwz1x3Fj07hDXKMb2SFEQkACshvG6MWWSvPioiccaYVHvcocmrR0UGB7D5wclNfRi3GtJSmDJlCgsWLGDOnDksWLCAyy+/vJGjU0q1JPHRobw8awSDH/ySnMJTPRv5xaXsSM3i6ufXsOHPkxrlWN6YfSTAi8AOY8w8l6c+AhLt3xOBxc0dW0tz7bXXMmbMGHbt2kV8fDwvvvgiAHPmzGHJkiX06dOHpUuXMmfOHC9HqpRqaiLCoK5RVdY7jXWldKMdxwuzj84BvgG2AGXXdv8v1rjCO0B3IBn4pTEmvaZ91Tb7SLmnfyOlWqfqxkEfvmJQncZBa5p91OzdR8aY1UB1oyITmzMWpZRqTZpyHLSMV2cfKaWU8lxzjIO2+TIXSimlTtGkoJRSqpwmBaWUUuU0KSillCqnScFZCt/+Cx7rCWuetpZbkNLSUoYOHcqll15avm7//v2MGjWK3r17M23aNIqKmr+SolLKN7XtpJC2F+aPhxWPQn46LH8E5k+w1rcQTz31VJVrCu677z7uvfde9uzZQ3R0dPlFbUop1VC+PSX18zlwZEv1zx9cD87iU8vFeXBkMzw7CrqNcv+azoPhoprvX9BYpbNTUlL49NNP+eMf/8i8edbF38YYvv76a9544w0AEhMTefDBB7n99ttr3Z9SStXGt5NCbQJCoLDYzfrQBu96z549vPvuu7z00kuMGDGivHT2Rx99xCOPPMLdd99da0G8e+65h3/84x9kZ2eXP5+Wlka7du3w97f+6eLj4zl06FCD41VKKfD1pFDLGT0/vQ2fzoainFPrAsPh4n/CmdMadOiGls7+5JNPiI2NZdiwYaxYsaJBsSillKd8OynUpt+F8PnvK65z+FnrG6ihpbO//fZbPvroIz777DMKCgrIysrihhtu4NVXXyUzM5OSkhL8/f1JSUmha9euDY5XKaWgrSeF4CiYc8Arh66tpfDoo4/y6KOPArBixQoef/xxXnvttfLXvvfee0yfPl1LZyulGlXbnn3USj322GPMmzeP3r17k5aWxk033eTtkJRSPqLZS2c3Ji2dXT/6N1KqbaupdLa2FJRSSpXTpKCUUqqcJgWllFLlNCkopZQqp0lBKaVUOU0KSimlyrX5pFDqLOXlrS9zzlvn8Mq2VyhtQaWzf/WrXxEbG8ugQYMqrE9PT2fSpEn06dOHSZMmkZGRAVjF8u666y569+7NGWecwcaNG70RtlKqFWvTSSE5K5lpn0zj+Z+e52ThSZ7b9BzTP51Oclayt0MDYNasWXzxxRdV1s+dO5eJEyeye/duJk6cyNy5Vo2nzz//nN27d7N7927mz5+vlVOVUnXm02UuHvvuMXam76z2+U3HNlFiSsqX80vy2Zm+k6kfTmVI7BC3r+kf05/7Rt5X43Ebq3T2uHHjSEpKqrJ+8eLF5UXyEhMTmTBhAo899hiLFy9m5syZiAijR48mMzOT1NRU4uLiaj2WUkqBjyeF2oT4h5BdnO12fUM1Runs6hw9erT8i75z584cPXoUgEOHDtGtW7fy7crKamtSUEp5yqeTQm1n9B/v/ZiH1z1MXkle+bpQ/1DuH3U/l512WYOO3dDS2Z4SEUSkwftRSilo42MKE7pNwE/8KqzzEz8mdJvQ4H17Ujp7yJAhVX7Gjh1b6747depEamoqAKmpqcTGxgLQtWtXDh48WL6dltVWStWVT7cUahMRGMGa62ruqmkqDWkpTJkyhQULFjBnzpwKpbOnTJnCM888w/Tp01m/fj1RUVHadaSUqpM23VJo6a699lrGjBnDrl27iI+P58UXXwRgzpw5LFmyhD59+rB06VLmzJkDwMUXX0yvXr3o3bs3t9xyC88995w3w1dKtUJaOrsN0r+RUm2bls5WSinlEU0KSimlyvlkUmjNXWJNTf82Sqma+FxSCA4OJi0tTb/83DDGkJaWRnBwsLdDUUq1UD43JTU+Pp6UlBSOHz/u7VBapODgYOLj470dhlKqhfK5pBAQEEDPnj29HYZSSrVKLar7SEQuFJFdIrJHROZ4Ox6llGprWkxSEBE/4FngImAAcK2IDPBuVEop1ba0mKQAjAT2GGP2GWOKgLeAy70ck1JKtSktaUyhK3DQZTkFGFV5IxG5FbjVXswRkV31PF4H4EQ9X6tUbfTzpZpaQz5jPap7oiUlBY8YY+YD8xu6HxH5obrLvJVqKP18qabWVJ+xltR9dAjo5rIcb69TSinVTFpSUvge6CMiPUUkEJgOfOTlmJRSqk1pMd1HxpgSEfk18CXgB7xkjNnWhIdscBeUUjXQz5dqak3yGWvVpbOVUko1rpbUfaSUUsrLNCkopZQq5/NJobbSGSIyTkQ2ikiJiFztjRhV6+XB52u2iGwXkc0iskxEqp0frlRlHny+bhORLSKySURWN0YVCJ8eU7BLZ/wMTMK6GO574FpjzHaXbRKASOB3wEfGmPeaP1LVGnn4+foFsN4YkycitwMTjDHTvBKwalU8/HxFGmOy7N+nAHcYYy5syHF9vaVQa+kMY0ySMWYz4PRGgKpV8+TztdwYk2cvrsO6/kYpT3jy+cpyWQwDGnyW7+tJwV3pjK5eikX5nrp+vm4CPm/SiJQv8ejzJSJ3ishe4B/AXQ09qK8nBaVaBBG5ARgO/NPbsSjfYox51hhzGnAf8KeG7s/Xk4KWzlBNyaPPl4icD/wRmGKMKWym2FTrV9fvr7eAqQ09qK8nBS2doZpSrZ8vERkKvICVEI55IUbVenny+erjsngJsLuhB/XppGCMKQHKSmfsAN4xxmwTkYfskXpEZISIpADXAC+ISFOW1lA+xJPPF1Z3UTjwrj1tUE9KlEc8/Hz9WkS2icgmYDaQ2NDj+vSUVKWUUnXj0y0FpZRSdaNJQSmlVDlNCkoppcppUlBKKVVOk4JSSqlymhSUagIiMqWsqqWIPCgiv/N2TEp5osXcjlMpX2KM+Qi9UFK1QtpSUMoNEblBRL6zLzh7QUT8RCRHRJ6wLxZaJiId7W3vcrlnwlv2ulki8oyb/Q4RkXX2th+ISLS9foWIPGYf82cRObd537FSFk0KSlUiIqcD04CzjTFDgFLgeqzSxD8YYwYCK4EH7JfMAYYaY84Abqtl9wuB++xtt7jsA8DfGDMSuKfSeqWajXYfKVXVRGAY8L2IAIQAx7DuufG2vc1rwCL7983A6yLyIfBhdTsVkSignTFmpb1qAfCuyyZl+9sAJDT4XShVD9pSUKoqARYYY4bYP/2MMQ+62a6sRswlwLPAWViJpL4nW2UVVEvREzblJZoUlKpqGXC1iMQCiEiMfW9lB1B2H+/rgNUi4gC6GWOWY9Wzj8IqgFeFMeYkkOEyXjADqxtKqRZDz0aUqsQYs11E/gR8ZX/pFwN3ArnASPu5Y1jjDn7Aa3bXkAD/MsZk2t1O7iQC/xaRUGAfcGPTvhul6karpCrlIRHJMca4bQUo5Su0+0gppVQ5bSkopZQqpy0FpZRS5TQpKKWUKqdJQSmlVDlNCkoppcppUlBKKVXu/wEuAr7z2/3uGAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Accuracy: 99.02%, Robust Accuracy: 93.17%\n",
            "Accuracy: 99.02%, Robust Accuracy: 56.40%\n",
            "Accuracy: 99.02%, Robust Accuracy: 8.35%\n",
            "Accuracy: 99.10%, Robust Accuracy: 97.03%\n",
            "Accuracy: 99.10%, Robust Accuracy: 92.01%\n",
            "Accuracy: 99.10%, Robust Accuracy: 79.31%\n",
            "Accuracy: 98.79%, Robust Accuracy: 96.37%\n",
            "Accuracy: 98.79%, Robust Accuracy: 89.76%\n",
            "Accuracy: 98.79%, Robust Accuracy: 71.03%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3hUZfbA8e/JpIeQhISEQCiB0HsHKaJIsaHuqmDH3hVcC1t+u67rWlkQC/aCvSuKSlWkI02QHnoL6SGB9OT9/XEnYYCUIclkJsn5PM88yS1z75kwzJm3izEGpZRSCsDL3QEopZTyHJoUlFJKldKkoJRSqpQmBaWUUqU0KSillCqlSUEppVQpTQpKKaVKaVJQLiUii0UkXUT83B2LpxORx0WkQESOi0iGiKwQkcEOx6NF5E0ROWI/Z4+IvCcinezH24iIsR87LiKJIjJHREZVct9uIjJPRFJE5IyBSyLSRES+EZETIrJfRK497fi19v0nRORbEWlSU38TVfs0KSiXEZE2wDDAAONq+d7etXm/GvSZMaYR0BRYBnwtlnBgBRCI9TcNBvoAvwKnf+iH2q/RE1gAfCMiEyu4ZwHwOXBrOcdfAfKBKOA64FUR6Qpg//k6cIP9eDYw82xesPIwxhh96MMlD+CfwHJgGjDntGMtga+BZCAVeNnh2O3ANiAL2Ar0se83QJzDee8BT9p/HwEcAh4DjgIfAGHAHPs90u2/xzg8vwnwLnDEfvxb+/7NwKUO5/kAKUDvcl7n7cAuIA34DmjucMwAdwHxQAbWB6yUc53HgQ8dtrvanx8BPAlsBLwq+Hu3sZ/vfdr+h4HEip5rPy/O+kg4ZV8QVkLo4LDvA+AZ++9PAR87HGtnPz/Y3e8/fVTtoSUF5Uo3Ah/ZH2NEJApARGxYH9D7sT7IWgCf2o9dhfXheCPQGKuEkerk/ZphfdC3Bu7AKgm/a99uBeQALzuc/wHWN++uQCQw3b7/feB6h/MuAhKMMRtOv6GInA88DVwNRNtf06ennXYJ0B/oYT9vTGUvxF7dNhE4aIxJAS4AvjHGFFf23DJ8jfX6OlbhuR2AQmPMTod9G7H+Zth/biw5YIzZjT2JVOFeygNoUlAuISJDsT6MPzfGrAN2AyV10QOA5sAjxpgTxphcY8wy+7HbgOeMMWuMZZcxZr+Tty0G/mWMyTPG5BhjUo0xXxljso0xWcB/gXPt8UUDFwJ3GWPSjTEFxphf7df5ELhIRBrbt2/ASiBluQ54xxiz3hiTB/wVGGyvOivxjDEmwxhzAPgF6FXBa7haRDKAg0Bf4Ar7/gisEhD2+MfZ2x2yRGR+JX+XI/afVanrbwRknrbvGFb1VcnxYxUcV3WMJgXlKjcB8+3fcgE+tu8Dq+povzGmsIzntcRKIFWRbIzJLdkQkUARed3eCJoJLAFC7SWVlkCaMSb99IsYY45gVXv9WURCsZLHR+XcszlW6aDkucexSjYtHM456vB7NtYHaXk+N8aEGmMijTHn2xMq9mtGO9znO2NMKDAZ8K3gejjEkiYi1zk0RP9UyfMAjmOV2Bw1xqrac+a4qmPqamOc8mAiEoBVTWITkZIPRD+sD+SeWN+CW4mIdxmJ4SBWvXRZsrGqe0o0w2pHKHF6z5m/YFWZDDTGHBWRXsAGQOz3aSIiocaYjDLuNQur1OINrDTGHC4npiNYJSIARCQICAfKO7+qFgGXi8i/q1CFdAWQBOwwxmyj/ARXlp2At4i0N8bE2/f1BLbYf99i3wZARNpi/VvvRNVJWlJQrnA5UAR0waoq6QV0BpZitRX8BiQAz4hIkIj4i8gQ+3PfAh4Wkb72XjdxIlLyofs7cK2I2ERkLPaqoAoEY7UjZNi7Sf6r5IAxJgH4CZgpImEi4iMiwx2e+y1W754HsdoYyvMJcLOI9LK3AzwFrDbG7KsktrM1Davh/AMRaWf/2wRTQVWUiESJyH1Yr/uv5SUT+7X8sZc47P8efgDGmBNYbRJP2P+thgCXcbI67SPgUhEZZk+ITwBf26vrVB2kSUG5wk3Au8aYA8aYoyUPrEbe67C+qV+K1dvlANa3/fEAxpgvsOr+P8aqgviWk3XhD9qfl2G/zreVxPECEIDVc2gVMPe04zdgdcfcjvVNelLJAWNMDvAVEIv1oVgmY8xC4P/s5yZglXImVBLXWbNXww0CcrG6qmZhJclg4O7TTs8QkRPAH1iN5FcZY96p4PKtsZJnybf/HGCHw/F7sP6OSVhJ8G5jzBZ7XFuweld9ZD8ebD9f1VFijC6yo1RZROSfWF0xr6/0ZKXqCW1TUKoM9uqmW7FKE0o1GC6rPhKRd0QkSUQ2O+xrIiILRCTe/jPMvl9E5EUR2SUim0Skj6viUqoyInI7VkP0T8aYJe6OR6na5Mo2hfeAsaftmwIsMsa0x+pNMcW+/0Kgvf1xB/CqC+NSqkLGmDeNMUHGmLvcHYtStc1lScH+DSvttN2XYXX1w/7zcof979sHK63C6roYjVJKqVpV220KUfaugGAN6Imy/94Cq7he4pB9XwKnEZE7sEoTBAUF9e3UqZProlVKqXpo3bp1KcaYpmUdc1tDszHGlDVNrxPPewN4A6Bfv35m7dq1NR6bUkrVZyJS7tQxtT1OIbGkWsj+M8m+/zDWtAMlYqj5EaFKKaUqUdtJ4TtOzn9zEzDbYf+N9l5Ig4BjDtVMSimlaokru6R+AqwEOorIIRG5FXgGGCUi8VhTAT9jP/1HYA/WnPRv4soRkcVFsPxFeDYWVrxkbSullALq+Ijms25TSN1N0Rc38n7eEd4ODuC2rBxu8GuB7apZEF7eHGxKKVW/iMg6Y0y/so41qBHN+98bw1+CbRxoHEiOlxczGwfyQ0EyU984h9YdL4OAJhAQZj0Cm0BA6Kn7/ENAxN0vQymlXKZBJYUbwoM4RhHF9g/2HC8vdvj6cG2zCL45uJIm2Rl4552+nogDsdkTRVgZCSTs1IfjPr/GmkyUUnVCg0oKcY1asubEqT2xjAiZIowMBQkNI8wvlnC/MCJ8g4mwBRDh5U+E2IgohojCIiIK8gjPz6ZxTiZy/CgkbYOcdMivYKZgsZWRLByTimNCcUwmwZpMlFK1qkElhSu63ciWlU+Q7XXyg9a/2HBlp/HENulIak4qKTkpJOckk5qTyv7jB0jJSSG/OP+Ma/l6+RIRHkFETEvCA8KJ8GtChE8QETZ/IvAmwggRRcWEF+Thl5cF2WlW8shJh8zDkLjFnkyOlx+wl/eZyaI0qYSWX1rxbaTJRClVJQ0qKYxoeyFPr38BCk5+q/fxa8w9fScR7Fv2krLGGLIKskjJSSElO8X6mZNCSm4KqTmpJGcnc+j4ITYmbyQt9/RZPSyNfRsTERBBREgE4c06Wb8HRNA0oCnhvo2JEF8ijBehRQV45R47NYHkOPx+7BAc/cP6veBE+S/Uy6eckkloGdVdjskkSJOJUg1cw+p95GIFxQWk56afTBwVPHIKc854vk1shPuHExEYUZo4wv3DrQQS2NTa5x9BeEA4gXhBbkb5CeSU/Q77yrjvyQB8na/ackw2PgGaTJSqQ7T3kYOiYsPby/Ywc/Fu7hnRjluHtsXmVTMfaD5ePkQGRhIZGFnpudkF2aUJIjknmZSclNLqq5ScFJKzk9mWuo3U3FSKy1hFMdA7sDRxnPKIbEV4QG+aBlhJJMw/DG8vh3/mghzIySgngTjuS4f0fXBkg7W/MLf8F2PzK7/Bvdx2FHsyUUp5lAZVUtibcoJ7P1rP3pQT5BQUEeBjo23TIF6+tg+xEUEujLTqioqLyMjLKE0aJQmkNInkppRWbWUVnNnYLQhh/mFlJ5DTHo18GiHlfeMvyKkggZRRKinZX5RX/ovz9j8tWYSWn0AcSys+/jX011WqYaqopNCgkkLf/ywgPTufYoeX7CUQFujLuv8b5YIIa1duYS6puamnJI0zkoj994LigjOe72fzs6qsAsKJ8LeqrMIDwkurrRwTiI/Np/KAjLEnk/ISSMnvGacmm+w0KCO+Ut4B5Te6l9deEtgEvP2q8ddVqv7Q6iO79lGNWLXn1MbgYgPtmjZyU0Q1y9/bnxaNWtCiUYsKzzPGkJmfWWGbx4GsA2xI2kB6XnqZ1wjxCylNFCWJo2mAQxKxP0L8QvDyjYGQGOdfiDFQkF12Asl2SCQl+1LiT+6vKJn4BJ7a6F5RAnGs/jqbZFJcBCtfgWXTYdhDMOge8LI5/3yl3KxBlRS+2XCIf3yzmRP5p8531MjPxsOjO3LNwFb4eet/YEcFRQWk5qaeUspw7LbruC+36Mx2B28v79LG8tLG84Dw0jYPx6QS4F3NNgZjIP9EGQmkJKlklF9aKS4s/7o+QQ4j3CtIIPk5sORZyDhoNej7BEJ4HFz1nk6jojyKVh/ZZeYWMPSZn8nMPfkBEOhro0vzYNbuy6BFaACTLmjPFb1b4G2r7Qlk6zZjDNmF2SRnJ5/SZbek0dxxOy03rczG8yCfoDJLG+H+4Sd7XwVEEOYXhq0mv30bA3lZ5fTiSq+gtJIOxokJFX0C4cp3IKqbVWLSnlrKzTQpVMIYw7JdKTw/bwebDh2jXdMgHh7dkbHdmpXf8KqqrKi4iPS8M7vuOraBlCSQ4wVnDu7zEi/C/MJOtnn4l994HuQT5Lp/Q2MgL/Nkgph9rzUosSJ+IRDVFZp1s35GdYfITtYYEaVqiSYFJxljmLflKFPn72RX0nG6twjhkTEdGdY+QpODm+QU5pxRTVXWIzUnlUJzZhWQv82/wjYPx9KIU43nFdn4Gfzw0Kmj1H0bwej/QmRnSPzDSholj9LzxKpeKkkSJUkjpKWWKpRLaFI4S0XFhm82HGb6gp0czshhYGwTHh3bib6tw2r8XqpmFJtiMvMyyx734dBtNyU3hWN5x8q8RqhfaIVtHiU9shr7Ni77S0LuMYpe6M77vsW8HdqY2zIyuSHfC9ukP6wZdk8JuBgy9kPiZitBHLUnjPS9J88pKVWUPJp1t5KLlipUNWlSqKK8wiI+/e0gL/28i5TjeVzQOZK/jO5I5+jGLruncr38onzSctMqHDxY8sgrY5yFt5f3Kd10wwOsNg8vvJi9azYpudbzArwDaN24NVPPnUrrxq2dCy4vy5pk8ejppYqSMSgCTdqeTBJRXa22itBWWqpQTtOkUE3Z+YW8u3wfr/+6m6y8Qi7t0ZyHRnWgjYcOeFM1wxjD8YLjlY77SM5JJj03HUPZ/5eCfIL48U8/0sS/SdUCKS6GYwfgqL1UUVINlbYXSu7p1xgiu5zWVtEZ/OpHd2tVszQp1JBj2QW8vmQ37y7fR35RMVf3a8mDI9vTLERH2DZ0hcWFTJw7kY3JG8s9p1OTTgyOHsyg6EH0ieqDv3c13zd5x61SReLmk9VQiVusxm/AKlXEltFW0Qq8tHddQ6ZJoYYlZeXyys+7+Pi3A3iJcOPg1tw9Io4mQb61HovyHN/v/p4nVz1JdmF26b5A70Amdp2IzcvGqoRVbEjaQGFxIb5evvSO7M2g5oMYHD2YTk061Uw3W2Mg48CZbRVpeygtVfgGl91W4Vf2TMGq/tGk4CIH07J5YWE832w4RKCvN7cNi+W2YW1p5NegBooru6z8LMZ8OeaUOaiCfYKZd+W80qnZswuyWZ+0npVHVrIqYRU703cC1gjxAc0GMLi5VZJoGdyyZoPLP3GyVHHUsVTh0OgeFltGW0VrLVXUQ5oUXCw+MYv/zd/J3C1HaRLkyz0j2nH9oNb4++joaFWxlJwUViesZuWRlaxMWElSdhIAMY1iSksRA6MHEuIXUsmVqsAYOHbQIUnYq6FSd3OyVNHo1FJFVHeI6qKlijpOk0It2Xgwg6nzd7A0PoXoEH8eGNmeq/rG6Oho5RRjDHsz97LqyCpWJqxkzdE1nCg4gSB0Ce9SWoroFdkLP5sLJ/fLzz6zreLo5tNKFW2skkRUt5NtFaFttFRRR2hSqGUrdlujozccyCA2IojJozpwSfdovGpo3QbVMBQWF7I5ZTMrE1ay6sgqNiVvotAU4m/zp09UH6vRuvkgOoR1wEtc/GFsjLXyn2Pvp6ObIW03lExZ4tvI6gFVOmK7m7Xtr124PY0mBTcwxrBwWxJT5+1gR2IWnaMb88iYDpzXMVJHR6sqOVFwgrVH17IqYRUrj6xk97HdADTxb8LAZgNLSxLRjaJrL6j8bEjedjJJlCSNXIdSRWhrK0GUdpftZrVfaKnCbTQpuFFRsWHOpiNMW7CT/anZ9GsdxiNjOjKwbbi7Q1N1XFJ2EqsSVpVWN6XkpADQpnEbBkZbSaJ/s/409q3lb+rGQObhU3s/JW6G1F0nSxU+QVbbREmSiOpmbZ8+8lu5hCYFD1BQVMznaw/y4qJ4EjPzOLdDUx4Z05FuLfQ/gao+Ywy7M3ZbVU0Jq1hzdA05hTl4iRfdIroxKNpqtO7ZtGf153iqqoIce1vFllO7zOZmnDwntNVpbRXdrfYLXZOiRmlS8CC5BUW8v3IfMxfvJiO7gIu7RzN5VAfiInXkqao5BUUFbErZVNr1dXPKZopMEQHeAfSL6mclieaDiQuNc291pjGQeeTMtorUeIdSRaBDW4W9u2xkF2t9C1UlmhQ8UGZuAW8t3cvbS/eQU1DElX1jePCCDrQI1cXsVc3Lys9izdE1pUliX+Y+ACICIhgUPaj0ERUU5d5ASxTkQPL2k+Mpjv5hlS5yHFYCDGl15jTkTWK1VOEETQoeLPV4HjMX7+aDVfvBwHWDWnHveXFENNL1hJXrJBxPsBqsE1ayOmE1abnWMrVtQ9qWNlj3b9afIB8Pmt/LGMhKOLOtIiX+5GJHPoHW6GzHqT2iumqp4jSaFOqAIxk5vLgoni/WHcLP24tbh1qjo0MC3FT/qxqMYlNMfHp8aa+mdYnryC3KxVu86d60e2nX124R3fDx8sD3Y0EupOw4tffT0c3WSnklQlo6NGrbq6GatG2wpQpNCnXI7uTjTFuwkx82JRAS4MNd57Zj4jltCPBtmG9eVfvyivLYmLSxNElsSd2CwRDkE0T/qP6lI61jQ2I9t3u1MZB19My2ipSdJ0sV3gEnSxXNHEsV9X/dFE0KddDmw8f43/wd/LIjmchgP+4f2Z7x/Vri6619u1XtOpZ3jN+O/lba9fVg1kEAIgMjSxusB0UPIiIgws2ROqEwr+y2iuzUk+c0jjmzrSK8Xb0qVWhSqMN+25vG8/O2s2ZfOq2aBDJ5VHvG9WyBTUdHKzc5lHWotBSx+ujq0pXs2oe1L+362jeqL4E+gW6O1EnGwPHE0yYLtJcqiu1LvHr7l91WEVjFNTLcTJNCHWeMYfHOZJ6fu4OtCZl0jArmL6M7MKpLlOcW31WDUGyK2Za2rbQUsSFxA/nF+Xh7edOraa/SkkSX8C54e9Wx2YML86zEcLRkDih70shOOXlO4xZltFW0A5tnv1ZNCvVEcbHhx80JTJu/kz0pJ+jVMpRHx3TknLg6UGxXDUJuYS7rk9aXjrTelrYNsKYQHxA9oDRJtApuVXe/0GQlnrmwUfL2U0sVTTudObWHB5UqPC4piMhk4Das+Xn/AG4GooFPgXBgHXCDMSa/ous0tKRQorComK/WH2LGwniOHMtlSFw4j4zpRK+W2u1OeZa03DR+S/ittLrpyIkjAEQHRZe2RQyMHlj1pUo9RWG+VapwLFEkboETSSfPCW5+6mSBUd0gPM4tpQqPSgoi0gJYBnQxxuSIyOfAj8BFwNfGmE9F5DVgozHm1Yqu1VCTQoncgiI+Wn2AV37ZRdqJfEZ3ieIvozvSsZnOda88jzGGg1kHSwfQrT66mqx8a0GiGl+q1FMcTzp1+vHSUkWBddzmB5Gdzpzao6JSRXERrHwFlk2HYQ/BoHvOuhHcE5PCKqAnkAl8C7wEfAQ0M8YUishg4HFjzJiKrtXQk0KJ43mFvLNsL28u2cPx/EKu6NWCSRd0oFV4HWnoUw1SUXERW1O3ls7X5PKlSj1FYb41jYdjW0XiFquxu0Rw9KmTBTazlyoyDsAXN1kLIRVkW4P1wuPgqvesHlJO8qikACAiDwL/BXKA+cCDwCpjTJz9eEvgJ2NMtzKeewdwB0CrVq367t+/v9bi9nTpJ/J57dfdvLdiH8XGMKF/K+4/P47IxvXkW5eq19y2VKmnOJ7s0E5hTxbJO6DIXotu87VKCaaY0pXxAMQLAprAo7udvpVHJQURCQO+AsYDGcAXwJdYJYNKk4IjLSmULTEzlxcXxfPZmoN424SJ58Ry17ltCQ30dXdoSjnNrUuVeoqiAmsaj5IksW7WqbPKlmgzDCbOcfqynpYUrgLGGmNutW/fCAwGrkKrj2rU/tQTTF+wk9kbj9DIz5s7h7fl5iGxBPl5dnc5pU7nMUuVutvGz+CHhyD/+Ml9vo3g4mnQc7zTl/G0pDAQeAfoj1V99B6wFhgOfOXQ0LzJGDOzomtpUnDO9qOZTJ23k4XbEolo5Mu958Vx7cBW+HnXo3pa1aBUtlRpSdfXWlmqtDblHoMXup+6sp1/CEz646wWKPKopAAgIv/Gqj4qBDZgdU9tgdUltYl93/XGmLyKrqNJ4eys25/O8/O2s2pPGi1CA3jwgvb8qXcLvG316D+NapAqW6q0pLqpVpcq9WAelxRqiiaFs2eMYfmuVJ6ft52Nh47RrmkQfxndkbFdm+GlU2eoesJjlyr1EJoU1BmMMczbksj/5u8gPuk43Vo05pExnRjePqLujjRVqgzOLFU6KHoQvZr2ct9SpbVMk4IqV1Gx4dsNh5m+cCeH0nMYENuEx8Z2pG/rOj7CVKlyVLRUad+ovgyOHuwZS5W6kCYFVam8wiI+W3OQFxftIuV4Hud3iuTh0R3p0rxhFq9Vw1HnliqtAZoUlNOy8wt5b8U+Xlu8m8zcQsb1bM7kUR2IjfCgZRmVcqE6uVTpWdKkoM7asewC3li6m3eW7SO/qJir+8XwwMj2RIcEuDs0pWpNyVKlJaWIOrdUaTk0KagqS8rKZeYvu/lo9X5EhBsHteae8+JoEqSjo1XDUy+WKkWTgqoBB9OymbEonq/XHyLQ15vbhsVy69BYgv3rzrcjpWpayVKlJSWJurJUqSYFVWPiE7OYtmAnP20+SligD/eeF8f1g1rj76Ojo5WqK0uValJQNW7ToQyen7eDpfEpNGvszwMj23NVvxh8dHS0UoBnL1WqSUG5zMrdqTw3bzsbDmTQJjyQyaM6cGmP5jo6WqnTeNJSpZoUlEsZY1i0LYmp83ew/WgWnZoF88iYjpzfKdKjG9uUcqeqLlVaVFzE+1vf5+3Nb3Nb99u4ofMNZ70IkSYFVSuKiw3fbzrCtAU72Z+aTd/WYTwypiOD2oa7OzSlPJqzS5VGBETwt2V/40DWAXIKcwjwDqB149ZMPXcqrRu3dvp+mhRUrSooKubztQd5cVE8iZl5DO/QlEdGd6R7TD1eDEWpGlTeUqVl8RIvQvxCWDJ+idPX16Sg3CK3oIj3V+5j5uLdZGQXcFH3Zjw0qiNxkY3cHZpSdUrJUqWPr3icxOzEM44PaDaAt8e87fT1KkoK2lVEuYy/j407hrdjyaPn8cDI9vy6I5nR03/lkS82cig9293hKVVnBPoEMrTFUB7s8yCB3qd2Zw30DuTyuMtr7F6aFJTLNfb34aFRHVjy6HncPCSW2RuPcP7UX3n8uy0kZ1W4jpJSysGIliOwyamNyjaxMaLliBq7h1YfqVp3JCOHFxfF88W6Q/h5e3HLkFhuH96WkAAdHa1UbdA2BeWR9iQfZ9qCnczZlEBIgA93nduOiee0IcBXR0cr5UqaFJRH23LkGFPn7eCXHck0DfbjgfPjGN+/Fb7eWruplCtoUlB1wpp9aTw/dwe/7UujZZMAJl/Qgct6tcCmo6OVqlHa+0jVCf3bNOGzOwfx3s39rcbpzzdy4YwlzNtylLr85UWpukSTgvIoIsKIjpF8f99QXrm2D4VFhjs/WMflM1ewfFeKu8NTqt7TpKA8kpeXcHGPaOZPHs5zf+5BcmYu1721mmvfXMWGA+nuDk+pekvbFFSdkFtQxMerD/DKL7tIPZHPqC5RPDy6Ix2bBbs7NKXqHG1oVvXG8bxC3l22lzeW7OF4fiGX92rB5As60CrcvYuWKFWXaFJQ9U76iXxeW7Kb95bvo6jYMGFAS+4/vz1Rjf3dHZpSHk+Tgqq3EjNzeenneD797SDeNuGmc9pw97ntCA30dXdoSnksTQqq3tufeoIXFsbz7e+HaeTrzR3D23LL0FiC/Gp3mUOl6gJNCqrB2H40k//N38mCrYmEB/ly73lxXDuwFf4+OnWGUiU0KagGZ/2BdJ6fu4OVe1JpHuLPpAs68Kc+LfC2aS9spXREs2pw+rQK45M7BvHhrQNpGuzHo19tYvQLS/hhUwLFxXX3i5BSrqZJQdVrQ9tH8O29Q3jt+r7YRLj34/WMe2UZi3ck6dQZSpXB6aQgInEi8qGIfCUig10ZlFI1SUQY260ZcycN539X9SQju4CJ765h/BurWLsvzd3hKeVRym1TEBF/Y0yuw/YnwKP2ze+NMb1qIb4KaZuCqor8wmI+XXOAFxftIuV4Hud1bMrDYzrStXmIu0NTqlZUtU3hexG50WG7AGgDtAaKqhlQqIh8KSLbRWSbiAwWkSYiskBE4u0/w6pzD6XK4+vtxY2D27Dk0RE8OrYj6/anc/GLy7j/kw3sTTnh7vCUcquKksJYoLGIzBWR4cDDwBjgCuC6at53BjDXGNMJ6AlsA6YAi4wx7YFF9m2lXCbQ15t7RsSx9LHzufe8dizcmsgF037lr19v4khGjrvDU8otKu2SKiIhwP8BLYB/GGN2V+uG1vV+B9oah5uLyA5ghDEmQUSigcXGmI4VXUurj1RNSs7K45VfdvHR6v2ICDcMas09I9oR3sjP3aEpVaOqNE5BRAYCjwD5wFNADvBf4DDwH2NMRhWD6QW8AWzFKiWsAx4EDhtjQu3nCJBesn3a8+8A7gBo1apV3/3791clDKXKdTAtmxmL4vl6/SECfGzcOqwttw+LJdjfx92hKVUjqpoUfgcuAhoB7xpjhtj3nwv8zRgzporB9ANWAUOMMci2h98AAB7cSURBVKtFZAaQCdzvmAREJN0YU2G7gpYUlCvtSsrif/N38tPmo4QF+nDPiDhuGNxaR0erOq+qDc2FnGxYzi/ZaYz5taoJwe4QcMgYs9q+/SXQB0i0Vxth/5lUjXsoVW1xkcG8en1fvrtvCN1ahPDfH7cx4vnFfLz6AAVFxe4OTymXqCgpXAv8GTgfuLGC886KMeYocFBEStoLRmJVJX0H3GTfdxMwu6buqVR19IgJ5YNbB/LJ7YNoHurP3775gwum/crs3w/r6GhV77hl7iN7u8JbgC+wB7gZK0F9DrQC9gNXG2MqHFmk1Ueqthlj+Hl7Es/P28H2o1l0ahbMI2M6cn6nSKymMKU8n06Ip1QNKy42fL/pCNMW7GR/ajZ9WoXyyJhODIhtwtvL9jBz8W7uGdGOW4e2xealyUJ5Fk0KSrlIQVExX6w9xIxFO0nMzCPIz0ZhkSGvsJgAHxttmwbx8rV9iI0IcneoSpWq1iypInKpiOjEeUqVwcfmxbUDW/HrI+cR6GvjRF4ReYVWI3ROQRHbEjK58tUVbo5SKec582E/HogXkedEpJOrA1KqLvL3sdEj5sy5k4oNdIhq5IaIlKqaSpOCMeZ6oDewG3hPRFaKyB0iEuzy6JSqQ8b3b0mQ75ljGLLzi0g7kV/GM5TyPE5VCxljMrHGE3wKRGPNf7ReRO53YWxK1SkjO0ed0ajs5+3FliPHGD19CYu2JbopMqWc58zcR+OwuozGAe8Ds4wxSSISCGw1xrRxeZTl0IZmVRdsS8hk8me/s/1oFuP7teQfl3TWKTOUW1V3Oc4/A9ONMd2NMc8bY5IAjDHZwK01GKdS9VLn6MbMvm8I94xoxxfrDnLhjKWs2pPq7rCUKpMzSeFx4LeSDREJEJE2AMaYRS6JSql6xs/bxqNjO/HFXYOxeQnXvLmKJ+dsJbegWkuTKFXjnEkKXwCOE70U2fcppc5S39ZN+PGBYVw3sBVvLdvLpS8t449Dx9wdllKlnEkK3sYYxwnx8rGmp1BKVUGQnzdPXt6dWbcMIDO3gCtmLmfGwnidZE95BGeSQrK9sRkAEbkMSHFdSEo1DOd2aMr8SedycY9opi/cyZWvrmB38nF3h6UaOGeSwl3A30TkgIgcBB4D7nRtWEo1DCGBPsyY0JtXru3D/rRsLpqxlHeX79XZV5XbOD33kYg0AjDGeMxXGe2SquqTpMxcpnz9Bz9vT+KcduE8f1VPWoQGuDssVQ9Ve0I8EbkY6Ar4l+wzxjxRYxFWkSYFVd8YY/hszUH+M2crXiL8a1xX/tynhU7LrWpUdSfEew1r/qP7AQGuwlqNTSlVw0SECQNaMXfScDpHN+bhLzZy5wfrSDme5+7QVAPhTJvCOcaYG4F0Y8y/gcFAB9eGpVTD1rJJIJ/cMYi/X9SZxTuSGTN9CfO2HHV3WKoBcCYp5Np/ZotIc6AAa/4jpZQL2byE24e35fv7h9IsxJ87P1jHXz7fSGZugbtDU/WYM0nhexEJBZ4H1gP7gI9dGZRS6qSOzYL55p4h3H9+HN/+fpix05ewYpf2CleuUWFSsC+us8gYk2GM+QqrLaGTMeaftRKdUgoAX28v/jK6I1/eNRh/HxvXvrWax7/bQk6+TpOhalaFScEYUwy84rCdZ4zRMflKuUnvVmH88MAwJp7ThvdW7OPil5ay8WCGu8NS9Ygz1UeLROTPon3ilPIIAb42Hh/XlY9uG0hOfhF/enUF0xbs1GkyVI1wJinciTUBXp6IZIpIlohkujgupVQlhsRFMHfScC7r1ZwXF8VzxczlxCdmuTssVcc5sxxnsDHGyxjja4xpbN9uXBvBKaUqFhLgw7Sre/Ha9X05kpHLxS8t462le3SaDFVl3pWdICLDy9pvjFlS8+EopapibLdm9G0dxt+++YMnf9jGgq2JTL2qJy2bBLo7NFXHOLMc5/cOm/7AAGCdMeZ8VwbmDJ3mQqlTGWP4ct0h/v39Vowx/PPSLlzdr6VOk6FOUa1pLowxlzo8RgHdgPSaDlIpVX0iwlX9WjJ30jC6x4Tw2Fd/cNustSRl5Vb+ZKVwrqH5dIeAzjUdiFKq5sSEBfLxbYP4v0u6sGxXCmOmL+HHPxLcHZaqA5xpU3gJKKlj8gJ6YY1sVkp5MC8v4dahsZzbIYKHPt/IPR+t5/Jezfn3uG6EBPq4OzzloSpNCoBjpX0h8IkxZrmL4lFK1bC4yGC+uvscXvllFy/9vItVe9J4/qoeDGvf1N2hKQ/kTENzEJBrjCmyb9sAP2NMdi3EVyFtaFbq7Gw6lMFDn29kV9JxbhzcmikXdiLQ15nvhqo+qVZDM7AIcFz+KQBYWBOBKaVqV4+YUObcP5Rbh8bywar9XDRjKev2a78RdZIzScHfcQlO++/a+VmpOsrfx8b/XdKFj28bREGR4arXVvD8vO3kF+o0Gcq5pHBCRPqUbIhIXyDHdSEppWrD4HbhzJ00jCv7xvDKL7u57JXlbD+qM9g0dM4khUnAFyKyVESWAZ8B97k2LKVUbQj29+G5K3vy5o39SM7KZdxLy3nt190U6TQZDValDc0AIuIDdLRv7jDGeMTST9rQrFTNST2ex9+/2czcLUfp1zqM/13dk9bhQe4OS7lAtRqaReReIMgYs9kYsxloJCL31EBQNhHZICJz7NuxIrJaRHaJyGci4lvdeyilnBfeyI9Xr+/D9PE92ZGYxYUzlvLR6v0488VR1R/OVB/dbowpXcXDGJMO3F4D934Q2Oaw/Sww3RgThzWNxq01cA+l1FkQEa7oHcO8ScPp0yqMv3+zmYnvriExU6fJaCicSQo2xwV27OMUqvUtXkRigIuBt+zbApwPfGk/ZRZweXXuoZSquuahAbx/ywCeuKwrq/emMnr6Er7feMTdYala4ExSmAt8JiIjRWQk8Il9X3W8ADwKlPSBCwcyjDGF9u1DQIuynigid4jIWhFZm5ycXM0wlFLl8fISbhzchh8fGEZsRBD3f7KB+z/ZQEZ2vrtDUy7kTFJ4DPgZuNv+WAQ8UtUbisglQJIxZl1Vnm+MecMY088Y069pUx2mr5SrtW3aiC/vGszDozvw0x8JjJ6+hF92JLk7LOUizkydXWyMec0Yc6Ux5kpgK/BSNe45BBgnIvuAT7GqjWYAoSJSMt4+BjhcjXsopWqQt82L+85vz7f3DiEs0Jeb313D3775gxN5hZU/WdUpTk2dLSK9ReQ5+wf5E8D2qt7QGPNXY0yMMaYNMAH42RhzHfALcKX9tJuA2VW9h1LKNbq1CGH2fUO4c3hbPvntABfOWMqafWnuDkvVoHKTgoh0EJF/ich2rJLBQaxxDecZY6pTUijPY8BDIrILq43hbRfcQylVTf4+Nv56UWc+u2MwBsPVr6/k6Z+2kVdY5O7QVA0od/CaiBQDS4FbjTG77Pv2GGPa1mJ8FdLBa0q51/G8Qv77wzY++e0AHaOCmTa+J12bh7g7LFWJqg5e+xOQAPwiIm/aex7pQq9KqVKN/Lx5+k/deXdif9Ky87n8leW88ssuCot0cr26qtykYIz51hgzAeiEVd8/CYgUkVdFZHRtBaiU8nzndYpk/qThjO7ajOfn7eCq11eyN+WEu8NSVeBM76MTxpiPjTGXYvUK2oBV/6+UUqXCgnx55do+vHhNb/Ykn+DCGUt4f+U+inVyvTrFqd5HJYwx6fZxAiNdFZBSqm4b17M58yYNZ0BsOP+cvYWb3v2NhGM6235dcVZJQSmlnNEsxJ9ZN/fnv1d0Y+2+dEZPX8K3Gw7r5Hp1gCYFpZRLiAjXDWzNTw8Oo0NUMJM++517PlpP2gmdJsOTaVJQSrlUm4ggPr9zMI+N7cTCbYmMnr6EhVsT3R2WKocmBaWUy9m8hLtHtOO7+4YS0ciX295fy2NfbiIr1yPW61IONCkopWpN5+jGzL5vCPeMaMcX6w4y9oWlrNqT6u6wlANNCkqpWuXnbePRsZ344q7BeNuEa95cxZNztpJboNNkeAJNCkopt+jbugk/PTiM6we25q1le7nkpWX8ceiYu8Nq8DQpKKXcJtDXm/9c3o1ZtwwgK7eAK2YuZ8bCeAp0mgy30aSglHK7czs0Zf6kc7mkRzTTF+7kyldXsCvpuLvDapA0KSilPEJIoA8vTOjNK9f24UBaNhe/uJR3l+/VaTJqmSYFpZRHubhHNPMmD2doXAT//n4r1721msMZOk1GbdGkoJTyOJHB/rx1Uz+e/XN3Nh3KYOz0JXy57pBOk1ELNCkopTySiDC+fyvmThpO5+jGPPzFRu74YB0px/PcHVq9pklBKeXRWjYJ5JM7BvH3izrz645kxkxfwtzNR90dVr2lSUEp5fFsXsLtw9sy54GhNAvx564P1/HQ57+TqdNk1DhNCkqpOqNDVDDf3DOEB86PY/bvRxg7fQnLd6W4O6x6RZOCUqpO8fX24qHRHfnq7nPw97Fx3Vurefy7LeTk6zQZNUGTglKqTurVMpQfHhjGxHPa8N6KfVz80lJ+P5jh7rDqPE0KSqk6K8DXxuPjuvLRbQPJzS/iz6+uYNr8HTpNRjVoUlBK1XlD4iKYO3k4l/dqwYs/7+KKmcvZmZjl7rDqJE0KSql6obG/D/+7uievXd+XIxm5XPLSMt5csocinSbjrGhSUErVK2O7NWPepOGc26Ep//1xG9e8uYqDadnuDqvO0KSglKp3mgb78cYNfXn+yh5sPZLJ2BeW8NmaAzpNhhM0KSil6iUR4ap+LZk7aRg9YkJ57Ks/uHXWWpKyct0dmkeTupw5+/XrZ9auXXvKvoKCAg4dOkRurv7Dl8Xf35+YmBh8fHzcHYpStaa42PDein08O3c7gb42/ntFdy7qHu3usNxGRNYZY/qVeay+JYW9e/cSHBxMeHg4IuKmyDyTMYbU1FSysrKIjY11dzhK1bpdScd56PPf2XToGJf1as4T47oREtjwviBVlBTqXfVRbm6uJoRyiAjh4eFailINVlxkI766+xwmX9CBHzYlMOaFJSzZmezusDxKvUsKgCaECujfRjV0PjYvHrygPd/cM4RG/t7c+M5v/N+3m8nOL3R3aB6hXiYFpZSqTPeYEObcP5Tbhsby4er9XDRjKev2p7s7LLdr8EmhqNjwxpLd9HpiPm8s2e1RA11uueUWIiMj6dat2yn709LSGDVqFO3bt2fUqFGkp+sbWamq8Pex8Y9LuvDxbYMoKDJc9doKnpu7nfzChjtNRq0nBRFpKSK/iMhWEdkiIg/a9zcRkQUiEm//GebqWPamnODSl5YxfUE8GdkFTF8Qz7iXl7E35YSrb+2UiRMnMnfu3DP2P/PMM4wcOZL4+HhGjhzJM88844bolKo/BrcLZ+6kYVzZN4aZi3dz2SvL2ZaQ6e6w3KLWex+JSDQQbYxZLyLBwDrgcmAikGaMeUZEpgBhxpjHKrpWWb2Ptm3bRufOnQH49/db2Hqk/H/YdfvTKSyjZODtJfRtXXZO6tK8Mf+6tGtFYbFv3z7Gjh3LoEGDWLFiBf379+fmm2/mX//6F0lJSXz00UcMGDCgwms4XuuSSy5h8+bNpfs6duzI4sWLiY6OJiEhgREjRrBjxw6nrgen/o2UUqdauDWRKV9vIjOnkMmjOnDH8LbYvOpXW5xH9T4yxiQYY9bbf88CtgEtgMuAWfbTZmElCpcK8LWd1f6zsWvXLv7yl7+wfft2tm/fzscff8yyZcuYOnUqTz31FL/88gu9evU643HOOedUeu3ExESio60+1s2aNSMxMbHa8SqlLBd0iWLepOGc3ymSZ+duZ/zrK9mf6hm1B7XB2503F5E2QG9gNRBljEmwHzoKRJXznDuAOwBatWpV4fUr+0b/zYZD/OObzZxwWJwjyNfGE5d15YreMU69hvLExsbSvXt3ALp27crIkSMREbp3786+ffs477zz+P3336t1D7B6E2mPIqVqVngjP169vg/f/n6Yf87ewoUzlvK3izpz3cBW9f7/m9samkWkEfAVMMkYc0odj7HqtMqs1zLGvGGM6WeM6de0adNqxTCyc9QZxUKblzCyc5n56Kz4+fmV/u7l5VW67eXlRWFhYbVKClFRUSQkWPkzISGByMjIaserlDqViHBF7xjmTRpOn1Zh/OPbzUx8dw1Hj9XvcT5uKSmIiA9WQvjIGPO1fXeiiEQbYxLs7Q5Jro6jsb8Pmx4f4+rblKk6JYVx48Yxa9YspkyZwqxZs7jssstqODqlVInmoQG8f8sAPly9n6d+3MaYF5bwn8u7Ma5nc3eH5hLu6H0kwNvANmPMNIdD3wE32X+/CZhd27F5mmuuuYbBgwezY8cOYmJiePvttwGYMmUKCxYsoH379ixcuJApU6a4OVKl6jcvL+HGwW348YFhtG0axAOfbOC+j9eTfiLf3aHVOHf0PhoKLAX+AEo6A/8Nq13hc6AVsB+42hiTVtG1Kut9pMqmfyOlqq6wqJjXl+zhhYU7CQv05dkre3Bex7pVhVtR76Narz4yxiwDymupGVmbsSil1Nnytnlx73lxjOjYlIc+28jN767hmgGt+MfFnQnyc2vfnRrR4Ec0K6VUVXRtHsLs+4Zw5/C2fLrmABfOWMqafRVWbtQJmhSUUqqK/H1s/PWiznx2x2AMhqtfX8nTP24jt6Co8id7KE0KSilVTQNim/DTg8OZ0L8Vry/Zw2UvL2fLkWPuDqtKNCkopVQNaOTnzdN/6s67E/uTnp3PZS8v5+Wf4yksqluT62lSUEqpGnRep0jmTRrO2G7NmDp/J1e+tpI9ycfdHZbTNCkUF8HyF+HZWFjxkrXtQYqKiujduzeXXHJJ6b69e/cycOBA4uLiGD9+PPn59a+vtFJ1WViQLy9f24cXr+nN3pQTXPTiUt5fuY9iD5qavzwNOymk7oY3zoXFT0NOGvzyFLwxwtrvIWbMmHHGmILHHnuMyZMns2vXLsLCwkoHtSmlPMu4ns2ZP3k4A2PD+efsLdz07m8kHMtxd1gVqvXBazWp0sFrP02Bo3+Uf4GDq6G44Mz9Xj7QcmDZz2nWHS6seP2Cmpo6+9ChQ9x00038/e9/Z9q0acyZMwdjDE2bNuXo0aN4e3uzcuVKHn/8cebNm1fp9Uro4DWlapcxho9/O8CTc7bhbROeuKwrl/dq4bbJ9Txq6myP4hNQzv7Aal+6JqbOnjRpEs899xxeXif/mVJTUwkNDcXb2xokExMTw+HDh6sdr1LKdUSE6wa25qcHh9EhKpjJn23kno/Wk3o8z92hnaHuD7+rSCXf6Nn4GfzwEOQ7NAL5NoKLnoee46t16+pOnT1nzhwiIyPp27cvixcvrlYsSinP0CYiiM/vHMybS/cwbf5O1uxbwtN/6sGoLtWfmbmmNOySQsex4HXagjpeNmt/NVV36uzly5fz3Xff0aZNGyZMmMDPP//M9ddfT3h4OBkZGRQWFgJWFVOLFi2qHa9SqnbYvIS7zm3H7PuGENHIj9vfX8ujX24kK7eMqmw3qN8lhcr4h8CUA265dWUlhaeffpqnn34agMWLFzN16lQ+/PDD0ud++eWXTJgwQafOVqqO6hzdmO/uG8qMRTt5dfFulu9KZepVPRncLtytcTXskkId9eyzzzJt2jTi4uJITU3l1ltvdXdISqkq8PX24pExnfjirnPwsQnXvLmK/8zZ6tZpMup37yNVJv0bKeV5svMLefrH7Xywaj9xkY2YdnVPesSEuuRe2vtIKaU8XKCvN/+5vBvv3zKA47mFXDFzBS8s3ElBLU+ToUlBKaU8yPAOTZk3aTiX9ojmhYXx/PnVFexKqr1pMjQpKKWUhwkJ9OGFCb2ZeV0fDqZlc/GLS3ln2d5amSZDk4JSSnmoi7pHM2/ycIbGRfDEnK1c99Zq9qee4I0lu+n1xHzeWLKbohpOFA27S6pSSnm4yGB/3rqpH5+vPcjj323hvKmLsXkJBUWG6Qvimf37EV6+tg+xEUE1cj8tKSillIcTEcb3b4Wfj41iAwVFVukgp6CIbQmZXPnqihq7V4NPCkXFRby7+V2GfjqU97a8R5EHTZ19yy23EBkZSbdu3U7Zn5aWxqhRo2jfvj2jRo0iPT0dsCbdeuCBB4iLi6NHjx6sX7/eHWErpVykU7PgM/YVG+gQdeb+qmrQSWF/5n7GzxnPqxtf5VjeMWb+PpMJP0xgf+Z+d4cGwMSJE5k7d+4Z+5955hlGjhxJfHw8I0eO5JlnrDmefvrpJ+Lj44mPj+eNN97g7rvvru2QlVIuNL5/S4J8T52aJ8jXxtX9Y2rsHvW6TeHZ355le9r2co//nvQ7haawdDunMIftadu5/NvL6RXZq8zndGrSiccGPFbhfWtq6uzhw4ezb9++M/bPnj27dJK8m266iREjRvDss88ye/ZsbrzxRkSEQYMGkZGRQUJCAtHR0ZXeSynl+UZ2juJfs7ecss/mJYzsXHMT6tXrpFCZAO8AsgqyytxfXbt27eKLL77gnXfeoX///qVTZ3/33Xc89dRTPPjgg0yePPmM5wUGBrJiRcX1g4mJiaUf9M2aNSMxMRGAw4cP07Jly9LzSqbV1qSgVP3Q2N+HTY+Pcek96nVSqOwb/fe7v+fJVU+SXZhdui/QO5C/Dvwrl7a7tFr3ru7U2c4SEbct1KGUqn8adJvCiJYjsMmp9XM2sTGi5YhqX7u6U2dXJCoqioSEBAASEhKIjIwEoEWLFhw8eLD0PJ1WWyl1tup1SaEywb7BrLi25rpynY3qlBTGjRvHrFmzmDJlyilTZ48bN46XX36ZCRMmsHr1akJCQrTqSCl1Vhp0ScHTXXPNNQwePJgdO3YQExPD22+/DcCUKVNYsGAB7du3Z+HChUyZMgWAiy66iLZt2xIXF8ftt9/OzJkz3Rm+UqoO0qmzGyD9GynVsOnU2UoppZyiSUEppVSpepkU6nKVmKvp30YpVZF6lxT8/f1JTU3VD78yGGNITU3F39/f3aEopTxUveuSGhMTw6FDh0hOTnZ3KB7J39+fmJiamydFKVW/1Luk4OPjQ2xsrLvDUEqpOsmjqo9EZKyI7BCRXSIyxd3xKKVUQ+MxSUFEbMArwIVAF+AaEeni3qiUUqph8ZikAAwAdhlj9hhj8oFPgcvcHJNSSjUontSm0AI46LB9CBh4+kkicgdwh33zuIjsqOL9IoCUKj5Xqcro+0u5WnXeY63LO+BJScEpxpg3gDeqex0RWVveMG+lqkvfX8rVXPUe86Tqo8NAS4ftGPs+pZRStcSTksIaoL2IxIqILzAB+M7NMSmlVIPiMdVHxphCEbkPmAfYgHeMMVsqeVp1VLsKSqkK6PtLuZpL3mN1eupspZRSNcuTqo+UUkq5mSYFpZRSpep9Uqhs6gwRGS4i60WkUESudEeMqu5y4v31kIhsFZFNIrJIRMrtH67U6Zx4f90lIn+IyO8isqwmZoGo120K9qkzdgKjsAbDrQGuMcZsdTinDdAYeBj4zhjzZe1HquoiJ99f5wGrjTHZInI3MMIYM94tAas6xcn3V2NjTKb993HAPcaYsdW5b30vKVQ6dYYxZp8xZhNQ7I4AVZ3mzPvrF2NMtn1zFdb4G6Wc4cz7K9NhMwio9rf8+p4Uypo6o4WbYlH1z9m+v24FfnJpRKo+cer9JSL3ishu4DnggeretL4nBaU8gohcD/QDnnd3LKp+Mca8YoxpBzwG/KO616vvSUGnzlCu5NT7S0QuAP4OjDPG5NVSbKruO9vPr0+By6t70/qeFHTqDOVKlb6/RKQ38DpWQkhyQ4yq7nLm/dXeYfNiIL66N63XScEYUwiUTJ2xDfjcGLNFRJ6wt9QjIv1F5BBwFfC6iLhyag1Vjzjz/sKqLmoEfGHvNqhfSpRTnHx/3SciW0Tkd+Ah4Kbq3rded0lVSil1dup1SUEppdTZ0aSglFKqlCYFpZRSpTQpKKWUKqVJQSmlVClNCkq5gIiMK5nVUkQeF5GH3R2TUs7wmOU4lapPjDHfoQMlVR2kJQWlyiAi14vIb/YBZ6+LiE1EjovIdPtgoUUi0tR+7gMOayZ8at83UUReLuO6vURklf3cb0QkzL5/sYg8a7/nThEZVruvWCmLJgWlTiMinYHxwBBjTC+gCLgOa2ritcaYrsCvwL/sT5kC9DbG9ADuquTy7wOP2c/9w+EaAN7GmAHApNP2K1VrtPpIqTONBPoCa0QEIABIwlpz4zP7OR8CX9t/3wR8JCLfAt+Wd1ERCQFCjTG/2nfNAr5wOKXkeuuANtV+FUpVgZYUlDqTALOMMb3sj47GmMfLOK9kjpiLgVeAPliJpKpftkpmUC1Cv7ApN9GkoNSZFgFXikgkgIg0sa+t7AWUrON9LbBMRLyAlsaYX7Dmsw/BmgDvDMaYY0C6Q3vBDVjVUEp5DP02otRpjDFbReQfwHz7h34BcC9wAhhgP5aE1e5gAz60Vw0J8KIxJsNe7VSWm4DXRCQQ2APc7NpXo9TZ0VlSlXKSiBw3xpRZClCqvtDqI6WUUqW0pKCUUqqUlhSUUkqV0qSglFKqlCYFpZRSpTQpKKWUKqVJQSmlVKn/BxNcwBwzKSs2AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 896
        },
        "id": "cpZE4_NUAaP2",
        "outputId": "96445369-ebbb-458f-af9f-75d3e6d7ec3c"
      },
      "source": [
        "column = ['total_epochs', 'replay', 'lr_type', 'attack_type', 'attack_n_itr', 'eps', 'adv_acc','clean_acc']\n",
        "df = pd.DataFrame(pgd_report, columns= column)\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>total_epochs</th>\n",
              "      <th>replay</th>\n",
              "      <th>lr_type</th>\n",
              "      <th>attack_type</th>\n",
              "      <th>attack_n_itr</th>\n",
              "      <th>eps</th>\n",
              "      <th>adv_acc</th>\n",
              "      <th>clean_acc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1000</td>\n",
              "      <td>10</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.1</td>\n",
              "      <td>95.35</td>\n",
              "      <td>99.02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1000</td>\n",
              "      <td>10</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.2</td>\n",
              "      <td>93.66</td>\n",
              "      <td>99.02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1000</td>\n",
              "      <td>10</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.3</td>\n",
              "      <td>91.40</td>\n",
              "      <td>99.02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.1</td>\n",
              "      <td>97.54</td>\n",
              "      <td>99.10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.2</td>\n",
              "      <td>97.27</td>\n",
              "      <td>99.10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.3</td>\n",
              "      <td>97.11</td>\n",
              "      <td>99.10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>1000</td>\n",
              "      <td>100</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.1</td>\n",
              "      <td>96.97</td>\n",
              "      <td>98.79</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>1000</td>\n",
              "      <td>100</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.2</td>\n",
              "      <td>96.48</td>\n",
              "      <td>98.79</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>1000</td>\n",
              "      <td>100</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.3</td>\n",
              "      <td>95.99</td>\n",
              "      <td>98.79</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>1000</td>\n",
              "      <td>10</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.1</td>\n",
              "      <td>93.20</td>\n",
              "      <td>99.02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>1000</td>\n",
              "      <td>10</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.2</td>\n",
              "      <td>63.50</td>\n",
              "      <td>99.02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>1000</td>\n",
              "      <td>10</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.3</td>\n",
              "      <td>18.84</td>\n",
              "      <td>99.02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.1</td>\n",
              "      <td>96.99</td>\n",
              "      <td>99.10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.2</td>\n",
              "      <td>92.92</td>\n",
              "      <td>99.10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.3</td>\n",
              "      <td>86.79</td>\n",
              "      <td>99.10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>1000</td>\n",
              "      <td>100</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.1</td>\n",
              "      <td>96.40</td>\n",
              "      <td>98.79</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>1000</td>\n",
              "      <td>100</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.2</td>\n",
              "      <td>90.51</td>\n",
              "      <td>98.79</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>1000</td>\n",
              "      <td>100</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.3</td>\n",
              "      <td>81.43</td>\n",
              "      <td>98.79</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>1000</td>\n",
              "      <td>10</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.1</td>\n",
              "      <td>93.17</td>\n",
              "      <td>99.02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>1000</td>\n",
              "      <td>10</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.2</td>\n",
              "      <td>56.40</td>\n",
              "      <td>99.02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>1000</td>\n",
              "      <td>10</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.3</td>\n",
              "      <td>8.35</td>\n",
              "      <td>99.02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.1</td>\n",
              "      <td>97.03</td>\n",
              "      <td>99.10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.2</td>\n",
              "      <td>92.01</td>\n",
              "      <td>99.10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.3</td>\n",
              "      <td>79.31</td>\n",
              "      <td>99.10</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>1000</td>\n",
              "      <td>100</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.1</td>\n",
              "      <td>96.37</td>\n",
              "      <td>98.79</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>1000</td>\n",
              "      <td>100</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.2</td>\n",
              "      <td>89.76</td>\n",
              "      <td>98.79</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>1000</td>\n",
              "      <td>100</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.3</td>\n",
              "      <td>71.03</td>\n",
              "      <td>98.79</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    total_epochs  replay lr_type  ...  eps  adv_acc  clean_acc\n",
              "0           1000      10    flat  ...  0.1    95.35      99.02\n",
              "1           1000      10    flat  ...  0.2    93.66      99.02\n",
              "2           1000      10    flat  ...  0.3    91.40      99.02\n",
              "3           1000      40    flat  ...  0.1    97.54      99.10\n",
              "4           1000      40    flat  ...  0.2    97.27      99.10\n",
              "5           1000      40    flat  ...  0.3    97.11      99.10\n",
              "6           1000     100    flat  ...  0.1    96.97      98.79\n",
              "7           1000     100    flat  ...  0.2    96.48      98.79\n",
              "8           1000     100    flat  ...  0.3    95.99      98.79\n",
              "9           1000      10    flat  ...  0.1    93.20      99.02\n",
              "10          1000      10    flat  ...  0.2    63.50      99.02\n",
              "11          1000      10    flat  ...  0.3    18.84      99.02\n",
              "12          1000      40    flat  ...  0.1    96.99      99.10\n",
              "13          1000      40    flat  ...  0.2    92.92      99.10\n",
              "14          1000      40    flat  ...  0.3    86.79      99.10\n",
              "15          1000     100    flat  ...  0.1    96.40      98.79\n",
              "16          1000     100    flat  ...  0.2    90.51      98.79\n",
              "17          1000     100    flat  ...  0.3    81.43      98.79\n",
              "18          1000      10    flat  ...  0.1    93.17      99.02\n",
              "19          1000      10    flat  ...  0.2    56.40      99.02\n",
              "20          1000      10    flat  ...  0.3     8.35      99.02\n",
              "21          1000      40    flat  ...  0.1    97.03      99.10\n",
              "22          1000      40    flat  ...  0.2    92.01      99.10\n",
              "23          1000      40    flat  ...  0.3    79.31      99.10\n",
              "24          1000     100    flat  ...  0.1    96.37      98.79\n",
              "25          1000     100    flat  ...  0.2    89.76      98.79\n",
              "26          1000     100    flat  ...  0.3    71.03      98.79\n",
              "\n",
              "[27 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8iNTzQqkGz_X"
      },
      "source": [
        "### Training Model with 100K Total Epochs\n",
        "\n",
        "In this section, I trained a model with 100000 total epochs like Madry paper. I usded 40 as the number of replays, at it performed well for the smaller number of epochs. Also, I did checkpoint saving to continue from latest checkpoint when colab reached the limit. If you want to use the latest checkpoint you should change the **resume** to true. Also you can change the path if your checkpoint is saved elsewhere."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oYMt6e8FfxEs",
        "outputId": "557c0d98-8b0b-4a43-9ce2-bce4010ae75d"
      },
      "source": [
        "batch_size=100\n",
        "total_epochs=100000\n",
        "replays= 40\n",
        "epochs = total_epochs / replays\n",
        "start_epoch = 0\n",
        "epsilon=0.3\n",
        "alpha=0.01\n",
        "lr_type='flat'\n",
        "fname= '/content/mnist_model'\n",
        "seed=0\n",
        "resume= True\n",
        "\n",
        "# It automatically converts NumPy arrays and Python numerical values into PyTorch Tensors.\n",
        "# It outputs a dictionary with the same set of keys but batched Tensors as values \n",
        "train_loader = torch.utils.data.DataLoader(mnist_data, batch_size, shuffle=True)\n",
        "\n",
        "\n",
        "if resume:\n",
        "    path = '/content/mnist_model-latest_checkpoint-' + str(total_epochs) + '_' + str(replays)\n",
        "    checkpoint = torch.load(path)\n",
        "    model = mnist_net().cuda()\n",
        "    lr=.0001\n",
        "    opt = torch.optim.Adam(model.parameters(), lr=.0001)\n",
        "    model.load_state_dict(checkpoint['model_state_dict'])\n",
        "    opt.load_state_dict(checkpoint['optimizer_state_dict'])\n",
        "    start_epoch = checkpoint['epoch'] + 1\n",
        "    loss = checkpoint['loss']\n",
        "    model.train()\n",
        "else:\n",
        "    print(\"No check point found! We train from the first epoch\") \n",
        "    model = mnist_net().cuda()\n",
        "    lr=.0001\n",
        "    opt = torch.optim.Adam(model.parameters(), lr=.0001)\n",
        "    start_epoch = 0\n",
        "    model.train()\n",
        "\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "logger = logging.getLogger(__name__)\n",
        "logging.basicConfig(\n",
        "    format='[%(asctime)s] - %(message)s',\n",
        "    datefmt='%Y/%m/%d %H:%M:%S',\n",
        "    level=logging.DEBUG)\n",
        "\n",
        "logger.info('Epoch \\t Time \\t LR \\t \\t Train Loss \\t Train Acc')\n",
        "global_noise_data = torch.zeros([batch_size, 1, 28, 28]).cuda()\n",
        "for epoch in range(start_epoch, int(epochs)):\n",
        "    start_time = time.time()\n",
        "    train_loss = 0\n",
        "    train_acc = 0\n",
        "    train_n = 0\n",
        "    for i, (X, y) in enumerate(train_loader):\n",
        "        input, target = X.cuda(), y.cuda()\n",
        "        for j in range(replays):\n",
        "            # Ascend on the global noise\n",
        "            noise_batch = Variable(global_noise_data[0:input.size(0)], requires_grad=True).cuda()\n",
        "            in1 = input + noise_batch\n",
        "            in1.clamp_(0, 1.0)\n",
        "            #in1.sub_(mean).div_(std)\n",
        "            output = model(in1)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # compute gradient\n",
        "            opt.zero_grad()\n",
        "            loss.backward()\n",
        "            \n",
        "            # Update the noise for the next iteration\n",
        "            pert = alpha * torch.sign(noise_batch.grad)\n",
        "            global_noise_data[0:input.size(0)] += pert.data\n",
        "            global_noise_data.clamp_(-epsilon, epsilon)\n",
        "          \n",
        "            #finally do SGD step\n",
        "            opt.step()\n",
        "            # measure elapsed time\n",
        "            #batch_time.update(time.time() - end)\n",
        "            #end = time.time()\n",
        "        train_loss += loss.item() * target.size(0)\n",
        "        train_acc += (output.max(1)[1] == target).sum().item()\n",
        "        train_n += target.size(0)\n",
        "\n",
        "    model_path = './mnist_model-latest_checkpoint-' + str(total_epochs) + '_' + str(replays)\n",
        "    torch.save(model.state_dict(),  model_path)\n",
        "    torch.save({\n",
        "            'epoch': epoch,\n",
        "            'model_state_dict': model.state_dict(),\n",
        "            'optimizer_state_dict': opt.state_dict(),\n",
        "            'loss': loss,\n",
        "            }, model_path)\n",
        "\n",
        "    train_time = time.time()\n",
        "    logger.info('%d \\t %.1f \\t %.4f \\t %.6f \\t %.4f',\n",
        "        epoch, train_time - start_time, lr, train_loss/train_n, train_acc/train_n)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[2021/04/22 21:08:55] - Epoch \t Time \t LR \t \t Train Loss \t Train Acc\n",
            "[2021/04/22 21:09:50] - 2465 \t 55.1 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:10:45] - 2466 \t 54.9 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:11:40] - 2467 \t 54.8 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:12:35] - 2468 \t 54.8 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:13:30] - 2469 \t 54.8 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:14:25] - 2470 \t 55.1 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:15:20] - 2471 \t 55.1 \t 0.0001 \t 0.000006 \t 1.0000\n",
            "[2021/04/22 21:16:15] - 2472 \t 55.5 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:17:11] - 2473 \t 55.4 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:18:06] - 2474 \t 55.6 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:19:02] - 2475 \t 55.7 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:19:58] - 2476 \t 55.6 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:20:53] - 2477 \t 55.0 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:21:47] - 2478 \t 55.0 \t 0.0001 \t 0.000001 \t 1.0000\n",
            "[2021/04/22 21:22:42] - 2479 \t 54.9 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:23:37] - 2480 \t 54.8 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:24:32] - 2481 \t 54.6 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:25:26] - 2482 \t 54.7 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:26:21] - 2483 \t 54.5 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:27:16] - 2484 \t 54.7 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:28:11] - 2485 \t 55.2 \t 0.0001 \t 0.000001 \t 1.0000\n",
            "[2021/04/22 21:29:06] - 2486 \t 54.7 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:30:00] - 2487 \t 54.9 \t 0.0001 \t 0.000579 \t 1.0000\n",
            "[2021/04/22 21:30:56] - 2488 \t 55.1 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:31:50] - 2489 \t 54.9 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:32:45] - 2490 \t 54.6 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:33:40] - 2491 \t 54.6 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:34:34] - 2492 \t 54.6 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:35:29] - 2493 \t 54.7 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:36:23] - 2494 \t 54.4 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:37:18] - 2495 \t 54.5 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:38:12] - 2496 \t 54.4 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:39:07] - 2497 \t 54.4 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:40:01] - 2498 \t 54.4 \t 0.0001 \t 0.000000 \t 1.0000\n",
            "[2021/04/22 21:40:56] - 2499 \t 55.2 \t 0.0001 \t 0.000000 \t 1.0000\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2GrRbYjvCIs4"
      },
      "source": [
        "Now we test our robust model against different attacks, please note that I renamed the latest checkpoint manually."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 488
        },
        "id": "czHIGT2A7V2o",
        "outputId": "f06e7f1f-f8e1-41a0-9912-43dbc6b58ff4"
      },
      "source": [
        "pgd_report = []\n",
        "\n",
        "# MNIST Test dataset and dataloader declaration\n",
        "loader = torch.utils.data.DataLoader(\n",
        "    datasets.MNIST('./data', train=False, download=False, transform=transforms.Compose([\n",
        "            transforms.ToTensor(),\n",
        "            ])),\n",
        "        batch_size=200, shuffle=True)\n",
        "\n",
        "n_replay = 40\n",
        "attack_n_itrs_list = [10, 40, 100]\n",
        "epsilons = [.1, .2, .3]\n",
        "total_epochs = 100000\n",
        "# Define what device we are using\n",
        "device = torch.device('cuda')\n",
        "# Load model and put it in evaluation mode\n",
        "path = '/content/mnist_model-100000_40'\n",
        "checkpoint = torch.load(path)\n",
        "model = mnist_net().cuda()\n",
        "opt = torch.optim.Adam(model.parameters(), lr=.0001)\n",
        "model.load_state_dict(checkpoint['model_state_dict'])\n",
        "opt.load_state_dict(checkpoint['optimizer_state_dict'])\n",
        "last_epoch = checkpoint['epoch']\n",
        "loss = checkpoint['loss']\n",
        "model.eval()\n",
        "for attack_n_itrs in attack_n_itrs_list:\n",
        "    acc_list = []\n",
        "    for epsilon in epsilons: \n",
        "\n",
        "        adversary = LinfPGDAttack(\n",
        "            model, loss_fn=nn.CrossEntropyLoss(reduction=\"sum\"), eps=epsilon,\n",
        "            nb_iter=attack_n_itrs, eps_iter=0.01, rand_init=True, clip_min=0.0, clip_max=1.0,\n",
        "            targeted=False)\n",
        "\n",
        "        _, label, pred, advpred = attack_whole_dataset(\n",
        "            adversary, loader, device=\"cuda\")\n",
        "        \n",
        "        clean_acc = 100. * (label == pred).sum().item() / len(label)\n",
        "        adv_acc = 100. * (label == advpred).sum().item() / len(label)\n",
        "        acc_list.append(adv_acc)\n",
        "        pgd_report.append([total_epochs, n_replay, 'flat', 'LinfPGD', attack_n_itrs, epsilon, adv_acc, clean_acc])\n",
        "\n",
        "        print(\"Accuracy: {:.2f}%, Robust Accuracy: {:.2f}%\".format(\n",
        "            100. * (label == pred).sum().item() / len(label),\n",
        "            100. * (label == advpred).sum().item() / len(label)))\n",
        "column = ['total_epochs', 'replay', 'lr_type', 'attack_type', 'attack_n_itr', 'eps', 'adv_acc','clean_acc']\n",
        "df = pd.DataFrame(pgd_report, columns= column)\n",
        "df\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy: 98.86%, Robust Accuracy: 98.31%\n",
            "Accuracy: 98.86%, Robust Accuracy: 98.14%\n",
            "Accuracy: 98.86%, Robust Accuracy: 98.11%\n",
            "Accuracy: 98.86%, Robust Accuracy: 98.25%\n",
            "Accuracy: 98.86%, Robust Accuracy: 98.18%\n",
            "Accuracy: 98.86%, Robust Accuracy: 98.16%\n",
            "Accuracy: 98.86%, Robust Accuracy: 98.21%\n",
            "Accuracy: 98.86%, Robust Accuracy: 98.22%\n",
            "Accuracy: 98.86%, Robust Accuracy: 98.11%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>total_epochs</th>\n",
              "      <th>replay</th>\n",
              "      <th>lr_type</th>\n",
              "      <th>attack_type</th>\n",
              "      <th>attack_n_itr</th>\n",
              "      <th>eps</th>\n",
              "      <th>adv_acc</th>\n",
              "      <th>clean_acc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>100000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.1</td>\n",
              "      <td>98.31</td>\n",
              "      <td>98.86</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>100000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.2</td>\n",
              "      <td>98.14</td>\n",
              "      <td>98.86</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>100000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.3</td>\n",
              "      <td>98.11</td>\n",
              "      <td>98.86</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>100000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.1</td>\n",
              "      <td>98.25</td>\n",
              "      <td>98.86</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>100000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.2</td>\n",
              "      <td>98.18</td>\n",
              "      <td>98.86</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>100000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.3</td>\n",
              "      <td>98.16</td>\n",
              "      <td>98.86</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>100000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.1</td>\n",
              "      <td>98.21</td>\n",
              "      <td>98.86</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>100000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.2</td>\n",
              "      <td>98.22</td>\n",
              "      <td>98.86</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>100000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.3</td>\n",
              "      <td>98.11</td>\n",
              "      <td>98.86</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   total_epochs  replay lr_type  ...  eps  adv_acc  clean_acc\n",
              "0        100000      40    flat  ...  0.1    98.31      98.86\n",
              "1        100000      40    flat  ...  0.2    98.14      98.86\n",
              "2        100000      40    flat  ...  0.3    98.11      98.86\n",
              "3        100000      40    flat  ...  0.1    98.25      98.86\n",
              "4        100000      40    flat  ...  0.2    98.18      98.86\n",
              "5        100000      40    flat  ...  0.3    98.16      98.86\n",
              "6        100000      40    flat  ...  0.1    98.21      98.86\n",
              "7        100000      40    flat  ...  0.2    98.22      98.86\n",
              "8        100000      40    flat  ...  0.3    98.11      98.86\n",
              "\n",
              "[9 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 103
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v6usuMgBD0BV"
      },
      "source": [
        "## Effect of Faster Convergence\n",
        "\n",
        "In the extended-abstract, I explained I want use cyclic learning rate because it has been shown that it make the convergence rate faster. But during doing the experiment I observed that although convergence rate gets better, it actually worsen the robust accuracy. Also, I observed that batch-normalization made the robsut accuracy lower. After training and testing following model with 1000 total epochs, we use another strategy to make the model better. Please see the project report to see why faster convergence makes robust accuracy worser. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n1tu3c3uIu2I"
      },
      "source": [
        "### Cyclic Learning Rate"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zSN2lFh3Imkv",
        "outputId": "b7e731d7-3f28-48ad-993d-fc56d2346092"
      },
      "source": [
        "train_loader = torch.utils.data.DataLoader(mnist_data, batch_size, shuffle=True)\n",
        "# we use cyclic learning rate without normalization\n",
        "model = mnist_net().cuda()\n",
        "model.train()\n",
        "\n",
        "opt = torch.optim.Adam(model.parameters(), lr=.0001)\n",
        "lr_schedule = lambda t: np.interp([t], [0, epochs * 2//5, epochs], [0.0001,lr_max, 0])[0]\n",
        "\n",
        "batch_size=100\n",
        "total_epochs=1000\n",
        "replays= 40\n",
        "epochs = total_epochs / replays\n",
        "epsilon=0.3\n",
        "alpha=0.01\n",
        "seed=0\n",
        "\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "logger = logging.getLogger(__name__)\n",
        "logging.basicConfig(\n",
        "    format='[%(asctime)s] - %(message)s',\n",
        "    datefmt='%Y/%m/%d %H:%M:%S',\n",
        "    level=logging.DEBUG)\n",
        "\n",
        "logger.info('Epoch \\t Time \\t LR \\t \\t Train Loss \\t Train Acc')\n",
        "global_noise_data = torch.zeros([batch_size, 1, 28, 28]).cuda()\n",
        "for epoch in range(int(epochs)):\n",
        "    start_time = time.time()\n",
        "    train_loss = 0\n",
        "    train_acc = 0\n",
        "    train_n = 0\n",
        "    # switch to the train mode\n",
        "    model.train()\n",
        "    for i, (X, y) in enumerate(train_loader):\n",
        "        input, target = X.cuda(), y.cuda()\n",
        "        lr = lr_schedule(epoch + (i+1)/len(train_loader))\n",
        "        opt.param_groups[0].update(lr=lr)\n",
        "        for j in range(replays):\n",
        "\n",
        "            # Ascend on the global noise\n",
        "            noise_batch = Variable(global_noise_data[0:input.size(0)], requires_grad=True).cuda()\n",
        "            in1 = input + noise_batch\n",
        "            in1.clamp_(0, 1.0)\n",
        "            in1.sub_(mean).div_(std)\n",
        "            output = model(in1)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # compute gradient\n",
        "            opt.zero_grad()\n",
        "            loss.backward()\n",
        "            \n",
        "            # Update the noise for the next iteration\n",
        "            pert = alpha * torch.sign(noise_batch.grad)\n",
        "            global_noise_data[0:input.size(0)] += pert.data\n",
        "            global_noise_data.clamp_(-epsilon, epsilon)\n",
        "          \n",
        "            #finally do SGD step\n",
        "            opt.step()\n",
        "\n",
        "        train_loss += loss.item() * target.size(0)\n",
        "        train_acc += (output.max(1)[1] == target).sum().item()\n",
        "        train_n += target.size(0)\n",
        "\n",
        "    train_time = time.time()\n",
        "    logger.info('%d \\t %.1f \\t %.4f \\t %.4f \\t %.4f',\n",
        "        epoch, train_time - start_time, lr, train_loss/train_n, train_acc/train_n)\n",
        "    model_name= '/content/mnist_model-Cyclic-'+ str(total_epochs) + '_' + str(replays)\n",
        "    torch.save(model.state_dict(),  model_name)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[2021/04/22 18:03:11] - Epoch \t Time \t LR \t \t Train Loss \t Train Acc\n",
            "[2021/04/22 18:04:08] - 0 \t 56.5 \t 0.0006 \t 0.1092 \t 0.9684\n",
            "[2021/04/22 18:05:04] - 1 \t 56.6 \t 0.0011 \t 0.0021 \t 1.0000\n",
            "[2021/04/22 18:06:01] - 2 \t 56.4 \t 0.0016 \t 0.0013 \t 1.0000\n",
            "[2021/04/22 18:06:57] - 3 \t 56.5 \t 0.0021 \t 0.0014 \t 0.9999\n",
            "[2021/04/22 18:07:54] - 4 \t 56.5 \t 0.0025 \t 0.0018 \t 0.9996\n",
            "[2021/04/22 18:08:52] - 5 \t 58.0 \t 0.0030 \t 0.0027 \t 0.9992\n",
            "[2021/04/22 18:09:49] - 6 \t 57.3 \t 0.0035 \t 0.0044 \t 0.9988\n",
            "[2021/04/22 18:10:47] - 7 \t 57.5 \t 0.0040 \t 0.0102 \t 0.9974\n",
            "[2021/04/22 18:11:44] - 8 \t 57.2 \t 0.0045 \t 0.0251 \t 0.9936\n",
            "[2021/04/22 18:12:41] - 9 \t 57.2 \t 0.0050 \t 0.0493 \t 0.9867\n",
            "[2021/04/22 18:13:38] - 10 \t 56.8 \t 0.0047 \t 0.0826 \t 0.9793\n",
            "[2021/04/22 18:14:35] - 11 \t 57.2 \t 0.0043 \t 0.1028 \t 0.9729\n",
            "[2021/04/22 18:15:32] - 12 \t 57.1 \t 0.0040 \t 0.1104 \t 0.9703\n",
            "[2021/04/22 18:16:29] - 13 \t 56.9 \t 0.0037 \t 0.1206 \t 0.9688\n",
            "[2021/04/22 18:17:26] - 14 \t 57.1 \t 0.0033 \t 0.1307 \t 0.9664\n",
            "[2021/04/22 18:18:23] - 15 \t 56.9 \t 0.0030 \t 0.1402 \t 0.9598\n",
            "[2021/04/22 18:19:19] - 16 \t 56.6 \t 0.0027 \t 0.1469 \t 0.9570\n",
            "[2021/04/22 18:20:16] - 17 \t 56.8 \t 0.0023 \t 0.1523 \t 0.9529\n",
            "[2021/04/22 18:21:13] - 18 \t 56.8 \t 0.0020 \t 0.1624 \t 0.9490\n",
            "[2021/04/22 18:22:10] - 19 \t 56.5 \t 0.0017 \t 0.1676 \t 0.9457\n",
            "[2021/04/22 18:23:06] - 20 \t 56.3 \t 0.0013 \t 0.1822 \t 0.9386\n",
            "[2021/04/22 18:24:03] - 21 \t 57.1 \t 0.0010 \t 0.2119 \t 0.9285\n",
            "[2021/04/22 18:25:00] - 22 \t 57.3 \t 0.0007 \t 0.2362 \t 0.9204\n",
            "[2021/04/22 18:25:57] - 23 \t 56.8 \t 0.0003 \t 0.2923 \t 0.8977\n",
            "[2021/04/22 18:26:54] - 24 \t 56.6 \t 0.0000 \t 0.4185 \t 0.8685\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o7vNerwHz8yg"
      },
      "source": [
        "Now we test the trained model with cyclic learning rate."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 488
        },
        "id": "cw8NPfjeOqz8",
        "outputId": "9e4c62a0-aef1-47d6-b99b-3b5aa8b5b2bb"
      },
      "source": [
        "pgd_report = []\n",
        "\n",
        "# MNIST Test dataset and dataloader declaration\n",
        "loader = torch.utils.data.DataLoader(\n",
        "    datasets.MNIST('./data', train=False, download=False, transform=transforms.Compose([\n",
        "            transforms.ToTensor(),\n",
        "            ])),\n",
        "        batch_size=200, shuffle=True)\n",
        "\n",
        "n_replay = 40\n",
        "attack_n_itrs_list = [10, 40, 100]\n",
        "epsilons = [.1, .2, .3]\n",
        "total_epochs = 1000\n",
        "# load the model and put it in evaluation mode\n",
        "model = mnist_net().cuda()\n",
        "model.load_state_dict(torch.load('/content/mnist_model-Cyclic-1000_40'))\n",
        "model.eval()\n",
        "\n",
        "for attack_n_itrs in attack_n_itrs_list:\n",
        "    acc_list = []\n",
        "    for epsilon in epsilons: \n",
        "\n",
        "        adversary = LinfPGDAttack(\n",
        "            model, loss_fn=nn.CrossEntropyLoss(reduction=\"sum\"), eps=epsilon,\n",
        "            nb_iter=attack_n_itrs, eps_iter=0.01, rand_init=True, clip_min=0.0, clip_max=1.0,\n",
        "            targeted=False)\n",
        "\n",
        "        _, label, pred, advpred = attack_whole_dataset(\n",
        "            adversary, loader, device=\"cuda\")\n",
        "        \n",
        "        clean_acc = 100. * (label == pred).sum().item() / len(label)\n",
        "        adv_acc = 100. * (label == advpred).sum().item() / len(label)\n",
        "        acc_list.append(adv_acc)\n",
        "        pgd_report.append([total_epochs, n_replay, 'cyclic', 'LinfPGD', attack_n_itrs, epsilon, adv_acc, clean_acc])\n",
        "\n",
        "        print(\"Accuracy: {:.2f}%, Robust Accuracy: {:.2f}%\".format(\n",
        "            100. * (label == pred).sum().item() / len(label),\n",
        "            100. * (label == advpred).sum().item() / len(label)))\n",
        "column = ['total_epochs', 'replay', 'lr_type', 'attack_type', 'attack_n_itr', 'eps', 'adv_acc','clean_acc']\n",
        "df = pd.DataFrame(pgd_report, columns= column)\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy: 94.46%, Robust Accuracy: 66.60%\n",
            "Accuracy: 94.46%, Robust Accuracy: 51.98%\n",
            "Accuracy: 94.46%, Robust Accuracy: 39.49%\n",
            "Accuracy: 94.46%, Robust Accuracy: 58.85%\n",
            "Accuracy: 94.46%, Robust Accuracy: 14.34%\n",
            "Accuracy: 94.46%, Robust Accuracy: 1.92%\n",
            "Accuracy: 94.46%, Robust Accuracy: 58.71%\n",
            "Accuracy: 94.46%, Robust Accuracy: 11.77%\n",
            "Accuracy: 94.46%, Robust Accuracy: 0.91%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>total_epochs</th>\n",
              "      <th>replay</th>\n",
              "      <th>lr_type</th>\n",
              "      <th>attack_type</th>\n",
              "      <th>attack_n_itr</th>\n",
              "      <th>eps</th>\n",
              "      <th>adv_acc</th>\n",
              "      <th>clean_acc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.1</td>\n",
              "      <td>66.60</td>\n",
              "      <td>94.46</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.2</td>\n",
              "      <td>51.98</td>\n",
              "      <td>94.46</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.3</td>\n",
              "      <td>39.49</td>\n",
              "      <td>94.46</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.1</td>\n",
              "      <td>58.85</td>\n",
              "      <td>94.46</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.2</td>\n",
              "      <td>14.34</td>\n",
              "      <td>94.46</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.3</td>\n",
              "      <td>1.92</td>\n",
              "      <td>94.46</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.1</td>\n",
              "      <td>58.71</td>\n",
              "      <td>94.46</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.2</td>\n",
              "      <td>11.77</td>\n",
              "      <td>94.46</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.3</td>\n",
              "      <td>0.91</td>\n",
              "      <td>94.46</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   total_epochs  replay lr_type  ...  eps  adv_acc  clean_acc\n",
              "0          1000      40    flat  ...  0.1    66.60      94.46\n",
              "1          1000      40    flat  ...  0.2    51.98      94.46\n",
              "2          1000      40    flat  ...  0.3    39.49      94.46\n",
              "3          1000      40    flat  ...  0.1    58.85      94.46\n",
              "4          1000      40    flat  ...  0.2    14.34      94.46\n",
              "5          1000      40    flat  ...  0.3     1.92      94.46\n",
              "6          1000      40    flat  ...  0.1    58.71      94.46\n",
              "7          1000      40    flat  ...  0.2    11.77      94.46\n",
              "8          1000      40    flat  ...  0.3     0.91      94.46\n",
              "\n",
              "[9 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 95
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DIblXD4bLAje"
      },
      "source": [
        "## Improving Model Using Curriculum Learning\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sWye6AMMLsI_"
      },
      "source": [
        "### 1000 total epochs\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eUhpxMmvyTDw",
        "outputId": "66f6e488-7172-4d95-cf7f-b6a5011eb35d"
      },
      "source": [
        "# It automatically converts NumPy arrays and Python numerical values into PyTorch Tensors.\n",
        "# It outputs a dictionary with the same set of keys but batched Tensors as values \n",
        "train_loader = torch.utils.data.DataLoader(mnist_data, batch_size, shuffle=True)\n",
        "\n",
        "model = mnist_net().cuda()\n",
        "model.train()\n",
        "\n",
        "lr=.0001\n",
        "opt = torch.optim.Adam(model.parameters(), lr=.0001)\n",
        "\n",
        "batch_size=100\n",
        "total_epochs=1000\n",
        "replays= 40\n",
        "epochs = total_epochs / replays\n",
        "epsilon=0.3\n",
        "alpha=0.01\n",
        "lr_max=5e-3\n",
        "lr_type='cyclic'\n",
        "seed=0\n",
        "\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "logger = logging.getLogger(__name__)\n",
        "logging.basicConfig(\n",
        "    format='[%(asctime)s] - %(message)s',\n",
        "    datefmt='%Y/%m/%d %H:%M:%S',\n",
        "    level=logging.DEBUG)\n",
        "\n",
        "logger.info('Epoch \\t Time \\t LR \\t \\t Train Loss \\t Train Acc')\n",
        "global_noise_data = torch.zeros([batch_size, 1, 28, 28]).cuda()\n",
        "lr_schedule = lambda t: np.interp([t], [0, epochs * 2//5, epochs], [0, lr_max, 0])[0]\n",
        "for epoch in range(int(epochs)):\n",
        "    start_time = time.time()\n",
        "    train_loss = 0\n",
        "    train_acc = 0\n",
        "    train_n = 0\n",
        "    mean = 0.1307\n",
        "    std= 0.3081\n",
        "    # switch to the train mode\n",
        "    model.train()\n",
        "    if epoch < (epochs/ 4):\n",
        "        alpha = .1\n",
        "    elif epoch < (epochs/ 2):\n",
        "        alpha = .2\n",
        "    else:\n",
        "        alpha = .3\n",
        "\n",
        "    for i, (X, y) in enumerate(train_loader):\n",
        "        input, target = X.cuda(), y.cuda()\n",
        "        lr = lr_schedule(epoch + (i+1)/len(train_loader))\n",
        "        opt.param_groups[0].update(lr=lr)\n",
        "        for j in range(replays):\n",
        "\n",
        "            # Ascend on the global noise\n",
        "            noise_batch = Variable(global_noise_data[0:input.size(0)], requires_grad=True).cuda()\n",
        "            in1 = input + noise_batch\n",
        "            in1.clamp_(0, 1.0)\n",
        "            output = model(in1)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # compute gradient\n",
        "            opt.zero_grad()\n",
        "            loss.backward()\n",
        "            \n",
        "            # Update the noise for the next iteration\n",
        "            pert = alpha * torch.sign(noise_batch.grad)\n",
        "            global_noise_data[0:input.size(0)] += pert.data\n",
        "            global_noise_data.clamp_(-epsilon, epsilon)\n",
        "          \n",
        "            #finally do SGD step\n",
        "            opt.step()\n",
        "\n",
        "        train_loss += loss.item() * target.size(0)\n",
        "        train_acc += (output.max(1)[1] == target).sum().item()\n",
        "        train_n += target.size(0)\n",
        "\n",
        "    train_time = time.time()\n",
        "    logger.info('%d \\t %.1f \\t %.4f \\t %.4f \\t %.4f',\n",
        "        epoch, train_time - start_time, lr, train_loss/train_n, train_acc/train_n)\n",
        "    model_name= '/content/mnist_model-improved-'+ str(total_epochs) + '_' + str(replays)\n",
        "    torch.save(model.state_dict(),  model_name)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[2021/04/22 17:27:47] - Epoch \t Time \t LR \t \t Train Loss \t Train Acc\n",
            "[2021/04/22 17:28:43] - 0 \t 56.1 \t 0.0005 \t 0.3380 \t 0.8931\n",
            "[2021/04/22 17:29:39] - 1 \t 56.0 \t 0.0010 \t 0.0058 \t 0.9999\n",
            "[2021/04/22 17:30:35] - 2 \t 56.2 \t 0.0015 \t 0.0047 \t 0.9999\n",
            "[2021/04/22 17:31:30] - 3 \t 55.5 \t 0.0020 \t 0.0061 \t 0.9998\n",
            "[2021/04/22 17:32:26] - 4 \t 55.5 \t 0.0025 \t 0.0071 \t 0.9993\n",
            "[2021/04/22 17:33:21] - 5 \t 55.0 \t 0.0030 \t 0.0107 \t 0.9983\n",
            "[2021/04/22 17:34:16] - 6 \t 55.3 \t 0.0035 \t 0.0390 \t 0.9909\n",
            "[2021/04/22 17:35:12] - 7 \t 55.6 \t 0.0040 \t 0.0238 \t 0.9960\n",
            "[2021/04/22 17:36:08] - 8 \t 56.0 \t 0.0045 \t 0.0265 \t 0.9949\n",
            "[2021/04/22 17:37:04] - 9 \t 55.8 \t 0.0050 \t 0.0321 \t 0.9927\n",
            "[2021/04/22 17:37:59] - 10 \t 55.5 \t 0.0047 \t 0.0696 \t 0.9817\n",
            "[2021/04/22 17:38:55] - 11 \t 55.8 \t 0.0043 \t 0.0653 \t 0.9839\n",
            "[2021/04/22 17:39:51] - 12 \t 55.6 \t 0.0040 \t 0.0599 \t 0.9859\n",
            "[2021/04/22 17:40:46] - 13 \t 55.8 \t 0.0037 \t 0.0237 \t 0.9957\n",
            "[2021/04/22 17:41:42] - 14 \t 55.5 \t 0.0033 \t 0.0203 \t 0.9957\n",
            "[2021/04/22 17:42:37] - 15 \t 55.5 \t 0.0030 \t 0.0184 \t 0.9958\n",
            "[2021/04/22 17:43:33] - 16 \t 55.6 \t 0.0027 \t 0.0183 \t 0.9961\n",
            "[2021/04/22 17:44:28] - 17 \t 55.1 \t 0.0023 \t 0.0198 \t 0.9955\n",
            "[2021/04/22 17:45:24] - 18 \t 55.9 \t 0.0020 \t 0.0206 \t 0.9952\n",
            "[2021/04/22 17:46:20] - 19 \t 56.2 \t 0.0017 \t 0.0205 \t 0.9957\n",
            "[2021/04/22 17:47:16] - 20 \t 55.5 \t 0.0013 \t 0.0248 \t 0.9941\n",
            "[2021/04/22 17:48:11] - 21 \t 55.7 \t 0.0010 \t 0.0282 \t 0.9930\n",
            "[2021/04/22 17:49:07] - 22 \t 55.4 \t 0.0007 \t 0.0374 \t 0.9902\n",
            "[2021/04/22 17:50:02] - 23 \t 55.5 \t 0.0003 \t 0.0534 \t 0.9847\n",
            "[2021/04/22 17:50:58] - 24 \t 56.2 \t 0.0000 \t 0.0832 \t 0.9738\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 488
        },
        "id": "fIKqByJaJ_iv",
        "outputId": "acff41d0-85fc-4306-8858-e061211f3d9d"
      },
      "source": [
        "pgd_report = []\n",
        "\n",
        "# MNIST Test dataset and dataloader declaration\n",
        "loader = torch.utils.data.DataLoader(\n",
        "    datasets.MNIST('./data', train=False, download=False, transform=transforms.Compose([\n",
        "            transforms.ToTensor(),\n",
        "            ])),\n",
        "        batch_size=200, shuffle=True)\n",
        "\n",
        "n_replay = 40\n",
        "attack_n_itrs_list = [10, 40, 100]\n",
        "epsilons = [.1, .2, .3]\n",
        "total_epochs = 1000\n",
        "# load the model and put it in evaluation mode\n",
        "model = mnist_net().cuda()\n",
        "model.load_state_dict(torch.load('/content/mnist_model-improved-1000_40'))\n",
        "model.eval()\n",
        "\n",
        "for attack_n_itrs in attack_n_itrs_list:\n",
        "    acc_list = []\n",
        "    for epsilon in epsilons: \n",
        "\n",
        "        adversary = LinfPGDAttack(\n",
        "            model, loss_fn=nn.CrossEntropyLoss(reduction=\"sum\"), eps=epsilon,\n",
        "            nb_iter=attack_n_itrs, eps_iter=0.01, rand_init=True, clip_min=0.0, clip_max=1.0,\n",
        "            targeted=False)\n",
        "\n",
        "        _, label, pred, advpred = attack_whole_dataset(\n",
        "            adversary, loader, device=\"cuda\")\n",
        "        \n",
        "        clean_acc = 100. * (label == pred).sum().item() / len(label)\n",
        "        adv_acc = 100. * (label == advpred).sum().item() / len(label)\n",
        "        acc_list.append(adv_acc)\n",
        "        pgd_report.append([total_epochs, n_replay, 'cyclic', 'LinfPGD', attack_n_itrs, epsilon, adv_acc, clean_acc])\n",
        "\n",
        "        print(\"Accuracy: {:.2f}%, Robust Accuracy: {:.2f}%\".format(\n",
        "            100. * (label == pred).sum().item() / len(label),\n",
        "            100. * (label == advpred).sum().item() / len(label)))\n",
        "column = ['total_epochs', 'replay', 'lr_type', 'attack_type', 'attack_n_itr', 'eps', 'adv_acc','clean_acc']\n",
        "df = pd.DataFrame(pgd_report, columns= column)\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy: 97.33%, Robust Accuracy: 95.04%\n",
            "Accuracy: 97.33%, Robust Accuracy: 94.34%\n",
            "Accuracy: 97.33%, Robust Accuracy: 93.75%\n",
            "Accuracy: 97.33%, Robust Accuracy: 94.41%\n",
            "Accuracy: 97.33%, Robust Accuracy: 91.07%\n",
            "Accuracy: 97.33%, Robust Accuracy: 86.69%\n",
            "Accuracy: 97.33%, Robust Accuracy: 94.51%\n",
            "Accuracy: 97.33%, Robust Accuracy: 90.74%\n",
            "Accuracy: 97.33%, Robust Accuracy: 84.52%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>total_epochs</th>\n",
              "      <th>replay</th>\n",
              "      <th>lr_type</th>\n",
              "      <th>attack_type</th>\n",
              "      <th>attack_n_itr</th>\n",
              "      <th>eps</th>\n",
              "      <th>adv_acc</th>\n",
              "      <th>clean_acc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.1</td>\n",
              "      <td>95.04</td>\n",
              "      <td>97.33</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.2</td>\n",
              "      <td>94.34</td>\n",
              "      <td>97.33</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.3</td>\n",
              "      <td>93.75</td>\n",
              "      <td>97.33</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.1</td>\n",
              "      <td>94.41</td>\n",
              "      <td>97.33</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.2</td>\n",
              "      <td>91.07</td>\n",
              "      <td>97.33</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.3</td>\n",
              "      <td>86.69</td>\n",
              "      <td>97.33</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.1</td>\n",
              "      <td>94.51</td>\n",
              "      <td>97.33</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.2</td>\n",
              "      <td>90.74</td>\n",
              "      <td>97.33</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>1000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.3</td>\n",
              "      <td>84.52</td>\n",
              "      <td>97.33</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   total_epochs  replay lr_type  ...  eps  adv_acc  clean_acc\n",
              "0          1000      40    flat  ...  0.1    95.04      97.33\n",
              "1          1000      40    flat  ...  0.2    94.34      97.33\n",
              "2          1000      40    flat  ...  0.3    93.75      97.33\n",
              "3          1000      40    flat  ...  0.1    94.41      97.33\n",
              "4          1000      40    flat  ...  0.2    91.07      97.33\n",
              "5          1000      40    flat  ...  0.3    86.69      97.33\n",
              "6          1000      40    flat  ...  0.1    94.51      97.33\n",
              "7          1000      40    flat  ...  0.2    90.74      97.33\n",
              "8          1000      40    flat  ...  0.3    84.52      97.33\n",
              "\n",
              "[9 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 96
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1GBpDVTSM63J"
      },
      "source": [
        "### 2000 total epochs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4nYin2GvK9WA",
        "outputId": "36fcc9de-6f87-4186-d103-4c7900a4df2f"
      },
      "source": [
        "# It automatically converts NumPy arrays and Python numerical values into PyTorch Tensors.\n",
        "# It outputs a dictionary with the same set of keys but batched Tensors as values \n",
        "train_loader = torch.utils.data.DataLoader(mnist_data, batch_size, shuffle=True)\n",
        "\n",
        "model = mnist_net().cuda()\n",
        "model.train()\n",
        "\n",
        "lr=.0001\n",
        "opt = torch.optim.Adam(model.parameters(), lr=.0001)\n",
        "\n",
        "batch_size=100\n",
        "total_epochs=2000\n",
        "replays= 40\n",
        "epochs = total_epochs / replays\n",
        "epsilon=0.3\n",
        "alpha=0.01\n",
        "lr_max=5e-3\n",
        "lr_type='cyclic'\n",
        "seed=0\n",
        "\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "logger = logging.getLogger(__name__)\n",
        "logging.basicConfig(\n",
        "    format='[%(asctime)s] - %(message)s',\n",
        "    datefmt='%Y/%m/%d %H:%M:%S',\n",
        "    level=logging.DEBUG)\n",
        "\n",
        "logger.info('Epoch \\t Time \\t LR \\t \\t Train Loss \\t Train Acc')\n",
        "global_noise_data = torch.zeros([batch_size, 1, 28, 28]).cuda()\n",
        "lr_schedule = lambda t: np.interp([t], [0, epochs * 2//5, epochs], [0, lr_max, 0])[0]\n",
        "for epoch in range(int(epochs)):\n",
        "    start_time = time.time()\n",
        "    train_loss = 0\n",
        "    train_acc = 0\n",
        "    train_n = 0\n",
        "    if epoch < (epochs/ 4):\n",
        "        alpha = .1\n",
        "    elif epoch < (epochs/ 2):\n",
        "        alpha = .2\n",
        "    else:\n",
        "        alpha = .3\n",
        "    \n",
        "    for i, (X, y) in enumerate(train_loader):\n",
        "        input, target = X.cuda(), y.cuda()\n",
        "        lr = lr_schedule(epoch + (i+1)/len(train_loader))\n",
        "        opt.param_groups[0].update(lr=lr)\n",
        "        for j in range(replays):\n",
        "\n",
        "            # Ascend on the global noise\n",
        "            noise_batch = Variable(global_noise_data[0:input.size(0)], requires_grad=True).cuda()\n",
        "            in1 = input + noise_batch\n",
        "            in1.clamp_(0, 1.0)\n",
        "            #in1.sub_(mean).div_(std)\n",
        "            output = model(in1)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # compute gradient\n",
        "            opt.zero_grad()\n",
        "            loss.backward()\n",
        "            \n",
        "            # Update the noise for the next iteration\n",
        "            pert = alpha * torch.sign(noise_batch.grad)\n",
        "            global_noise_data[0:input.size(0)] += pert.data\n",
        "            global_noise_data.clamp_(-epsilon, epsilon)\n",
        "          \n",
        "            #finally do SGD step\n",
        "            opt.step()\n",
        "\n",
        "        train_loss += loss.item() * target.size(0)\n",
        "        train_acc += (output.max(1)[1] == target).sum().item()\n",
        "        train_n += target.size(0)\n",
        "\n",
        "    train_time = time.time()\n",
        "    logger.info('%d \\t %.1f \\t %.4f \\t %.4f \\t %.4f',\n",
        "        epoch, train_time - start_time, lr, train_loss/train_n, train_acc/train_n)\n",
        "    model_name= '/content/mnist_model-improved-'+ str(total_epochs) + '_' + str(replays)\n",
        "    torch.save(model.state_dict(),  model_name)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[2021/04/22 20:11:27] - Epoch \t Time \t LR \t \t Train Loss \t Train Acc\n",
            "[2021/04/22 20:12:24] - 0 \t 56.2 \t 0.0003 \t 0.4905 \t 0.8405\n",
            "[2021/04/22 20:13:19] - 1 \t 55.7 \t 0.0005 \t 0.0172 \t 0.9996\n",
            "[2021/04/22 20:14:15] - 2 \t 55.5 \t 0.0008 \t 0.0053 \t 1.0000\n",
            "[2021/04/22 20:15:10] - 3 \t 55.5 \t 0.0010 \t 0.0037 \t 0.9999\n",
            "[2021/04/22 20:16:06] - 4 \t 55.6 \t 0.0013 \t 0.0033 \t 0.9999\n",
            "[2021/04/22 20:17:01] - 5 \t 55.2 \t 0.0015 \t 0.0030 \t 1.0000\n",
            "[2021/04/22 20:17:57] - 6 \t 55.7 \t 0.0018 \t 0.0038 \t 0.9997\n",
            "[2021/04/22 20:18:53] - 7 \t 55.8 \t 0.0020 \t 0.0050 \t 0.9994\n",
            "[2021/04/22 20:19:48] - 8 \t 55.4 \t 0.0023 \t 0.0076 \t 0.9990\n",
            "[2021/04/22 20:20:44] - 9 \t 56.0 \t 0.0025 \t 0.0237 \t 0.9957\n",
            "[2021/04/22 20:21:40] - 10 \t 55.7 \t 0.0027 \t 0.0305 \t 0.9938\n",
            "[2021/04/22 20:22:35] - 11 \t 55.7 \t 0.0030 \t 0.0347 \t 0.9923\n",
            "[2021/04/22 20:23:31] - 12 \t 55.6 \t 0.0033 \t 0.0354 \t 0.9923\n",
            "[2021/04/22 20:24:27] - 13 \t 55.7 \t 0.0035 \t 0.0246 \t 0.9952\n",
            "[2021/04/22 20:25:22] - 14 \t 55.5 \t 0.0037 \t 0.0246 \t 0.9953\n",
            "[2021/04/22 20:26:18] - 15 \t 56.0 \t 0.0040 \t 0.0285 \t 0.9940\n",
            "[2021/04/22 20:27:14] - 16 \t 56.0 \t 0.0043 \t 0.0370 \t 0.9912\n",
            "[2021/04/22 20:28:10] - 17 \t 55.8 \t 0.0045 \t 0.0719 \t 0.9809\n",
            "[2021/04/22 20:29:06] - 18 \t 56.2 \t 0.0047 \t 0.0678 \t 0.9832\n",
            "[2021/04/22 20:30:02] - 19 \t 55.8 \t 0.0050 \t 0.0652 \t 0.9835\n",
            "[2021/04/22 20:30:58] - 20 \t 56.0 \t 0.0048 \t 0.0699 \t 0.9817\n",
            "[2021/04/22 20:31:55] - 21 \t 56.5 \t 0.0047 \t 0.0671 \t 0.9827\n",
            "[2021/04/22 20:32:50] - 22 \t 55.7 \t 0.0045 \t 0.0622 \t 0.9844\n",
            "[2021/04/22 20:33:45] - 23 \t 55.2 \t 0.0043 \t 0.0489 \t 0.9895\n",
            "[2021/04/22 20:34:40] - 24 \t 54.8 \t 0.0042 \t 0.0480 \t 0.9888\n",
            "[2021/04/22 20:35:35] - 25 \t 55.0 \t 0.0040 \t 0.0176 \t 0.9964\n",
            "[2021/04/22 20:36:30] - 26 \t 54.8 \t 0.0038 \t 0.0166 \t 0.9962\n",
            "[2021/04/22 20:37:24] - 27 \t 54.4 \t 0.0037 \t 0.0137 \t 0.9970\n",
            "[2021/04/22 20:38:19] - 28 \t 54.3 \t 0.0035 \t 0.0150 \t 0.9966\n",
            "[2021/04/22 20:39:13] - 29 \t 54.4 \t 0.0033 \t 0.0144 \t 0.9965\n",
            "[2021/04/22 20:40:08] - 30 \t 54.9 \t 0.0032 \t 0.0147 \t 0.9964\n",
            "[2021/04/22 20:41:03] - 31 \t 54.6 \t 0.0030 \t 0.0138 \t 0.9966\n",
            "[2021/04/22 20:41:58] - 32 \t 54.8 \t 0.0028 \t 0.0131 \t 0.9967\n",
            "[2021/04/22 20:42:53] - 33 \t 54.9 \t 0.0027 \t 0.0138 \t 0.9966\n",
            "[2021/04/22 20:43:47] - 34 \t 54.8 \t 0.0025 \t 0.0135 \t 0.9966\n",
            "[2021/04/22 20:44:43] - 35 \t 55.1 \t 0.0023 \t 0.0131 \t 0.9969\n",
            "[2021/04/22 20:45:38] - 36 \t 55.1 \t 0.0022 \t 0.0138 \t 0.9963\n",
            "[2021/04/22 20:46:33] - 37 \t 55.1 \t 0.0020 \t 0.0147 \t 0.9961\n",
            "[2021/04/22 20:47:28] - 38 \t 55.3 \t 0.0018 \t 0.0147 \t 0.9963\n",
            "[2021/04/22 20:48:23] - 39 \t 55.2 \t 0.0017 \t 0.0156 \t 0.9963\n",
            "[2021/04/22 20:49:18] - 40 \t 54.9 \t 0.0015 \t 0.0175 \t 0.9951\n",
            "[2021/04/22 20:50:13] - 41 \t 54.7 \t 0.0013 \t 0.0180 \t 0.9953\n",
            "[2021/04/22 20:51:08] - 42 \t 54.9 \t 0.0012 \t 0.0212 \t 0.9939\n",
            "[2021/04/22 20:52:03] - 43 \t 54.9 \t 0.0010 \t 0.0216 \t 0.9941\n",
            "[2021/04/22 20:52:58] - 44 \t 55.1 \t 0.0008 \t 0.0235 \t 0.9937\n",
            "[2021/04/22 20:53:53] - 45 \t 55.2 \t 0.0007 \t 0.0265 \t 0.9923\n",
            "[2021/04/22 20:54:48] - 46 \t 55.0 \t 0.0005 \t 0.0299 \t 0.9915\n",
            "[2021/04/22 20:55:43] - 47 \t 54.8 \t 0.0003 \t 0.0330 \t 0.9902\n",
            "[2021/04/22 20:56:37] - 48 \t 54.6 \t 0.0002 \t 0.0410 \t 0.9870\n",
            "[2021/04/22 20:57:32] - 49 \t 54.4 \t 0.0000 \t 0.0534 \t 0.9823\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 488
        },
        "id": "eUvLAV4YNYth",
        "outputId": "6d712cfb-50a6-413e-8fe8-9c225f5ce66a"
      },
      "source": [
        "pgd_report = []\n",
        "\n",
        "# MNIST Test dataset and dataloader declaration\n",
        "loader = torch.utils.data.DataLoader(\n",
        "    datasets.MNIST('./data', train=False, download=False, transform=transforms.Compose([\n",
        "            transforms.ToTensor(),\n",
        "            ])),\n",
        "        batch_size=200, shuffle=True)\n",
        "\n",
        "n_replay = 40\n",
        "attack_n_itrs_list = [10, 40, 100]\n",
        "epsilons = [.1, .2, .3]\n",
        "total_epochs = 2000\n",
        "# load the model and put it in evaluation mode\n",
        "model = mnist_net().cuda()\n",
        "model.load_state_dict(torch.load('/content/mnist_model-improved-2000_40'))\n",
        "model.eval()\n",
        "\n",
        "for attack_n_itrs in attack_n_itrs_list:\n",
        "    acc_list = []\n",
        "    for epsilon in epsilons: \n",
        "\n",
        "        adversary = LinfPGDAttack(\n",
        "            model, loss_fn=nn.CrossEntropyLoss(reduction=\"sum\"), eps=epsilon,\n",
        "            nb_iter=attack_n_itrs, eps_iter=0.01, rand_init=True, clip_min=0.0, clip_max=1.0,\n",
        "            targeted=False)\n",
        "\n",
        "        _, label, pred, advpred = attack_whole_dataset(\n",
        "            adversary, loader, device=\"cuda\")\n",
        "        \n",
        "        clean_acc = 100. * (label == pred).sum().item() / len(label)\n",
        "        adv_acc = 100. * (label == advpred).sum().item() / len(label)\n",
        "        acc_list.append(adv_acc)\n",
        "        pgd_report.append([total_epochs, n_replay, 'cyclic', 'LinfPGD', attack_n_itrs, epsilon, adv_acc, clean_acc])\n",
        "\n",
        "        print(\"Accuracy: {:.2f}%, Robust Accuracy: {:.2f}%\".format(\n",
        "            100. * (label == pred).sum().item() / len(label),\n",
        "            100. * (label == advpred).sum().item() / len(label)))\n",
        "column = ['total_epochs', 'replay', 'lr_type', 'attack_type', 'attack_n_itr', 'eps', 'adv_acc','clean_acc']\n",
        "df = pd.DataFrame(pgd_report, columns= column)\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy: 97.20%, Robust Accuracy: 96.03%\n",
            "Accuracy: 97.20%, Robust Accuracy: 95.52%\n",
            "Accuracy: 97.20%, Robust Accuracy: 95.34%\n",
            "Accuracy: 97.20%, Robust Accuracy: 95.77%\n",
            "Accuracy: 97.20%, Robust Accuracy: 94.25%\n",
            "Accuracy: 97.20%, Robust Accuracy: 92.11%\n",
            "Accuracy: 97.20%, Robust Accuracy: 95.64%\n",
            "Accuracy: 97.20%, Robust Accuracy: 93.97%\n",
            "Accuracy: 97.20%, Robust Accuracy: 91.18%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>total_epochs</th>\n",
              "      <th>replay</th>\n",
              "      <th>lr_type</th>\n",
              "      <th>attack_type</th>\n",
              "      <th>attack_n_itr</th>\n",
              "      <th>eps</th>\n",
              "      <th>adv_acc</th>\n",
              "      <th>clean_acc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.1</td>\n",
              "      <td>96.03</td>\n",
              "      <td>97.2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.2</td>\n",
              "      <td>95.52</td>\n",
              "      <td>97.2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.3</td>\n",
              "      <td>95.34</td>\n",
              "      <td>97.2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.1</td>\n",
              "      <td>95.77</td>\n",
              "      <td>97.2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.2</td>\n",
              "      <td>94.25</td>\n",
              "      <td>97.2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>2000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.3</td>\n",
              "      <td>92.11</td>\n",
              "      <td>97.2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>2000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.1</td>\n",
              "      <td>95.64</td>\n",
              "      <td>97.2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>2000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.2</td>\n",
              "      <td>93.97</td>\n",
              "      <td>97.2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>2000</td>\n",
              "      <td>40</td>\n",
              "      <td>flat</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.3</td>\n",
              "      <td>91.18</td>\n",
              "      <td>97.2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   total_epochs  replay lr_type  ...  eps  adv_acc  clean_acc\n",
              "0          2000      40    flat  ...  0.1    96.03       97.2\n",
              "1          2000      40    flat  ...  0.2    95.52       97.2\n",
              "2          2000      40    flat  ...  0.3    95.34       97.2\n",
              "3          2000      40    flat  ...  0.1    95.77       97.2\n",
              "4          2000      40    flat  ...  0.2    94.25       97.2\n",
              "5          2000      40    flat  ...  0.3    92.11       97.2\n",
              "6          2000      40    flat  ...  0.1    95.64       97.2\n",
              "7          2000      40    flat  ...  0.2    93.97       97.2\n",
              "8          2000      40    flat  ...  0.3    91.18       97.2\n",
              "\n",
              "[9 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 98
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RLcbclCc1U70"
      },
      "source": [
        "## Some Visualization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o11vlfcrJY1B"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "\n",
        "import os\n",
        "import argparse\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "\n",
        "from advertorch.utils import predict_from_logits\n",
        "from advertorch_examples.utils import get_mnist_test_loader\n",
        "from advertorch_examples.utils import _imshow\n",
        "\n",
        "torch.manual_seed(0)\n",
        "use_cuda = torch.cuda.is_available()\n",
        "device = torch.device(\"cuda\" if use_cuda else \"cpu\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vHzhVbov13TG"
      },
      "source": [
        "### Adversarial for Free with 100k epochs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 448
        },
        "id": "QmzF9QiE2AKo",
        "outputId": "f9979a57-8ae6-4b82-9aff-08db3e39fb6b"
      },
      "source": [
        "# Load model and put it in evaluation mode\n",
        "path = '/content/mnist_model-100000_40'\n",
        "checkpoint = torch.load(path)\n",
        "model = mnist_net().cuda()\n",
        "opt = torch.optim.Adam(model.parameters(), lr=.0001)\n",
        "model.load_state_dict(checkpoint['model_state_dict'])\n",
        "opt.load_state_dict(checkpoint['optimizer_state_dict'])\n",
        "last_epoch = checkpoint['epoch']\n",
        "loss = checkpoint['loss']\n",
        "model.eval()\n",
        "batch_size = 5\n",
        "loader = get_mnist_test_loader(batch_size=batch_size)\n",
        "for cln_data, true_label in loader:\n",
        "    break\n",
        "cln_data, true_label = cln_data.to(device), true_label.to(device)\n",
        "adv_untargeted = adversary.perturb(cln_data, true_label)\n",
        "pred_cln = predict_from_logits(model(cln_data))\n",
        "pred_untargeted_adv = predict_from_logits(model(adv_untargeted))\n",
        "plt.figure(figsize=(10, 8))\n",
        "for ii in range(batch_size):\n",
        "    plt.subplot(2, batch_size, ii + 1)\n",
        "    _imshow(cln_data[ii])\n",
        "    plt.title(\"clean \\n pred: {}\".format(pred_cln[ii]))\n",
        "    plt.subplot(2, batch_size, ii + 1 + batch_size)\n",
        "    _imshow(adv_untargeted[ii])\n",
        "    plt.title(\"untargeted \\n adv \\n pred: {}\".format(\n",
        "        pred_untargeted_adv[ii]))\n",
        "\n",
        "plt.tight_layout()\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsgAAAGvCAYAAABGsiMJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5hcxZX38V8hlElCEkISCiChLCRhRPISvASDQCwZDNiEfQ322tiPMThiovFiWMPai43XvEtwwGSDScKsecFmLbIkUARJKIAkUAJlULjvH92s61adUdfc6ZnpGX0/zzMPqjPVt6u7q28XPXXucVmWCQAAAEDJds09AAAAAKCWsEAGAAAAPCyQAQAAAA8LZAAAAMDDAhkAAADwsEAGAAAAPNvsAtk5d55z7vnmHgdaDuYMimDeoAjmDYpg3lTPNrtABgAAACwskAEAAABPq18gO+f6OOcecs4tdc4td87dUke/Ic65p51zK5xzs5xzp3u/O845N8k5t8o5t9A5d5X3u/7Oucw5d65zboFzbplz7vtN8NDQSJgzKIJ5gyKYNyiCedP4WvUC2TnXRtJjkuZL6i+pt6R7jH6dJT0t6W5Ju0k6U9IvnHPDyl3WSvqCpF0kHSfpy865E4PD/IOkwZKOkHSFc25otR8PGh9zBkUwb1AE8wZFMG+aRqteIEvaX1IvSZdlWbY2y7INWZZZm9ePlzQvy7I7sizblGXZJEkPSjpNkrIsezbLsjeyLNuSZdnrkn4v6bDgGFdnWbY+y7IpkqZIGtVojwqNiTmDIpg3KIJ5gyKYN01g++YeQCPrI2l+lmWbKvTrJ+kA59wHXmx7Sb+RJOfcAZKulzRCUjtJ7SXdHxxjiffvdZJ2aMC40XyYMyiCeYMimDcognnTBFr7AnmhpL7Oue0rTKSFkp7LsuyoOn5/t6RbJB2bZdkG59y/S+pW5bGiNjBnUATzBkUwb1AE86YJtPYtFi9JWizpeudcZ+dcB+fcp41+j0ka5Jz7vHOubflnrLfXZkdJK8oTaH9JZzXR+NH0mDMognmDIpg3KIJ50wRa9QI5y7LNksZLGihpgaR3JJ1h9Fst6WiVNrAvUulPCj9W6c8NkvQvkq5xzq2WdIWk+xp98GgWzBkUwbxBEcwbFMG8aRouy7LmHgMAAABQM1r1N8gAAABAfbFABgAAADwskAEAAAAPC2QAAADAwwK5oHKN8oHNPQ60LMwbFMG8QRHMGxTBvClhgdwEnHPTnHNrvJ9NzrlHm3tcqG3OuX9zzr3lnFvtnJvpnPtCc48Jtc85d7pz7m/OuXXOuWebezxoGZxz7Z1ztzvnVjnnljjnLmnuMaHlcM7t6pxb6pyzSl63SK29kl6ShGo0DZJl2XDvvpykuYrLOaKFaex5I2mtSte6fFPSWEkTnHOzsyz7WyPeJxpZE8ybFZL+XdIQSf/YiPeDJtQE8+YqSXurVJ54d0n/zzk3PcuyCY14n2hkTTBvPvFjSTPUir54bTUPJFT+E8HXnHNznXPLnHM3Oue2K//uPOfc/zjnbnbOLZd0Vfn/nv/NObfAOfeec+6XzrmO3vEuc84tds4tcs5d0IChHapSKccHG/YI0Rhqad5kWXZllmUzsyzbkmXZi5L+Kumgqj5gVEWNzZv/zrLsPpUKA6CG1dK8kXSupGuzLFuZZdkMSbdJOq9ajxXVU2PzRs65gyWNkHRH1R5kDWi1C+SykyTtJ2lfSf8kyX/hD1Dpm9wekq6TdL2kQZJGq1SdprdKlWXknDtG0qWSjlLp/7CP9O/EOXeWc+71xDGdK+nBLMvWFntIaAI1N2/KJ7OxkqYVfVBodDU3b9AiNPu8cc51kdRT0hQvPEXScKs/akKzz5vy79tIukXSVyW1rspzWZa1yh+VXqhjvPa/SPpz+d/nSVrg/c6p9OfsAV7sIElvl/99u6Trvd8NKh9/YD3H1EnSKkmHN/fzw0/LmTfl294laYLK1S/5qa2fWpw3kv6PpGeb+7nhp/bnjaQ+5b4dvNhRkuY193PET+3Om3L/b0i61bvv55v7+anWT2vfg7zQ+/d8Sb3q+F13lRavr5a2CEsqTao25X/3kvRqcKwiTlZpf+BzBW+PplFT88Y5d6NKf776TFY+C6Em1dS8QYtRC/NmTfm/O0na4P17dT2OgabV7PPGOddL0tckfSr1Ni1Ja18g99Hf/yTdV/k9ef5CY5mk9ZKGZ1n2rnGcxeVjfaJvwfGcK+nXLHJqXs3MG+fc1ZKOlXRYlmWr6nt7NKmamTdoUZp93mRZttI5t1jSKElPl8OjxJauWtbs80bS/iptzZleXnx3lNTRObdEUu8syzbX41g1p7XvQb7MOdfFOddH0tcl3Wt1yrJsi0oJCTc753aTJOdcb+fcZ8td7pN0nnNumHOuk6Qr6zsQ59wekj6j0p/KUdtqYt44574r6SxJR2ZZtrzgY0HTqZV508Y510GlL0C2c851cM61LfiY0PhqYt5I+rWky8tjGSLpi5LurP/DQROphXnzpKT+Ku1tHq3SvuZJkka39MWx1PoXyI+o9KeDyZIel/RfW+n7bUmzJb3gnFsl6b8lDZakLMueVOmySc+U+zzj39A5d7ZzrtL/aX9e0sQsy+YUeBxoWrUyb36k0v/Nz3Z/v4b294o9JDSBWpk3n1fpG6NbJR1S/vdtBR4PmkatzJsrJc1R6U/sz0m6MeMSb7Ws2edNlmUfZVm25JMfSR9K2lj+d4vnWutf+51zmaS9syyb3dxjQcvBvEERzBsUwbxBEcybptHav0EGAAAA6oUFMgAAAOBptVssAAAAgCL4BhkAAADwsEAGAAAAPFstFFLOlAQkSVmWucq9mDfIY96gCOYNimDeoAhr3vANMgAAAOBhgQwAAAB4WCADAAAAHhbIAAAAgIcFMgAAAOBhgQwAAAB4WCADAAAAHhbIAAAAgIcFMgAAAOBhgQwAAAB4WCADAAAAHhbIAAAAgGf75h4AUEsuvfTSKNaxY8cots8+++Tap556atLxb7311lx74sSJUZ/f/OY3SccCAACNg2+QAQAAAA8LZAAAAMDDAhkAAADwsEAGAAAAPC7Lsrp/6Vzdv8Q2J8syl9Kvpcybe++9N4qlJttVy5w5c6LYkUceGcUWLFjQFMNpFK1t3tSCQYMGRbGZM2dGsa9//etR7D/+4z8aZUzVxrz5u86dO+faN954Y9TnoosuimKvvvpqrn3aaadFfebPn9/A0dUW5g2KsOYN3yADAAAAHhbIAAAAgIcFMgAAAOBhgQwAAAB4qKSHbUaYlNeQhLwwIeqpp56K+uy1115RbPz48bn2gAEDoj7nnHNOFPvRj35U3yGiFRszZkwU27JlSxR79913m2I4aGS9evXKtb/4xS9GfazX/1Of+lSuffzxx0d9fv7znzdwdGhq++67bxR76KGHolj//v2bYDRbd/TRR+faM2bMiPosXLiwqYZTL3yDDAAAAHhYIAMAAAAeFsgAAACAhz3IaJX222+/KHbSSSdVvN20adOi2AknnBDFli1blmuvWbMm6tOuXbso9sILL+Tao0aNivrsuuuuFceJbdvo0aOj2Nq1a6OYtS8Rta179+5R7M4772z6gaBmffazn41i7du3b4aRVBZ+fl5wwQVRnzPPPLOphlMvfIMMAAAAeFggAwAAAB4WyAAAAICHBTIAAADgqakkPatwg3VB9EWLFuXaGzZsiPr87ne/i2JLliyJYrNnz67PENFC9OzZM4o553JtKyHPSn5YvHhxoTFceumlUWzYsGEVb/f4448Xuj+0XiNHjsy1L7744qjPr3/966YaDqrka1/7WhQ78cQTo9j+++9flfs79NBDo9h228Xfk02ZMiWK/eUvf6nKGFA/228fL9PGjRvXDCMp5pVXXsm1L7nkkqhP586do5iVdNzU+AYZAAAA8LBABgAAADwskAEAAAAPC2QAAADAU1NJejfccEMU69+/f6FjXXTRRVFs9erVUcxK1Gpu77zzThSznptw8zv+7tFHH41iAwcOzLWt+bBixYqqjeGMM86IYm3btq3a8bHtGDx4cK7dqVOnqM+9997bVMNBldx8881RbMuWLY12fyeffHJSbP78+VEsPJ+9+uqr1RsY6vSZz3wmih100EFRzFoj1IKwMqyVqG6dz0jSAwAAAGoMC2QAAADAwwIZAAAA8NTUHmSrKMioUaOi2PTp03Nta0/LmDFjotjhhx8exQ488MBce+HChVGfPn36RLEUmzZtimJLly6NYlZRi9CCBQuiGHuQ68faV1ctl112WRQbNGhQxdu9+OKLSTFs2771rW/l2tZc5nxQ25544okoZhXpqKbly5fn2mvWrIn69OvXL4rtueeeUeyll17Ktdu0adPA0cESFgX6/e9/H/WZM2dOFPvRj37UaGNqiBNOOKG5h1AY3yADAAAAHhbIAAAAgIcFMgAAAOBhgQwAAAB4aipJ789//nNSLDRhwoSk43fp0iWKhcl8VqLL2LFjk44f2rBhQxR78803o9iMGTNy7fDC2pI0d+7cQmNA4zj++ONz7WuuuSbq065duyj2/vvv59rf/e53oz7r1q1r4OjQklnFkfbbb79c2zqP1MKF9fF3hx12WK4dFnuR7KIgRQuF/PKXv4xif/rTn3LtDz/8MOrzj//4j1Hs+9//fsX7+/KXvxzFbr311oq3w9aFz33nzp2jPscee2wUsxIwm5q1dgnfB41ZCKfa+AYZAAAA8LBABgAAADwskAEAAAAPC2QAAADAU1NJeo1t5cqVUeyZZ56peLuURMFUp5xyShQLkwffeOONqM8999xTtTGg4cKkKSshz3Lvvffm2s8991zVxoTWIUxqsVgVOdF8rMTK8JzdrVu3wscPKyc++OCDUZ+rr746iqUk/FpVGS+88MIo1r1791z7hhtuiPp06NAhit1yyy259saNGyuOaVtx6qmnRrFx48bl2rNnz476vPzyy402poawkjvDpLxnn3026vPBBx801pAahG+QAQAAAA8LZAAAAMDDAhkAAADwsEAGAAAAPNtUkl5T22233aLYL37xiyi23Xb5/0+xqrKtWLGiegNDvTz88MNR7Oijj654u1//+tdR7PLLL6/KmNB6jRw5smIfK0EKzadt27ZRrGhSnpW4e+aZZ+bay5YtK3Rsi5Wk96//+q9R7Kabbsq1O3XqFPWx5uUf//jHXHvOnDn1HWKrddppp0Wx8Hmt1eqEVmLq2WefHcU2b96ca1933XVRn1pN3OQbZAAAAMDDAhkAAADwsEAGAAAAPOxBbkRf+cpXolh4sXUpLmAya9asRhsTtq5nz55R7OCDD45i7du3z7WtPYE//OEPo9iaNWsaMDq0NgcddFAUO//886PYpEmTcu2nn3660caEpvPKK69EsQsuuCCKVXPPcYpw37AU7y8dO3ZsUw2nVdh5552j2IEHHljxdlbeUi2wislY++5nzJiRa6cUZ6sVfIMMAAAAeFggAwAAAB4WyAAAAICHBTIAAADgIUmvSj796U9Hse985ztJtz3xxBNz7alTp1ZlTKi/Bx98MIp17dq14u1++9vfRjEuiI9KjjjiiCi26667RrEJEybk2hs2bGi0MaE6wgJQlgMOOKAJRlJ/zrkoFj6elMcnSVdffXWufc455xQfWAsWJnZLUu/evaPYPffc0xTDabABAwYk9WvJ6xm+QQYAAAA8LJABAAAADwtkAAAAwMMCGQAAAPCQpFcl48aNi2Jt27aNYn/+85+j2MSJExtlTNi6E044IYrtu+++Sbd99tlnc+0rr7yyGkPCNmbUqFFRLMuyKPbAAw80xXBQ0EUXXRTFtmzZ0gwjqY7x48dHsTFjxuTa1uOzYpwbS1avXh3FJk+eHMVGjhyZa1tJuytWrKjewBLstttuUezUU09Nuu3zzz9f7eE0Gb5BBgAAADwskAEAAAAPC2QAAADAwx7kgjp27JhrH3PMMVGfjz/+OIpZ+7E2btxYvYGhTmHBj+9973tRH2vfuCXcO7ZmzZriA8M2Y/fdd8+1DznkkKjPrFmzotgf/vCHRhsTGs7as1uLunfvHsWGDRsWxaxzY4qlS5dGMT7fStavXx/FrGJSp5xySq79+OOPR31uuummqo1rxIgRUSwsAtKvX7+oj5UrYUntV4v4BhkAAADwsEAGAAAAPCyQAQAAAA8LZAAAAMBDkl5Bl112Wa4dXkRdkiZMmBDF/va3vzXamLB13/zmN3PtsWPHJt3u4YcfjmJc/B5FnHfeebm2dQH+J598solGg23N97///Sj2la98pdCx5s2bF8XC+S1JCxYsKHT8bcFVV10VxZxzufZxxx0X9fn9739ftTEsW7YsioWJdd26dSt8/DvuuKPwbZsb3yADAAAAHhbIAAAAgIcFMgAAAOBhgQwAAAB4SNJLYG2S/8EPfpBrr1q1Kupz7bXXNtqYUH+XXHJJodt99atfjWJUzkMRVkWq0MqVK5tgJNgWPPHEE7n24MGDq3bsGTNmRLG//vWvVTv+tsB6Dk8//fRc27oAQFjpriEeeOCBin3uuuuuKHb22WcnHd+qINhS8A0yAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhyS9QNeuXaPYz372syjWpk2bXDtMhpCkiRMnVm9gaDa77rprFNu4cWNVjv3hhx8mHbtt27ZRbOedd654/C5dukSxb3zjG4mjy9u8eXMU+/a3v51rr1u3rtCxtxXjx4+v2Oexxx5rgpGgmsLqZ5K03XaVv3869thjk45/22235do9e/ZMul04hi1btiTdLsXxxx9ftWOhbpMmTUqKNaa5c+cWvu3IkSNz7TfeeKOhw2kyfIMMAAAAeFggAwAAAB4WyAAAAIBnm9+DHO4lnjBhQtRnzz33jGJz5szJtcPCIWg9Xn/99UY79v333x/FFi9eHMV69OgRxc4444xGGVN9LFmyJNe+7rrrmmkkteeQQw6JYtbriJbv1ltvjWI33HBDxdtZ+81T9gkX3UvckD3Iv/zlLwvfFi2btcfeilla0p7jEN8gAwAAAB4WyAAAAICHBTIAAADgYYEMAAAAeLb5JL0BAwbk2p/61KeSbnfJJZfk2mHSHmpPWMzln/7pn5ppJH932mmnVe1YmzZtimIpSTl//OMfo9grr7ySdJ/PP/98Ur9t0YknnhjFwqRg64L/zz33XKONCY3joYceimKXXXZZrt29e/emGk6dli5dGsVmzJgRxS688MIoZiUPY9uQZVlSrLXhG2QAAADAwwIZAAAA8LBABgAAADwskAEAAADPNpWk169fvyj2pz/9qeLtwmQLya6AhNp28skn59rf+ta3oj5t27YtdOzhw4dHsaKV7m6//fYoNm/evIq3sxKFrAQcVF+nTp2i2Lhx4yre7oEHHohimzdvrsqY0HTmz58fxc4888xc20ra/PrXv95oY7JYlS5//vOfN+kY0PJ06NAhqd+GDRsaeSRNi2+QAQAAAA8LZAAAAMDDAhkAAADwuK1d7Nk516quBG3tv/rud79b8Xb7779/FEstpNCaZFnmUvq1tnmDhtkW5o21d90q+PH+++/n2meddVbUZ926ddUbWAu2LcybY445JopZRTrGjx+fa1vFfX71q19FMefyT+H06dOjPgsWLKg4zpZkW5g3TW3JkiVRbPvt4xS2a6+9Nor99Kc/bZQxVZs1b/gGGQAAAPCwQAYAAAA8LJABAAAADwtkAAAAwNNqk/QOOeSQKPb4449HsR122KHisUjSKyH5AUUwb1AE8wZFMG+q79FHH41iN998cxR75plnmmI4jYIkPQAAAKACFsgAAACAhwUyAAAA4GGBDAAAAHjiUiitxD/8wz9EsZSEvDlz5kSxNWvWVGVMAAAALUlYyXFbwTfIAAAAgIcFMgAAAOBhgQwAAAB4WCADAAAAnlabpJdqypQpufYRRxwR9VmxYkVTDQcAAADNjG+QAQAAAA8LZAAAAMDDAhkAAADwuCzL6v6lc3X/EtucLMtcSj/mDXzMGxTBvEERzBsUYc0bvkEGAAAAPCyQAQAAAA8LZAAAAMDDAhkAAADwbDVJDwAAANjW8A0yAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBXIOcc4c7595p7nGgZWHeoAjmDYpg3qCIljRvWuUC2Tl3p3Puh9vq/aOY5n7dmvv+UUxzv27Nff8oprlft+a+fxTT3K9bc99/U2qVC+SGcs5t39xjQMvDvEERzBsUwbxBEcybesiyrCZ/JGWSBnrtOyX9sPzvwyW9I+mbkt6XtFjS+eXfXShpo6SPJa2R9Gg5/h1JcyStljRd0knesc+T9D+Sbpa0XNIPJXWV9KikVZJeLsee924zRNLTklZImiXp9Ar330vSg5KWSnpb0te8Y3UsP76V5bFdJumd5n4NWuIP84Z5w7xh3jBvmDe1/MO8aRnzptkH0IAJtEnSNZLaShonaZ2kLmFf7/anlV/E7SSdIWmtpJ7eBNok6WJJ25df0HvKP50kDZO08JMJJKlzuX1+uf8YScskDbPuv3yfr0q6QlI7SXtJmivps+XfXy/pr5J2ldRH0tSWMoFq7Yd5w7xh3jBvmDfMm1r+Yd60jHnTkrdYbJR0TZZlG7Mse0Kl/5sZXFfnLMvuz7JsUZZlW7Isu1fSW5L297osyrLsP7Is26TS/x2dIunKLMvWZVk2XdJdXt/jJc3LsuyOLMs2ZVk2SaX/ezqtjrsfK6l7lmXXZFn2cZZlcyXdJunM8u9Pl3RdlmUrsixbKOln9XwukI55gyKYNyiCeYMimDc1oCXvRVlefrE/sU7SDnV1ds59QdIlkvqXQztI6uZ1Wej9u7tKz83COn7fT9IBzrkPvNj2kn5Tx933k9Qr6N9Gpf+rkkr/5+cff35djwMNxrxBEcwbFMG8QRHMmxpQywvkdSp9/f+J3VXal5Mi8xvOuX4q/R/NEZImZlm22Tk3WZKr4zZLVfqTxB6S3izH+ni/XyjpuSzLjkq5/3L/t7Ms27uO/ovLx59Wbvetox8qY96gCOYNimDeoAjmTQtQy1ssJks6yznXxjl3jKTD6nHb91TaB/OJziq9qEslyTl3vqQRdd04y7LNkh6SdJVzrpNzboikL3hdHpM0yDn3eedc2/LPWOfc0Dru/yVJq51z33bOdSw/phHOubHl398n6bvOuS7OuT1U2iuEYpg3KIJ5gyKYNyiCedMC1PIC+euSxkv6QNLZkh6ux23/S9Iw59wHzrmHy3tsfiJpokov7kiVsjq35quSdpa0RKU/Lfxe0keSlGXZaklHq7THZlG5z48lta/j/jertK9ntEoZnssk/d/y8SXpapX+7PC2pD+p7j9loDLmDYpg3qAI5g2KYN60AC7Lwm/LYXHO/VjS7lmWndvcY0HLwbxBEcwbFMG8QRHMG1stf4PcrJxzQ5xz+7iS/SX9s6Q/NPe4UNuYNyiCeYMimDcognmTppaT9Jrbjir92aGXSn+2+ImkR5p1RGgJmDcognmDIpg3KIJ5k4AtFgAAAICHLRYAAACAhwVyIufc4c651OsUApKYN6g/5gyKYN6gCOZN3VggAwAAAB4WyAAAAIBnm14gO+d+6pxb6Jxb5Zx71Tl3iPe7js65O51zK51z0yWN9X73befcA8axftaEw0czYd6gvpgzKIJ5gyKYN9WxTS+QJb2sUvWXXSXdLel+51yH8u+ulDSg/PNZSf4FtO+RNM45t6MkOefaSDq9fAy0fswb1BdzBkUwb1AE86Yasizjp/wjaaWkUeV/z5V0jPe7CyW947Wfl/SF8r+PkjSnucfPD/OGn5bxw5zhp8gP84afIj/Mm2I/2/Q3yM65S51zM5xzHzrnPlCpdni38q97SVrodZ8f3PxuSZ8r//ssbav/h7UNYt6gvpgzKIJ5gyKYN9WxzS6Qy3tyvqXSnw+6ZFm2i6QPJblyl8WS+ng36Rsc4n5Jhzvn9pB0krbhSbQtYd6gvpgzKIJ5gyKYN9WzzS6QVSq1uEnSUknbO+eukLST9/v7JH3XOdelPFEu9m+cZdlSSc9KukPS21mWzWiSUaO5MW9QX8wZFMG8QRHMmyrZlhfIT0maIOlNlf7EsEH5PztcXY6/LelPkn5jHONuSUdqG/4/rG0Q8wb1xZxBEcwbFMG8qRJX3ogNAAAAQNv2N8gAAABAhAUyAAAA4GGBDAAAAHhYIAMAAAAeFsgFOecy59zA5h4HWhbmDYpg3qAI5g2KYN6UsEBuAs65ac65Nd7PJufco809LtQ259y/Oefecs6tds7NdM59obnHhNrnnDvdOfc359w659yzzT0etAzOufbOududc6ucc0ucc5c095jQcjjndnXOLXXOPd/cY6mW7Zt7ALXAObd9lmWbGuv4WZYN9+7LqVQL/f7Guj80jcaeN5LWShqv0vUsx0qa4JybnWXZ37lW/4cAACAASURBVBrxPtHImmDerJD075KGSPrHRrwfNKEmmDdXSdpbUj9Ju0v6f8656VmWTWjE+0Qja4J584kfS5qhVvTFa6t5IKHynwi+5pyb65xb5py70Tm3Xfl35znn/sc5d7Nzbrmkq8r/9/xvzrkFzrn3nHO/dM519I53mXNusXNukXPuggYM7VCVaqI/2LBHiMZQS/Mmy7IrsyybmWXZlizLXpT0V0kHVfUBoypqbN78d5Zl90laVN1HiWqrpXkj6VxJ12ZZtrJcPe02SedV67Giemps3sg5d7CkESpV32s1Wu0CuewkSftJ2lfSP0nyX/gDVPomt4ek6yRdL2mQpNGSBkrqLekKSXLOHSPpUklHqfR/2Ef6d+KcO8s593rimM6V9GCWZWuLPSQ0gZqbN+WT2VhJ04o+KDS6mps3aBGafd4457pI6ilpiheeImm41R81odnnTfn3bSTdIumrklpX5bksy1rlj0ov1DFe+18k/bn87/MkLfB+51T6c/YAL3aQSnXIJel2Sdd7vxtUPv7Aeo6pk6RVkg5v7ueHn5Yzb8q3vUul8qGuuZ8jflrGvJH0fyQ929zPDT+1P28k9Sn37eDFjpI0r7mfI35qd96U+39D0q3efT/f3M9PtX5a+x5kv/74fEm96vhdd5UWr6+WtghLKk2qNuV/95L0anCsIk5WaX/gcwVvj6ZRU/PGOXejSn+++kxWPguhJtXUvEGLUQvzZk35vztJ2uD9e3U9joGm1ezzxjnXS9LXJH0q9TYtSWtfIPfR3/8k3Vf5PXn+QmOZpPWShmdZ9q5xnMXlY32ib8HxnCvp1yxyal7NzBvn3NWSjpV0WJZlq+p7ezSpmpk3aFGafd5kWbbSObdY0ihJT5fDo8SWrlrW7PNG0v4qbc2ZXl58d5TU0Tm3RFLvLMs21+NYNae170G+zDnXxTnXR9LXJd1rdcqybItKCQk3O+d2kyTnXG/n3GfLXe6TdJ5zbphzrpOkK+s7EOfcHpI+o9KfylHbamLeOOe+K+ksSUdmWba84GNB06mVedPGOddBpS9AtnPOdXDOtS34mND4amLeSPq1pMvLYxki6YuS7qz/w0ETqYV586Sk/irtbR6t0r7mSZJGt/TFsdT6F8iPqPSng8mSHpf0X1vp+21JsyW94JxbJem/JQ2WpCzLnlTpsknPlPs849/QOXe2c67S/2l/XtLELMvmFHgcaFq1Mm9+pNL/zc92f7+G9veKPSQ0gVqZN59X6RujWyUdUv73bQUeD5pGrcybKyXNUelP7M9JujHjEm+1rNnnTZZlH2VZtuSTH0kfStpY/neL51rrX/udc5mkvbMsm93cY0HLwbxBEcwbFMG8QRHMm6bR2r9BBgAAAOqFBTIAAADgabVbLAAAAIAi+AYZAAAA8LBABgAAADxbLRRSzpTM2WeffXLtdevWRbd7++23o9jmzfEl8XbYYYdce88994z6vPHGG1sboiRpr732imKLFy+OYj179oxic+fOrXj8FLvttlsUe//996ty7LqMGDEi1546dWrVjr3HHntEsYULFzqja8SaN0X1798/inXt2rXi7V599dWKfbYFn/pU5QJH1nNlPe+77rprFHvttdcqHj/LsqrNm5EjR0axlHNE+fjhuJJuF+rVq1cUW7RoURSz3kPvvPNOrh2+hyX7fTxgwIBce86c+GqR1rGsfoMHD861P/zww6iPdQ5P0a9fvyg2f37lwlx9+8a1CebPn1+1edOpU6cott128fdDa9asiWLDhw/PtT/66KOoz+zZ1buYwE477RTFVq3K1whq37591McaV1HW+z+8zzZt2kR9pk+fHsXC+TZr1qykMQwdOjSKhZ/rH3zwQdSnIeeb8D0bvl/rI/ycWr48vpR9jx49oth7771X8dg777xzFGvXrl0UW7p0acVjWe+D7bfPLw2t+WCNs3v37lFs5cqVubb1PFhSnr8hQ4ZEMet5ePfdfH0U61jWvOEbZAAAAMDDAhkAAADwsEAGAAAAPFu9zFs195JaUvZHpqjmftPUMU2ePDnXDvepSdLrr7+edKxwz1m3bt2iPuF+bev4Y8aMifpMmjQpaQzh3qG1a9dGfdauXZu0t6tjx47RvNmwYUOunbqX1Nr3Gu5Vt15/63UsOk+s/YvW3vtqKTr2ou+nLVu2RLHUeRMaPXq0dazCewLD/b7WXl/LoEGDotibb76ZMoYoVs1LYVZrH7T1vgjfY5K9J3jGjBkVj58ylxoyb8I9jUuWxJVp169f3+Q5D5bw/BzuB05l7Um3zrNWTk0o9Xm2XqNQ0fOGNYaU+7P2jc6cObPQGCype5AHDBgQzZtq5SRJTb++KZpvkmLgwIFRzNoHXVRTr+GsvIu33nqLPcgAAADA1rBABgAAADwskAEAAADPVq+DXJR1vWHrusQpGvt6tkX3CYXXdbb2G1vXF+zcuXMUC/ch7rLLLlGflOvNbty4MYpZ+9msfVbh9RJT9sHVZxzDhg3Lta19fNZ1sK3rsa5YsaLiGKo5b6zrcabcXzX3QaccK3UMIeu9ae3h7dixYxSbMmVKrm1dR7YhUvYcW/uerev/hvuZrf2S1l7YoqzrxFrXg0+R8jpa++Kt/fPV0pDXOtwDaF3Xt5qsc7H1+luvWXhNWEuXLl2iWLjneNq0aVGfHXfcMYqlnCNSH0+fPn1y7YbMhwULFhS+rS91v7F1Xe+wtkBDag2k7De2cn+sa/0WfW5q9Xr9e++9d65tnVtS9yA35mMsun5LvW453yADAAAAHhbIAAAAgIcFMgAAAOBhgQwAAAB46p2k165du1z7448/jvq89957USy82LqlOQp+VEu4qV2yEylmzZpV8Vjz58+PYgMGDIhiYSKSldC0cuXKivcnxYUHGnLBdCtZZPr06bm2VQxl2bJlUcxKBAgvWl503liJNSeccEIUO+mkk6LYPvvsk2tbCYZWQYYwCeyBBx6I+vziF7+IB5ug6Jy3Eiat1zClQELRJLS69O3bN9e2EkCtxDqraEaYaDJ48OCkYxVVzYS8cI5bfcJEW8kuFBImAVvnFkv4nL7zzjtJt+vatWsUC89LVlJoNVlJx1aijvWahe8F67yRep4NWQULUoTnH8lOFN9tt90KHd9KpAvPu9Z8s4Sfg1ZRLatI1NSpU6NYWGDGmlvVZCWiWrG2bdtWPFbRRO6GrGWKfjaGj9FKhtx9992rdn+WxlzD9ejRI6kf3yADAAAAHhbIAAAAgIcFMgAAAOBhgQwAAAB4XFjFLfdL5+r+ZX3vyLkotrX7ro/GTsizNp6HiQZWlSRLmOQoxVXmrOSujz76KIqFFd6s2xVlVbbasGFD/CIaqjlvOnToYI2jKsd+7LHHothxxx2XdNuXX3451x47dmzUx6quFCadWaz3Soqi74PGruaUZVnSA+rQoUM0b8JkimpV86o2KxnKSmwJk2Gt5M4UVlLb3XffHcXuvPPOKHbhhRfm2r179476WOebMGa9N1MqH6ZKnTcp5xurQp51Ll6/fn0UGzlyZK5tJSdaSYCbNm3KtcOqk1L652KYEPXQQw9Fffbff/+Kxz/ttNOiPg8//HAUs5J0U1iJ6WPGjMm1rfONVWHWSsAL3/9WJcJVq1Y16ueUVenWGqtVzbOI5kjSa2rNsYYLWecbvkEGAAAAPCyQAQAAAA8LZAAAAMDDAhkAAADw1LuSXjU3U4dViyZPnhz1Sdn0b5k0aVIUCysBSVKnTp1ybSuZI+Uxpz4vVuWk8D5HjBiRdKzwMaZU5UllJemksqo8hRXQrIRCq7JZSuXBVN/61rdybSuxKky+q0tYacpK7rESd1KO/+CDD0axU045JWlcRQwZMiSKWclDViW9d999t2rjsOZcSlJeSpVJSdp++/zpLkyikuwqaWFSaJhUW5c+ffok9UsRVkmzjv3SSy9FMSsBK6z6ZSUYpST8rlixIopZCX/hOVaS3nrrrYrHb4gweWv16tVRH+s9awmrvI0ePTrqY51nwyqg3bt3j/osX748illJegcccECubZ1jrc+uhQsX5tpW9TOL9bkbPh7r/GYJqwymflamfHZZr2sqK2E65XxjVfO0zjfh4yxaSS/1M9yaN0UTvsMKnFbyrTUHrbGG55dDDjkk6hPOU+tYDVl7hnM39dzMN8gAAACAhwUyAAAA4GGBDAAAAHi2ugfZuhh9isa+OHW499bak2zFUsZl7XOx9sGuWbOm4rGaWjWf97CgQX1Y+9fCPY3WvkQrVk0//vGPc21rP7BV8MUqKLJs2bJc25oPViGCCy64INceNWpU1OeBBx6IYpbGLAwS7tmS7IvyNzVr3+CSJUuimFXoJtzjPm/evKiPtS+1aGGaoucb63bhvuFwP7UkrV27Noo98sgjFceQWmBo3bp1uba139gaQzX3qVus/YThOKw+1r7HFFauTMp7cf78+VGsW7duUcwqMHPEEUfk2lZhjblz50axHXbYIdcO95bWxTp3hTkbVp6PpVevXhX7FP3s6tixY6HbSfZ+43DdYD1Ga45bUh5T0cd9ww03RLHHH3+80LEsH3zwQa6dOm8sJ5xwQq5tzfmU92Lqc2V9doW5X9b7x8I3yAAAAICHBTIAAADgYYEMAAAAeFggAwAAAJ6tJum9+eabUaxaF7+W4mSuPffcM+rz4osvRrHwotU/+9nPoj6//e1vo5iVEBVeBN7alL9ly5Yo1tSqWQQk5fgNOXZKUpNVpMF67q1EoJSL3Y8fPz6KhXMpvPi+JF1zzTVRbPHixRXvz3q+jj766CiWUmSimskWlpTX2ipoY10gPxQmk9VHmFAkSf3798+1p06dmnSsoUOHRrEwkbJDhw5RH6sYSoqGnAdThM/9xRdfHPWxHk+KMPmuLuHjCc+dkvTxxx8nHWv48OG5tpUcmyolwceau9ZcTZnjRZNjrSRX6zPpoosuimJ77713xeNbBV/Cz/Bjjjkm6vPKK69EMev8bCUBhqyiQ9OnT8+133nnnYrHSZXyetVHSgKelcBuJQWHYwsTuxvCSqI87rjjCh0rJakt9XaWcH5dcsklUZ8bb7wx6VgprIJwodR5wzfIAAAAgIcFMgAAAOBhgQwAAAB4WCADAAAAHpdlWZ2/HDp0aPTLmTNnVu3OwyS9+++/P+pz8sknR7HNmzfn2qkVfawN+AcddFCu/atf/SrpWI3JSmIIq8BJ0ujRo3Ntq8KTJSXJxEomyrLMpRzfOVf3pNoKq8LOypUro1j4uC0pyVBHHXVUFFuxYkXF26Xe3+233x7FRowYkWuPHTs26mMlP1jJY9WqpLfTTjsl3V9RDZk3YVJjmPBTF6vSXJhIY1UKTT2XNLUwgfm+++6L+hx++OFRLLXqV6hoZTgrEclKmEyp3tfY5xtLmzZtoljK+SaFNbes5yH8fJPiuWqNs6jVq1dHMet9Fn42/uEPf4j6WMld1UxyT0kwTp03/fv3j+ZNSiJd0fdUUVZS4PLly6PYqaeeGsUefPDBise3Kt+mVE1NSeSTpMMOOyzXfvbZZ6M+ziW9ZJFqVpO15g3fIAMAAAAeFsgAAACAhwUyAAAA4NlqoZClS5dGsWoWCtlnn31y7UceeSTqY+33mTVrVq7dt2/fqE+fPn2imHUR+9dffz3X/upXvxr1SWEVVdlrr72i2JNPPhnFwufBuoi1dYH3z3zmMxXHVXSPTufOnQvdri7hPirrNbP2UFt77apVIOXpp5+uynEk+7Ww9riGbrnllihWdL9x0efFKrxi7UvcbbfdolhYZMJ636Wy9kKn7Dnu2bNnFOvevXsUC9/rYeGQVKnnt2oWCnn44Ydz7X79+iXd31/+8peKxy56jkgtfGDtcQyFBWEaKtxDaRWhsQoAWft/w9cs9fkKb3fttddGfaz3WdHXwxLmcFgFLTp16hTFrD384R7kO++8M+ozZsyYeo6wJPV9Uc3iWNYe+nAvbK9evaI+1prE2mcfPq+pYz/xxBNz7d/85jdRn9mzZ0exCRMmVDy2NbesY4XFNqwiMdb72tpv/qUvfSnXfumll6I+1vom5fE0Nr5BBgAAADwskAEAAAAPC2QAAADAwwIZAAAA8Gy1UMiIESOiX06bNq1qdx4mboUXw69LeMH1hlyIvEuXLrm2dVF4q5hDSkGR9evXR7GPPvooik2dOjXXthIDjj322Cj24osv5trVTO6wEphmzpyZdDVvq8BMmCBpbfq3Eo/atm0bxaykghQpCaaprrzyylzbKjry6U9/Ooo9/vjjufYZZ5wR9RkyZEjSGKqZsJJi3333jWLvv/9+rm0VuUm9cP+YMWOieRMmMVkJMtZctWJhIph17rMSBd944414sE3s3XffzbWtc0TqxfardZ4IEzQl+/xmJYGF5zxLQwqFhAmyVhJ1jx49oth7770XxYo+X2GBhAMPPDDqc9pppyUdK3yured04sSJUSwsHvHBBx9EfS699NIodvnll0exMCnryCOPjPqkJIVWs3CIJXXe7L333tG8CefvwoULC48jPJdYCZLWhRAuu+yyXPuGG26I+hxyyCFR7Pnnn684JiuxzkqsTPlsSUm+leLn0HrfWUmhoWqubxYtWmTFKBQCAAAAbA0LZAAAAMDDAhkAAADwsEAGAAAAPFvdGf3WW29FsWpW0gurqYXJd1JaZR4rsS6smiXZlc3CinFWFTNrk3zRhK8BAwZEseHDh1e83bx585KOXy1Wsk2qmTNnRrFwE/6IESOiPlbijlXlLVTNimWpiQD77bdfrm0lTVmVAcPEMCshz5q7YbXFatpll12imJXMM2XKlCgWVh5rSCW9OXPmRLG99967Yh+L9ZjCmPX6WPMyTLYJxyQVPw9ar7+VzBXOLysBuJrvgxTWOckalyVMmKz2+S2cJ7179476hImPdUmppGdVAvzJT36SdPwUYVKelUR33XXXRbFwPi9ZsiTqk5IwKUmvvPJKrn3VVVdFfawk3XPOOSfX7tatW9QntSpjyEp8S2Ul5IYXIbAuHPD2229HMSuBOUyatd6L1vHHjRsXDzaQkpAnxc+1lQhf9BxhJVueffbZUSxMyrPO4dY574UXXig0rhRWFU0L3yADAAAAHhbIAAAAgIcFMgAAAOBhgQwAAAB4tpqkN3jw4CiWsqE7ddN3mIxkbd63EvcqHUeyN5CHCXmW1LEXrTRz6623VrzdNddcE8WsCk/VFD6elOS4+giT01ITQ6ykkpREumr1kaSHH344ioVJRRs3boz6PPbYY1HsoYceyrVT5rdU3WSr8HFbz/EOO+wQxazEhjApL7WamyWsmidJGzZsKHQsK8kw5VjWvAzPS9U8R1jnqXPPPbfi7S644IJC99cQKeeI1CS9sIpqQ5L0unfvHsU6duyYa1tJYO3atYtiYcVPKe08YVX8TLFy5cooZr2HzjzzzFz7qaeeSjr+9OnTK/axzm9XXHFFFDvxxBNzbaua38CBAyveX9GEPCn+TG3I59SmTZuiWJicbr2u1kUBXnvttUJjsCophkn7VgJbUdU8R+y///5R7K677qp4O6tyY2Mm5EnFHzffIAMAAAAeFsgAAACAhwUyAAAA4NnqHmRrH19jsvYmWXv0QtYesdS9nWGxgGpebP+EE06IYtZ+udCVV15Z6P6qub/I2pfaEOG+qjfffDPqExYTkex93I2pZ8+eUWyPPfaoGHvyySejPuGF9aW0eZm6N7paUvfxWUUAFi5cmGtXe+xz586t2GfnnXeOYh9++GEUC/eXWntQrb3X4ZxIvdh+yrnEKjpg7WcP3XPPPRX71KVa54kwr0BKLzoTPm6rkEcq6xyxYMGCXHvUqFFRn0WLFkWxpUuXFh5H6MUXX8y1DzjggKiPVazC2pca7jlO/cwLC22lvvbXX399FAvzDcaOHRv1sR5jNYWPJ7Xgg2XWrFlRLHw9rM8pa6526dIliu211165tvXcW/uLw/eGtVc6VXiuKnrusvpYRaGsfuFe5UcffdQebAWNnWNh4RtkAAAAwMMCGQAAAPCwQAYAAAA8LJABAAAAz1aT9MIEnIZoSCJdJdZxrGQ4q1+YBFh0I3j79u2j2He+850oZl1kPKWAiSVlI33RY73xxhuFj2UlAljFXFIMGTKk8DhC4fNjzckHH3ww6Vgvv/xyrv3uu+9GfR555JGKx7EuRN+YRUGqffzGNmDAgFzbSqxLPY+kJPytWLEiKZYi5Xk+4ogjolhYREOSvvnNbxYaQzVf//BYs2fPjvpYyZHWuTEskGEVzEi1efPmKBaeU61CS1byYK9evQqNwUqavOSSSyrezkpqq2Yxrmq+18NkSCtJLzwvStI3vvGNXPvmm29Our+UhF8r0TJVWBTEig0aNCjqYyXu9e3bN4qlPPdWwl84l1Jfw6IJ0kXPEeG5uS6XXXZZvcdUbUXXSnyDDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4Nlqkl6nTp2i2Lp163JtK6HESmqxNkWHFaqsqjhWpaHwdlb1q6KVrVI3rIcJC6effnrUx0rAsJJRwuc0VcpG87CCnSRNmzYtis2cOTPXDquO1cf8+fOjWFjJKrw/ya7CZfWrVnKilSBlJYFZwuQNKxEhZS5ZVdNS52A1kzSLqmblPKsaVZs2bXLt9evXR306dOgQxTZs2FBoDFbSTJiAWc3EN+scMXLkyChmnWdTWOMK566VdGSNITyWVZ1uzpw5Uax///5RLEzm69q1a9QnlVWBNRy/lXRsVQKzEtPDzxvrOf3c5z4XxVKqwFYrYbLax7Je2y996UsVj2Ul7oWV1KzEtx133LHisaXqVjpMsXbt2qR+VrJ1yvk5rOQr2e+9aqnmZ8Spp56a1G/YsGFVu8/QiBEjotjUqVOjWFhZs1u3bknH5xtkAAAAwMMCGQAAAPCwQAYAAAA8W92DnLI3NvUi+lYxjPC2VoGJ7bePhxjuX5s1a1bSGKp5AfbQYYcdltTv8ssvL3R8S7jHafny5VGfefPmRTFrP2NDLtSfYsqUKbn2wIEDoz6p+8uqtY/qyCOPLHzbcO/YmjVroj5Fx1nNIgCNXXSkmqw5GO5Bti7uX3S/scUq+BKq5nMaFlGQ7H3w1TxvWLkloZRCQeF7ui5W8ZBwr29DilJZz1fK+K3b7b777lEs5fUeP358FAv3l7/44osVj5OqaI6N5dBDD41iN910U8XbWUVBrIIsodT9xtZeUuv9X0177LFHrm3tQbb21Ft771NYtzvllFNy7VtuuSXqY70+p512WsX7s86x1j7esAiItTbLsiyKhfvNJeniiy+uOK6irDVqat5VCr5BBgAAADwskAEAAAAPC2QAAADAwwIZAAAA8Dhro/X//tK56JfhBnvrIv2pyVZDhgzJta2iEL169YpiixYtyrVTkxO6d+8exYpeeHzcuHG59tVXXx316du3bxTr0aNHxWMXTYayCq2Ez1VDZFnmUvp17NgxmjcpiVRjxoyJYpMmTYpiKRdgt5JCwySGO+64o+KYJLvwSZioUbS4h5U0YRXMqIVCIUWTgFLnzfDhw6N5M3369JSbNqrwHGclgVjPg1X45oQTTsi177rrrqiPlWC2zz77VBxnUWEipCRt3ry54u3CAhqSfQ6yhAVZrOTI1HljfU6Fnxup58HUwlehu+++O4rttddeubZVFMa5+CGmnP9T3+thQasvfOELUZ/U82CK888/P4rdeeedFW+39957RzFrXobndetzZerUqUnzpn379tG8aUiBrCLOOeecKBa+N66//vqkY7300ksV+1jJdtaaJOX9b70+FmuOV4v1WVn0ggPW+YZvkAEAAAAPC2QAAADAwwIZAAAA8LBABgAAADxbTdLbcccdo1+GlUusKiyW119/vZ5DK6lmYl1RVnWlhx9+ONe2EjCsSk0bN26MYmH1npREPktq4oa1UT9MRBs1alTUZ/LkyYWTZkJW8mXbtm2jmFUd0KpaF7KqhVlVkUJWdSirOlCKoklttaBooqj1+BqSbGX0SYpt2bIliu2www659k477RT1qWZSa5ggJUlXXXVVrv2DH/wg6mMl7vzud78rNIaUObjvvvtGfV577bUoFiZIpbyfGiJ13tRCspWVpPe5z30u17Yq6R144IFRrJpJemGC17e//e2k21ljDT/jlixZEvWxEjdD1vvCYiVth6wEyrlz5ybNm3bt2kXzxvp8Dg0dOjSKWWuSZcuWpQwjEr7+77//ftTHqjCc4sILL4xiDzzwQBQLz4M//elPoz4plfuk4kl64fNgVR205tuMGTOiWMeOHXPtPffcM+ozbdo0kvQAAACArWGBDAAAAHhYIAMAAAAeFsgAAACAZ6s7va1kqEGDBuXaVhJVaqJLSjUlK5krTH6zkuiKJkMdffTRUeypp56qeLs333wzilnVr6xxFU3KK8qqDLfbbrvl2lOmTCl8/PBYUpxo0JBkqJQKcmEyqST97W9/y7UPPvjgqM/YsWMLjytUqwl5//zP/1zodlZC7+23397Q4fwvK+E3TEaZPHly0rgs4fksJdnTkpp8aSUKWsmvIeucV5Q1rp133jnXthLyLF27ds21repn1uszcuTIKBZWC7QSdFMVTcgLH49kz4mURPQwIU+KE36t5DTrXLlw4cIodtttt+XaYUVGSdq0aVMUC98/1nnR+ty1XttQSkKeJSX5LpVVMTVVWA1Vij+nPvzww6iP9XytWrUqiqV8TqWcS4pWabVYCXlW1bzwHPTkk09GfVKT9MJ5X/T1X79+fRSzEvJSbptS2VfiG2QAAAAghwUyAAAA4GGBDAAAAHi2ugfZusCztdc2hVU0I9zv1aVLl6iPtRc2PJa1J6ioCRMmFLqddcH/xt6DGh5/9OjRUR9rT+CwYcOimLU3rShrP1aKHXfcMYqtXr06ijXm82oVCnnppZca7f7uv//+KLZ48eIoZu1TP+OMMwrdZ7jPdSpcBQAACn5JREFU2nrMFuv9GRYLeOKJJwqNSZKmTp1asY9V5Mbavzp79uwottdee+Xac+fOjfoMGTIkis2cOTPXTp1/4cXpJXvvaGMqWqzGKqKyYMGCXNt6b1rC/caWlAINdQnzYqS0z6k+ffpEMet8meKwww6LYjfccEPF2z322GNRzNq7HhbpsIomDBgwoOL9ha+hZM95y/nnn5/ULxTueW+IcO5anxmp3nrrrSgWPhfWHuQOHTpEsfbt21e8v6LvxWp+3lnnResxhvuGrfeYxfosKbrnOHzc1vy23gcprHO/hW+QAQAAAA8LZAAAAMDDAhkAAADwsEAGAAAAPPVO0uvWrVuuvXTp0qQ7si7mnnKBfysBJ7yd1cdK+rA2yYebz63HbPnmN7+Za993331Jt7NUaxP+O++8E8WsxCrrMc6bN68qY5Dsi3APHz4817ZeMys55fXXXy80Buu1DhPIGrtQiCVMYrAutt7YYwjtuuuuUey6666LYp/97Gej2AsvvNAoY/pE+P4sWtRASkvMCBPyJGnPPffMta3ny3oPn3TSSVHslVdeybX322+/qM+NN95YcZwpRQEawkq0TSnsVJRVMCNV0WIRVkJeeJ6Sip+fw8/G448/Pul2VqGQF198MdceOnRooTFZCXnWfHv66aejWGpxhVCYBFbNpLPURNFU4fvfSr4Li4lIdsJnUxeKqub9hZ/FViEcq8BI3759o1iYGGr1sYRrF+u1tpIHly1bFsXCokBhAZ268A0yAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAZ6s7ldu0aRPFws3oa9eujfpUsyqbtRE8ZCV3WawqXHfffXfF2x1++OFRLEwe69mzZ9THqoiWmlCYIkzUsSriWJVzVq5cGcX23XffXPu1114rNKa6TJs2Lde2qvlNnz49illJEh999FHF+0tJWJg4cWIUs16fFCNGjIhiRSvdTZo0KYpZSZT/+Z//mWs/9dRThe4vrJpUl7vuuiuKpb73UnTq1CmKhRXRrIqc69evj2JWIsiiRYty7e7du0d9rKTWcF6+/fbbUR/LuHHjoliYlHf00UcnHSt8r1vzu7ET98LzupWsaM0l6zMifM2sxKdU1vslTGJ77733oj4rVqyIYinnllTPPPNMrm1VJzz00EOjWFhh1mK976zPkfB8dscdd0R9rPd1eL5uiJS52xzHShFW35SkGTNmRDHrvWAlWza1kSNH5topVS2l+Hm2qgdarPPn8uXLc+0ddtgh6mO9F8P1jJWYaq1RreqKVtXEFHyDDAAAAHhYIAMAAAAeFsgAAACAZ6t7kK2Lpod7k6w9OtZ+FWufWMjab2ztMQn7pe6htC6SHu6Ps/YN9+jRI4rtvvvuWz2OZO8JLFr4whLuObP2Gw8cODCKWRf4Tyna0hDhc28VNbFYewKL7scM+6X0Se1nXZz8Jz/5SdKxUhTdc5oy9jFjxhS+v2ruAbSKCYV7icP9bJJdwMDKg9i0aVOu3bFjx6iPtQ8+lPo8WEVgrrjiilzbKshgSZm7qcJiT9Z+bSsHoVevXrm2db629pGn5DNY+wZTWZ9Tc+bMybWtzwirWIB1Dg2lvv5/+ctfttqW7Hl68sknR7Hx48fn2inFZCTptttuy7VTC3tV872ecrs99tgjilmfEbNmzSo0hlS77LJLrp1aHGXKlCkV+1jFcFL23lv5OtYay8qNSNlzbL3W4Xn2qKOOSrq/Sy65pOL97bzzzlHM2oMcss7XRXOlrHOGhW+QAQAAAA8LZAAAAMDDAhkAAADwsEAGAAAAPG5ryVnOueiX4Sb2Dz74IOmO+vfvH8UWLFiQa1ub0a0CCSmFSA444IAodv3110exT3/607m2lXhy8MEHR7Ew8SC1wIS1Ud9KmksRJttYiWIWK+kwvPC8lcyRZZlLOb41b3r37p1rW8+DlchpXVx/1apVKcNAjWjIvCmqaIGZWlXN5M4w8dkqcpCazBWyEnCsAkbh50EdhQKS5s3w4cOjeZOSbBkmHUpxMRmpegVYrPNb0UI7qWMKX38rIdMqmJNyrFTVLO4RFmSaOnVq1Kch55swmbMhhZDC59UqaJRSFMZKYLSKbYRFlaR4/EXncpgkKklPPPFEFHvllVcqjiH1HNGYrIs/bNq0KZo3fIMMAAAAeFggAwAAAB4WyAAAAICHBTIAAADgqXeSXlgpyUpqC5PHJKldu3ZRLKyUYyVuWRu6w83ozsV78r/85S9HsZtuuimKhSZOnBjFDjnkkCgWbua3Nn1brGS+rl275toLFy5MOlZR1gb/MFnAStxZvnx51ZKtrKRNK/nOSmywYqhdqUkzY8aMiebN3Llzc+0wSViyE11SKjOlGjRoUK5tJcNUU7WSwuoSnnetamHW50KY5GidR6xqiNZ5PfwcsSp+pc6boUOHRoMNk36seWNVZbQqmzXm61G0QmZL9t5770Uxq2qeVfl25syZFY+fOm86duwYzZuUynmjRo2KYlYlvbCKrfV+mTx5csX7s1jrqS5dukSx8Lmu5tyqZhXVFGGCpmQnaVp69uyZa1sVk615wzfIAAAAgIcFMgAAAOBhgQwAAAB4WCADAAAAnu3re4MwecOqRrRkyZIoZiVWhYluVlU+azN1irBCniStXr06ij311FO59llnnRX1CZPopDgRMUwmkuwkA2tzfbjZ3Ur4s5IhQ1aVpLBaoZRWvSclWaE+OnfunGtbFRKtJINwc71kP9cpqlnRCSXVfE7D84EUV1K0klythDyr38aNGyuOYd99941ir732WsXbpWrqBCzr9Rg6dGiuPWPGjKjP7rvvHsXCpKPU58U671rnxqJSErdSP0f69evX0OH8r6Y+v1Qz4a+xjxUqmoBlJfGnsi4mECZ3WnPXSsizzjezZ88uPLZKrGTY1Gq+RTTHZ2U43xpyHi66juQbZAAAAMDDAhkAAADwsEAGAAAAPFstFNK5c+fol927d8+158+fn3RHVmEIa39sKGVfbeoFxffYY48oZl2gPEV4n9YF5tetWxfFrIuFW7dNEe7/XrlyZdSnd+/eSccK90tZ+0E3b96cdAH2nj17RvPG2pfe3MK5LNkFBdauXRvFevTokWtPmjQp6jNy5MgotnTp0opjsPabT58+PYqFBRiseTR48OAoNmvWrCiWwtoPHo7/9ddfj/qkXrh/l112qVjwYfTo0dHtrHOQ9V4ItW/fPopZ+xI3bdqUa4f76aX0PfthDoK1F98S5iVYe2WtvflWwZ+Qtc8yfMySXTwkFBZVkezCKuG5y8rNePvttwvPmy1btuTa1mNMFb6HrPfPjjvuGMVS7tOab8uWLat4O6tYxbRp06JY+P5M3YtpvR7hvlcrZ8TKIwrPg9Y5ok+fPlHMKpgVfu6+9dZbUZ9NmzZVraCVdb6xinsMGzYsilnn7CKsz/B333230LHCAj2SvU4Jn2crLyr8LJPsc4TVL2Ttnx4wYECube3pts5TRVEoBAAAAKiABTIAAADgYYEMAAAAeFggAwAAAJ6tJukBAAAA2xq+QQYAAAA8LJABAAAADwtkAAAAwMMCGQAAAPCwQAYAAAA8LJABAAAAz/8H3rLc4tzSm/QAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 720x576 with 10 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GOR5rCumBaNc"
      },
      "source": [
        "### Imroved with Curriculum Learning"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 448
        },
        "id": "bo9R-tZY8kHK",
        "outputId": "503f820b-129f-45aa-a0b0-bfb07f4a5e68"
      },
      "source": [
        "# load the model and put it in evaluation mode\n",
        "model = mnist_net().cuda()\n",
        "model.load_state_dict(torch.load('/content/mnist_model-improved-2000_40'))\n",
        "model.eval()\n",
        "batch_size = 5\n",
        "loader = get_mnist_test_loader(batch_size=batch_size)\n",
        "for cln_data, true_label in loader:\n",
        "    break\n",
        "cln_data, true_label = cln_data.to(device), true_label.to(device)\n",
        "adv_untargeted = adversary.perturb(cln_data, true_label)\n",
        "pred_cln = predict_from_logits(model(cln_data))\n",
        "pred_untargeted_adv = predict_from_logits(model(adv_untargeted))\n",
        "plt.figure(figsize=(10, 8))\n",
        "for ii in range(batch_size):\n",
        "    plt.subplot(2, batch_size, ii + 1)\n",
        "    _imshow(cln_data[ii])\n",
        "    plt.title(\"clean \\n pred: {}\".format(pred_cln[ii]))\n",
        "    plt.subplot(2, batch_size, ii + 1 + batch_size)\n",
        "    _imshow(adv_untargeted[ii])\n",
        "    plt.title(\"untargeted \\n adv \\n pred: {}\".format(\n",
        "        pred_untargeted_adv[ii]))\n",
        "\n",
        "plt.tight_layout()\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsgAAAGvCAYAAABGsiMJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZxcVZ3///chJGRhSyAhC1kgCyEkJCwBAdHIJgaCCkT4IQgyCjMi+AAFQUYRF2RRmQWEkRlFdARkGYZFIgwM28hONpKGkD1kIftOICH3+0cVP88959Op07eru6s7r+fj0Q9yPn3q1qmqU7cO1edzPy7LMgEAAAAo2aGlBwAAAADUEhbIAAAAgIcFMgAAAOBhgQwAAAB4WCADAAAAHhbIAAAAgGe7XSA7585zzr3Q0uNA68GcQRHMGxTBvEERzJvq2W4XyAAAAICFBTIAAADgafMLZOdcX+fcg865Zc65Fc65W+rpN9Q596RzbqVz7m3n3Je8353knJvonFvrnFvgnPuh97sBzrnMOXeuc26+c265c+7qZnhoaCLMGRTBvEERzBsUwbxpem16geycayfpUUnzJA2Q1EfSPUa/LpKelPRHST0knSnpV865YeUuGyR9RdLukk6S9A/OuS8Eh/mkpP0kHSvpB865/av9eND0mDMognmDIpg3KIJ50zza9AJZ0mGSeku6PMuyDVmWbcqyzNq8frKkuVmW/TbLsi1Zlk2U9ICk8ZKUZdkzWZZNzbJsa5ZlUyTdLenTwTGuzbLs/SzLJkuaLGlkkz0qNCXmDIpg3qAI5g2KYN40gx1begBNrK+keVmWbanQr7+kw51zq73YjpJ+L0nOucMlXS9puKQOknaSdF9wjCXevzdK2rkR40bLYc6gCOYNimDeoAjmTTNo6wvkBZL6Oed2rDCRFkh6Nsuy4+v5/R8l3SLpc1mWbXLO/ZOkPas8VtQG5gyKYN6gCOYNimDeNIO2vsXiFUmLJV3vnOvinOvonDvK6PeopCHOuXOcc+3LP6O9vTa7SFpZnkCHSTqrmcaP5secQRHMGxTBvEERzJtm0KYXyFmWfSRpnKRBkuZLelfSGUa/dZJOUGkD+yKV/qRwg0p/bpCkb0j6kXNunaQfSPpTkw8eLYI5gyKYNyiCeYMimDfNw2VZ1tJjAAAAAGpGm/4GGQAAAGgoFsgAAACAhwUyAAAA4GGBDAAAAHhYIBdUrlE+qKXHgdaFeYMimDcognmDIpg3JSyQm4Fzbppzbr33s8U590hLjwu1zTn3c+fcO865dc65t5xzX2npMaH2Oee+5Jz7q3Nuo3PumZYeD1oH59xOzrnfOOfWOueWOOcua+kxofVwznVzzi1zzlklr1ultl5JL0lCNZpGybLsAO++nKTZiss5opVp6nkjaYNK17qcIWm0pAnOuZlZlv21Ce8TTawZ5s1KSf8kaaikY5rwftCMmmHe/FDSYJXKE/eU9L/OuelZlk1owvtEE2uGefOxGyTVqQ198dpmHkio/CeCS5xzs51zy51zNznndij/7jzn3P855252zq2Q9MPy/z3/3Dk33zn3nnPududcJ+94lzvnFjvnFjnnzm/E0D6lUinHBxr3CNEUamneZFl2TZZlb2VZtjXLspclPS/piKo+YFRFjc2b/8my7E8qFQZADauleSPpXEk/zrJsVZZldZLukHRetR4rqqfG5o2cc0dKGi7pt1V7kDWgzS6Qy74o6VBJB0v6vCT/hT9cpW9y95L0U0nXSxoiaZRK1Wn6qFRZRs65EyV9R9LxKv0f9nH+nTjnznLOTUkc07mSHsiybEOxh4RmUHPzpnwyGy1pWtEHhSZXc/MGrUKLzxvnXFdJvSRN9sKTJR1g9UdNaPF5U/59O0m3SPqmpLZVeS7Lsjb5o9ILdaLX/oakp8r/Pk/SfO93TqU/Zw/0YkdImlP+928kXe/9bkj5+IMaOKbOktZKGtPSzw8/rWfelG/7O0kTVK5+yU9t/dTivJH0NUnPtPRzw0/tzxtJfct9O3qx4yXNbenniJ/anTfl/pdKus277xda+vmp1k9b34O8wPv3PEm96/ldd5UWr6+XtghLKk2qduV/95b0enCsIk5VaX/gswVvj+ZRU/PGOXeTSn+++kxWPguhJtXUvEGrUQvzZn35v7tK2uT9e10DjoHm1eLzxjnXW9Ilkg5JvU1r0tYXyH31tz9J91N+T56/0Fgu6X1JB2RZttA4zuLysT7Wr+B4zpV0F4ucmlcz88Y5d62kz0n6dJZlaxt6ezSrmpk3aFVafN5kWbbKObdY0khJT5bDI8WWrlrW4vNG0mEqbc2ZXl58d5LUyTm3RFKfLMs+asCxak5b34N8uXOuq3Our6RvSbrX6pRl2VaVEhJuds71kCTnXB/n3GfLXf4k6Tzn3DDnXGdJ1zR0IM65vSV9RqU/laO21cS8cc5dJeksScdlWbai4GNB86mVedPOOddRpS9AdnDOdXTOtS/4mND0amLeSLpL0j+WxzJU0tcl3dnwh4NmUgvz5nFJA1Ta2zxKpX3NEyWNau2LY6ntL5D/W6U/HUyS9Jik/9hG3+9KminpJefcWkn/I2k/Scqy7HGVLpv0dLnP0/4NnXNfds5V+j/tcyS9mGXZrAKPA82rVubNdSr93/xM97draH+v2ENCM6iVeXOOSt8Y3Sbp6PK/7yjweNA8amXeXCNplkp/Yn9W0k0Zl3irZS0+b7Is+yDLsiUf/0haI2lz+d+tnmurf+13zmWSBmdZNrOlx4LWg3mDIpg3KIJ5gyKYN82jrX+DDAAAADQIC2QAAADA02a3WAAAAABF8A0yAAAA4GGBDAAAAHi2WSiknCkJSJKyLHOVezFvkMe8QRHMGxTBvEER1rzhG2QAAADAwwIZAAAA8LBABgAAADwskAEAAAAPC2QAAADAwwIZAAAA8LBABgAAADwskAEAAAAPC2QAAADAwwIZAAAA8LBABgAAADwskAEAAADPji09AKCWfOc734linTp1imIHHnhgrn366acnHf+2227LtV988cWoz+9///ukYwEAgKbBN8gAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAel2VZ/b90rv5fYruTZZlL6dda5s29994bxVKT7apl1qxZUey4446LYvPnz2+O4TSJtjZvasGQIUOi2FtvvRXFvvWtb0Wxf/3Xf22SMVUb8+ZvunTpkmvfdNNNUZ8LL7wwir3++uu59vjx46M+8+bNa+ToagvzBkVY84ZvkAEAAAAPC2QAAADAwwIZAAAA8LBABgAAADxU0sN2I0zKa0xCXpgQ9Ze//CXqs++++0axcePG5doDBw6M+px99tlR7LrrrmvoENGGHXTQQVFs69atUWzhwoXNMRw0sd69e+faX//616M+1ut/yCGH5Nonn3xy1OfWW29t5OjQ3A4++OAo9uCDD0axAQMGNMNotu2EE07Itevq6qI+CxYsaK7hNAjfIAMAAAAeFsgAAACAhwUyAAAA4GEPMtqkQw89NIp98YtfrHi7adOmRbFTTjklii1fvjzXXr9+fdSnQ4cOUeyll17KtUeOHBn16datW8VxYvs2atSoKLZhw4YoZu1LRG3r3r17FLvzzjubfyCoWZ/97Gej2E477dQCI6ks/Pw8//zzoz5nnnlmcw2nQfgGGQAAAPCwQAYAAAA8LJABAAAADwtkAAAAwFNTSXpW4QbrguiLFi3KtTdt2hT1+c///M8otmTJkig2c+bMhgwRrUSvXr2imHMu17YS8qzkh8WLFxcaw3e+850oNmzYsIq3e+yxxwrdH9quESNG5NoXX3xx1Oeuu+5qruGgSi655JIo9oUvfCGKHXbYYVW5v0996lNRbIcd4u/JJk+eHMWee+65qowBDbPjjvEybezYsS0wkmJee+21XPuyyy6L+nTp0iWKWUnHzY1vkAEAAAAPC2QAAADAwwIZAAAA8LBABgAAADw1laR34403RrEBAwYUOtaFF14YxdatWxfFrEStlvbuu+9GMeu5CTe/428eeeSRKDZo0KBc25oPK1eurNoYzjjjjCjWvn37qh0f24/99tsv1+7cuXPU5957722u4aBKbr755ii2devWJru/U089NSk2b968KBaez15//fXqDQz1+sxnPhPFjjjiiChmrRFqQVgZ1kpUt85nJOkBAAAANYYFMgAAAOBhgQwAAAB4amoPslUUZOTIkVFs+vTpuba1p+Wggw6KYmPGjIlin/jEJ3LtBQsWRH369u0bxVJs2bIlii1btiyKWUUtQvPnz49i7EFuGGtfXbVcfvnlUWzIkCEVb/fyyy8nxbB9u+KKK3Jtay5zPqhtf/7zn6OYVaSjmlasWJFrr1+/PurTv3//KLbPPvtEsVdeeSXXbteuXSNHB0tYFOjuu++O+syaNSuKXXfddU02psY45ZRTWnoIhfENMgAAAOBhgQwAAAB4WCADAAAAHhbIAAAAgKemkvSeeuqppFhowoQJScfv2rVrFAuT+axEl9GjRycdP7Rp06YoNmPGjChWV1eXa4cX1pak2bNnFxoDmsbJJ5+ca//oRz+K+nTo0CGKLV26NNe+6qqroj4bN25s5OjQmlnFkQ499NBc2zqP1MKF9fE3n/70p3PtsNiLZBcFKVoo5Pbbb49iTzzxRK69Zs2aqM8xxxwTxa6++uqK9/cP//APUey2226reDtsW/jcd+nSJerzuc99LopZCZjNzVq7hO+DpiyEU218gwwAAAB4WCADAAAAHhbIAAAAgIcFMgAAAOCpqSS9prZq1aoo9vTTT1e8XUqiYKrTTjstioXJg1OnTo363HPPPVUbAxovTJqyEvIs9957b6797LPPVm1MaBvCpBaLVZETLcdKrAzP2XvuuWfh44eVEx944IGoz7XXXhvFUhJ+raqMF1xwQRTr3r17rn3jjTdGfTp27BjFbrnlllx78+bNFce0vTj99NOj2NixY3PtmTNnRn1effXVJhtTY1jJnWFS3jPPPBP1Wb16dVMNqVH4BhkAAADwsEAGAAAAPCyQAQAAAA8LZAAAAMCzXSXpNbcePXpEsV/96ldRbIcd8v+fYlVlW7lyZfUGhgZ56KGHotgJJ5xQ8XZ33XVXFPvHf/zHqowJbdeIESMq9rESpNBy2rdvH8WKJuVZibtnnnlmrr18+fJCx7ZYSXo/+9nPotgvf/nLXLtz585RH2tePvzww7n2rFmzGjrENmv8+PFRLHxea7U6oZWY+uUvfzmKffTRR7n2T3/606hPrSZu8g0yAAAA4GGBDAAAAHhYIAMAAAAe9iA3oYsuuiiKhRdbl+ICJm+//XaTjQnb1qtXryh25JFHRrGddtop17b2BP7kJz+JYuvXr2/E6NDWHHHEEVHsq1/9ahSbOHFirv3kk0822ZjQfF577bUodv7550exau45ThHuG5bi/aWjR49uruG0CbvttlsU+8QnPlHxdlbeUi2wislY++7r6upy7ZTibLWCb5ABAAAADwtkAAAAwMMCGQAAAPCwQAYAAAA8JOlVyVFHHRXFrrzyyqTbfuELX8i133zzzaqMCQ33wAMPRLE99tij4u3+8Ic/RDEuiI9Kjj322CjWrVu3KDZhwoRce9OmTU02JlRHWADKcvjhhzfDSBrOORfFwseT8vgk6dprr821zz777OIDa8XCxG5J6tOnTxS75557mmM4jTZw4MCkfq15PcM3yAAAAICHBTIAAADgYYEMAAAAeFggAwAAAB6S9Kpk7NixUax9+/ZR7KmnnopiL774YpOMCdt2yimnRLGDDz446bbPPPNMrn3NNddUY0jYzowcOTKKZVkWxe6///7mGA4KuvDCC6PY1q1bW2Ak1TFu3LgodtBBB+Xa1uOzYpwbS9atWxfFJk2aFMVGjBiRa1tJuytXrqzewBL06NEjip1++ulJt33hhReqPZxmwzfIAAAAgIcFMgAAAOBhgQwAAAB42INcUKdOnXLtE088Merz4YcfRjFrP9bmzZurNzDUKyz48b3vfS/qY+0bt4R7x9avX198YNhu9OzZM9c++uijoz5vv/12FPuv//qvJhsTGs/as1uLunfvHsWGDRsWxaxzY4ply5ZFMT7fSt5///0oZhWTOu2003Ltxx57LOrzy1/+smrjGj58eBQLi4D0798/6mPlSlhS+9UivkEGAAAAPCyQAQAAAA8LZAAAAMDDAhkAAADwkKRX0OWXX55rhxdRl6QJEyZEsb/+9a9NNiZs27e//e1ce/To0Um3e+ihh6IYF79HEeedd16ubV2A//HHH2+m0WB7c/XVV0exiy66qNCx5s6dG8XC+S1J8+fPL3T87cEPf/jDKOacy7VPOumkqM/dd99dtTEsX748ioWJdXvuuWfh4//2t78tfNuWxjfIAAAAgIcFMgAAAOBhgQwAAAB4WCADAAAAHpL0Elib5L///e/n2mvXro36/PjHP26yMaHhLrvsskK3++Y3vxnFqJyHIqyKVKFVq1Y1w0iwPfjzn/+ca++3335VO3ZdXV0Ue/7556t2/O2B9Rx+6UtfyrWtCwCEle4a4/7776/Y53e/+10U+/KXv5x0fKuCYGvBN8gAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAekvQCe+yxRxT7l3/5lyjWrl27XDtMhpCkF198sXoDQ4vp1q1bFNu8eXNVjr1mzZqkY7dv3z6K7bbbbhWP37Vr1yh26aWXJo4u76OPPopi3/3ud3PtjRs3Fjr29mLcuHEV+zz66KPNMBJUU1j9TJJ22KHy90+f+9znko5/xx135Nq9evVKul04hq1btybdLsXJJ59ctWOhfhMnTkyKNaXZs2cXvu2IESNy7alTpzZ2OM2Gb5ABAAAADwtkAAAAwMMCGQAAAPBs93uQw73EEyZMiPrss88+UWzWrFm5dlg4BG3HlClTmuzY9913XxRbvHhxFNtrr72i2BlnnNEkY2qIJUuW5No//elPW2gktefoo4+OYtbriNbvtttui2I33nhjxdtZ+81T9gkX3UvcmD3It99+e+HbonWz9thbMUtr2nMc4htkAAAAwMMCGQAAAPCwQAYAAAA8LJABAAAAz3afpDdw4MBc+5BDDkm63WWXXZZrh0l7qD1hMZfPf/7zLTSSvxk/fnzVjrVly5YolpKU8/DDD0ex1157Lek+X3jhhaR+26MvfOELUSxMCrYu+P/ss8822ZjQNB588MEodvnll+fa3bt3b67h1GvZsmVRrK6uLopdcMEFUcxKHsb2IcuypFhbwzfIAAAAgIcFMgAAAOBhgQwAAAB4WCADAAAAnu0qSa9///5R7Iknnqh4uzDZQrIrIKG2nXrqqbn2FVdcEfVp3759oWMfcMABUaxopbvf/OY3UWzu3LkVb2clClkJOKi+zp07R7GxY8dWvN39998fxT766KOqjAnNZ968eVHszDPPzLWtpM1vfetbTTYmi1Xp8tZbb23WMaD16dixY1K/TZs2NfFImhffIAMAAAAeFsgAAACAhwUyAAAA4HHbutizc65NXQna2n911VVXVbzdYYcdFsVSCym0JVmWuZR+bW3eoHG2h3lj7V23Cn4sXbo01z7rrLOiPhs3bqzewFqx7WHenHjiiVHMKtIxbty4XNsq7vPrX/86ijmXfwqnT58e9Zk/f37FcbYm28O8aW5LliyJYjvuGKew/fjHP45i//zP/9wkY6o2a97wDTIAAADgYYEMAAAAeFggAwAAAB4WyAAAAICnzSbpHX300VHssccei2I777xzxWORpFdC8gOKYN6gCOYNimDeVN8jjzwSxW6++eYo9vTTTzfHcJoESXoAAABABSyQAQAAAA8LZAAAAMDDAhkAAADwxKVQ2ohPfvKTUSwlIW/WrFlRbP369VUZEwAAQGsSVnLcXvANMgAAAOBhgQwAAAB4WCADAAAAHhbIAAAAgKfNJumlmjx5cq597LHHRn1WrlzZXMMBAABAC+MbZAAAAMDDAhkAAADwsEAGAAAAPC7Lsvp/6Vz9v8R2J8syl9KPeQMf8wZFMG9QBPMGRVjzhm+QAQAAAA8LZAAAAMDDAhkAAADwsEAGAAAAPNtM0gMAAAC2N3yDDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHhYINcg59wY59y7LT0OtC7MGxTBvEERzBsU0ZrmTZtcIDvn7nTO/WR7vX8U09KvW0vfP4pp6detpe8fxbT069bS949iWvp1a+n7b05tcoHcWM65HVt6DGh9mDcognmDIpg3KIJ50wBZltXkj6RM0iCvfaekn5T/PUbSu5K+LWmppMWSvlr+3QWSNkv6UNJ6SY+U41dKmiVpnaTpkr7oHfs8Sf8n6WZJKyT9RNIekh6RtFbSq+XYC95thkp6UtJKSW9L+lKF++8t6QFJyyTNkXSJd6xO5ce3qjy2yyW929KvQWv8Yd4wb5g3zBvmDfOmln+YN61j3rT4ABoxgbZI+pGk9pLGStooqWvY17v9+PKLuIOkMyRtkNTLm0BbJF0sacfyC3pP+aezpGGSFnw8gSR1Kbe/Wu5/kKTlkoZZ91++z9cl/UBSB0n7Spot6bPl318v6XlJ3ST1lfRma5lAtfbDvGHeMG+YN8wb5k0t/zBvWse8ac1bLDZL+lGWZZuzLPuzSv83s199nbMsuy/LskVZlm3NsuxeSe9IOszrsijLsn/NsmyLSv93dJqka7Is25hl2XRJv/P6nixpbpZlv82ybEuWZRNV+r+n8fXc/WhJ3bMs+1GWZR9mWTZb0h2Sziz//kuSfppl2cosyxZI+pcGPhdIx7xBEcwbFMG8QRHMmxrQmveirCi/2B/bKGnn+jo7574i6TJJA8qhnSXt6XVZ4P27u0rPzYJ6ft9f0uHOudVebEdJv6/n7vtL6h30b6fS/1VJpf/z848/r77HgUZj3qAI5g2KYN6gCOZNDajlBfJGlb7+/1hPlfblpMj8hnOuv0r/R3OspBezLPvIOTdJkqvnNstU+pPE3pJmlGN9vd8vkPRslmXHp9x/uf+cLMsG19N/cfn408rtfvX0Q2XMGxTBvEERzBsUwbxpBWp5i8UkSWc559o5506U9OkG3PY9lfbBfKyLSi/qMklyzn1V0vD6bpxl2UeSHpT0Q+dcZ+fcUElf8bo8KmmIc+4c51z78s9o59z+9dz/K5LWOee+65zrVH5Mw51zo8u//5Okq5xzXZ1ze6u0VwjFMG9QBPMGRTBvUATzphWo5QXytySNk7Ra0pclPdSA2/6HpGHOudXOuYfKe2x+IelFlV7cESpldW7LNyXtJmmJSn9auFvSB5KUZdk6SSeotMdmUbnPDZJ2quf+P1JpX88olTI8l0v69/LxJelalf7sMEfSE6r/TxmojHmDIpg3KIJ5gyKYN62Ay7Lw23JYnHM3SOqZZdm5LT0WtB7MGxTBvEERzBsUwbyx1fI3yC3KOTfUOXegKzlM0t9J+q+WHhdqG/MGRTBvUATzBkUwb9LUcpJeS9tFpT879Fbpzxa/kPTfLToitAbMGxTBvEERzBsUwbxJwBYLAAAAwMMWCwAAAMDDAjmRc26Mcy71OoWAJOYNGo45gyKYNyiCeVM/FsgAAACAhwUyAAAA4NmuF8jOuX92zi1wzq11zr3unDva+10n59ydzrlVzrnpkkZ7v/uuc+5+41j/0ozDRwth3qChmDMognmDIpg31bFdL5AlvapS9Zdukv4o6T7nXMfy766RNLD881lJ/gW075E01jm3iyQ559pJ+lL5GGj7mDdoKOYMimDeoAjmTTVkWcZP+UfSKkkjy/+eLelE73cXSHrXa78g6Svlfx8vaVZLj58f5g0/reOHOcNPkR/mDT9Ffpg3xX6262+QnXPfcc7VOefWOOdWq1Q7fM/yr3tLWuB1nxfc/I+S/r/yv8/S9vp/WNsh5g0aijmDIpg3KIJ5Ux3b7QK5vCfnCpX+fNA1y7LdJa2R5MpdFkvq692kX3CI+ySNcc7tLemL2o4n0faEeYOGYs6gCOYNimDeVM92u0BWqdTiFknLJO3onPuBpF293/9J0lXOua7liXKxf+Msy5ZJekbSbyXNybKsrllGjZbGvEFDMWdQBPMGRTBvqmR7XiD/RdIESTNU+hPDJuX/7HBtOT5H0hOSfm8c44+SjtN2/H9Y2yHmDRqKOYMimDcognlTJa68ERsAAACAtu9vkAEAAIAIC2QAAADAwwIZAAAA8LBABgAAADwskAtyzmXOuUEtPQ60LswbFMG8QRHMGxTBvClhgdwMnHPTnHPrvZ8tzrlHWnpcqG3OuZ87595xzq1zzr3lnPtKS48Jtc859yXn3F+dcxudc8+09HjQOjjndnLO/cY5t9Y5t8Q5d1lLjwmth3Oum3NumXPuhZYeS7Xs2NIDqAXOuR2zLNvSVMfPsuwA776cSrXQ72uq+0PzaOp5I2mDpHEqXc9ytKQJzk3a8KkAACAASURBVLmZWZb9tQnvE02sGebNSkn/JGmopGOa8H7QjJph3vxQ0mBJ/SX1lPS/zrnpWZZNaML7RBNrhnnzsRsk1akNffHaZh5IqPwngkucc7Odc8udczc553Yo/+4859z/Oeduds6tkPTD8v89/9w5N985955z7nbnXCfveJc75xY75xY5585vxNA+pVJN9Aca9wjRFGpp3mRZdk2WZW9lWbY1y7KXJT0v6YiqPmBURY3Nm//JsuxPkhZV91Gi2mpp3kg6V9KPsyxbVa6edoek86r1WFE9NTZv5Jw7UtJwlarvtRltdoFc9kVJh0o6WNLnJfkv/OEqfZO7l6SfSrpe0hBJoyQNktRH0g8kyTl3oqTvSDpepf/DPs6/E+fcWc65KYljOlfSA1mWbSj2kNAMam7elE9moyVNK/qg0ORqbt6gVWjxeeOc6yqpl6TJXniypAOs/qgJLT5vyr9vJ+kWSd+U1LYqz2VZ1iZ/VHqhTvTa35D0VPnf50ma7/3OqfTn7IFe7AiV6pBL0m8kXe/9bkj5+IMaOKbOktZKGtPSzw8/rWfelG/7O5XKh7qWfo74aR3zRtLXJD3T0s8NP7U/byT1Lfft6MWOlzS3pZ8jfmp33pT7XyrpNu++X2jp56daP219D7Jff3yepN71/K67SovX10tbhCWVJlW78r97S3o9OFYRp6q0P/DZgrdH86ipeeOcu0mlP199JiufhVCTamreoNWohXmzvvzfXSVt8v69rgHHQPNq8XnjnOst6RJJh6TepjVp6wvkvvrbn6T7Kb8nz19oLJf0vqQDsixbaBxncflYH+tXcDznSrqLRU7Nq5l545y7VtLnJH06y7K1Db09mlXNzBu0Ki0+b7IsW+WcWyxppKQny+GRYktXLWvxeSPpMJW25kwvL747SerknFsiqU+WZR814Fg1p63vQb7cOdfVOddX0rck3Wt1yrJsq0oJCTc753pIknOuj3Pus+Uuf5J0nnNumHOus6RrGjoQ59zekj6j0p/KUdtqYt44566SdJak47IsW1HwsaD51Mq8aeec66jSFyA7OOc6OufaF3xMaHo1MW8k3SXpH8tjGSrp65LubPjDQTOphXnzuKQBKu1tHqXSvuaJkka19sWx1PYXyP+t0p8OJkl6TNJ/bKPvdyXNlPSSc26tpP+RtJ8kZVn2uEqXTXq63Odp/4bOuS875yr9n/Y5kl7MsmxWgceB5lUr8+Y6lf5vfqb72zW0v1fsIaEZ1Mq8OUelb4xuk3R0+d93FHg8aB61Mm+ukTRLpT+xPyvppoxLvNWyFp83WZZ9kGXZko9/JK2RtLn871bPtdW/9jvnMkmDsyyb2dJjQevBvEERzBsUwbxBEcyb5tHWv0EGAAAAGoQFMgAAAOBps1ssAAAAgCL4BhkAAADwsEAGAAAAPNssFFLOlMzp2rVrrr1q1arCdz5o0KBce/PmzVGfLl26RLHp06dXPHa3bt2i2MqVK6PY4MGDc+2ZM+Ok0GHDhkWxxYsXVzy2ZejQoVHsrbfeyrXbtWsX9Rk5cmQUe+ONN5Lus1qyLHOVe0l9+vSJ5s2iRYusrlXRq1evKGY9h++++26ufcghacV/Xn/99cqdalTKY5w0aVIUs+b81KlTKx4rfD9J0owZM5LmzYABA6J507lz51y7rq4u5VDaYYf4//23bt2aa++9995Rn9133z2KzZgxI9f+8MMPk8ZgOfDAA3PtFSviy1svXBhfyz+83ZQpU6I+1muWcq7s1y+uCzB//vyKt0s9VocOHaJYeJ7dd999oz6zZs1Kmje9evWK5s2SJdW7ylQ4l6xtifWMv+KxO3bsGMU2bdoUxcJ5uc8++0R9Jk6cGMV22223XHvt2rjW0B577BHFPvjggyi2bl3lonojRoyIYinnDUs4dileD1iPZ926dUnzZuDAgdELucsuu+TakydPTjmUuU7p06dPrh2eR1KFY5Ls12LAgAFRbMGCBbn2Rx/Flya23rMbNmzIta3zYnhuloq/1tb7YM8998y1w89vKf25SWGtb/gGGQAAAPCwQAYAAAA8LJABAAAAzzYv82btQU7dt5ki3A8zb968qE///v2jWLi/1Np7ZQn3IFpSH9/777+fa6fs9ZPsfY/W3ppqsfYlzZ07N4qFe8esvUSpe5B33nnnaN6Ee5oao+gcDPcSW8ephf3GRcdVreelPilzt3fv3lGfhQsXJs2bnj17VtyDPGfOnJRDmfsqw73DRfeqpd6ftb84ZO2VTnHQQQdFsaJz1xq7dd4IWfkns2fPLjQGS+r5xvqcqkXV/Oys5nnK2j8d5hpJ8efzTjvtFPUJc3Ms1ufwwQcfHMWmTYurG/fs2TPXtubgmjVrCs+b8H3lXHwoK2ejffv2UWz48OEpw6go9bVOmV+px+rRo0eube2x3nnnnZOOFbJyEqo5n1OeB+s8tXLlSvYgAwAAANvCAhkAAADwsEAGAAAAPNu8DnJR1dxPsmbNmqodyzJq1KhCtwv3HHfv3j3qY+1LKrrf2BpnuBfKulahtd/YsnHjxkLjsljX8SzKup5xipQ5WAv7jVOl7Eu2Hk/KfixrH6x1bcyUudSY612/9957hW5nXRPW2gtpXds1FF5709K3b98oZuVBWPuxi16fN7xmvHU9U4t13givHWvtN01h7eOz9rNa+xeLXi81Vbg/NuW1b4xa3V8cXid2yJAhhY8VfkZYe5BTWHlF1jX9rc+zcB+0tX8+VbjPVorzEqy6CJai86tWP4PC82Bj3j9F3+vhZ791ziv6vkut38E3yAAAAICHBTIAAADgYYEMAAAAeFggAwAAAJ5GJ+mlJgZZ/cIEkilTpkR9rAvihwkx1oXHU5PviiaBhRcBf/PNN6M+VhJDWPhAipMFrGQo6+LkISthYfXq1VHMSjIKC580RkoCkVV0wkqQSnnc1Ux0GDNmTBTr1KlTFBs5cmSu/bOf/Szp+I8//niu/eyzz0Z9brjhhqRjhYomLFjvH+sxDx06NIqFiTvz588vNIb6hOOw5umWLVuiWEqiqFWcwEoWCi1fvrxiHyktYdFKkLSSDsPk3l69eiWNwTq/FU3Ks87PIav4ipVsFbISmlNZxRyaMikv9fMt7Dd58uSojzV3U4QFMyRp6dKlUSwlKc86X8+YMSOKDRw4MNdOKQpijSG1mIz12RVqTJKe9XyFMWv9YZ2DrDkYKrpWakwCaNHPxpTCZ2FxMam6ybdFL6CQIkx6rg/fIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHi2maRnbcKuZoWylKQPq0JVKDXppGhCnjWGjh07VrzdbrvtlnS7FStW5No77hi/LFYyR7jR3EoeCispSdKCBQuiWGMqLKXYb7/9cu2333476mM97g4dOkSxaiXlXXHFFVHs9NNPT7rt6NGjc22r4qP1+h911FG5djUrGBZlPcd1dXVJt7WqURVlJR6FlefCeSTZcylMKJLiBDIrIc86l6RUXTrwwAOjmJX8NG3atFzbSsiz3rNhMp+V0Golgd1yyy1R7L777otiIevcvHnz5oq3W7ZsWVIsTHK1xp4qy7IoFr73rPfnsGHDotg777wTxazXNpSSSFU0IU+KX++vf/3rUZ9LLrkkinXr1i3XHj9+fNQnrE4n2eel8DyxcuVKe7CBMHH3ww8/TLqdlSgcJndbyYSNEb7PrMqX1vnSeg5T1GolvfBiAtZ8qGZCnrXWDOeJ9bxbrPkVjjXlPS3xDTIAAACQwwIZAAAA8LBABgAAADwskAEAAACPsxIcPrb//vtHv+zSpUuu3ZhKeiGrqpRVYSklWcRibcwumrhXlJW4U3QMKRV3qpkEkGVZ5XJBkvr37x/Nm2pXWCsiTMpLTcizvPXWW7l2//79oz777rtvFEupPmUlUZ522mkVb1e04pJVGcqqYpVSGc6SOm+cc9G8SUm2StWvX7/w/qI+4flNiqtKNaayVQrruZ81a1au/cUvfjHq8/Of/zyKXXzxxVHsvffeqziGlPOGVSnUqmyWMm+sc+CWLVsKz5sU1ueNVVUyVLSSnsVK7rLOJWFy2kMPPVTx2FI8d2+99daoz+233550rDDhL7WiZFEpz5/1vDfmcyp871mJ4wMGDIhiViJ/WKmzmmsli7WWS6nwV1Tq2MNzwtFHHx31Cc9vUpxYZ91fasJnuI60ErStecM3yAAAAICHBTIAAADgYYEMAAAAeLZZKMRSzUIhIWv/V9H9xtaxqrnfuJp7e0eNGpVrT5o0qdBxauWi49a+rZRCIRaraEJY8CHVDTfckGu/+uqrUZ+wkIMkPfroo1Es3H+3fv36qI91YfPzzz8/1w4LJkjS/fffH8UsRffChvMknH+S/V6x+hWdqxarsEpj9hyHwiIgVj6AtVe+aBGdlPej9Zxa+/FC1uvz7rvvRjFrv3G1zhNWfkjqxfxDVpGYxgj3R/fu3TvqY+3P3H333aNYuF+xmudZa79x9+7do9i9995b8Vgvv/xyFDv88MNz7dTiDtbztWnTplzbKsaTMnctTb2v3xLuqZbiOW09Hqvgi7X/tynXSldeeWUUe+yxxwodyxK+Hql7fS2nnHJKrm0VObLybkKpz1U193XzDTIAAADgYYEMAAAAeFggAwAAAB4WyAAAAIBnm0l6ViJIuAG6MRe/7ty5c659xhlnRH1uuummKBZeeP43v/lN1GfFihVRzNpo/uCDD+baRZMCGyMl0amaG89TkrIak4iycePGKJaSbNWnT58oZiWspCRbjhs3LoqFSSxhAosk/ehHP4piKcU9rOfrhBNOiGLDhg2reKxqJltYUt7DFiuRYq+99sq1165dW3hcVnJnUVaSUZgYahUFsW4XFoVpTKGIsF/RxGGrAEjHjh2TxlVU+His93mYyCVJnTp1imKDBw/OtadMmdLI0eV98MEHufacOXOiPmHxDckuIBCq5rn4kksuiWLWnAiTDq3PqaFDh1a8vyOOOCKKWeepsMCIJD355JO5dmpCXksk4KUommBsFZix9OrVK9e2PkeGDx8excLzrPXZaSW1nnTSSUnjCo0YMSKKpRQhS03ce+2113Ltyy67LOpjrfOKssYVjj+1cBnfIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHicVQHm//+lc/X/sgrCpJIwCUCSPvnJT0axsALWO++8k3R/VhLDr371q1zbqsBXTVYCRvh4rIpYYRU4qXiyVUrSRJgIWY7FpacMRedNaqW2lPGnPBfHH398FFu5cmXF26Xen5U8GiZljB49OupjVZSzkt+qVUkvTACS4iQnSdp///2jWF1dXa594IEHRn0mT57cpPOmmqzqWkXnRDXts88+ufaf/vSnqM+YMWOi2IYNGwrdX8rcWrhwYRSzqutZsTCZzzrfZFnW7PPm4IMPto5frcNHbrvttigWJr5KUr9+/XLtmTNnRn0GDRpU8f7ef//9KGZVP1yyZEkUO/LII3Nt63Mq5fOzMUl74bkrfF9I0uzZs5NesEMOOSSaN2HiYWolTyu515rTobC6pxQ/h9aawUqQ/NrXvhbFHnjggYpjsFiJe6HUqoyf/vSnc+1nnnkm6lP0PWaNM6WaZ/i5JUkbNmyIBsE3yAAAAICHBTIAAADgYYEMAAAAeLZ5ZX7rQurh/qjGFAoJ9wT/+te/jvrMmDEjil155ZW5tlV84aCDDopixxxzTBT7xCc+kWtbe9CKeumll6LYsmXLophV1CJ04oknRrHPfOYzFW9XdL+XtQ821cCBA6NYWAQiLNogpV+4vVrFD6w970VZr8WQIUMq3u6WW26JYkX3Gxd9Xqz9xtbePqv4Tv/+/XPtahd8CFnjsgpWWBfSnzx5cq5t5SSERUGk+L1g7fVszHkwxV/+8pdcOyy0Ud/9PffccxWPXfQcYe1THTlyZBQLn/dqGzBgQBQL939axR169OgRxayCIqtWrcq1iz5ff/7zn5PGYAk/B1POLRZrr2dKcRRJuv/++3Ptww47LOqTMufDnBspvWBO+Dpar1eqadOmRTHrXBiyCvJY+43D58LaNx7OLSleD1x//fVRH2v///PPPx8PtsKYUqUWBbGcddZZufYrr7wS9bHWNxMmTKh47JT9xhareJGFb5ABAAAADwtkAAAAwMMCGQAAAPCwQAYAAAA8LVooJEyS6dKlS9SnaFKLtRl94sSJUSxMwLGKVVjFHKyEwpB1UXYrCWBbr8HHwmRCSXr55Zdz7cZcgD1kJRSsXr268IX7w2IBmzdvThrHrrvuGsWsJLYURQurWK655ppc2yo6ctRRR0Wxxx57LNc+44wzoj5W8pilWsmKlrCgiSS9+eabUSxMmttll12iPo0pFBImAc6bNy+6XZ8+faKYlbhnJcSEdt555yi2fv36irdramEykpWYlnqx/WqdJ5py/knphUK6du0azZswmcv6bLESj6wErD333DNlGJGwQMIvfvGLqM+rr74axazPm7CwijXOF198MYqdfvrpufbq1aujPscee2wUu/rqq6NY6JJLLoliVgJW0fOulViZUoikJQrMWIom915++eW59qmnnhr1+d3vfhfFbr/99oYOUVJ1k4ktr732WsX7Szl3Nea8Fb7XrSIn1rzhG2QAAADAwwIZAAAA8LBABgAAADwskAEAAADPNivpWRXqwgokjakgFVatsjblpyTbWVV4rIQ8a4N/mARkJYDdeOONFcdlPT7r8Zx33nlRLIWVzNGU1qxZU/i2I0aMiGLWpviQVR3KSnQMX6NqJhmkJgIceuihuXZYYVKyq7KFSWBWQp41d63KkE0prHwoSQcccEAUs6pRVVPXrl1zbStJz6oqZb33UlivY5iklzrfUvr169cv6mMlTYVJedb7oqmTbVJYleGWLl0axcJkznXr1hW+TythLbxPq3pb6jkunHPW82wlTYZJeQsWLIj6hJ8/Upx8bY1hjz32iPocd9xxUSwcq1XxLUygluzPwTBh2vpcfPfdd6PY2WefHcVCqe/X8PGkfK7Ux0ru7datW65tzZHOnTtHMes5DKu8We9FqzLo2LFjc20rQf+II46IYpbu3bvn2tb5pprniHPOOSeKha+Z9Zxaj9GqRFxU+FpYaxQL3yADAAAAHhbIAAAAgIcFMgAAAOBhgQwAAAB4WrSSXqhoYo0lpeKOFFfOmzRpUtXGYD0eK1kkdMwxx0Sx//3f/614u8ZUmgk36ltJJ3PmzKlahSIrOSGsFiZJgwcPjmJWdb2m9NBDD0WxuXPn5tph8qok3XXXXVEsrKRnmTVrVvrgCiha2ap3795RzEr6CaVWturYsWM0b6zKk8btophVmSlMrlmxYkXKsKKEjsYkBoUGDhwYxf793/89io0ZMybXtip+Tp48Oek+i54nmjvhL3XeDBo0KJo3Ke8hq1qklSyYUknPOk/dfffdubZ1frPm98MPPxzFvvvd7+baqa9h0dfMqvoXVgZNrXRnJWCFrGOlJCZbn6cTJ05Mmjd9+/aN5k2YPFpXVxfdbqeddopiKecpyxVXXBHFbrjhhlw7pYpuY4QV/6TiSfopVYGtx3zTTTdVvF011zfW58PWrVuppAcAAABsCwtkAAAAwMMCGQAAAPBss1BIU0vZc5yyl7gxF+4PWRf83rx5cxRL2bt8yimnRLE33ngjiq1evTrXTtlvbI0hZX9z6rHC4giNFe7tWrlyZdRn0KBBUeydd96JYo3Zi1RJr169otjee+9dMfb4449HfcJ9yhZrr2RTPj5LNfczNmbs1j6+8GL+VlGQ3XffPYotWbIkilnFNUIHHnhgFJsyZUquXc1CIfPnz4/6WOebUOp+Y0u19hKHRW8ke0+1NdawIIO1rzOV9R4Kcyis96I1H6xCNCl7kFNYeRfWOdsqkJDyvqrmHvFvf/vbUezWW2/NtUePHl3o2NZnrPW+S9GYnCGrqEmK1P3GKbke1v7iV155JdfesmVLA0a37TFYUs5dVp/TTjstioVjl6TDDjss107Zb2yp5vxO2Sst8Q0yAAAAkMMCGQAAAPCwQAYAAAA8LJABAAAAT7Ml6VmbxSdOnJhrWxcGT9mYbSU6pCbSVMtFF10Uxc4555yk244bN67QfbZr1y7XbkySXpiI1L9//8LHsm7bpUuXXHv69OlRn6IXJ08Vvv7WHHnggQeSjvXqq6/m2laSkZU8FMasRFVrnlrJTynFEJr7fdAYffv2jWILFiyoeDsrIS9F+P6R4veBJO2///65durzl9Lv2GOPjWJhAptkJ02lqObrn5K4YyW/DB06NIq99dZbhcZgsQoHffjhhxVvl5IM2RjhOcJKajv88MMLHTv87EzVmPkQnqusxxM+Zkm69NJLc+3nnnsu6f5SNCaBMiwAJMXzN0ygl6TFixdHMeuzN+V5DZOQJemee+5p8HGk4gnSReeE9Zlkufzyyxs8pmoLH2NqoiXfIAMAAAAeFsgAAACAhwUyAAAA4GGBDAAAAHganKR38MEH59pWZbg99tgjiqVs+rY2xFtVsjZs2JBrW8k2RStbpW5YD5N5rCpwVsWgp556Kopt3LgxiqUommwzePDgKBZWLJwzZ06hY0t2NSorATPUvXv3KJZSRajo82Ald7322mtR7Mgjj4xiQ4YMybU///nPR306duxYcQxWpciildpaIvmumlX/UhLyrGRIK7lv5syZFY+VmtRaV1eXa1cz8S2sYCnZiW5W4l4Ka1xFk3tXrVqVa++7775RHyvJ0RImPs6YMSPpdpa1a9cmxULWZ4tV2TBkPafh+UBKqwJbdN5Yny0p51iLNZ8vvvjiKBZWRLNYn8VFK9ZZwuREq5pjKutzKpw3zrmoT2oVtpTzs1UJ0EoerJZqfkacfvrpSf2GDRtWtfsMpSYAh4879SIEfIMMAAAAeFggAwAAAB4WyAAAAIBnm3uQd9tttygW7jm29tBZ+2r222+/KPb222/n2lbhg1122SWKDR8+PNeu5oX7i+7ROfHEE6OYdbH6uXPnFjq+pege1B13rLz1vDEXYO/du3cUS9kTumzZsqTjV2sf1fe+970oZu03toRzfP369VGfouOshflsqeZ+46KsC7ynFIWwWPsXrbkb7o+t5nN69NFHR7FDDz00illFGYoqWlDI2o8Z6tWrVxSzCiuE56DGFDnq2bNnFHvvvfdybWvfqFWkx9ovmfJ6W8WewiIgL7/8csXj1Kda7z0rz8N6zNb5f9OmTRWPH+YoSWl5JNZ7eNq0aVEs3Neduh/YYu2hX7p0aa69aNGiqI+VD7By5cooljJvrDXPaaedlmvfcsstUZ9f/vKXUWz8+PEV7y/MI5Di9ZQUFwGx9uxaz721T93az14tRQsOWfvPLXyDDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4NlmttaAAQOi2OTJk3NtayO9lTSxYsWKKBZe7LxDhw5RH2sjfXix8NQL96cmFKYIk/JGjhwZ9enXr18U+9WvflXx2EUTMqyLx2/evDmKLVy4MIodcMABubZVtCWVlXARFnexWMkC1mb68KL8qUlTX/nKV3Ltb3zjG0m3s8YwZsyYXDv1NQv7hXNZsi/4X6uFQpr7Pvv06RPFrMJEVsGHAw88MNd+//33oz4pCX+NKRRy7rnn5tpjx46N+kydOjXpWEWF77M333wz6XZhwQerEI6V2G0VcgkLEVnv/VRhYpUUFyKxinZYCVjWsYoKk/LCpL36NGUy7NVXXx3FrMJRKcW+7rvvvqhPOL+ltEIRRed8ly5dCt1Osl//zp0759pWQruVkGcVnUn5DLXG8OCDD+ba119/fdTnoosuimKvvPJKxfs7++yzo5h1Tn3ppZdybSv50lqvWRqTnOpLudBDqtSx8w0yAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAZ5tJelaVl7CKjLVhPdzoLkmzZ8+OYmFSmZXIlVKFJzVBpmhCnpV0+IMf/CDXtqpfJW8EDyo6TZkyJeoTJhhZrIS8QYMGRTErgclKtizKSshMqXhkzTcrkapoYtiFF15Ysc+rr74axazqQCljKprMVc1KekUVTTqs9pjChMyddtop6mMlOlpJRmGlRishy6roFibgNuYxHnPMMRX73HDDDYWPH7Jex6IJuGECnpWkZ51brKTTsCrb8uXLC41JspPAwsTDsDKYlJY43BjVrKSXwpqXYUXBZ599NupjVZTca6+9Kt5fXV1dFEtJyKvmOaJdu3aFb2tVCwzPCdbxrTVJ0Wqef/jDH6JY+J61Kljus88+he7vggsuiGL3339/FAtfo+effz7q88lPfrLQGFKlfLZYlU+XLFkSxcJzlXXusvANMgAAAOBhgQwAAAB4WCADAAAAHhbIAAAAgGebSXrr168vdNB169ZFse7du0exMGnGYiWPhRuze/XqFfUpmghwwgknRLG//OUvFW83Y8aMKGYl/FisJJamtPfee0exMInSqmCYKiUhz6o8GFZprE9TJoZ17dq1asdqicp2Kf7u7/4u17YSck466aQodscdd0SxO++8s2rjshInrPdVyErcsxJFQ2HCsSStXbs2ioXzsjGV9KyKWyEr8aSoonPQOl+HrETINWvWRDEriTLlWKmsCqKhMBFaSk/SSznfWIlUYcKvNYYePXpEsQULFkSx8L13yimn2IOtYPz48VHMSky2YmHSoVXNrblZ7/1U06dPj2KdOnXKta2kQ2uOWxchSJk3KeeS1PNNSmJ1SkKedazf/va3UZ/UJL2wgm01LwhgJUxaVTPDNaJ1QQAL3yADAAAAHhbIAAAAgIcFMgAAAODZ5h7kFNb+WWvPWXiReYu1j8vaz7po0aJtthtjwoQJhW73/e9/P4pZe2GqKdw7ZBUTsYqO9OnTJ4qFt7VuV01z585N6jd06NAo1pR7e63CKq+88kqT3d99990XxRYvXhzFrH3CZ5xxRqH7HD16dVNzpwAACqhJREFUdK49b968qI9VROMb3/hGFAvHmrJfvz5h8QiLtTfe2v9n5SWEY7WKHKVInX/hfkap+N7RolL2L44YMSLqM3Xq1Ci266675tphEQrJ3p+ZImXPeH2sAkkHH3xwrv3GG29U7CPZhU5SXu/bbrstit14440Vb/foo49GMetzI9z/a7H2VYb7rK3PiNTPqRdffDGpX6ia5+twPle72MuAAQNybWsts//++xc6dlMXjkqReqzwcVt5SxZr73rRPccpY009h4fF66xidha+QQYAAAA8LJABAAAADwtkAAAAwMMCGQAAAPC4bRV1cM5FvwwT6VKKQkj2Bb2LbnZPMWnSpChmbZIfMmRIrv3HP/4x6fhXX311rn3dddcl3c5KiGnXrl3SbSuxHnOXLl2imJXYECanWYlbH374YZxFabDmTZg0ZSWiDR48OIpZyWIpiUDWa33iiSfm2j/5yU8qHqfarCSGUJhE19Ss5J7XXnst6bYvvPBCrn3LLbdEfbIsS5o3Q4YMiebNO++8kzSOFMOHD8+1ly9fHvVZsmRJFOvQoUOubb2HrYSSs846K4pdeumlufahhx4a9bGSlUMpRQHqEybN9O/fP+pjJb+ESZRWUqWVMJfCSo595513Cp9vUoTzQbKLY6UmFIceeeSRXPvkk08udJzGWL16da5tJStac/evf/1rFAsTKa3PiI0bN0Yxa35Vi/X5MH/+/KrNG6uwj1XQaOedd45iM2fOTBlGq/Dwww9HsU996lNRzEpMDp8bK1HUEn7Opz6fo0aNimLh2shaF61fvz6aN3yDDAAAAHhYIAMAAAAeFsgAAACAhwUyAAAA4NlmJb2UalSpPvjgg0K3S6nolloJyEoWSEnKGzNmTBR79tlnk+4zFCb8SNJHH31U6Fjhc2MlD6VWsQkrSIUVharNSkxcsGBBFEuprmZJqcJjVYZq3759ofuzEn6KVrqzqjdZiUL/9m//lmtbVeyKVm9qCVaCV1j1ceHChYWP/+abb+baVqJG165do9guu+ySa6c+f2PHjo1iYVLeCSeckHSsxiTlhbp165Zrv/3221Ef63lYt25drh0+L5LUo0ePKLbHHntEsfDclVrZKlVYcdFKtrKSNK1KekU9/fTTuXZYiVCyE53CxDrJHn+K8LMxTC6X7NffqlgZSn2uwue5MXM5fO9Z8zSVVUkxTAq2XjPrcY8cObLwOJpS+FynnrvC21mJiVZC3pw5c6JYv379cu26urqoj5XcWZR1sYLQPvvsk3QsvkEGAAAAPCyQAQAAAA8LZAAAAMCzzT3Ie+65ZxRL2YO8ww7xujtln7C139i6qHTYz7o/i1WsZOrUqbm2tY936NChUez5559Pus+Qtd949uzZuba138za93TAAQfk2u+//37UZ++99046Vrg3upoFGiRpr732yrWteZS6FztlX1XK3tvU/bkp/az9jL/4xS+SjpUidVxFVPN5aAzrYvvhHnprP6s1xydPnlzx/qz3y447xqfE8P2S+jyMHz8+iv3gBz/ItZ988smK47SO35jXPnyM1j5Oa4/eqlWrcu1wb6Fk7yW0iq+EUnJNGiI8p1r7Tau539jy3HPP5do333xz1MfaE2wZN25crm0VbrCew5dffjnXPuqoo6I+L730UtIY3n333aR+lVjvlWHDhkWx6dOnVzxWOCcbwjpHhIXDrP2y1prEOlaYZ7PbbrtFfazCNB9++GE82IJSzs/WuSTM/Tn++OOT7u/KK6+MYuG6q+h+Y2sd9tZbbxU6VpiPUh++QQYAAAA8LJABAAAADwtkAAAAwMMCGQAAAPC4LMvq/eWuu+4a/TK8WHwqKwGnmheHDlkXnn/iiSei2GGHHZZrWwUs/v7v/z6K3XHHHRXHYG3mt4TPw4wZM5Jul8JKagoLBUjS0qVLc+2+fftGfaZOnepS7rNnz57RvAkTD6x5Z10gH61flmVJ88Y5F02KooVC9t9//ygWJtxYiVtWsk3RQj7VVM3kzvCcsGLFikLH7t27dxRbtmxZFLMKwIS3tZK4Fy9enDRvevToEc2bcBwDBw6Mbjdr1qyK45KkRYsWVRxDNQu5FE18tcYQFh0aPHhw1Mf6bG7KcTUmsde5/JSwipktXLgwad707t07mjdh8rhVMMsq2mQlrIZJoFaRsJSEPOtcZs3JNWvWRLHu3btXHGeKMElUsouCPPXUU1EsTO601iRFz0FWgS5r/RT2s5L0Pvjgg2je8A0yAAAA4GGBDAAAAHhYIAMAAAAeFsgAAACAZ5tJelbSTJjEZiW1pVaCGTVqVK5dtJKe1efQQw+NYrfffnvSuEJWda2wel9qQp5l5syZuXaYiCDZyZGDBg3a5nHqk5KcFFYUkqTp06cXTrZKYSUGLliwoMihUENSk/SGDRsWzZstW7bk2lblrDARRbIrYIXv43rmeBQLk5is6lfVVM2EL0uYJGUlplmVLrt06ZJrh+cfSZo0aVKhMVljSE226tOnTzRvwiSm1EptPXv2tI6fMow2JfWzJGTNiTAJ3XoPW4nj1usTVqMLzw+StH79+sKfU2FSXlhRTpI++OCDlMMXrvLXqVOnXNuq+FlULSSTtoTwog1WsmJdXR1JegAAAMC2sEAGAAAAPCyQAQAAAA8LZAAAAMATZ595wg3xUlytxaoy1b9//yi26667RjErKa9IHysh56qrrqp4O0m66667cu2vfe1rUZ8wIU8qnpRnJTUWTfoJEyms53jDhg1R7I033qh4bOs5TWU9xpRqZFZCXmpyTYpqVnRCSTWfU2vOtW/fPte2qrItX7486fhhIpg1j6wKnNVMymvqBLxQyuthJTpZydvh+cVKJi7KqqSXqmPHjhX7WK+1lahjJWQ3Jev1aco5YiXfWZX0rKps4biWLFmSdJ/h+3rvvfeO+livj5VEG1ZITL0gQKowAS/1MVr22muvXPu9996L+qRUlbOSRK1xNWXFz5b4rAznm1V5OXWdEiaPbuviFD6+QQYAAAA8LJABAAAADwtkAAAAwLPNQiE77LBD9MvwYvHW/jxrz8zChQut4ycNshY1pjBIKLy4vrUnzHqewz101nM8cODAKDZ37twolrKXK7XgQ//+/aN5E+4x3GWXXaLbdejQIYqFBUykeC+ptYdy6dKlUSzcG92jR4+oj1UgIcV+++0Xxaw9Z6tXry50fGu/Z/je3X///aM+1sXqw/1rXbt2TbqdJZxf1h7eKVOmFL5wf69evXJt6/VJ3fM+fPjwXPvNN99MGZb23XffXNt6n4XnRcneXxgWsLDen1beRbhv03rM8+bNi2KW8P0TjkmKixVIccGCsNCTZJ9HiuYMpJ5vRowYEc2blNc2Nb8h/DyzXusZM2ZUvD+LtX/aKk6RYp999olic+bMKXSs3XffPYqF5xtrn7Jl8ODBufbKlSsrHru+fuE5ztqDmjpvevToEd1peN6winQULdxhncOt97H1eRay1k5WkZbQrFmzopj1Pgg/r6dNm1bx2I2R8vlmfbZY+5Kt5yZ8PFZBoMWLF1MoBAAAANgWFsgAAACAhwUyAAAA4GGBDAAAAHi2maQHAAAAbG/4BhkAAADwsEAGAAAAPCyQAQAAAA8LZAAAAMDDAhkAAADwsEAGAAAAPP8PM1+a3BiW3UIAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 720x576 with 10 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BNUxz2BVzJB_"
      },
      "source": [
        "## Catastrophic Forgetting!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dsD_Aty2zVW6"
      },
      "source": [
        "Authors suggest that catastrophic forgetting can happen in some extreme cases where informative images of one class are in first minibatches. Therefore, model does not consider it in updating weights anymore. In the extended abstract I sugested that this can be improved by faster convergence rate which leads to less training steps.\n",
        "\n",
        "\n",
        "To check if the final model trained by mixture of free adversarial training, cyclic learning rate and curriculm learning is improved in such a case, I did this expermint.\n",
        "\n",
        "First, we need a trainloader that have the same images of only one class in first minibatches and then shows other images to our model in the training phase. Well, catastrophic forgetting stills a problem when all informative images of one class is just in first minibatches.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0VnwGT-2LFXe"
      },
      "source": [
        "T = torchvision.transforms.Compose([torchvision.transforms.ToTensor()])\n",
        "# in case you don't have the data change download to True\n",
        "dataset = torchvision.datasets.MNIST('./data', train=True, transform=T, download=False)\n",
        "idx = dataset.targets==1\n",
        "dataset.targets = dataset.targets[idx]\n",
        "dataset.data = dataset.data[idx]\n",
        "data_one = dataset"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KkmIg18QNj_g"
      },
      "source": [
        "T = torchvision.transforms.Compose([torchvision.transforms.ToTensor()])\n",
        "# in case you don't have the data change download to True\n",
        "dataset = torchvision.datasets.MNIST('./data', train=True, transform=T, download=False)\n",
        "idx = dataset.targets!=1\n",
        "dataset.targets = dataset.targets[idx]\n",
        "dataset.data = dataset.data[idx]\n",
        "data_other = dataset"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WQLkQPQL_h66",
        "outputId": "35c42d32-7b1d-4a04-b2eb-83910c8cf3ed"
      },
      "source": [
        "new_mnist = torch.utils.data.ConcatDataset([data_one, data_other])\n",
        "new_mnist"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<torch.utils.data.dataset.ConcatDataset at 0x7ff52bf610d0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 121
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HxmRfC9MAPFu"
      },
      "source": [
        "train_loader = torch.utils.data.DataLoader(new_mnist, batch_size=100, shuffle=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z0EfLXMCY9YU"
      },
      "source": [
        "In case to see batch orders. I just printed two batch for keeping it short, but I checked it myslef."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZIdJOPVBSTdu",
        "outputId": "90f7cea6-2c7a-47fe-dcf6-4c9be525236b"
      },
      "source": [
        "for i, (X, y) in enumerate(train_loader):\n",
        "    input, target = X.cuda(), y.cuda()\n",
        "    if(i==0):\n",
        "        print(target)\n",
        "    if(i==200):\n",
        "        print(target)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
            "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
            "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
            "        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
            "        1, 1, 1, 1], device='cuda:0')\n",
            "tensor([3, 0, 2, 4, 6, 8, 9, 0, 5, 4, 0, 9, 5, 6, 7, 5, 9, 4, 0, 6, 7, 2, 6, 0,\n",
            "        4, 8, 7, 5, 0, 4, 6, 9, 8, 4, 9, 2, 5, 4, 9, 7, 9, 9, 8, 9, 6, 4, 0, 8,\n",
            "        5, 2, 7, 7, 5, 3, 6, 2, 6, 3, 3, 3, 4, 5, 2, 3, 0, 2, 0, 9, 7, 8, 3, 4,\n",
            "        5, 6, 4, 0, 9, 2, 3, 6, 8, 4, 6, 7, 4, 2, 2, 5, 0, 6, 3, 0, 6, 9, 8, 6,\n",
            "        3, 3, 5, 2], device='cuda:0')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a6u5C_axYawM",
        "outputId": "b85658c2-16e3-4890-eb28-4fd663c2a78c"
      },
      "source": [
        "model = mnist_net().cuda()\n",
        "model.train()\n",
        "\n",
        "lr=.0001\n",
        "opt = torch.optim.Adam(model.parameters(), lr=.0001)\n",
        "\n",
        "batch_size=100\n",
        "total_epochs=2000\n",
        "replays= 40\n",
        "epochs = total_epochs / replays\n",
        "epsilon=0.3\n",
        "alpha=0.01\n",
        "lr_max=5e-3\n",
        "lr_type='cyclic'\n",
        "seed=0\n",
        "\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "logger = logging.getLogger(__name__)\n",
        "logging.basicConfig(\n",
        "    format='[%(asctime)s] - %(message)s',\n",
        "    datefmt='%Y/%m/%d %H:%M:%S',\n",
        "    level=logging.DEBUG)\n",
        "\n",
        "logger.info('Epoch \\t Time \\t LR \\t \\t Train Loss \\t Train Acc')\n",
        "global_noise_data = torch.zeros([batch_size, 1, 28, 28]).cuda()\n",
        "lr_schedule = lambda t: np.interp([t], [0, epochs * 2//5, epochs], [0, lr_max, 0])[0]\n",
        "for epoch in range(int(epochs)):\n",
        "    start_time = time.time()\n",
        "    train_loss = 0\n",
        "    train_acc = 0\n",
        "    train_n = 0\n",
        "    if epoch < (epochs/ 4):\n",
        "        alpha = .1\n",
        "    elif epoch < (epochs/ 2):\n",
        "        alpha = .2\n",
        "    else:\n",
        "        alpha = .3\n",
        "    \n",
        "    for i, (X, y) in enumerate(train_loader):\n",
        "        input, target = X.cuda(), y.cuda()\n",
        "        lr = lr_schedule(epoch + (i+1)/len(train_loader))\n",
        "        opt.param_groups[0].update(lr=lr)\n",
        "        for j in range(replays):\n",
        "\n",
        "            # Ascend on the global noise\n",
        "            noise_batch = Variable(global_noise_data[0:input.size(0)], requires_grad=True).cuda()\n",
        "            in1 = input + noise_batch\n",
        "            in1.clamp_(0, 1.0)\n",
        "            #in1.sub_(mean).div_(std)\n",
        "            output = model(in1)\n",
        "            loss = criterion(output, target)\n",
        "\n",
        "            # compute gradient\n",
        "            opt.zero_grad()\n",
        "            loss.backward()\n",
        "            \n",
        "            # Update the noise for the next iteration\n",
        "            pert = alpha * torch.sign(noise_batch.grad)\n",
        "            global_noise_data[0:input.size(0)] += pert.data\n",
        "            global_noise_data.clamp_(-epsilon, epsilon)\n",
        "          \n",
        "            #finally do SGD step\n",
        "            opt.step()\n",
        "\n",
        "        train_loss += loss.item() * target.size(0)\n",
        "        train_acc += (output.max(1)[1] == target).sum().item()\n",
        "        train_n += target.size(0)\n",
        "\n",
        "    train_time = time.time()\n",
        "    logger.info('%d \\t %.1f \\t %.4f \\t %.4f \\t %.4f',\n",
        "        epoch, train_time - start_time, lr, train_loss/train_n, train_acc/train_n)\n",
        "    model_name= '/content/mnist_model-catasCheck-'+ str(total_epochs) + '_' + str(replays)\n",
        "    torch.save(model.state_dict(),  model_name)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[2021/04/29 16:12:03] - Epoch \t Time \t LR \t \t Train Loss \t Train Acc\n",
            "[2021/04/29 16:13:18] - 0 \t 75.7 \t 0.0003 \t 0.4381 \t 0.8719\n",
            "[2021/04/29 16:14:34] - 1 \t 75.7 \t 0.0005 \t 0.0148 \t 0.9993\n",
            "[2021/04/29 16:15:50] - 2 \t 75.7 \t 0.0008 \t 0.0047 \t 0.9999\n",
            "[2021/04/29 16:17:05] - 3 \t 75.7 \t 0.0010 \t 0.0029 \t 1.0000\n",
            "[2021/04/29 16:18:21] - 4 \t 75.8 \t 0.0013 \t 0.0025 \t 0.9998\n",
            "[2021/04/29 16:19:37] - 5 \t 75.6 \t 0.0015 \t 0.0021 \t 0.9999\n",
            "[2021/04/29 16:20:53] - 6 \t 75.8 \t 0.0018 \t 0.0046 \t 0.9998\n",
            "[2021/04/29 16:22:08] - 7 \t 75.4 \t 0.0020 \t 0.0289 \t 0.9950\n",
            "[2021/04/29 16:23:23] - 8 \t 75.2 \t 0.0023 \t 0.1162 \t 0.9660\n",
            "[2021/04/29 16:24:38] - 9 \t 75.1 \t 0.0025 \t 0.0808 \t 0.9788\n",
            "[2021/04/29 16:25:54] - 10 \t 75.3 \t 0.0027 \t 0.0712 \t 0.9816\n",
            "[2021/04/29 16:27:09] - 11 \t 74.9 \t 0.0030 \t 0.0696 \t 0.9820\n",
            "[2021/04/29 16:28:24] - 12 \t 75.6 \t 0.0033 \t 0.0661 \t 0.9825\n",
            "[2021/04/29 16:29:40] - 13 \t 75.3 \t 0.0035 \t 0.0542 \t 0.9873\n",
            "[2021/04/29 16:30:55] - 14 \t 75.4 \t 0.0037 \t 0.0483 \t 0.9888\n",
            "[2021/04/29 16:32:10] - 15 \t 75.3 \t 0.0040 \t 0.0695 \t 0.9820\n",
            "[2021/04/29 16:33:25] - 16 \t 75.2 \t 0.0043 \t 0.0684 \t 0.9820\n",
            "[2021/04/29 16:34:41] - 17 \t 75.3 \t 0.0045 \t 0.0585 \t 0.9852\n",
            "[2021/04/29 16:35:56] - 18 \t 75.1 \t 0.0047 \t 1.9370 \t 0.2447\n",
            "[2021/04/29 16:37:11] - 19 \t 74.8 \t 0.0050 \t 2.1600 \t 0.2144\n",
            "[2021/04/29 16:38:26] - 20 \t 75.5 \t 0.0048 \t 2.0972 \t 0.2213\n",
            "[2021/04/29 16:39:42] - 21 \t 75.5 \t 0.0047 \t 2.1080 \t 0.2205\n",
            "[2021/04/29 16:40:57] - 22 \t 75.2 \t 0.0045 \t 2.1158 \t 0.2185\n",
            "[2021/04/29 16:42:12] - 23 \t 75.2 \t 0.0043 \t 2.1244 \t 0.2161\n",
            "[2021/04/29 16:43:27] - 24 \t 75.1 \t 0.0042 \t 2.1336 \t 0.2152\n",
            "[2021/04/29 16:44:42] - 25 \t 75.1 \t 0.0040 \t 2.1435 \t 0.2133\n",
            "[2021/04/29 16:45:57] - 26 \t 74.9 \t 0.0038 \t 2.1541 \t 0.2108\n",
            "[2021/04/29 16:47:12] - 27 \t 74.9 \t 0.0037 \t 2.1656 \t 0.2086\n",
            "[2021/04/29 16:48:27] - 28 \t 75.0 \t 0.0035 \t 2.1781 \t 0.2058\n",
            "[2021/04/29 16:49:42] - 29 \t 75.4 \t 0.0033 \t 2.1917 \t 0.2035\n",
            "[2021/04/29 16:50:57] - 30 \t 75.1 \t 0.0032 \t 2.2065 \t 0.2009\n",
            "[2021/04/29 16:52:12] - 31 \t 74.9 \t 0.0030 \t 2.2226 \t 0.1984\n",
            "[2021/04/29 16:53:27] - 32 \t 74.6 \t 0.0028 \t 2.2402 \t 0.1942\n",
            "[2021/04/29 16:54:42] - 33 \t 75.4 \t 0.0027 \t 2.2596 \t 0.1894\n",
            "[2021/04/29 16:55:58] - 34 \t 75.4 \t 0.0025 \t 2.2810 \t 0.1852\n",
            "[2021/04/29 16:57:13] - 35 \t 75.4 \t 0.0023 \t 2.3046 \t 0.1804\n",
            "[2021/04/29 16:58:29] - 36 \t 75.6 \t 0.0022 \t 2.3306 \t 0.1758\n",
            "[2021/04/29 16:59:44] - 37 \t 75.2 \t 0.0020 \t 2.3594 \t 0.1695\n",
            "[2021/04/29 17:00:59] - 38 \t 74.7 \t 0.0018 \t 2.3911 \t 0.1633\n",
            "[2021/04/29 17:02:14] - 39 \t 74.9 \t 0.0017 \t 2.4260 \t 0.1553\n",
            "[2021/04/29 17:03:28] - 40 \t 74.7 \t 0.0015 \t 2.4637 \t 0.1457\n",
            "[2021/04/29 17:04:43] - 41 \t 75.1 \t 0.0013 \t 2.5042 \t 0.1368\n",
            "[2021/04/29 17:05:58] - 42 \t 74.9 \t 0.0012 \t 2.5466 \t 0.1256\n",
            "[2021/04/29 17:07:13] - 43 \t 74.9 \t 0.0010 \t 2.5885 \t 0.1159\n",
            "[2021/04/29 17:08:28] - 44 \t 75.0 \t 0.0008 \t 2.6309 \t 0.1140\n",
            "[2021/04/29 17:09:43] - 45 \t 75.0 \t 0.0007 \t 2.6699 \t 0.1128\n",
            "[2021/04/29 17:10:58] - 46 \t 75.1 \t 0.0005 \t 2.7049 \t 0.1110\n",
            "[2021/04/29 17:12:13] - 47 \t 74.7 \t 0.0003 \t 2.7333 \t 0.1094\n",
            "[2021/04/29 17:13:28] - 48 \t 75.1 \t 0.0002 \t 2.7536 \t 0.1082\n",
            "[2021/04/29 17:14:43] - 49 \t 75.5 \t 0.0000 \t 2.7600 \t 0.1053\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 488
        },
        "id": "m8tS9iw3Y1Ci",
        "outputId": "0acdd490-f27f-4e95-da20-87cd6249cd5b"
      },
      "source": [
        "import pandas as pd\n",
        "# importing advertorch\n",
        "from advertorch.attacks import LinfPGDAttack\n",
        "from advertorch.attacks.utils import attack_whole_dataset\n",
        "\n",
        "pgd_report = []\n",
        "\n",
        "# MNIST Test dataset and dataloader declaration\n",
        "loader = torch.utils.data.DataLoader(\n",
        "    datasets.MNIST('./data', train=False, download=False, transform=transforms.Compose([\n",
        "            transforms.ToTensor(),\n",
        "            ])),\n",
        "        batch_size=200, shuffle=True)\n",
        "\n",
        "n_replay = 40\n",
        "attack_n_itrs_list = [10, 40, 100]\n",
        "epsilons = [.1, .2, .3]\n",
        "total_epochs = 2000\n",
        "# load the model and put it in evaluation mode\n",
        "model = mnist_net().cuda()\n",
        "model.load_state_dict(torch.load('/content/mnist_model-catasCheck-2000_40'))\n",
        "model.eval()\n",
        "\n",
        "for attack_n_itrs in attack_n_itrs_list:\n",
        "    acc_list = []\n",
        "    for epsilon in epsilons: \n",
        "\n",
        "        adversary = LinfPGDAttack(\n",
        "            model, loss_fn=nn.CrossEntropyLoss(reduction=\"sum\"), eps=epsilon,\n",
        "            nb_iter=attack_n_itrs, eps_iter=0.01, rand_init=True, clip_min=0.0, clip_max=1.0,\n",
        "            targeted=False)\n",
        "\n",
        "        _, label, pred, advpred = attack_whole_dataset(\n",
        "            adversary, loader, device=\"cuda\")\n",
        "        \n",
        "        clean_acc = 100. * (label == pred).sum().item() / len(label)\n",
        "        adv_acc = 100. * (label == advpred).sum().item() / len(label)\n",
        "        acc_list.append(adv_acc)\n",
        "        pgd_report.append([total_epochs, n_replay, 'cyclic', 'LinfPGD', attack_n_itrs, epsilon, adv_acc, clean_acc])\n",
        "\n",
        "        print(\"Accuracy: {:.2f}%, Robust Accuracy: {:.2f}%\".format(\n",
        "            100. * (label == pred).sum().item() / len(label),\n",
        "            100. * (label == advpred).sum().item() / len(label)))\n",
        "column = ['total_epochs', 'replay', 'lr_type', 'attack_type', 'attack_n_itr', 'eps', 'adv_acc','clean_acc']\n",
        "df = pd.DataFrame(pgd_report, columns= column)\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy: 10.28%, Robust Accuracy: 10.28%\n",
            "Accuracy: 10.28%, Robust Accuracy: 10.28%\n",
            "Accuracy: 10.28%, Robust Accuracy: 10.28%\n",
            "Accuracy: 10.28%, Robust Accuracy: 10.28%\n",
            "Accuracy: 10.28%, Robust Accuracy: 10.28%\n",
            "Accuracy: 10.28%, Robust Accuracy: 10.28%\n",
            "Accuracy: 10.28%, Robust Accuracy: 10.28%\n",
            "Accuracy: 10.28%, Robust Accuracy: 10.28%\n",
            "Accuracy: 10.28%, Robust Accuracy: 10.28%\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>total_epochs</th>\n",
              "      <th>replay</th>\n",
              "      <th>lr_type</th>\n",
              "      <th>attack_type</th>\n",
              "      <th>attack_n_itr</th>\n",
              "      <th>eps</th>\n",
              "      <th>adv_acc</th>\n",
              "      <th>clean_acc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2000</td>\n",
              "      <td>40</td>\n",
              "      <td>cyclic</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.1</td>\n",
              "      <td>10.28</td>\n",
              "      <td>10.28</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2000</td>\n",
              "      <td>40</td>\n",
              "      <td>cyclic</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.2</td>\n",
              "      <td>10.28</td>\n",
              "      <td>10.28</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2000</td>\n",
              "      <td>40</td>\n",
              "      <td>cyclic</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>10</td>\n",
              "      <td>0.3</td>\n",
              "      <td>10.28</td>\n",
              "      <td>10.28</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2000</td>\n",
              "      <td>40</td>\n",
              "      <td>cyclic</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.1</td>\n",
              "      <td>10.28</td>\n",
              "      <td>10.28</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2000</td>\n",
              "      <td>40</td>\n",
              "      <td>cyclic</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.2</td>\n",
              "      <td>10.28</td>\n",
              "      <td>10.28</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>2000</td>\n",
              "      <td>40</td>\n",
              "      <td>cyclic</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>40</td>\n",
              "      <td>0.3</td>\n",
              "      <td>10.28</td>\n",
              "      <td>10.28</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>2000</td>\n",
              "      <td>40</td>\n",
              "      <td>cyclic</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.1</td>\n",
              "      <td>10.28</td>\n",
              "      <td>10.28</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>2000</td>\n",
              "      <td>40</td>\n",
              "      <td>cyclic</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.2</td>\n",
              "      <td>10.28</td>\n",
              "      <td>10.28</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>2000</td>\n",
              "      <td>40</td>\n",
              "      <td>cyclic</td>\n",
              "      <td>LinfPGD</td>\n",
              "      <td>100</td>\n",
              "      <td>0.3</td>\n",
              "      <td>10.28</td>\n",
              "      <td>10.28</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   total_epochs  replay lr_type  ...  eps  adv_acc  clean_acc\n",
              "0          2000      40  cyclic  ...  0.1    10.28      10.28\n",
              "1          2000      40  cyclic  ...  0.2    10.28      10.28\n",
              "2          2000      40  cyclic  ...  0.3    10.28      10.28\n",
              "3          2000      40  cyclic  ...  0.1    10.28      10.28\n",
              "4          2000      40  cyclic  ...  0.2    10.28      10.28\n",
              "5          2000      40  cyclic  ...  0.3    10.28      10.28\n",
              "6          2000      40  cyclic  ...  0.1    10.28      10.28\n",
              "7          2000      40  cyclic  ...  0.2    10.28      10.28\n",
              "8          2000      40  cyclic  ...  0.3    10.28      10.28\n",
              "\n",
              "[9 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 138
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fAYug7ju9ivv"
      },
      "source": [
        "## Refrences\n",
        "\n",
        "https://github.com/mahyarnajibi/FreeAdversarialTraining\n",
        "\n",
        "https://github.com/BorealisAI/advertorch\n",
        "\n",
        "https://github.com/locuslab/fast_adversarial"
      ]
    }
  ]
}