CIFAR_100_with_ResNet.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "CIFAR-100 with ResNet",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python [default]",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.0"
    },
    "toc": {
      "base_numbering": 1,
      "nav_menu": {},
      "number_sections": true,
      "sideBar": true,
      "skip_h1_title": false,
      "title_cell": "Table of Contents",
      "title_sidebar": "Contents",
      "toc_cell": false,
      "toc_position": {},
      "toc_section_display": true,
      "toc_window_display": false
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "CuAvKKD5_CG0",
        "colab": {}
      },
      "source": [
        "!mkdir results"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "u8J03tMz_Nku",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import argparse\n",
        "import numpy as np\n",
        "import time\n",
        "from copy import deepcopy # Add Deepcopy for args\n",
        "import seaborn as sns \n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "aj0CoQLUAQkR"
      },
      "source": [
        "# Data Preparation\n",
        "."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "_U0zaT67_Plf",
        "outputId": "f672e046-f791-447f-a42e-9a6fbac30002",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "transform = transforms.Compose(\n",
        "    [transforms.ToTensor(),\n",
        "     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
        "\n",
        "trainset = torchvision.datasets.CIFAR100(root='./data', train=True,\n",
        "                                        download=True, transform=transform)\n",
        "trainset, valset = torch.utils.data.random_split(trainset, [40000, 10000])\n",
        "testset = torchvision.datasets.CIFAR100(root='./data', train=False,\n",
        "                                       download=True, transform=transform)\n",
        "partition = {'train': trainset, 'val':valset, 'test':testset}"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz to ./data/cifar-100-python.tar.gz\n",
            "Files already downloaded and verified\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "Ix5IJ8boAb1-"
      },
      "source": [
        "# Model Architecture\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "B9N12-ITAtCv"
      },
      "source": [
        "### conv3x3 and conv1x1 functions\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4ey5yMjX_S8d",
        "colab": {}
      },
      "source": [
        "def conv3x3(in_planes, out_planes, stride=1):\n",
        "    \"\"\"3x3 convolution with padding\"\"\"\n",
        "    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,\n",
        "                     padding=1, bias=False)\n",
        "\n",
        "\n",
        "def conv1x1(in_planes, out_planes, stride=1):\n",
        "    \"\"\"1x1 convolution\"\"\"\n",
        "    return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, \n",
        "                     bias=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "KTsbOiknBEBA"
      },
      "source": [
        "### BasicBlock Module\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "aa4d5OM7Amkd",
        "colab": {}
      },
      "source": [
        "class BasicBlock(nn.Module):\n",
        "    expansion = 1\n",
        "\n",
        "    def __init__(self, inplanes, planes, stride=1, downsample=None):\n",
        "        super(BasicBlock, self).__init__()\n",
        "        self.conv1 = conv3x3(inplanes, planes, stride)\n",
        "        self.bn1 = nn.BatchNorm2d(planes)\n",
        "        self.relu = nn.ReLU(inplace=True)\n",
        "        self.conv2 = conv3x3(planes, planes)\n",
        "        self.bn2 = nn.BatchNorm2d(planes)\n",
        "        self.downsample = downsample\n",
        "        self.stride = stride\n",
        "\n",
        "    def forward(self, x):\n",
        "        identity = x\n",
        "\n",
        "        out = self.conv1(x)\n",
        "        out = self.bn1(out)\n",
        "        out = self.relu(out)\n",
        "\n",
        "        out = self.conv2(out)\n",
        "        out = self.bn2(out)\n",
        "\n",
        "        if self.downsample is not None:\n",
        "            identity = self.downsample(x)\n",
        "\n",
        "        out += identity\n",
        "        out = self.relu(out)\n",
        "\n",
        "        return out"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "bOst0o4OBYYG"
      },
      "source": [
        "### Bottleneck Module\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "KgmSrlL4BXVq",
        "colab": {}
      },
      "source": [
        "class Bottleneck(nn.Module):\n",
        "    expansion = 4\n",
        "\n",
        "    def __init__(self, inplanes, planes, stride=1, downsample=None):\n",
        "        super(Bottleneck, self).__init__()\n",
        "        self.conv1 = conv1x1(inplanes, planes)\n",
        "        self.bn1 = nn.BatchNorm2d(planes)\n",
        "        self.conv2 = conv3x3(planes, planes, stride)\n",
        "        self.bn2 = nn.BatchNorm2d(planes)\n",
        "        self.conv3 = conv1x1(planes, planes * self.expansion)\n",
        "        self.bn3 = nn.BatchNorm2d(planes * self.expansion)\n",
        "        self.relu = nn.ReLU(inplace=True)\n",
        "        self.downsample = downsample\n",
        "        self.stride = stride\n",
        "\n",
        "    def forward(self, x):\n",
        "        identity = x\n",
        "\n",
        "        out = self.conv1(x)\n",
        "        out = self.bn1(out)\n",
        "        out = self.relu(out)\n",
        "\n",
        "        out = self.conv2(out)\n",
        "        out = self.bn2(out)\n",
        "        out = self.relu(out)\n",
        "\n",
        "        out = self.conv3(out)\n",
        "        out = self.bn3(out)\n",
        "\n",
        "        if self.downsample is not None:\n",
        "            identity = self.downsample(x)\n",
        "\n",
        "        out += identity\n",
        "        out = self.relu(out)\n",
        "\n",
        "        return out"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "n8Mna3wVDOFM"
      },
      "source": [
        "# ResNet Module\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "F56f7NMSDEYs",
        "colab": {}
      },
      "source": [
        "class ResNet(nn.Module):\n",
        "\n",
        "    def __init__(self, block, layers, num_classes=1000, zero_init_residual=False):\n",
        "        super(ResNet, self).__init__()\n",
        "        self.inplanes = 64\n",
        "        self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,\n",
        "                               bias=False)\n",
        "        self.bn1 = nn.BatchNorm2d(64)\n",
        "        self.relu = nn.ReLU(inplace=True)\n",
        "        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n",
        "        self.layer1 = self._make_layer(block, 64, layers[0])\n",
        "        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)\n",
        "        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)\n",
        "        self.layer4 = self._make_layer(block, 512, layers[3], stride=2)\n",
        "        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n",
        "        self.fc = nn.Linear(512 * block.expansion, num_classes)\n",
        "\n",
        "        for m in self.modules():\n",
        "            if isinstance(m, nn.Conv2d):\n",
        "                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')\n",
        "            elif isinstance(m, nn.BatchNorm2d):\n",
        "                nn.init.constant_(m.weight, 1)\n",
        "                nn.init.constant_(m.bias, 0)\n",
        "\n",
        "        # Zero-initialize the last BN in each residual branch,\n",
        "        # so that the residual branch starts with zeros, and each residual block behaves like an identity.\n",
        "        # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677\n",
        "        if zero_init_residual:\n",
        "            for m in self.modules():\n",
        "                if isinstance(m, Bottleneck):\n",
        "                    nn.init.constant_(m.bn3.weight, 0)\n",
        "                elif isinstance(m, BasicBlock):\n",
        "                    nn.init.constant_(m.bn2.weight, 0)\n",
        "\n",
        "    def _make_layer(self, block, planes, blocks, stride=1):\n",
        "        downsample = None\n",
        "        if stride != 1 or self.inplanes != planes * block.expansion:\n",
        "            downsample = nn.Sequential(\n",
        "                conv1x1(self.inplanes, planes * block.expansion, stride),\n",
        "                nn.BatchNorm2d(planes * block.expansion),\n",
        "            )\n",
        "\n",
        "        layers = []\n",
        "        layers.append(block(self.inplanes, planes, stride, downsample))\n",
        "        self.inplanes = planes * block.expansion\n",
        "        for _ in range(1, blocks):\n",
        "            layers.append(block(self.inplanes, planes))\n",
        "\n",
        "        return nn.Sequential(*layers)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.conv1(x)\n",
        "        x = self.bn1(x)\n",
        "        x = self.relu(x)\n",
        "        x = self.maxpool(x)\n",
        "\n",
        "        x = self.layer1(x)\n",
        "        x = self.layer2(x)\n",
        "        x = self.layer3(x)\n",
        "        x = self.layer4(x)\n",
        "\n",
        "        x = self.avgpool(x)\n",
        "        x = x.view(x.size(0), -1)\n",
        "        x = self.fc(x)\n",
        "\n",
        "        return x"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "PUkZeRF_D0y1"
      },
      "source": [
        "# Train, Validate, Test and Experiment"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "AodJ0AufDgmJ",
        "colab": {}
      },
      "source": [
        "def train(net, partition, optimizer, criterion, args):\n",
        "    trainloader = torch.utils.data.DataLoader(partition['train'], \n",
        "                                              batch_size=args.train_batch_size, \n",
        "                                              shuffle=True, num_workers=2)\n",
        "    net.train()\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    train_loss = 0.0\n",
        "    for i, data in enumerate(trainloader, 0):\n",
        "        # get the inputs\n",
        "        inputs, labels = data\n",
        "        inputs = inputs.cuda()\n",
        "        labels = labels.cuda()\n",
        "        outputs = net(inputs)\n",
        "\n",
        "        loss = criterion(outputs, labels)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        train_loss += loss.item()\n",
        "        _, predicted = torch.max(outputs.data, 1)\n",
        "        total += labels.size(0)\n",
        "        correct += (predicted == labels).sum().item()\n",
        "\n",
        "    train_loss = train_loss / len(trainloader)\n",
        "    train_acc = 100 * correct / total\n",
        "    return net, train_loss, train_acc"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "WDa7F5dyD0S9",
        "colab": {}
      },
      "source": [
        "def validate(net, partition, criterion, args):\n",
        "    valloader = torch.utils.data.DataLoader(partition['val'], \n",
        "                                            batch_size=args.test_batch_size, \n",
        "                                            shuffle=False, num_workers=2)\n",
        "    net.eval()\n",
        "\n",
        "    correct = 0\n",
        "    total = 0\n",
        "    val_loss = 0 \n",
        "    with torch.no_grad():\n",
        "        for data in valloader:\n",
        "            images, labels = data\n",
        "            images = images.cuda()\n",
        "            labels = labels.cuda()\n",
        "            outputs = net(images)\n",
        "\n",
        "            loss = criterion(outputs, labels)\n",
        "            \n",
        "            val_loss += loss.item()\n",
        "            _, predicted = torch.max(outputs.data, 1)\n",
        "            total += labels.size(0)\n",
        "            correct += (predicted == labels).sum().item()\n",
        "\n",
        "        val_loss = val_loss / len(valloader)\n",
        "        val_acc = 100 * correct / total\n",
        "    return val_loss, val_acc"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Z9aSUcjAD566",
        "colab": {}
      },
      "source": [
        "def test(net, partition, args):\n",
        "    testloader = torch.utils.data.DataLoader(partition['test'], \n",
        "                                             batch_size=args.test_batch_size, \n",
        "                                             shuffle=False, num_workers=2)\n",
        "    net.eval()\n",
        "    \n",
        "    correct = 0\n",
        "    total = 0\n",
        "    with torch.no_grad():\n",
        "        for data in testloader:\n",
        "            images, labels = data\n",
        "            images = images.cuda()\n",
        "            labels = labels.cuda()\n",
        "\n",
        "            outputs = net(images)\n",
        "            _, predicted = torch.max(outputs.data, 1)\n",
        "            total += labels.size(0)\n",
        "            correct += (predicted == labels).sum().item()\n",
        "\n",
        "        test_acc = 100 * correct / total\n",
        "    return test_acc"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "_QyjiV5yD7to",
        "colab": {}
      },
      "source": [
        "def experiment(partition, args):\n",
        "  \n",
        "    net = net = CNN('VGG19', 3)\n",
        "    net.cuda()\n",
        "\n",
        "    criterion = nn.CrossEntropyLoss()\n",
        "    if args.optim == 'SGD':\n",
        "        optimizer = optim.SGD(net.parameters(), lr=args.lr, weight_decay=args.l2)\n",
        "    elif args.optim == 'RMSprop':\n",
        "        optimizer = optim.RMSprop(net.parameters(), lr=args.lr, weight_decay=args.l2)\n",
        "    elif args.optim == 'Adam':\n",
        "        optimizer = optim.Adam(net.parameters(), lr=args.lr, weight_decay=args.l2)\n",
        "    else:\n",
        "        raise ValueError('In-valid optimizer choice')\n",
        "    \n",
        "    train_losses = []\n",
        "    val_losses = []\n",
        "    train_accs = []\n",
        "    val_accs = []\n",
        "        \n",
        "    for epoch in range(args.epoch):  # loop over the dataset multiple times\n",
        "        ts = time.time()\n",
        "        net, train_loss, train_acc = train(net, partition, optimizer, criterion, args)\n",
        "        val_loss, val_acc = validate(net, partition, criterion, args)\n",
        "        te = time.time()\n",
        "        \n",
        "        train_losses.append(train_loss)\n",
        "        val_losses.append(val_loss)\n",
        "        train_accs.append(train_acc)\n",
        "        val_accs.append(val_acc)\n",
        "        \n",
        "        print('Epoch {}, Acc(train/val): {:2.2f}/{:2.2f}, Loss(train/val) {:2.2f}/{:2.2f}. Took {:2.2f} sec'.format(epoch, train_acc, val_acc, train_loss, val_loss, te-ts))\n",
        "        \n",
        "    test_acc = test(net, partition, args)    \n",
        "    \n",
        "    result = {}\n",
        "    result['train_losses'] = train_losses\n",
        "    result['val_losses'] = val_losses\n",
        "    result['train_accs'] = train_accs\n",
        "    result['val_accs'] = val_accs\n",
        "    result['train_acc'] = train_acc\n",
        "    result['val_acc'] = val_acc\n",
        "    result['test_acc'] = test_acc\n",
        "    return vars(args), result"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "NGEg-JdZD_od"
      },
      "source": [
        "# Manage Experiment Result"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "tHNXD0FLD-RT",
        "colab": {}
      },
      "source": [
        "import hashlib\n",
        "import json\n",
        "from os import listdir\n",
        "from os.path import isfile, join\n",
        "import pandas as pd\n",
        "\n",
        "def save_exp_result(setting, result):\n",
        "    exp_name = setting['exp_name']\n",
        "    del setting['epoch']\n",
        "    del setting['test_batch_size']\n",
        "\n",
        "    hash_key = hashlib.sha1(str(setting).encode()).hexdigest()[:6]\n",
        "    filename = './results/{}-{}.json'.format(exp_name, hash_key)\n",
        "    result.update(setting)\n",
        "    with open(filename, 'w') as f:\n",
        "        json.dump(result, f)\n",
        "\n",
        "    \n",
        "def load_exp_result(exp_name):\n",
        "    dir_path = './results'\n",
        "    filenames = [f for f in listdir(dir_path) if isfile(join(dir_path, f)) if '.json' in f]\n",
        "    list_result = []\n",
        "    for filename in filenames:\n",
        "        if exp_name in filename:\n",
        "            with open(join(dir_path, filename), 'r') as infile:\n",
        "                results = json.load(infile)\n",
        "                list_result.append(results)\n",
        "    df = pd.DataFrame(list_result) # .drop(columns=[])\n",
        "    return df"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "4dedRUkNEGYM"
      },
      "source": [
        "# Visualizatin Utility"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "IBO5-j6pEBtd",
        "colab": {}
      },
      "source": [
        "def plot_acc(var1, var2, df):\n",
        "\n",
        "    fig, ax = plt.subplots(1, 3)\n",
        "    fig.set_size_inches(15, 6)\n",
        "    sns.set_style(\"darkgrid\", {\"axes.facecolor\": \".9\"})\n",
        "\n",
        "    sns.barplot(x=var1, y='train_acc', hue=var2, data=df, ax=ax[0])\n",
        "    sns.barplot(x=var1, y='val_acc', hue=var2, data=df, ax=ax[1])\n",
        "    sns.barplot(x=var1, y='test_acc', hue=var2, data=df, ax=ax[2])\n",
        "    \n",
        "    ax[0].set_title('Train Accuracy')\n",
        "    ax[1].set_title('Validation Accuracy')\n",
        "    ax[2].set_title('Test Accuracy')\n",
        "    \n",
        "def plot_loss_variation(var1, var2, df, **kwargs):\n",
        "\n",
        "    list_v1 = df[var1].unique()\n",
        "    list_v2 = df[var2].unique()\n",
        "    list_data = []\n",
        "\n",
        "    for value1 in list_v1:\n",
        "        for value2 in list_v2:\n",
        "            row = df.loc[df[var1]==value1]\n",
        "            row = row.loc[df[var2]==value2]\n",
        "\n",
        "            train_losses = list(row.train_losses)[0]\n",
        "            val_losses = list(row.val_losses)[0]\n",
        "\n",
        "            for epoch, train_loss in enumerate(train_losses):\n",
        "                list_data.append({'type':'train', 'loss':train_loss, 'epoch':epoch, var1:value1, var2:value2})\n",
        "            for epoch, val_loss in enumerate(val_losses):\n",
        "                list_data.append({'type':'val', 'loss':val_loss, 'epoch':epoch, var1:value1, var2:value2})\n",
        "\n",
        "    df = pd.DataFrame(list_data)\n",
        "    g = sns.FacetGrid(df, row=var2, col=var1, hue='type', **kwargs)\n",
        "    g = g.map(plt.plot, 'epoch', 'loss', marker='.')\n",
        "    g.add_legend()\n",
        "    g.fig.suptitle('Train loss vs Val loss')\n",
        "    plt.subplots_adjust(top=0.89) \n",
        "    \n",
        "def plot_acc_variation(var1, var2, df, **kwargs):\n",
        "    list_v1 = df[var1].unique()\n",
        "    list_v2 = df[var2].unique()\n",
        "    list_data = []\n",
        "\n",
        "    for value1 in list_v1:\n",
        "        for value2 in list_v2:\n",
        "            row = df.loc[df[var1]==value1]\n",
        "            row = row.loc[df[var2]==value2]\n",
        "\n",
        "            train_accs = list(row.train_accs)[0]\n",
        "            val_accs = list(row.val_accs)[0]\n",
        "            test_acc = list(row.test_acc)[0]\n",
        "\n",
        "            for epoch, train_acc in enumerate(train_accs):\n",
        "                list_data.append({'type':'train', 'Acc':train_acc, 'test_acc':test_acc, 'epoch':epoch, var1:value1, var2:value2})\n",
        "            for epoch, val_acc in enumerate(val_accs):\n",
        "                list_data.append({'type':'val', 'Acc':val_acc, 'test_acc':test_acc, 'epoch':epoch, var1:value1, var2:value2})\n",
        "\n",
        "    df = pd.DataFrame(list_data)\n",
        "    g = sns.FacetGrid(df, row=var2, col=var1, hue='type', **kwargs)\n",
        "    g = g.map(plt.plot, 'epoch', 'Acc', marker='.')\n",
        "\n",
        "    def show_acc(x, y, metric, **kwargs):\n",
        "        plt.scatter(x, y, alpha=0.3, s=1)\n",
        "        metric = \"Test Acc: {:1.3f}\".format(list(metric.values)[0])\n",
        "        plt.text(0.05, 0.95, metric,  horizontalalignment='left', verticalalignment='center', transform=plt.gca().transAxes, bbox=dict(facecolor='yellow', alpha=0.5, boxstyle=\"round,pad=0.1\"))\n",
        "    g = g.map(show_acc, 'epoch', 'Acc', 'test_acc')\n",
        "\n",
        "    g.add_legend()\n",
        "    g.fig.suptitle('Train Accuracy vs Val Accuracy')\n",
        "    plt.subplots_adjust(top=0.89)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "MeRl-4w3ESID",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}