fedpl-rsna.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a51f5d2a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-09T15:55:40.864147Z",
     "iopub.status.busy": "2022-11-09T15:55:40.863496Z",
     "iopub.status.idle": "2022-11-09T15:55:42.377188Z",
     "shell.execute_reply": "2022-11-09T15:55:42.376250Z"
    },
    "papermill": {
     "duration": 1.522102,
     "end_time": "2022-11-09T15:55:42.379688",
     "exception": false,
     "start_time": "2022-11-09T15:55:40.857586",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as func\n",
    "import torchvision\n",
    "\n",
    "class ResNetFc(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(ResNetFc, self).__init__()\n",
    "        model_resnet = torchvision.models.resnet18(pretrained=True)\n",
    "\n",
    "        self.conv1 = model_resnet.conv1\n",
    "        self.bn1 = model_resnet.bn1\n",
    "        self.relu = model_resnet.relu\n",
    "\n",
    "        self.layer1 = model_resnet.layer1\n",
    "        self.layer2 = model_resnet.layer2\n",
    "        self.layer3 = model_resnet.layer3\n",
    "        self.layer4 = model_resnet.layer4\n",
    "\n",
    "        self.avgpool = model_resnet.avgpool\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        x = self.bn1(x)\n",
    "        x = self.relu(x)\n",
    "\n",
    "        x = self.layer1(x)\n",
    "        x = self.layer2(x)\n",
    "        x = self.layer3(x)\n",
    "        x = self.layer4(x)\n",
    "        x = self.avgpool(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "\n",
    "        return x\n",
    "\n",
    "class PLRSNAModel(nn.Module):\n",
    "    def __init__(self, class_num=10):\n",
    "        super(PLRSNAModel, self).__init__()\n",
    "        self.class_num = class_num\n",
    "\n",
    "        self.backbone = ResNetFc()\n",
    "\n",
    "        self.clf = nn.Linear(512, class_num)\n",
    "\n",
    "    def forward(self, x, Pi, priors_corr, prior_test):\n",
    "        x = self.backbone(x)\n",
    "        x = self.clf(x)\n",
    "\n",
    "        return x\n",
    "\n",
    "    def predict(self, x):\n",
    "        x = self.backbone(x)\n",
    "        x = self.clf(x)\n",
    "\n",
    "        g = torch.softmax(x, dim=1)\n",
    "\n",
    "        return g\n",
    "\n",
    "    def server_forward(self, x):\n",
    "        x = self.backbone(x)\n",
    "        x = self.clf(x)\n",
    "\n",
    "        return x\n",
    "\n",
    "\n",
    "# class RSNAModel(nn.Module):\n",
    "#     def __init__(self, class_num=6):\n",
    "#         super(RSNAModel, self).__init__()\n",
    "#         self.class_num = class_num\n",
    "\n",
    "#         self.backbone = ResNetFc()\n",
    "\n",
    "#         self.clf = nn.Linear(512, class_num)\n",
    "\n",
    "#     def forward(self, x, Pi, priors_corr, prior_test):\n",
    "#         x = self.backbone(x)\n",
    "#         x = self.clf(x)\n",
    "\n",
    "#         g = torch.softmax(x, dim=1)\n",
    "#         x = self.QfunctionMulticlass(g, Pi, priors_corr)\n",
    "\n",
    "#         return x\n",
    "\n",
    "#     def QfunctionMulticlass(self, g, Pi, priors_corr):\n",
    "#         pi_ita = torch.mm(Pi, g.permute(1, 0))\n",
    "#         rou_pi_ita = torch.matmul(priors_corr, pi_ita)\n",
    "\n",
    "#         pi_corr = pi_ita.permute(1, 0) * priors_corr.unsqueeze(0)\n",
    "#         output = (pi_corr.permute(1, 0) / rou_pi_ita).permute(1, 0)\n",
    "\n",
    "#         return output\n",
    "\n",
    "#     def predict(self, x):\n",
    "#         x = self.backbone(x)\n",
    "#         x = self.clf(x)\n",
    "\n",
    "#         g = torch.softmax(x, dim=1)\n",
    "\n",
    "#         return g\n",
    "\n",
    "#     def server_forward(self, x):\n",
    "#         x = self.backbone(x)\n",
    "#         x = self.clf(x)\n",
    "\n",
    "#         return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "69c71472",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-09T15:55:42.387902Z",
     "iopub.status.busy": "2022-11-09T15:55:42.387487Z",
     "iopub.status.idle": "2022-11-09T15:55:42.532587Z",
     "shell.execute_reply": "2022-11-09T15:55:42.531624Z"
    },
    "papermill": {
     "duration": 0.1518,
     "end_time": "2022-11-09T15:55:42.535110",
     "exception": false,
     "start_time": "2022-11-09T15:55:42.383310",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "def read_csv(data_file_path):\n",
    "    data = []\n",
    "    with open(data_file_path, 'r') as f:\n",
    "        reader = csv.reader(f)\n",
    "        data = list(reader)\n",
    "        data = np.asarray(data)\n",
    "    return data\n",
    "\n",
    "label_file = '../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train.csv'\n",
    "\n",
    "import glob\n",
    "import joblib\n",
    "import numpy as np\n",
    "import PIL\n",
    "import pydicom\n",
    "import tqdm\n",
    "import torch\n",
    "import torchvision\n",
    "from torch.utils.data import Dataset\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "\n",
    "RESIZED_WIDTH, RESIZED_HEIGHT = 128, 128\n",
    "def get_first_of_dicom_field_as_int(x):\n",
    "    if type(x) == pydicom.multival.MultiValue:\n",
    "        return int(x[0])\n",
    "    return int(x)\n",
    "\n",
    "def get_id(img_dicom):\n",
    "    return str(img_dicom.SOPInstanceUID)\n",
    "\n",
    "def get_metadata_from_dicom(img_dicom):\n",
    "    metadata = {\n",
    "        \"window_center\": img_dicom.WindowCenter,\n",
    "        \"window_width\": img_dicom.WindowWidth,\n",
    "        \"intercept\": img_dicom.RescaleIntercept,\n",
    "        \"slope\": img_dicom.RescaleSlope,\n",
    "    }\n",
    "    return {k: get_first_of_dicom_field_as_int(v) for k, v in metadata.items()}\n",
    "\n",
    "def window_image(img, window_center, window_width, intercept, slope):\n",
    "    img = img * slope + intercept\n",
    "    img_min = window_center - window_width // 2\n",
    "    img_max = window_center + window_width // 2\n",
    "    img[img < img_min] = img_min\n",
    "    img[img > img_max] = img_max\n",
    "    return img \n",
    "\n",
    "def resize(img, new_w, new_h):\n",
    "    img = PIL.Image.fromarray(img.astype(np.int8), mode=\"L\")\n",
    "    return img.resize((new_w, new_h), resample=PIL.Image.BICUBIC)\n",
    "\n",
    "def normalize(img):\n",
    "    mi, ma = img.min(), img.max()\n",
    "    return (((img - mi) / (ma - mi)) - 0.5) * 2\n",
    "\n",
    "def prepare_image(img_path):\n",
    "    img_dicom = pydicom.read_file(img_path)\n",
    "    img_id = get_id(img_dicom)\n",
    "    metadata = get_metadata_from_dicom(img_dicom)\n",
    "    img = window_image(img_dicom.pixel_array, **metadata)\n",
    "    img = normalize(img)\n",
    "    img_pil = resize(img, RESIZED_WIDTH, RESIZED_HEIGHT)\n",
    "    return img_id, img_pil\n",
    "\n",
    "class Dataset_Early_Fusion(Dataset):\n",
    "    def __init__(self, label_file, transform=None):\n",
    "        self.transform = transform\n",
    "        self.data = []\n",
    "        self.targets = []\n",
    "        files = read_csv(label_file)\n",
    "        sub_labels = []\n",
    "        # 106000\n",
    "        count_sum = 0\n",
    "        for i in range(len(files)):\n",
    "            img_labels = files[(i + 1) * 6]\n",
    "            if img_labels[0].split('_')[-1] == 'any':\n",
    "                count_labels = 0\n",
    "                if files[(i + 1) * 6][1] == '0':\n",
    "                    if random.randint(1, 20) == 1:\n",
    "                        label = 0\n",
    "                        count_labels = count_labels + 1\n",
    "                else:\n",
    "                    for n in range(1,6):\n",
    "                        if files[i * 6 + n][1] == '1':\n",
    "                            count_labels = count_labels + 1\n",
    "                            label = n\n",
    "                if count_labels == 1:\n",
    "                    sub_labels.append([img_labels[0].split('_')[-3] + '_' + img_labels[0].split('_')[-2], label])\n",
    "                    count_sum = count_sum + 1\n",
    "            if count_sum == 106000: # !!!!!!! -0\n",
    "                break\n",
    "        for i in range(len(sub_labels)):\n",
    "            temp = sub_labels[i]\n",
    "            full_path = '../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train/' + temp[0] + '.dcm'\n",
    "            _, image = prepare_image(full_path)\n",
    "            if i % 10000 == 0:\n",
    "                print(i, '/ 106000')\n",
    "            self.data.append(np.array(image))\n",
    "            self.targets.append(temp[1])\n",
    "        self.data = np.array(self.data)\n",
    "        self.targets = np.array(self.targets)\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.data)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        img, target = self.data[index], self.targets[index]\n",
    "        img = Image.fromarray(img.numpy(), mode='L')\n",
    "\n",
    "        if self.transform is not None:\n",
    "            img = self.transform(img)\n",
    "#       img = transforms.ToTensor()(img)\n",
    "\n",
    "        return img, target\n",
    "\n",
    "\n",
    "import warnings\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31f0f2ae",
   "metadata": {
    "papermill": {
     "duration": 0.002727,
     "end_time": "2022-11-09T15:55:42.540920",
     "exception": false,
     "start_time": "2022-11-09T15:55:42.538193",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8443d0bb",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2022-11-09T15:55:42.548432Z",
     "iopub.status.busy": "2022-11-09T15:55:42.548044Z",
     "iopub.status.idle": "2022-11-09T15:55:42.589238Z",
     "shell.execute_reply": "2022-11-09T15:55:42.588413Z"
    },
    "papermill": {
     "duration": 0.047638,
     "end_time": "2022-11-09T15:55:42.591349",
     "exception": false,
     "start_time": "2022-11-09T15:55:42.543711",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "np.seterr(divide='ignore', invalid='ignore')\n",
    "import torch\n",
    "import torchvision\n",
    "from torch.utils.data import Dataset\n",
    "import torchvision.transforms as transforms\n",
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "def get_set_sizes(sets, data_len):\n",
    "    set_size = data_len // sets\n",
    "    set_sizes = np.ones(sets) * set_size\n",
    "    # numpy\n",
    "    # [n, n, n, n] sets\n",
    "    return set_sizes\n",
    "\n",
    "\n",
    "def get_Pi_Multiclass(sets, classnum=6, noniid=False):\n",
    "    Pi = []\n",
    "\n",
    "    for i in range(sets):\n",
    "        # randomly set prior\n",
    "        this_Pi = np.random.rand(classnum) * 0.9 + 0.1\n",
    "\n",
    "        this_Pi[i % classnum] *= 10\n",
    "\n",
    "        this_Pi = this_Pi / np.sum(this_Pi)\n",
    "        Pi.append(this_Pi)\n",
    "\n",
    "    Pi = np.array(Pi)\n",
    "    # numpy\n",
    "    # [[...],  num0 * 10\n",
    "    #  [...],  num1 * 10\n",
    "    #  [...]]  setnum * classnum  Normalization in row\n",
    "\n",
    "    return Pi\n",
    "\n",
    "\n",
    "def get_Pi_Multiclass_clientnoniid(sets, classnum=6, noniid=False, clientid=0):\n",
    "    # def get_Pi_Multiclass(sets, classnum=6, noniid=False):\n",
    "    Pi = []\n",
    "\n",
    "    for i in range(sets):\n",
    "        # randomly set prior\n",
    "        this_Pi = np.random.rand(classnum) * 0.9 + 0.1\n",
    "\n",
    "        # dominate step\n",
    "        if noniid:\n",
    "            this_Pi = np.random.rand(classnum) * 0.02 + 0.05\n",
    "            this_Pi[clientid] = np.random.rand(1) * 0.2 + 0.3\n",
    "\n",
    "            this_Pi = np.random.rand(classnum) * 0.02 + 0.05\n",
    "\n",
    "        this_Pi = this_Pi / np.sum(this_Pi)\n",
    "        Pi.append(this_Pi)\n",
    "\n",
    "    Pi = np.array(Pi)\n",
    "\n",
    "    return Pi\n",
    "\n",
    "\n",
    "def get_test_sets_Multiclass(y_test, classnum=6, clientnum=5, clientsize=2000):\n",
    "    class_idx = []\n",
    "    for cls in range(classnum):\n",
    "        this_idx = [i for i, x in enumerate(y_test) if x == cls]\n",
    "        class_idx.append(this_idx)\n",
    "    # class_dix: [[...], index of each class\n",
    "    #             [...]] classnum * ~\n",
    "\n",
    "    test_clients = ()\n",
    "    for i in range(clientnum):\n",
    "        for cls in range(classnum):\n",
    "            np.random.shuffle(class_idx[cls])\n",
    "\n",
    "            # uniformly distributed\n",
    "            n_this = int(clientsize / classnum)\n",
    "\n",
    "            if cls == 0:\n",
    "                cur_set = np.array(class_idx[cls][:n_this])\n",
    "            else:\n",
    "                cur_set = np.concatenate((cur_set, class_idx[cls][:n_this])).astype(int)\n",
    "\n",
    "            np.random.shuffle(cur_set)\n",
    "\n",
    "        test_clients = test_clients + (torch.from_numpy(cur_set),)\n",
    "        # shuffle balance class index ([...], [...], [...], [...], [...]) clientnum\n",
    "\n",
    "    return test_clients\n",
    "\n",
    "\n",
    "def get_U_sets_Multiclass(bags, y_train, y_indices, bag_sizes, thetas, classnum=6):\n",
    "    class_idx = []\n",
    "    for cls in range(classnum):\n",
    "        this_idx = [y_indices[i] for i, x in enumerate(y_train) if x == cls]\n",
    "        class_idx.append(this_idx)\n",
    "    # indices of each class [[...], [...]] class_num\n",
    "\n",
    "    U_sets = ()\n",
    "    size_bag = []\n",
    "    # for every bag\n",
    "    for i in range(bags):\n",
    "        size_cls = []\n",
    "        # for every class in a bag\n",
    "        for cls in range(classnum):\n",
    "            # shuffle data index list\n",
    "            np.random.shuffle(class_idx[cls])\n",
    "            # the number of data selected for this class: prior * setsize\n",
    "            n_this = int(bag_sizes[i] * thetas[i][cls])\n",
    "\n",
    "            # concatenate index\n",
    "            if cls == 0:\n",
    "                cur_set = np.array(class_idx[cls][:n_this])\n",
    "            else:\n",
    "                cur_set = np.concatenate((cur_set, class_idx[cls][:n_this])).astype(int)\n",
    "\n",
    "            size_cls.append(len(class_idx[cls][:n_this]))\n",
    "\n",
    "            # shuffle current set\n",
    "            np.random.shuffle(cur_set)\n",
    "\n",
    "        # concatenate different class data\n",
    "        U_sets = U_sets + (torch.from_numpy(cur_set),)\n",
    "        # shuffle indices of each U_set ([...], [...], [...]) bags\n",
    "\n",
    "        size_bag.append(np.array(size_cls) / sum(size_cls))\n",
    "        # num of each class in each U_set ([...], [...], [...]) bags\n",
    "\n",
    "    # calculate priors corr for every U set\n",
    "    sets_num_count = [len(U_sets[j]) for j in range(len(U_sets))]\n",
    "    # sample_num in each U_set [...] bags\n",
    "\n",
    "    priors_corr = torch.from_numpy(\n",
    "        np.array([sets_num_count[k] / sum(sets_num_count) for k in range(len(sets_num_count))]))\n",
    "    # tensor\n",
    "    # Normalized set_sum_count\n",
    "\n",
    "    bags_pi = np.array(size_bag)\n",
    "    # num of each class in each U_set [[...], [...], [...]] bags [Normalized]\n",
    "\n",
    "    return U_sets, priors_corr, bags_pi\n",
    "\n",
    "\n",
    "def get_iid_Pi(clientnum, setnum_perclient, classnum):\n",
    "    for _ in range(clientnum):\n",
    "        this_Pi = torch.from_numpy(get_Pi_Multiclass(setnum_perclient, classnum=classnum))\n",
    "        if _ == 0:\n",
    "            iid_Pi = this_Pi\n",
    "        else:\n",
    "            iid_Pi += this_Pi\n",
    "    iid_Pi = iid_Pi / torch.sum(iid_Pi, dim=0)\n",
    "\n",
    "    return iid_Pi\n",
    "\n",
    "\n",
    "def get_noniid_class_priority(client_num, classnum=6, dominate_rate=0.5):\n",
    "    priority = []\n",
    "\n",
    "    for client in range(client_num):\n",
    "        this_label_shift = np.random.rand(classnum) * 0.1 + 0.45\n",
    "        # [] .shape = 10\n",
    "        this_label_shift[(2 * client) % classnum] *= (4 / (1 - dominate_rate))\n",
    "        this_label_shift[(2 * client + 1) % classnum] *= (4 / (1 - dominate_rate))\n",
    "\n",
    "        this_label_shift = this_label_shift / np.sum(this_label_shift)\n",
    "        priority.append(this_label_shift)\n",
    "        # [[   ...    ],    [(2 * client) % classnum] * 8 (each row)\n",
    "        #  [   ...    ],    [(2 * client + 1) % classnum] * 8 (each row)\n",
    "        #  [   ...    ]]   client_num * classnum   Normalization in row\n",
    "\n",
    "    return priority\n",
    "\n",
    "\n",
    "def get_class_index(targets, classnum=6):\n",
    "    indexs = []\n",
    "\n",
    "    for cls in range(classnum):\n",
    "        this_index = [index for (index, value) in enumerate(targets) if value == cls]\n",
    "        indexs.append(this_index)\n",
    "\n",
    "    return indexs\n",
    "\n",
    "\n",
    "def noniid_split_dataset(oridata, lengths, classnum=6, dominate_rate=0.95):\n",
    "    subsets = []\n",
    "    priority = get_noniid_class_priority(len(lengths), classnum=classnum, dominate_rate=dominate_rate)\n",
    "\n",
    "    targets = oridata.targets.tolist()\n",
    "    class_index = get_class_index(targets, classnum=classnum)\n",
    "    # index of label 0 [[...],\n",
    "    # index of label 1  [...]] classnum row\n",
    "    class_count = [0 for _ in range(classnum)]\n",
    "\n",
    "    for l in range(len(lengths)):\n",
    "        this_indices = []\n",
    "        for cls in range(classnum):\n",
    "            cls_num = int(priority[l][cls] * lengths[l])\n",
    "\n",
    "            this_indices.extend(class_index[cls][class_count[cls]: class_count[cls] + cls_num])\n",
    "            class_count[cls] += cls_num\n",
    "\n",
    "        this_subset = torch.utils.data.Subset(oridata, this_indices)\n",
    "        # Dataset distribute with priority\n",
    "        # [[   ...    ],    [(2 * client) % classnum] * 8 (each row)\n",
    "        #  [   ...    ],    [(2 * client + 1) % classnum] * 8 (each row)\n",
    "        #  [   ...    ]]   client_num * classnum   Normalization in row\n",
    "        subsets.append(this_subset)\n",
    "        # [Dataset0, ..., Dataset4]\n",
    "\n",
    "    return subsets\n",
    "\n",
    "def RSNA_PL(data_path='../data', clientnum=5, setnum_perclient=12, classnum=6, noniid=False):\n",
    "    all_train_data = Dataset_Early_Fusion(label_file='../input/rsna-intracranial-hemorrhage-detection/rsna-intracranial-hemorrhage-detection/stage_2_train.csv')\n",
    "#     all_train_data.data = torch.from_numpy(all_train_data.data)\n",
    "#     all_train_data.targets = torch.from_numpy(np.array(all_train_data.targets))\n",
    "\n",
    "    validation_data = torch.from_numpy(all_train_data.data[84800:]) # !!!!!!!!!!\n",
    "    validation_targets = torch.from_numpy(all_train_data.targets[84800:])\n",
    "    test_data = torch.from_numpy(all_train_data.data[:21200])\n",
    "    test_targets = torch.from_numpy(all_train_data.targets[:21200])\n",
    "    all_train_data.data = torch.from_numpy(all_train_data.data[21200:84800])\n",
    "    all_train_data.targets = torch.from_numpy(all_train_data.targets[21200:84800])\n",
    "\n",
    "    # split client bags\n",
    "    client_train_size = len(all_train_data) // clientnum\n",
    "    client_validation_size = len(validation_data) // clientnum\n",
    "    client_test_size = len(test_data) // clientnum\n",
    "    if noniid:\n",
    "        client_train_sets = noniid_split_dataset(all_train_data, [client_train_size for _ in range(clientnum)])\n",
    "    else:\n",
    "        client_train_sets = torch.utils.data.random_split(all_train_data, [client_train_size for _ in range(clientnum)])\n",
    "\n",
    "    # get uniformly distributed test data index\n",
    "    validation_client_idxs = get_test_sets_Multiclass(validation_targets, classnum=classnum, clientnum=clientnum,\n",
    "                                                      clientsize=client_validation_size)\n",
    "    test_client_idxs = get_test_sets_Multiclass(test_targets, classnum=classnum, clientnum=clientnum,\n",
    "                                                clientsize=client_test_size)\n",
    "\n",
    "    client_train_data = []\n",
    "    client_validation_data = []\n",
    "    client_test_data = []\n",
    "\n",
    "    # get Pis, prior test, prior corr\n",
    "    client_Pi = []\n",
    "    client_prior_test = []\n",
    "    client_priors_corr = []\n",
    "\n",
    "    if not noniid:\n",
    "        iid_Pi = get_iid_Pi(clientnum, setnum_perclient, classnum)\n",
    "\n",
    "    print('Spliting U sets for', clientnum, 'clients, each with', setnum_perclient, 'U sets...')\n",
    "    # for every client\n",
    "    for n in tqdm(range(clientnum)):\n",
    "        if noniid:\n",
    "            this_Pi = torch.from_numpy(get_Pi_Multiclass(setnum_perclient, classnum=classnum))\n",
    "        if not noniid:\n",
    "            this_Pi = iid_Pi\n",
    "\n",
    "        this_labels = torch.max(this_Pi, dim=1).indices\n",
    "\n",
    "        # w/o repeat\n",
    "        this_set_sizes = get_set_sizes(setnum_perclient, len(client_train_sets[n]))\n",
    "        this_U_sets, this_priors_corr, this_Pi = get_U_sets_Multiclass(setnum_perclient,\n",
    "                                                                       client_train_sets[n].dataset.targets[\n",
    "                                                                           client_train_sets[n].indices],\n",
    "                                                                       client_train_sets[n].indices,\n",
    "                                                                       this_set_sizes, this_Pi, classnum=classnum)\n",
    "\n",
    "        client_Pi.append(this_Pi)\n",
    "        client_priors_corr.append(this_priors_corr)\n",
    "\n",
    "        # get prior test\n",
    "        this_prior_test = None\n",
    "        client_prior_test.append(this_prior_test)\n",
    "\n",
    "        # set subsets labels, for every set in every client\n",
    "        client_set_temp_data = None\n",
    "        client_set_temp_targets = None\n",
    "        for i in range(setnum_perclient):\n",
    "            # w/o repeat\n",
    "            this_set_temp_data = client_train_sets[n].dataset.data[this_U_sets[i]]\n",
    "\n",
    "            # surrogate label as set index\n",
    "            this_set_temp_targets = torch.ones(len(this_U_sets[i])) * this_labels[i]\n",
    "\n",
    "            # concatenate data and labels\n",
    "            if i == 0:\n",
    "                client_set_temp_data = this_set_temp_data\n",
    "                client_set_temp_targets = this_set_temp_targets\n",
    "            else:\n",
    "                client_set_temp_data = torch.cat((client_set_temp_data, this_set_temp_data))\n",
    "                client_set_temp_targets = torch.cat((client_set_temp_targets, this_set_temp_targets))\n",
    "\n",
    "        # store different clients' data and labels in a dict, for further load\n",
    "        client_train_data.append({'images': client_set_temp_data, 'labels': client_set_temp_targets})\n",
    "        client_test_data.append({'images': test_data[test_client_idxs[n]],\n",
    "                                 'labels': test_targets[test_client_idxs[n]]})\n",
    "        client_validation_data.append({'images': validation_data[validation_client_idxs[n]],\n",
    "                                       'labels': validation_targets[validation_client_idxs[n]]})\n",
    "\n",
    "    return client_train_data, client_validation_data, client_test_data, client_prior_test, client_priors_corr, client_Pi\n",
    "\n",
    "class BaiscDataset(Dataset):\n",
    "    def __init__(self, data, transform=None):\n",
    "        self.transform = transform\n",
    "        self.images = data['images']\n",
    "        self.labels = data['labels']\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.images.shape[0]\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        image = self.images[idx]\n",
    "        label = self.labels[idx]\n",
    "\n",
    "        if self.transform is not None:\n",
    "            image = self.transform(transforms.ToPILImage()(image))\n",
    "\n",
    "        return image, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dd61bde8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-09T15:55:42.598865Z",
     "iopub.status.busy": "2022-11-09T15:55:42.598089Z",
     "iopub.status.idle": "2022-11-09T15:55:42.620632Z",
     "shell.execute_reply": "2022-11-09T15:55:42.619687Z"
    },
    "papermill": {
     "duration": 0.028504,
     "end_time": "2022-11-09T15:55:42.622610",
     "exception": false,
     "start_time": "2022-11-09T15:55:42.594106",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "\n",
    "\n",
    "import torch\n",
    "from torch import nn, optim\n",
    "import copy\n",
    "\n",
    "import argparse\n",
    "import numpy as np\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "\n",
    "\n",
    "def setup_seed(seed):\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "\n",
    "\n",
    "def prepare_data(args):\n",
    "    # Data Augmentation\n",
    "    train_transform = transforms.Compose([\n",
    "        transforms.RandomHorizontalFlip(p=0.5),\n",
    "        transforms.Grayscale(num_output_channels=3),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n",
    "    ])\n",
    "    test_transform = transforms.Compose([\n",
    "        transforms.Grayscale(num_output_channels=3),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n",
    "    ])\n",
    "\n",
    "    # Get splited data\n",
    "    client_train_data, client_validation_data, client_test_data, prior_test, client_priors_corr, client_Pi = \\\n",
    "        RSNA_PL(\n",
    "            data_path=\"./data\",\n",
    "            clientnum=args.clientnum,\n",
    "            setnum_perclient=args.setnum,\n",
    "            noniid=args.noniid)\n",
    "\n",
    "    # get dataloaders\n",
    "    train_loaders = []\n",
    "    validation_loaders = []\n",
    "    test_loaders = []\n",
    "\n",
    "    for i, this_client_data in enumerate(client_train_data):\n",
    "        this_train_set = BaiscDataset(client_train_data[i], transform=train_transform)\n",
    "        this_validation_set = BaiscDataset(client_validation_data[i], transform=test_transform)\n",
    "        this_test_set = BaiscDataset(client_test_data[i], transform=test_transform)\n",
    "\n",
    "        train_loaders.append(torch.utils.data.DataLoader(this_train_set, batch_size=args.batch, shuffle=True))\n",
    "        validation_loaders.append(\n",
    "            torch.utils.data.DataLoader(this_validation_set, batch_size=args.batch * 5, shuffle=False))\n",
    "        test_loaders.append(torch.utils.data.DataLoader(this_test_set, batch_size=args.batch * 5, shuffle=False))\n",
    "\n",
    "    return train_loaders, validation_loaders, test_loaders, prior_test, client_priors_corr, client_Pi\n",
    "\n",
    "\n",
    "def train(model, train_loader, optimizer, loss_fun, client_num, device, Pi, priors_corr, prior_test, a_iter):\n",
    "    model.train()\n",
    "    num_data = 0\n",
    "    correct = 0\n",
    "    loss_all = 0\n",
    "    train_iter = iter(train_loader)\n",
    "    for step in range(len(train_iter)):\n",
    "        optimizer.zero_grad()\n",
    "        x, y = next(train_iter)\n",
    "        num_data += y.size(0)\n",
    "        x = x.to(device).float()\n",
    "        y = y.to(device).long()\n",
    "        output = model(x, Pi, priors_corr, prior_test)\n",
    "\n",
    "        if a_iter >= 5:\n",
    "            tau = 0.4\n",
    "            lam = np.random.beta(0.75, 0.75)\n",
    "\n",
    "            y = torch.softmax(output, dim=1).max(1).indices\n",
    "            prob = torch.softmax(output, dim=1).max(1).values\n",
    "\n",
    "            output_plus = output[prob >= tau]\n",
    "            y_plus = y[prob >= tau]\n",
    "            output_minus = output[prob < tau]\n",
    "            y_minus = y[prob < tau]\n",
    "\n",
    "            max_samples = min(len(y_plus), len(y_minus))\n",
    "\n",
    "            if max_samples < 1:\n",
    "                continue\n",
    "\n",
    "            output = lam * output_plus[:max_samples] + (1 - lam) * output_minus[:max_samples]\n",
    "\n",
    "            l_fix = loss_fun(output_plus, y_plus)\n",
    "            l_mix = lam * loss_fun(output, y_plus[:max_samples]) + (1 - lam) * loss_fun(output, y_minus[:max_samples])\n",
    "\n",
    "            loss = l_fix + l_mix * 0.3\n",
    "            loss.backward()\n",
    "            loss_all += loss.item()\n",
    "\n",
    "            optimizer.step()\n",
    "\n",
    "        else:\n",
    "            loss = loss_fun(output, y)\n",
    "            loss.backward()\n",
    "            loss_all += loss.item()\n",
    "\n",
    "            optimizer.step()\n",
    "\n",
    "    return loss_all / len(train_iter)\n",
    "\n",
    "\n",
    "def test(model, test_loader, loss_fun, device, classnum=10):\n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for data, target in test_loader:\n",
    "            data = data.to(device).float()\n",
    "            target = target.to(device).long()\n",
    "\n",
    "            output = model.predict(data)\n",
    "\n",
    "            pred = output.data.max(1)[1]\n",
    "            correct += pred.eq(target.view(-1)).sum().item()\n",
    "            total += target.size(0)\n",
    "\n",
    "    test_error = (total - correct) / total\n",
    "\n",
    "    return test_error\n",
    "\n",
    "\n",
    "def communication(args, server_model, models, client_weights):\n",
    "    with torch.no_grad():\n",
    "        # aggregate params\n",
    "        for key in server_model.state_dict().keys():\n",
    "            # num_batches_tracked is a non trainable LongTensor and\n",
    "            # num_batches_tracked are the same for all clients for the given datasets\n",
    "            if 'num_batches_tracked' in key:\n",
    "                server_model.state_dict()[key].data.copy_(models[0].state_dict()[key])\n",
    "            else:\n",
    "                temp = torch.zeros_like(server_model.state_dict()[key])\n",
    "                for client_idx in range(len(client_weights)):\n",
    "                    temp += client_weights[client_idx] * models[client_idx].state_dict()[key]\n",
    "                server_model.state_dict()[key].data.copy_(temp)\n",
    "                for client_idx in range(len(client_weights)):\n",
    "                    models[client_idx].state_dict()[key].data.copy_(server_model.state_dict()[key])\n",
    "    return server_model, models\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a630fcf0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-09T15:55:42.630010Z",
     "iopub.status.busy": "2022-11-09T15:55:42.629739Z",
     "iopub.status.idle": "2022-11-09T18:56:57.637434Z",
     "shell.execute_reply": "2022-11-09T18:56:57.636152Z"
    },
    "papermill": {
     "duration": 10875.014587,
     "end_time": "2022-11-09T18:56:57.640087",
     "exception": false,
     "start_time": "2022-11-09T15:55:42.625500",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device: cuda \n",
      "\n",
      "Namespace(batch=128, classnum=6, clientnum=5, iters=100, lr=0.0001, mode='fedavg', noniid=False, save_path='./checkpoint12/RSNA', seed=0, setnum=12, test=False, wdecay=2e-06, wk_iters=1)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://download.pytorch.org/models/resnet18-f37072fd.pth\" to /root/.cache/torch/hub/checkpoints/resnet18-f37072fd.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a28a74e430447ccab17d6242ea88fb9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0.00/44.7M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 / 106000\n",
      "10000 / 106000\n",
      "20000 / 106000\n",
      "30000 / 106000\n",
      "40000 / 106000\n",
      "50000 / 106000\n",
      "60000 / 106000\n",
      "70000 / 106000\n",
      "80000 / 106000\n",
      "90000 / 106000\n",
      "100000 / 106000\n",
      "Spliting U sets for 5 clients, each with 12 U sets...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5/5 [00:08<00:00,  1.71s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Data prepared, start training...\n",
      "\n",
      "============ Train epoch 1 ============\n",
      " client1 | Train Loss: 1.7577\n",
      " client2 | Train Loss: 1.7598\n",
      " client3 | Train Loss: 1.7673\n",
      " client4 | Train Loss: 1.7627\n",
      " client5 | Train Loss: 1.7620\n",
      " client1 | Error Rate: 64.84 %\n",
      " client2 | Error Rate: 64.92 %\n",
      " client3 | Error Rate: 65.36 %\n",
      " client4 | Error Rate: 64.19 %\n",
      " client5 | Error Rate: 64.66 %\n",
      "\n",
      " Saving checkpoints to ./checkpoint12/RSNA/rsna_fedpl/fedavgclient5sets12seed0False\n",
      " Best Validation Error Rate: 64.80 %, Current Validation Error Rate: 64.80 %\n",
      "\n",
      "============ Train epoch 2 ============\n",
      " client1 | Train Loss: 1.7240\n",
      " client2 | Train Loss: 1.7349\n",
      " client3 | Train Loss: 1.7365\n",
      " client4 | Train Loss: 1.7371\n",
      " client5 | Train Loss: 1.7309\n",
      " client1 | Error Rate: 64.53 %\n",
      " client2 | Error Rate: 64.30 %\n",
      " client3 | Error Rate: 65.02 %\n",
      " client4 | Error Rate: 63.47 %\n",
      " client5 | Error Rate: 64.79 %\n",
      "\n",
      " Saving checkpoints to ./checkpoint12/RSNA/rsna_fedpl/fedavgclient5sets12seed0False\n",
      " Best Validation Error Rate: 64.42 %, Current Validation Error Rate: 64.42 %\n",
      "\n",
      "============ Train epoch 3 ============\n",
      " client1 | Train Loss: 1.7149\n",
      " client2 | Train Loss: 1.7140\n",
      " client3 | Train Loss: 1.7227\n",
      " client4 | Train Loss: 1.7170\n",
      " client5 | Train Loss: 1.7144\n",
      " client1 | Error Rate: 63.41 %\n",
      " client2 | Error Rate: 63.08 %\n",
      " client3 | Error Rate: 64.09 %\n",
      " client4 | Error Rate: 62.40 %\n",
      " client5 | Error Rate: 63.67 %\n",
      "\n",
      " Saving checkpoints to ./checkpoint12/RSNA/rsna_fedpl/fedavgclient5sets12seed0False\n",
      " Best Validation Error Rate: 63.33 %, Current Validation Error Rate: 63.33 %\n",
      "\n",
      "============ Train epoch 4 ============\n",
      " client1 | Train Loss: 1.6954\n",
      " client2 | Train Loss: 1.7001\n",
      " client3 | Train Loss: 1.7137\n",
      " client4 | Train Loss: 1.6994\n",
      " client5 | Train Loss: 1.7039\n",
      " client1 | Error Rate: 62.04 %\n",
      " client2 | Error Rate: 62.77 %\n",
      " client3 | Error Rate: 62.38 %\n",
      " client4 | Error Rate: 61.60 %\n",
      " client5 | Error Rate: 61.83 %\n",
      "\n",
      " Saving checkpoints to ./checkpoint12/RSNA/rsna_fedpl/fedavgclient5sets12seed0False\n",
      " Best Validation Error Rate: 62.12 %, Current Validation Error Rate: 62.12 %\n",
      "\n",
      "============ Train epoch 5 ============\n",
      " client1 | Train Loss: 1.6798\n",
      " client2 | Train Loss: 1.6899\n",
      " client3 | Train Loss: 1.7050\n",
      " client4 | Train Loss: 1.6991\n",
      " client5 | Train Loss: 1.6923\n",
      " client1 | Error Rate: 60.35 %\n",
      " client2 | Error Rate: 60.72 %\n",
      " client3 | Error Rate: 61.29 %\n",
      " client4 | Error Rate: 60.09 %\n",
      " client5 | Error Rate: 61.44 %\n",
      "\n",
      " Saving checkpoints to ./checkpoint12/RSNA/rsna_fedpl/fedavgclient5sets12seed0False\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 60.78 %\n",
      "\n",
      "============ Train epoch 6 ============\n",
      " client1 | Train Loss: 0.3072\n",
      " client2 | Train Loss: 0.2779\n",
      " client3 | Train Loss: 0.3195\n",
      " client4 | Train Loss: 0.3006\n",
      " client5 | Train Loss: 0.2384\n",
      " client1 | Error Rate: 72.63 %\n",
      " client2 | Error Rate: 72.83 %\n",
      " client3 | Error Rate: 72.65 %\n",
      " client4 | Error Rate: 73.20 %\n",
      " client5 | Error Rate: 72.94 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.85 %\n",
      "\n",
      "============ Train epoch 7 ============\n",
      " client1 | Train Loss: 0.1443\n",
      " client2 | Train Loss: 0.0617\n",
      " client3 | Train Loss: 0.1414\n",
      " client4 | Train Loss: 0.1457\n",
      " client5 | Train Loss: 0.1128\n",
      " client1 | Error Rate: 71.95 %\n",
      " client2 | Error Rate: 72.60 %\n",
      " client3 | Error Rate: 72.52 %\n",
      " client4 | Error Rate: 72.86 %\n",
      " client5 | Error Rate: 72.39 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.46 %\n",
      "\n",
      "============ Train epoch 8 ============\n",
      " client1 | Train Loss: 0.1074\n",
      " client2 | Train Loss: 0.0789\n",
      " client3 | Train Loss: 0.0461\n",
      " client4 | Train Loss: 0.0951\n",
      " client5 | Train Loss: 0.1739\n",
      " client1 | Error Rate: 73.27 %\n",
      " client2 | Error Rate: 73.14 %\n",
      " client3 | Error Rate: 72.63 %\n",
      " client4 | Error Rate: 72.96 %\n",
      " client5 | Error Rate: 72.86 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.97 %\n",
      "\n",
      "============ Train epoch 9 ============\n",
      " client1 | Train Loss: 0.0191\n",
      " client2 | Train Loss: 0.0714\n",
      " client3 | Train Loss: 0.0859\n",
      " client4 | Train Loss: 0.0703\n",
      " client5 | Train Loss: 0.0098\n",
      " client1 | Error Rate: 72.11 %\n",
      " client2 | Error Rate: 72.68 %\n",
      " client3 | Error Rate: 71.93 %\n",
      " client4 | Error Rate: 72.31 %\n",
      " client5 | Error Rate: 71.69 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.14 %\n",
      "\n",
      "============ Train epoch 10 ============\n",
      " client1 | Train Loss: 0.0332\n",
      " client2 | Train Loss: 0.0951\n",
      " client3 | Train Loss: 0.0130\n",
      " client4 | Train Loss: 0.0211\n",
      " client5 | Train Loss: 0.0461\n",
      " client1 | Error Rate: 72.83 %\n",
      " client2 | Error Rate: 72.73 %\n",
      " client3 | Error Rate: 72.63 %\n",
      " client4 | Error Rate: 72.96 %\n",
      " client5 | Error Rate: 72.42 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.71 %\n",
      "\n",
      "============ Train epoch 11 ============\n",
      " client1 | Train Loss: 0.0337\n",
      " client2 | Train Loss: 0.0143\n",
      " client3 | Train Loss: 0.0240\n",
      " client4 | Train Loss: 0.0431\n",
      " client5 | Train Loss: 0.0103\n",
      " client1 | Error Rate: 72.29 %\n",
      " client2 | Error Rate: 72.50 %\n",
      " client3 | Error Rate: 72.21 %\n",
      " client4 | Error Rate: 72.91 %\n",
      " client5 | Error Rate: 72.26 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.43 %\n",
      "\n",
      "============ Train epoch 12 ============\n",
      " client1 | Train Loss: 0.0227\n",
      " client2 | Train Loss: 0.0546\n",
      " client3 | Train Loss: 0.0549\n",
      " client4 | Train Loss: 0.0084\n",
      " client5 | Train Loss: 0.0460\n",
      " client1 | Error Rate: 72.94 %\n",
      " client2 | Error Rate: 72.86 %\n",
      " client3 | Error Rate: 72.63 %\n",
      " client4 | Error Rate: 73.17 %\n",
      " client5 | Error Rate: 72.26 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.77 %\n",
      "\n",
      "============ Train epoch 13 ============\n",
      " client1 | Train Loss: 0.0069\n",
      " client2 | Train Loss: 0.0420\n",
      " client3 | Train Loss: 0.0161\n",
      " client4 | Train Loss: 0.0338\n",
      " client5 | Train Loss: 0.0616\n",
      " client1 | Error Rate: 72.42 %\n",
      " client2 | Error Rate: 72.70 %\n",
      " client3 | Error Rate: 71.93 %\n",
      " client4 | Error Rate: 72.78 %\n",
      " client5 | Error Rate: 72.29 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.42 %\n",
      "\n",
      "============ Train epoch 14 ============\n",
      " client1 | Train Loss: 0.0338\n",
      " client2 | Train Loss: 0.0946\n",
      " client3 | Train Loss: 0.0261\n",
      " client4 | Train Loss: 0.0325\n",
      " client5 | Train Loss: 0.0220\n",
      " client1 | Error Rate: 72.52 %\n",
      " client2 | Error Rate: 72.63 %\n",
      " client3 | Error Rate: 71.87 %\n",
      " client4 | Error Rate: 72.86 %\n",
      " client5 | Error Rate: 72.06 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.39 %\n",
      "\n",
      "============ Train epoch 15 ============\n",
      " client1 | Train Loss: 0.0923\n",
      " client2 | Train Loss: 0.0461\n",
      " client3 | Train Loss: 0.0415\n",
      " client4 | Train Loss: 0.0041\n",
      " client5 | Train Loss: 0.0016\n",
      " client1 | Error Rate: 72.16 %\n",
      " client2 | Error Rate: 72.44 %\n",
      " client3 | Error Rate: 72.21 %\n",
      " client4 | Error Rate: 72.83 %\n",
      " client5 | Error Rate: 72.34 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.40 %\n",
      "\n",
      "============ Train epoch 16 ============\n",
      " client1 | Train Loss: 0.0394\n",
      " client2 | Train Loss: 0.0418\n",
      " client3 | Train Loss: 0.0399\n",
      " client4 | Train Loss: 0.0652\n",
      " client5 | Train Loss: 0.0038\n",
      " client1 | Error Rate: 71.87 %\n",
      " client2 | Error Rate: 72.13 %\n",
      " client3 | Error Rate: 71.59 %\n",
      " client4 | Error Rate: 72.39 %\n",
      " client5 | Error Rate: 71.90 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 71.98 %\n",
      "\n",
      "============ Train epoch 17 ============\n",
      " client1 | Train Loss: 0.0293\n",
      " client2 | Train Loss: 0.0151\n",
      " client3 | Train Loss: 0.0298\n",
      " client4 | Train Loss: 0.0658\n",
      " client5 | Train Loss: 0.0370\n",
      " client1 | Error Rate: 73.09 %\n",
      " client2 | Error Rate: 74.03 %\n",
      " client3 | Error Rate: 73.38 %\n",
      " client4 | Error Rate: 74.21 %\n",
      " client5 | Error Rate: 73.48 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 73.64 %\n",
      "\n",
      "============ Train epoch 18 ============\n",
      " client1 | Train Loss: 0.0334\n",
      " client2 | Train Loss: 0.0649\n",
      " client3 | Train Loss: 0.0168\n",
      " client4 | Train Loss: 0.0224\n",
      " client5 | Train Loss: 0.0524\n",
      " client1 | Error Rate: 73.22 %\n",
      " client2 | Error Rate: 74.10 %\n",
      " client3 | Error Rate: 73.27 %\n",
      " client4 | Error Rate: 74.10 %\n",
      " client5 | Error Rate: 73.46 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 73.63 %\n",
      "\n",
      "============ Train epoch 19 ============\n",
      " client1 | Train Loss: 0.0576\n",
      " client2 | Train Loss: 0.0187\n",
      " client3 | Train Loss: 0.0571\n",
      " client4 | Train Loss: 0.0199\n",
      " client5 | Train Loss: 0.0217\n",
      " client1 | Error Rate: 72.94 %\n",
      " client2 | Error Rate: 73.69 %\n",
      " client3 | Error Rate: 72.99 %\n",
      " client4 | Error Rate: 73.77 %\n",
      " client5 | Error Rate: 72.99 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 73.27 %\n",
      "\n",
      "============ Train epoch 20 ============\n",
      " client1 | Train Loss: 0.0134\n",
      " client2 | Train Loss: 0.0484\n",
      " client3 | Train Loss: 0.0299\n",
      " client4 | Train Loss: 0.0583\n",
      " client5 | Train Loss: 0.0793\n",
      " client1 | Error Rate: 71.82 %\n",
      " client2 | Error Rate: 71.95 %\n",
      " client3 | Error Rate: 71.85 %\n",
      " client4 | Error Rate: 72.42 %\n",
      " client5 | Error Rate: 71.69 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 71.95 %\n",
      "\n",
      "============ Train epoch 21 ============\n",
      " client1 | Train Loss: 0.0614\n",
      " client2 | Train Loss: 0.0381\n",
      " client3 | Train Loss: 0.0219\n",
      " client4 | Train Loss: 0.0657\n",
      " client5 | Train Loss: 0.0385\n",
      " client1 | Error Rate: 71.59 %\n",
      " client2 | Error Rate: 72.03 %\n",
      " client3 | Error Rate: 71.48 %\n",
      " client4 | Error Rate: 72.16 %\n",
      " client5 | Error Rate: 71.61 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 71.77 %\n",
      "\n",
      "============ Train epoch 22 ============\n",
      " client1 | Train Loss: 0.0175\n",
      " client2 | Train Loss: 0.0323\n",
      " client3 | Train Loss: 0.0126\n",
      " client4 | Train Loss: 0.0235\n",
      " client5 | Train Loss: 0.0242\n",
      " client1 | Error Rate: 72.37 %\n",
      " client2 | Error Rate: 72.60 %\n",
      " client3 | Error Rate: 71.93 %\n",
      " client4 | Error Rate: 72.83 %\n",
      " client5 | Error Rate: 72.24 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.39 %\n",
      "\n",
      "============ Train epoch 23 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0311\n",
      " client3 | Train Loss: 0.0370\n",
      " client4 | Train Loss: 0.0269\n",
      " client5 | Train Loss: 0.0706\n",
      " client1 | Error Rate: 72.21 %\n",
      " client2 | Error Rate: 72.60 %\n",
      " client3 | Error Rate: 71.74 %\n",
      " client4 | Error Rate: 72.52 %\n",
      " client5 | Error Rate: 72.16 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.25 %\n",
      "\n",
      "============ Train epoch 24 ============\n",
      " client1 | Train Loss: 0.0071\n",
      " client2 | Train Loss: 0.0267\n",
      " client3 | Train Loss: 0.0214\n",
      " client4 | Train Loss: 0.0321\n",
      " client5 | Train Loss: 0.0483\n",
      " client1 | Error Rate: 71.33 %\n",
      " client2 | Error Rate: 72.11 %\n",
      " client3 | Error Rate: 71.59 %\n",
      " client4 | Error Rate: 72.24 %\n",
      " client5 | Error Rate: 71.93 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 71.84 %\n",
      "\n",
      "============ Train epoch 25 ============\n",
      " client1 | Train Loss: 0.0238\n",
      " client2 | Train Loss: 0.0480\n",
      " client3 | Train Loss: 0.0393\n",
      " client4 | Train Loss: 0.0166\n",
      " client5 | Train Loss: 0.0082\n",
      " client1 | Error Rate: 71.61 %\n",
      " client2 | Error Rate: 72.73 %\n",
      " client3 | Error Rate: 71.54 %\n",
      " client4 | Error Rate: 72.11 %\n",
      " client5 | Error Rate: 72.21 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.04 %\n",
      "\n",
      "============ Train epoch 26 ============\n",
      " client1 | Train Loss: 0.0051\n",
      " client2 | Train Loss: 0.0777\n",
      " client3 | Train Loss: 0.0114\n",
      " client4 | Train Loss: 0.0184\n",
      " client5 | Train Loss: 0.0153\n",
      " client1 | Error Rate: 71.56 %\n",
      " client2 | Error Rate: 72.47 %\n",
      " client3 | Error Rate: 71.90 %\n",
      " client4 | Error Rate: 72.18 %\n",
      " client5 | Error Rate: 72.34 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.09 %\n",
      "\n",
      "============ Train epoch 27 ============\n",
      " client1 | Train Loss: 0.0166\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0558\n",
      " client4 | Train Loss: 0.0194\n",
      " client5 | Train Loss: 0.0152\n",
      " client1 | Error Rate: 71.74 %\n",
      " client2 | Error Rate: 72.47 %\n",
      " client3 | Error Rate: 72.00 %\n",
      " client4 | Error Rate: 72.37 %\n",
      " client5 | Error Rate: 72.39 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.20 %\n",
      "\n",
      "============ Train epoch 28 ============\n",
      " client1 | Train Loss: 0.0226\n",
      " client2 | Train Loss: 0.0439\n",
      " client3 | Train Loss: 0.0018\n",
      " client4 | Train Loss: 0.0241\n",
      " client5 | Train Loss: 0.0086\n",
      " client1 | Error Rate: 71.69 %\n",
      " client2 | Error Rate: 72.44 %\n",
      " client3 | Error Rate: 71.80 %\n",
      " client4 | Error Rate: 72.31 %\n",
      " client5 | Error Rate: 72.37 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.12 %\n",
      "\n",
      "============ Train epoch 29 ============\n",
      " client1 | Train Loss: 0.0053\n",
      " client2 | Train Loss: 0.0127\n",
      " client3 | Train Loss: 0.0115\n",
      " client4 | Train Loss: 0.0334\n",
      " client5 | Train Loss: 0.0553\n",
      " client1 | Error Rate: 72.08 %\n",
      " client2 | Error Rate: 72.76 %\n",
      " client3 | Error Rate: 71.80 %\n",
      " client4 | Error Rate: 72.34 %\n",
      " client5 | Error Rate: 72.18 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.23 %\n",
      "\n",
      "============ Train epoch 30 ============\n",
      " client1 | Train Loss: 0.0133\n",
      " client2 | Train Loss: 0.0485\n",
      " client3 | Train Loss: 0.0258\n",
      " client4 | Train Loss: 0.0631\n",
      " client5 | Train Loss: 0.0286\n",
      " client1 | Error Rate: 72.08 %\n",
      " client2 | Error Rate: 72.76 %\n",
      " client3 | Error Rate: 72.18 %\n",
      " client4 | Error Rate: 72.39 %\n",
      " client5 | Error Rate: 72.21 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.32 %\n",
      "\n",
      "============ Train epoch 31 ============\n",
      " client1 | Train Loss: 0.0285\n",
      " client2 | Train Loss: 0.0361\n",
      " client3 | Train Loss: 0.0460\n",
      " client4 | Train Loss: 0.0121\n",
      " client5 | Train Loss: 0.0389\n",
      " client1 | Error Rate: 71.90 %\n",
      " client2 | Error Rate: 72.94 %\n",
      " client3 | Error Rate: 72.47 %\n",
      " client4 | Error Rate: 72.68 %\n",
      " client5 | Error Rate: 72.47 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.49 %\n",
      "\n",
      "============ Train epoch 32 ============\n",
      " client1 | Train Loss: 0.0073\n",
      " client2 | Train Loss: 0.0234\n",
      " client3 | Train Loss: 0.0360\n",
      " client4 | Train Loss: 0.0206\n",
      " client5 | Train Loss: 0.0220\n",
      " client1 | Error Rate: 71.93 %\n",
      " client2 | Error Rate: 73.12 %\n",
      " client3 | Error Rate: 72.26 %\n",
      " client4 | Error Rate: 72.68 %\n",
      " client5 | Error Rate: 72.37 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.47 %\n",
      "\n",
      "============ Train epoch 33 ============\n",
      " client1 | Train Loss: 0.0240\n",
      " client2 | Train Loss: 0.0320\n",
      " client3 | Train Loss: 0.0424\n",
      " client4 | Train Loss: 0.0133\n",
      " client5 | Train Loss: 0.0276\n",
      " client1 | Error Rate: 71.85 %\n",
      " client2 | Error Rate: 73.02 %\n",
      " client3 | Error Rate: 72.31 %\n",
      " client4 | Error Rate: 72.68 %\n",
      " client5 | Error Rate: 72.13 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.40 %\n",
      "\n",
      "============ Train epoch 34 ============\n",
      " client1 | Train Loss: 0.0228\n",
      " client2 | Train Loss: 0.0720\n",
      " client3 | Train Loss: 0.0580\n",
      " client4 | Train Loss: 0.0129\n",
      " client5 | Train Loss: 0.0118\n",
      " client1 | Error Rate: 71.59 %\n",
      " client2 | Error Rate: 72.96 %\n",
      " client3 | Error Rate: 71.87 %\n",
      " client4 | Error Rate: 72.39 %\n",
      " client5 | Error Rate: 71.80 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.12 %\n",
      "\n",
      "============ Train epoch 35 ============\n",
      " client1 | Train Loss: 0.0219\n",
      " client2 | Train Loss: 0.0276\n",
      " client3 | Train Loss: 0.0235\n",
      " client4 | Train Loss: 0.0445\n",
      " client5 | Train Loss: 0.0114\n",
      " client1 | Error Rate: 70.94 %\n",
      " client2 | Error Rate: 72.39 %\n",
      " client3 | Error Rate: 71.38 %\n",
      " client4 | Error Rate: 72.13 %\n",
      " client5 | Error Rate: 71.46 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 71.66 %\n",
      "\n",
      "============ Train epoch 36 ============\n",
      " client1 | Train Loss: 0.0283\n",
      " client2 | Train Loss: 0.0812\n",
      " client3 | Train Loss: 0.0230\n",
      " client4 | Train Loss: 0.0419\n",
      " client5 | Train Loss: 0.0542\n",
      " client1 | Error Rate: 70.76 %\n",
      " client2 | Error Rate: 73.12 %\n",
      " client3 | Error Rate: 71.90 %\n",
      " client4 | Error Rate: 72.70 %\n",
      " client5 | Error Rate: 72.21 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.14 %\n",
      "\n",
      "============ Train epoch 37 ============\n",
      " client1 | Train Loss: 0.0154\n",
      " client2 | Train Loss: 0.0119\n",
      " client3 | Train Loss: 0.0499\n",
      " client4 | Train Loss: 0.0074\n",
      " client5 | Train Loss: 0.0355\n",
      " client1 | Error Rate: 71.69 %\n",
      " client2 | Error Rate: 73.33 %\n",
      " client3 | Error Rate: 72.83 %\n",
      " client4 | Error Rate: 73.02 %\n",
      " client5 | Error Rate: 72.31 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.64 %\n",
      "\n",
      "============ Train epoch 38 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0128\n",
      " client3 | Train Loss: 0.0129\n",
      " client4 | Train Loss: 0.0330\n",
      " client5 | Train Loss: 0.0341\n",
      " client1 | Error Rate: 71.61 %\n",
      " client2 | Error Rate: 73.07 %\n",
      " client3 | Error Rate: 72.50 %\n",
      " client4 | Error Rate: 72.89 %\n",
      " client5 | Error Rate: 72.26 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.46 %\n",
      "\n",
      "============ Train epoch 39 ============\n",
      " client1 | Train Loss: 0.0149\n",
      " client2 | Train Loss: 0.0163\n",
      " client3 | Train Loss: 0.0340\n",
      " client4 | Train Loss: 0.0014\n",
      " client5 | Train Loss: 0.0262\n",
      " client1 | Error Rate: 71.17 %\n",
      " client2 | Error Rate: 72.99 %\n",
      " client3 | Error Rate: 72.18 %\n",
      " client4 | Error Rate: 72.63 %\n",
      " client5 | Error Rate: 72.13 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.22 %\n",
      "\n",
      "============ Train epoch 40 ============\n",
      " client1 | Train Loss: 0.0344\n",
      " client2 | Train Loss: 0.0462\n",
      " client3 | Train Loss: 0.0540\n",
      " client4 | Train Loss: 0.0120\n",
      " client5 | Train Loss: 0.0051\n",
      " client1 | Error Rate: 71.82 %\n",
      " client2 | Error Rate: 73.46 %\n",
      " client3 | Error Rate: 72.89 %\n",
      " client4 | Error Rate: 73.09 %\n",
      " client5 | Error Rate: 72.50 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.75 %\n",
      "\n",
      "============ Train epoch 41 ============\n",
      " client1 | Train Loss: 0.0258\n",
      " client2 | Train Loss: 0.0017\n",
      " client3 | Train Loss: 0.0385\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0591\n",
      " client1 | Error Rate: 72.60 %\n",
      " client2 | Error Rate: 74.16 %\n",
      " client3 | Error Rate: 73.48 %\n",
      " client4 | Error Rate: 73.90 %\n",
      " client5 | Error Rate: 73.38 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 73.50 %\n",
      "\n",
      "============ Train epoch 42 ============\n",
      " client1 | Train Loss: 0.0359\n",
      " client2 | Train Loss: 0.0459\n",
      " client3 | Train Loss: 0.0116\n",
      " client4 | Train Loss: 0.0141\n",
      " client5 | Train Loss: 0.0437\n",
      " client1 | Error Rate: 72.47 %\n",
      " client2 | Error Rate: 73.48 %\n",
      " client3 | Error Rate: 72.91 %\n",
      " client4 | Error Rate: 73.02 %\n",
      " client5 | Error Rate: 72.99 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 72.97 %\n",
      "\n",
      "============ Train epoch 43 ============\n",
      " client1 | Train Loss: 0.0358\n",
      " client2 | Train Loss: 0.0016\n",
      " client3 | Train Loss: 0.0309\n",
      " client4 | Train Loss: 0.0269\n",
      " client5 | Train Loss: 0.0230\n",
      " client1 | Error Rate: 72.73 %\n",
      " client2 | Error Rate: 73.51 %\n",
      " client3 | Error Rate: 73.12 %\n",
      " client4 | Error Rate: 73.25 %\n",
      " client5 | Error Rate: 73.07 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 73.13 %\n",
      "\n",
      "============ Train epoch 44 ============\n",
      " client1 | Train Loss: 0.0229\n",
      " client2 | Train Loss: 0.0252\n",
      " client3 | Train Loss: 0.0212\n",
      " client4 | Train Loss: 0.0225\n",
      " client5 | Train Loss: 0.0426\n",
      " client1 | Error Rate: 73.04 %\n",
      " client2 | Error Rate: 73.61 %\n",
      " client3 | Error Rate: 73.40 %\n",
      " client4 | Error Rate: 73.72 %\n",
      " client5 | Error Rate: 73.38 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 73.43 %\n",
      "\n",
      "============ Train epoch 45 ============\n",
      " client1 | Train Loss: 0.0178\n",
      " client2 | Train Loss: 0.0418\n",
      " client3 | Train Loss: 0.0146\n",
      " client4 | Train Loss: 0.0012\n",
      " client5 | Train Loss: 0.0807\n",
      " client1 | Error Rate: 73.33 %\n",
      " client2 | Error Rate: 73.95 %\n",
      " client3 | Error Rate: 73.66 %\n",
      " client4 | Error Rate: 74.23 %\n",
      " client5 | Error Rate: 73.38 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 73.71 %\n",
      "\n",
      "============ Train epoch 46 ============\n",
      " client1 | Train Loss: 0.0291\n",
      " client2 | Train Loss: 0.0066\n",
      " client3 | Train Loss: 0.0086\n",
      " client4 | Train Loss: 0.0578\n",
      " client5 | Train Loss: 0.0124\n",
      " client1 | Error Rate: 73.69 %\n",
      " client2 | Error Rate: 74.03 %\n",
      " client3 | Error Rate: 73.77 %\n",
      " client4 | Error Rate: 74.44 %\n",
      " client5 | Error Rate: 73.77 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 73.94 %\n",
      "\n",
      "============ Train epoch 47 ============\n",
      " client1 | Train Loss: 0.0145\n",
      " client2 | Train Loss: 0.0036\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0066\n",
      " client5 | Train Loss: 0.0273\n",
      " client1 | Error Rate: 75.38 %\n",
      " client2 | Error Rate: 75.40 %\n",
      " client3 | Error Rate: 74.57 %\n",
      " client4 | Error Rate: 75.35 %\n",
      " client5 | Error Rate: 75.01 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 75.14 %\n",
      "\n",
      "============ Train epoch 48 ============\n",
      " client1 | Train Loss: 0.0229\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0165\n",
      " client4 | Train Loss: 0.0070\n",
      " client5 | Train Loss: 0.0044\n",
      " client1 | Error Rate: 74.55 %\n",
      " client2 | Error Rate: 74.55 %\n",
      " client3 | Error Rate: 74.29 %\n",
      " client4 | Error Rate: 74.39 %\n",
      " client5 | Error Rate: 74.47 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 74.45 %\n",
      "\n",
      "============ Train epoch 49 ============\n",
      " client1 | Train Loss: 0.0269\n",
      " client2 | Train Loss: 0.0314\n",
      " client3 | Train Loss: 0.0160\n",
      " client4 | Train Loss: 0.0091\n",
      " client5 | Train Loss: 0.0083\n",
      " client1 | Error Rate: 74.47 %\n",
      " client2 | Error Rate: 74.70 %\n",
      " client3 | Error Rate: 74.16 %\n",
      " client4 | Error Rate: 75.01 %\n",
      " client5 | Error Rate: 74.34 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 74.54 %\n",
      "\n",
      "============ Train epoch 50 ============\n",
      " client1 | Train Loss: 0.0044\n",
      " client2 | Train Loss: 0.0105\n",
      " client3 | Train Loss: 0.0043\n",
      " client4 | Train Loss: 0.0089\n",
      " client5 | Train Loss: 0.0366\n",
      " client1 | Error Rate: 74.62 %\n",
      " client2 | Error Rate: 74.75 %\n",
      " client3 | Error Rate: 74.23 %\n",
      " client4 | Error Rate: 74.75 %\n",
      " client5 | Error Rate: 74.86 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 74.64 %\n",
      "\n",
      "============ Train epoch 51 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0116\n",
      " client3 | Train Loss: 0.0350\n",
      " client4 | Train Loss: 0.0268\n",
      " client5 | Train Loss: 0.0229\n",
      " client1 | Error Rate: 74.16 %\n",
      " client2 | Error Rate: 74.29 %\n",
      " client3 | Error Rate: 73.90 %\n",
      " client4 | Error Rate: 74.60 %\n",
      " client5 | Error Rate: 74.34 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 74.26 %\n",
      "\n",
      "============ Train epoch 52 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0515\n",
      " client3 | Train Loss: 0.0193\n",
      " client4 | Train Loss: 0.0164\n",
      " client5 | Train Loss: 0.0398\n",
      " client1 | Error Rate: 75.09 %\n",
      " client2 | Error Rate: 75.30 %\n",
      " client3 | Error Rate: 74.55 %\n",
      " client4 | Error Rate: 75.17 %\n",
      " client5 | Error Rate: 75.40 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 75.10 %\n",
      "\n",
      "============ Train epoch 53 ============\n",
      " client1 | Train Loss: 0.0033\n",
      " client2 | Train Loss: 0.0582\n",
      " client3 | Train Loss: 0.0013\n",
      " client4 | Train Loss: 0.0229\n",
      " client5 | Train Loss: 0.0350\n",
      " client1 | Error Rate: 76.10 %\n",
      " client2 | Error Rate: 76.31 %\n",
      " client3 | Error Rate: 75.40 %\n",
      " client4 | Error Rate: 76.23 %\n",
      " client5 | Error Rate: 76.31 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 76.07 %\n",
      "\n",
      "============ Train epoch 54 ============\n",
      " client1 | Train Loss: 0.0583\n",
      " client2 | Train Loss: 0.0199\n",
      " client3 | Train Loss: 0.0159\n",
      " client4 | Train Loss: 0.0034\n",
      " client5 | Train Loss: 0.0370\n",
      " client1 | Error Rate: 74.36 %\n",
      " client2 | Error Rate: 74.68 %\n",
      " client3 | Error Rate: 74.08 %\n",
      " client4 | Error Rate: 74.62 %\n",
      " client5 | Error Rate: 74.94 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 74.54 %\n",
      "\n",
      "============ Train epoch 55 ============\n",
      " client1 | Train Loss: 0.0342\n",
      " client2 | Train Loss: 0.0041\n",
      " client3 | Train Loss: 0.0406\n",
      " client4 | Train Loss: 0.0270\n",
      " client5 | Train Loss: 0.0046\n",
      " client1 | Error Rate: 75.30 %\n",
      " client2 | Error Rate: 75.69 %\n",
      " client3 | Error Rate: 74.68 %\n",
      " client4 | Error Rate: 75.38 %\n",
      " client5 | Error Rate: 75.51 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 75.31 %\n",
      "\n",
      "============ Train epoch 56 ============\n",
      " client1 | Train Loss: 0.0777\n",
      " client2 | Train Loss: 0.0122\n",
      " client3 | Train Loss: 0.0377\n",
      " client4 | Train Loss: 0.0054\n",
      " client5 | Train Loss: 0.0369\n",
      " client1 | Error Rate: 74.70 %\n",
      " client2 | Error Rate: 75.27 %\n",
      " client3 | Error Rate: 74.23 %\n",
      " client4 | Error Rate: 74.55 %\n",
      " client5 | Error Rate: 74.96 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 74.74 %\n",
      "\n",
      "============ Train epoch 57 ============\n",
      " client1 | Train Loss: 0.0014\n",
      " client2 | Train Loss: 0.0121\n",
      " client3 | Train Loss: 0.0088\n",
      " client4 | Train Loss: 0.0295\n",
      " client5 | Train Loss: 0.0581\n",
      " client1 | Error Rate: 74.73 %\n",
      " client2 | Error Rate: 75.43 %\n",
      " client3 | Error Rate: 74.55 %\n",
      " client4 | Error Rate: 75.30 %\n",
      " client5 | Error Rate: 75.17 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 75.03 %\n",
      "\n",
      "============ Train epoch 58 ============\n",
      " client1 | Train Loss: 0.0242\n",
      " client2 | Train Loss: 0.0465\n",
      " client3 | Train Loss: 0.0439\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0373\n",
      " client1 | Error Rate: 73.98 %\n",
      " client2 | Error Rate: 74.42 %\n",
      " client3 | Error Rate: 73.79 %\n",
      " client4 | Error Rate: 74.29 %\n",
      " client5 | Error Rate: 74.29 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 74.15 %\n",
      "\n",
      "============ Train epoch 59 ============\n",
      " client1 | Train Loss: 0.0158\n",
      " client2 | Train Loss: 0.0074\n",
      " client3 | Train Loss: 0.0159\n",
      " client4 | Train Loss: 0.0105\n",
      " client5 | Train Loss: 0.0314\n",
      " client1 | Error Rate: 74.91 %\n",
      " client2 | Error Rate: 75.64 %\n",
      " client3 | Error Rate: 74.52 %\n",
      " client4 | Error Rate: 75.19 %\n",
      " client5 | Error Rate: 74.88 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 75.03 %\n",
      "\n",
      "============ Train epoch 60 ============\n",
      " client1 | Train Loss: 0.0252\n",
      " client2 | Train Loss: 0.0291\n",
      " client3 | Train Loss: 0.0329\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0095\n",
      " client1 | Error Rate: 75.82 %\n",
      " client2 | Error Rate: 76.39 %\n",
      " client3 | Error Rate: 75.17 %\n",
      " client4 | Error Rate: 75.82 %\n",
      " client5 | Error Rate: 75.74 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 75.79 %\n",
      "\n",
      "============ Train epoch 61 ============\n",
      " client1 | Train Loss: 0.0221\n",
      " client2 | Train Loss: 0.0400\n",
      " client3 | Train Loss: 0.0085\n",
      " client4 | Train Loss: 0.0199\n",
      " client5 | Train Loss: 0.0098\n",
      " client1 | Error Rate: 75.97 %\n",
      " client2 | Error Rate: 76.26 %\n",
      " client3 | Error Rate: 75.09 %\n",
      " client4 | Error Rate: 75.64 %\n",
      " client5 | Error Rate: 76.28 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 75.85 %\n",
      "\n",
      "============ Train epoch 62 ============\n",
      " client1 | Train Loss: 0.0081\n",
      " client2 | Train Loss: 0.0204\n",
      " client3 | Train Loss: 0.0181\n",
      " client4 | Train Loss: 0.0077\n",
      " client5 | Train Loss: 0.0383\n",
      " client1 | Error Rate: 75.61 %\n",
      " client2 | Error Rate: 76.36 %\n",
      " client3 | Error Rate: 75.61 %\n",
      " client4 | Error Rate: 76.52 %\n",
      " client5 | Error Rate: 76.13 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 76.05 %\n",
      "\n",
      "============ Train epoch 63 ============\n",
      " client1 | Train Loss: 0.0149\n",
      " client2 | Train Loss: 0.0139\n",
      " client3 | Train Loss: 0.0149\n",
      " client4 | Train Loss: 0.0460\n",
      " client5 | Train Loss: 0.0175\n",
      " client1 | Error Rate: 75.61 %\n",
      " client2 | Error Rate: 75.95 %\n",
      " client3 | Error Rate: 75.48 %\n",
      " client4 | Error Rate: 76.15 %\n",
      " client5 | Error Rate: 75.97 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 75.83 %\n",
      "\n",
      "============ Train epoch 64 ============\n",
      " client1 | Train Loss: 0.0498\n",
      " client2 | Train Loss: 0.0202\n",
      " client3 | Train Loss: 0.0161\n",
      " client4 | Train Loss: 0.0340\n",
      " client5 | Train Loss: 0.0180\n",
      " client1 | Error Rate: 76.47 %\n",
      " client2 | Error Rate: 77.22 %\n",
      " client3 | Error Rate: 76.67 %\n",
      " client4 | Error Rate: 77.09 %\n",
      " client5 | Error Rate: 77.19 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 76.93 %\n",
      "\n",
      "============ Train epoch 65 ============\n",
      " client1 | Train Loss: 0.0132\n",
      " client2 | Train Loss: 0.0083\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0012\n",
      " client5 | Train Loss: 0.0209\n",
      " client1 | Error Rate: 76.13 %\n",
      " client2 | Error Rate: 76.70 %\n",
      " client3 | Error Rate: 75.79 %\n",
      " client4 | Error Rate: 76.57 %\n",
      " client5 | Error Rate: 76.21 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 76.28 %\n",
      "\n",
      "============ Train epoch 66 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0009\n",
      " client3 | Train Loss: 0.0170\n",
      " client4 | Train Loss: 0.0134\n",
      " client5 | Train Loss: 0.0285\n",
      " client1 | Error Rate: 76.21 %\n",
      " client2 | Error Rate: 76.75 %\n",
      " client3 | Error Rate: 75.97 %\n",
      " client4 | Error Rate: 76.67 %\n",
      " client5 | Error Rate: 76.28 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 76.38 %\n",
      "\n",
      "============ Train epoch 67 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0153\n",
      " client3 | Train Loss: 0.0014\n",
      " client4 | Train Loss: 0.0169\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.19 %\n",
      " client2 | Error Rate: 77.58 %\n",
      " client3 | Error Rate: 77.04 %\n",
      " client4 | Error Rate: 77.35 %\n",
      " client5 | Error Rate: 77.45 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 77.32 %\n",
      "\n",
      "============ Train epoch 68 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0113\n",
      " client3 | Train Loss: 0.0079\n",
      " client4 | Train Loss: 0.0198\n",
      " client5 | Train Loss: 0.0014\n",
      " client1 | Error Rate: 76.88 %\n",
      " client2 | Error Rate: 77.06 %\n",
      " client3 | Error Rate: 76.60 %\n",
      " client4 | Error Rate: 77.22 %\n",
      " client5 | Error Rate: 77.09 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 76.97 %\n",
      "\n",
      "============ Train epoch 69 ============\n",
      " client1 | Train Loss: 0.0062\n",
      " client2 | Train Loss: 0.0676\n",
      " client3 | Train Loss: 0.0103\n",
      " client4 | Train Loss: 0.0183\n",
      " client5 | Train Loss: 0.0057\n",
      " client1 | Error Rate: 77.37 %\n",
      " client2 | Error Rate: 77.87 %\n",
      " client3 | Error Rate: 77.27 %\n",
      " client4 | Error Rate: 77.82 %\n",
      " client5 | Error Rate: 77.61 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 77.59 %\n",
      "\n",
      "============ Train epoch 70 ============\n",
      " client1 | Train Loss: 0.0033\n",
      " client2 | Train Loss: 0.0231\n",
      " client3 | Train Loss: 0.0050\n",
      " client4 | Train Loss: 0.0299\n",
      " client5 | Train Loss: 0.0120\n",
      " client1 | Error Rate: 77.35 %\n",
      " client2 | Error Rate: 78.23 %\n",
      " client3 | Error Rate: 77.43 %\n",
      " client4 | Error Rate: 77.94 %\n",
      " client5 | Error Rate: 77.66 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 77.72 %\n",
      "\n",
      "============ Train epoch 71 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0069\n",
      " client4 | Train Loss: 0.0018\n",
      " client5 | Train Loss: 0.0085\n",
      " client1 | Error Rate: 77.17 %\n",
      " client2 | Error Rate: 78.05 %\n",
      " client3 | Error Rate: 77.27 %\n",
      " client4 | Error Rate: 77.87 %\n",
      " client5 | Error Rate: 77.50 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 77.57 %\n",
      "\n",
      "============ Train epoch 72 ============\n",
      " client1 | Train Loss: 0.0010\n",
      " client2 | Train Loss: 0.0185\n",
      " client3 | Train Loss: 0.0190\n",
      " client4 | Train Loss: 0.0135\n",
      " client5 | Train Loss: 0.0150\n",
      " client1 | Error Rate: 77.63 %\n",
      " client2 | Error Rate: 78.41 %\n",
      " client3 | Error Rate: 77.63 %\n",
      " client4 | Error Rate: 78.07 %\n",
      " client5 | Error Rate: 78.18 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 77.99 %\n",
      "\n",
      "============ Train epoch 73 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0054\n",
      " client3 | Train Loss: 0.0103\n",
      " client4 | Train Loss: 0.0387\n",
      " client5 | Train Loss: 0.0130\n",
      " client1 | Error Rate: 78.13 %\n",
      " client2 | Error Rate: 78.80 %\n",
      " client3 | Error Rate: 78.10 %\n",
      " client4 | Error Rate: 78.57 %\n",
      " client5 | Error Rate: 78.52 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.42 %\n",
      "\n",
      "============ Train epoch 74 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0095\n",
      " client4 | Train Loss: 0.0102\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 78.28 %\n",
      " client2 | Error Rate: 79.01 %\n",
      " client3 | Error Rate: 78.28 %\n",
      " client4 | Error Rate: 78.65 %\n",
      " client5 | Error Rate: 78.78 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.60 %\n",
      "\n",
      "============ Train epoch 75 ============\n",
      " client1 | Train Loss: 0.0207\n",
      " client2 | Train Loss: 0.0011\n",
      " client3 | Train Loss: 0.0154\n",
      " client4 | Train Loss: 0.0023\n",
      " client5 | Train Loss: 0.0256\n",
      " client1 | Error Rate: 78.07 %\n",
      " client2 | Error Rate: 78.72 %\n",
      " client3 | Error Rate: 78.02 %\n",
      " client4 | Error Rate: 78.57 %\n",
      " client5 | Error Rate: 78.57 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.39 %\n",
      "\n",
      "============ Train epoch 76 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0169\n",
      " client3 | Train Loss: 0.0220\n",
      " client4 | Train Loss: 0.0179\n",
      " client5 | Train Loss: 0.0221\n",
      " client1 | Error Rate: 78.00 %\n",
      " client2 | Error Rate: 78.62 %\n",
      " client3 | Error Rate: 77.94 %\n",
      " client4 | Error Rate: 78.46 %\n",
      " client5 | Error Rate: 78.49 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.30 %\n",
      "\n",
      "============ Train epoch 77 ============\n",
      " client1 | Train Loss: 0.0034\n",
      " client2 | Train Loss: 0.0190\n",
      " client3 | Train Loss: 0.0132\n",
      " client4 | Train Loss: 0.0134\n",
      " client5 | Train Loss: 0.0219\n",
      " client1 | Error Rate: 77.76 %\n",
      " client2 | Error Rate: 78.46 %\n",
      " client3 | Error Rate: 77.79 %\n",
      " client4 | Error Rate: 78.33 %\n",
      " client5 | Error Rate: 78.20 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.11 %\n",
      "\n",
      "============ Train epoch 78 ============\n",
      " client1 | Train Loss: 0.0421\n",
      " client2 | Train Loss: 0.0141\n",
      " client3 | Train Loss: 0.0053\n",
      " client4 | Train Loss: 0.0105\n",
      " client5 | Train Loss: 0.0142\n",
      " client1 | Error Rate: 77.94 %\n",
      " client2 | Error Rate: 78.36 %\n",
      " client3 | Error Rate: 77.84 %\n",
      " client4 | Error Rate: 78.41 %\n",
      " client5 | Error Rate: 78.44 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.20 %\n",
      "\n",
      "============ Train epoch 79 ============\n",
      " client1 | Train Loss: 0.0199\n",
      " client2 | Train Loss: 0.0195\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0349\n",
      " client5 | Train Loss: 0.0260\n",
      " client1 | Error Rate: 77.58 %\n",
      " client2 | Error Rate: 78.15 %\n",
      " client3 | Error Rate: 77.22 %\n",
      " client4 | Error Rate: 78.02 %\n",
      " client5 | Error Rate: 77.76 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 77.75 %\n",
      "\n",
      "============ Train epoch 80 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0060\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0222\n",
      " client5 | Train Loss: 0.0010\n",
      " client1 | Error Rate: 77.71 %\n",
      " client2 | Error Rate: 78.23 %\n",
      " client3 | Error Rate: 77.53 %\n",
      " client4 | Error Rate: 78.20 %\n",
      " client5 | Error Rate: 78.26 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 77.99 %\n",
      "\n",
      "============ Train epoch 81 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0117\n",
      " client3 | Train Loss: 0.0095\n",
      " client4 | Train Loss: 0.0241\n",
      " client5 | Train Loss: 0.0026\n",
      " client1 | Error Rate: 77.61 %\n",
      " client2 | Error Rate: 78.13 %\n",
      " client3 | Error Rate: 77.66 %\n",
      " client4 | Error Rate: 78.26 %\n",
      " client5 | Error Rate: 77.87 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 77.90 %\n",
      "\n",
      "============ Train epoch 82 ============\n",
      " client1 | Train Loss: 0.0130\n",
      " client2 | Train Loss: 0.0428\n",
      " client3 | Train Loss: 0.0015\n",
      " client4 | Train Loss: 0.0409\n",
      " client5 | Train Loss: 0.0259\n",
      " client1 | Error Rate: 78.28 %\n",
      " client2 | Error Rate: 78.72 %\n",
      " client3 | Error Rate: 78.10 %\n",
      " client4 | Error Rate: 78.70 %\n",
      " client5 | Error Rate: 78.52 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.46 %\n",
      "\n",
      "============ Train epoch 83 ============\n",
      " client1 | Train Loss: 0.0232\n",
      " client2 | Train Loss: 0.0324\n",
      " client3 | Train Loss: 0.0065\n",
      " client4 | Train Loss: 0.0227\n",
      " client5 | Train Loss: 0.0377\n",
      " client1 | Error Rate: 78.15 %\n",
      " client2 | Error Rate: 78.80 %\n",
      " client3 | Error Rate: 78.07 %\n",
      " client4 | Error Rate: 78.75 %\n",
      " client5 | Error Rate: 78.49 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.45 %\n",
      "\n",
      "============ Train epoch 84 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0238\n",
      " client3 | Train Loss: 0.0106\n",
      " client4 | Train Loss: 0.0318\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 78.23 %\n",
      " client2 | Error Rate: 78.85 %\n",
      " client3 | Error Rate: 78.18 %\n",
      " client4 | Error Rate: 78.83 %\n",
      " client5 | Error Rate: 78.54 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.53 %\n",
      "\n",
      "============ Train epoch 85 ============\n",
      " client1 | Train Loss: 0.0219\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0133\n",
      " client4 | Train Loss: 0.0044\n",
      " client5 | Train Loss: 0.0132\n",
      " client1 | Error Rate: 78.07 %\n",
      " client2 | Error Rate: 78.80 %\n",
      " client3 | Error Rate: 78.07 %\n",
      " client4 | Error Rate: 78.57 %\n",
      " client5 | Error Rate: 78.28 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.36 %\n",
      "\n",
      "============ Train epoch 86 ============\n",
      " client1 | Train Loss: 0.0017\n",
      " client2 | Train Loss: 0.0106\n",
      " client3 | Train Loss: 0.0236\n",
      " client4 | Train Loss: 0.0147\n",
      " client5 | Train Loss: 0.0382\n",
      " client1 | Error Rate: 77.76 %\n",
      " client2 | Error Rate: 78.46 %\n",
      " client3 | Error Rate: 78.20 %\n",
      " client4 | Error Rate: 78.46 %\n",
      " client5 | Error Rate: 77.94 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.17 %\n",
      "\n",
      "============ Train epoch 87 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0046\n",
      " client3 | Train Loss: 0.0110\n",
      " client4 | Train Loss: 0.0344\n",
      " client5 | Train Loss: 0.0086\n",
      " client1 | Error Rate: 78.57 %\n",
      " client2 | Error Rate: 79.35 %\n",
      " client3 | Error Rate: 78.52 %\n",
      " client4 | Error Rate: 79.19 %\n",
      " client5 | Error Rate: 78.59 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.84 %\n",
      "\n",
      "============ Train epoch 88 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0149\n",
      " client3 | Train Loss: 0.0111\n",
      " client4 | Train Loss: 0.0042\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 78.62 %\n",
      " client2 | Error Rate: 79.27 %\n",
      " client3 | Error Rate: 78.46 %\n",
      " client4 | Error Rate: 79.53 %\n",
      " client5 | Error Rate: 78.54 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.88 %\n",
      "\n",
      "============ Train epoch 89 ============\n",
      " client1 | Train Loss: 0.0176\n",
      " client2 | Train Loss: 0.0431\n",
      " client3 | Train Loss: 0.0170\n",
      " client4 | Train Loss: 0.0146\n",
      " client5 | Train Loss: 0.0112\n",
      " client1 | Error Rate: 78.57 %\n",
      " client2 | Error Rate: 78.80 %\n",
      " client3 | Error Rate: 78.57 %\n",
      " client4 | Error Rate: 79.48 %\n",
      " client5 | Error Rate: 78.65 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.81 %\n",
      "\n",
      "============ Train epoch 90 ============\n",
      " client1 | Train Loss: 0.0130\n",
      " client2 | Train Loss: 0.0177\n",
      " client3 | Train Loss: 0.0262\n",
      " client4 | Train Loss: 0.0070\n",
      " client5 | Train Loss: 0.0145\n",
      " client1 | Error Rate: 78.65 %\n",
      " client2 | Error Rate: 79.03 %\n",
      " client3 | Error Rate: 78.98 %\n",
      " client4 | Error Rate: 79.48 %\n",
      " client5 | Error Rate: 78.98 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 79.02 %\n",
      "\n",
      "============ Train epoch 91 ============\n",
      " client1 | Train Loss: 0.0109\n",
      " client2 | Train Loss: 0.0109\n",
      " client3 | Train Loss: 0.0074\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0050\n",
      " client1 | Error Rate: 78.70 %\n",
      " client2 | Error Rate: 79.27 %\n",
      " client3 | Error Rate: 78.98 %\n",
      " client4 | Error Rate: 79.48 %\n",
      " client5 | Error Rate: 78.78 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 79.04 %\n",
      "\n",
      "============ Train epoch 92 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0270\n",
      " client3 | Train Loss: 0.0504\n",
      " client4 | Train Loss: 0.0094\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 78.28 %\n",
      " client2 | Error Rate: 79.11 %\n",
      " client3 | Error Rate: 78.75 %\n",
      " client4 | Error Rate: 79.09 %\n",
      " client5 | Error Rate: 78.54 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.75 %\n",
      "\n",
      "============ Train epoch 93 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0023\n",
      " client4 | Train Loss: 0.0074\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 78.26 %\n",
      " client2 | Error Rate: 78.98 %\n",
      " client3 | Error Rate: 78.65 %\n",
      " client4 | Error Rate: 79.24 %\n",
      " client5 | Error Rate: 78.46 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.72 %\n",
      "\n",
      "============ Train epoch 94 ============\n",
      " client1 | Train Loss: 0.0322\n",
      " client2 | Train Loss: 0.0190\n",
      " client3 | Train Loss: 0.0057\n",
      " client4 | Train Loss: 0.0060\n",
      " client5 | Train Loss: 0.0086\n",
      " client1 | Error Rate: 78.13 %\n",
      " client2 | Error Rate: 79.03 %\n",
      " client3 | Error Rate: 78.57 %\n",
      " client4 | Error Rate: 79.19 %\n",
      " client5 | Error Rate: 78.46 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.68 %\n",
      "\n",
      "============ Train epoch 95 ============\n",
      " client1 | Train Loss: 0.0259\n",
      " client2 | Train Loss: 0.0011\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 78.33 %\n",
      " client2 | Error Rate: 79.29 %\n",
      " client3 | Error Rate: 78.75 %\n",
      " client4 | Error Rate: 79.24 %\n",
      " client5 | Error Rate: 78.72 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.87 %\n",
      "\n",
      "============ Train epoch 96 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0100\n",
      " client1 | Error Rate: 78.57 %\n",
      " client2 | Error Rate: 79.29 %\n",
      " client3 | Error Rate: 78.91 %\n",
      " client4 | Error Rate: 79.42 %\n",
      " client5 | Error Rate: 78.62 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.96 %\n",
      "\n",
      "============ Train epoch 97 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0077\n",
      " client3 | Train Loss: 0.0193\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 78.33 %\n",
      " client2 | Error Rate: 79.11 %\n",
      " client3 | Error Rate: 78.65 %\n",
      " client4 | Error Rate: 79.14 %\n",
      " client5 | Error Rate: 78.52 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.75 %\n",
      "\n",
      "============ Train epoch 98 ============\n",
      " client1 | Train Loss: 0.0087\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0130\n",
      " client4 | Train Loss: 0.0185\n",
      " client5 | Train Loss: 0.0034\n",
      " client1 | Error Rate: 78.52 %\n",
      " client2 | Error Rate: 79.29 %\n",
      " client3 | Error Rate: 78.83 %\n",
      " client4 | Error Rate: 79.37 %\n",
      " client5 | Error Rate: 78.59 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.92 %\n",
      "\n",
      "============ Train epoch 99 ============\n",
      " client1 | Train Loss: 0.0074\n",
      " client2 | Train Loss: 0.0154\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 78.46 %\n",
      " client2 | Error Rate: 79.19 %\n",
      " client3 | Error Rate: 78.80 %\n",
      " client4 | Error Rate: 79.27 %\n",
      " client5 | Error Rate: 78.46 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.84 %\n",
      "\n",
      "============ Train epoch 100 ============\n",
      " client1 | Train Loss: 0.0108\n",
      " client2 | Train Loss: 0.0009\n",
      " client3 | Train Loss: 0.0200\n",
      " client4 | Train Loss: 0.0094\n",
      " client5 | Train Loss: 0.0210\n",
      " client1 | Error Rate: 78.07 %\n",
      " client2 | Error Rate: 78.75 %\n",
      " client3 | Error Rate: 78.33 %\n",
      " client4 | Error Rate: 78.59 %\n",
      " client5 | Error Rate: 78.00 %\n",
      "\n",
      " Best Validation Error Rate: 60.78 %, Current Validation Error Rate: 78.35 %\n",
      "\n",
      " Start final testing\n",
      "\n",
      " client1 | Error Rate: 60.54 %\n",
      " client2 | Error Rate: 61.54 %\n",
      " client3 | Error Rate: 63.37 %\n",
      " client4 | Error Rate: 61.74 %\n",
      " client5 | Error Rate: 61.87 %\n",
      " Best Test Error: 61.81 %\n"
     ]
    }
   ],
   "source": [
    "\n",
    "if __name__ == '__main__':\n",
    "    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "    print('Device:', device, '\\n')\n",
    "    parser = argparse.ArgumentParser()\n",
    "    parser.add_argument('--test', action='store_true', help='test the pretrained model')\n",
    "    parser.add_argument('--lr', type=float, default=1e-4, help='learning rate')\n",
    "    parser.add_argument('--wdecay', type=float, default=2e-6, help='learning rate')\n",
    "    parser.add_argument('--batch', type=int, default=128, help='batch size')\n",
    "    parser.add_argument('--iters', type=int, default=100, help='iterations for communication')\n",
    "    parser.add_argument('--wk_iters', type=int, default=1,\n",
    "                        help='optimization iters in local worker between communication')\n",
    "    parser.add_argument('--mode', type=str, default='fedavg', help='fedavg')\n",
    "    parser.add_argument('--save_path', type=str, default='./checkpoint12/RSNA', help='path to save the checkpoint')\n",
    "    parser.add_argument('--clientnum', type=int, default=5, help='client number')\n",
    "    parser.add_argument('--setnum', type=int, default=12, help='set number per client has')\n",
    "\n",
    "    parser.add_argument('--classnum', type=int, default=6, help='class num')\n",
    "    parser.add_argument('--seed', type=int, default=0, help='random seed')\n",
    "\n",
    "    parser.add_argument('--noniid', action='store_true', help='noniid sampling')\n",
    "\n",
    "    args = parser.parse_known_args()[0] # args = parser.parse_args()\n",
    "    print(args)\n",
    "\n",
    "    setup_seed(args.seed)\n",
    "\n",
    "    exp_folder = 'rsna_fedpl'\n",
    "\n",
    "    args.save_path = os.path.join(args.save_path, exp_folder)\n",
    "\n",
    "    if not os.path.exists(args.save_path):\n",
    "        os.makedirs(args.save_path)\n",
    "    SAVE_PATH = os.path.join(args.save_path,\n",
    "                             args.mode + 'client' + str(args.clientnum) + 'sets' + str(args.setnum) + 'seed' + str(\n",
    "                                 args.seed) + str(args.noniid))\n",
    "\n",
    "    # server model and ce loss\n",
    "    server_model = PLRSNAModel(class_num=args.classnum).to(device)\n",
    "    loss_fun = nn.CrossEntropyLoss()\n",
    "\n",
    "    # prepare the data\n",
    "    train_loaders, validation_loaders, test_loaders, prior_test, client_priors_corr, client_Pi = prepare_data(args)\n",
    "\n",
    "    print('\\nData prepared, start training...\\n')\n",
    "\n",
    "    # federated setting\n",
    "    client_num = args.clientnum\n",
    "    clients = ['client' + str(_) for _ in range(1, client_num + 1)]\n",
    "    client_weights = [1 / client_num for i in range(client_num)]\n",
    "    models = [copy.deepcopy(server_model).to(device) for idx in range(client_num)]\n",
    "\n",
    "    if args.test:\n",
    "        checkpoint = torch.load(SAVE_PATH)\n",
    "        server_model.load_state_dict(checkpoint['server_model'])\n",
    "        this_test_error = []\n",
    "        for test_idx, test_loader in enumerate(test_loaders):\n",
    "            test_loss = test(server_model, test_loader, loss_fun, device, classnum=args.classnum)\n",
    "            this_test_error.append(test_loss)\n",
    "            print(' {:<8s}| Error Rate: {:.2f} %'.format(clients[test_idx], test_loss * 100.))\n",
    "        print('Best Test Error: {:.2f} %'.format(100. * sum(this_test_error) / len(this_test_error)))\n",
    "\n",
    "        exit(0)\n",
    "\n",
    "    best_test_error = 1.\n",
    "    training_loss_log = []\n",
    "    error_rate_log = []\n",
    "\n",
    "    # start training\n",
    "    for a_iter in range(args.iters):\n",
    "        # record training loss and test error rate\n",
    "        this_test_error = []\n",
    "        this_train_loss = []\n",
    "\n",
    "        optimizers = [optim.Adam(params=models[idx].parameters(), lr=args.lr, weight_decay=args.wdecay)\n",
    "                      for idx in range(client_num)]\n",
    "\n",
    "        for wi in range(args.wk_iters):\n",
    "            print(\"============ Train epoch {} ============\".format(wi + 1 + a_iter * args.wk_iters))\n",
    "\n",
    "            for client_idx in range(client_num):\n",
    "                model, train_loader, optimizer = models[client_idx], train_loaders[client_idx], optimizers[client_idx]\n",
    "                train_loss = train(model, train_loader, optimizer, loss_fun, client_num, device,\n",
    "                                   client_Pi[client_idx], client_priors_corr[client_idx], prior_test, a_iter)\n",
    "                print(' {:<8s}| Train Loss: {:.4f}'.format(clients[client_idx], train_loss))\n",
    "\n",
    "                this_train_loss.append(train_loss)\n",
    "\n",
    "        # aggregation\n",
    "        server_model, models = communication(args, server_model, models, client_weights)\n",
    "\n",
    "        # start testing\n",
    "        for test_idx, test_loader in enumerate(validation_loaders):\n",
    "            test_loss = test(models[test_idx], test_loader, loss_fun, device, classnum=args.classnum)\n",
    "            this_test_error.append(test_loss)\n",
    "            print(' {:<8s}| Error Rate: {:.2f} %'.format(clients[test_idx], test_loss * 100.))\n",
    "\n",
    "        print()\n",
    "\n",
    "        # error rate after this communication\n",
    "        this_test_error = sum(this_test_error) / len(this_test_error)\n",
    "        if this_test_error < best_test_error:\n",
    "            best_test_error = this_test_error\n",
    "\n",
    "            # Save checkpoint\n",
    "            print(' Saving checkpoints to {}'.format(SAVE_PATH))\n",
    "            torch.save({\n",
    "                'server_model': server_model.state_dict(),\n",
    "                'a_iter': a_iter,\n",
    "            }, SAVE_PATH)\n",
    "\n",
    "        # Best Validation Error Rate\n",
    "        print(' Best Validation Error Rate: {:.2f} %, Current Validation Error Rate: {:.2f} %\\n'.format(\n",
    "            best_test_error * 100.,\n",
    "            this_test_error * 100.\n",
    "        ))\n",
    "\n",
    "        training_loss_log.append(sum(this_train_loss) / len(this_train_loss))\n",
    "        error_rate_log.append(this_test_error)\n",
    "\n",
    "        if not os.path.exists(os.path.join('./logs12/rsna_fedpl', args.mode)):\n",
    "            os.makedirs(os.path.join('./logs12/rsna_fedpl', args.mode))\n",
    "\n",
    "    print(' Start final testing\\n')\n",
    "    checkpoint = torch.load(SAVE_PATH)\n",
    "    server_model.load_state_dict(checkpoint['server_model'])\n",
    "    this_test_error = []\n",
    "    for test_idx, test_loader in enumerate(test_loaders):\n",
    "        test_loss = test(server_model, test_loader, loss_fun, device, classnum=args.classnum)\n",
    "        this_test_error.append(test_loss)\n",
    "        print(' {:<8s}| Error Rate: {:.2f} %'.format(clients[test_idx], test_loss * 100.))\n",
    "    print(' Best Test Error: {:.2f} %'.format(100. * sum(this_test_error) / len(this_test_error)))\n",
    "\n",
    "    error_rate_log.append(sum(this_test_error) / len(this_test_error))\n",
    "    # save record\n",
    "    np.savetxt(os.path.join('./logs12/rsna_fedpl', args.mode, 'client' + str(args.clientnum) +\n",
    "                            'sets' + str(args.setnum) + 'seed' + str(args.seed) + str(args.noniid) + 'train_loss.txt'),\n",
    "               training_loss_log, newline=\"\\r\\n\")\n",
    "    np.savetxt(os.path.join('./logs12/rsna_fedpl', args.mode, 'client' + str(args.clientnum) +\n",
    "                            'sets' + str(args.setnum) + 'seed' + str(args.seed) + str(args.noniid) + 'error_rate.txt'),\n",
    "               error_rate_log, newline=\"\\r\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6940fc38",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-11-09T18:56:57.753739Z",
     "iopub.status.busy": "2022-11-09T18:56:57.753438Z",
     "iopub.status.idle": "2022-11-09T21:02:37.213830Z",
     "shell.execute_reply": "2022-11-09T21:02:37.212807Z"
    },
    "papermill": {
     "duration": 7539.620055,
     "end_time": "2022-11-09T21:02:37.318069",
     "exception": false,
     "start_time": "2022-11-09T18:56:57.698014",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device: cuda \n",
      "\n",
      "Namespace(batch=128, classnum=6, clientnum=5, iters=100, lr=0.0001, mode='fedavg', noniid=False, save_path='./checkpoint24/RSNA', seed=0, setnum=24, test=False, wdecay=8e-07, wk_iters=1)\n",
      "0 / 106000\n",
      "10000 / 106000\n",
      "20000 / 106000\n",
      "30000 / 106000\n",
      "40000 / 106000\n",
      "50000 / 106000\n",
      "60000 / 106000\n",
      "70000 / 106000\n",
      "80000 / 106000\n",
      "90000 / 106000\n",
      "100000 / 106000\n",
      "Spliting U sets for 5 clients, each with 24 U sets...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5/5 [00:06<00:00,  1.28s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Data prepared, start training...\n",
      "\n",
      "============ Train epoch 1 ============\n",
      " client1 | Train Loss: 1.7724\n",
      " client2 | Train Loss: 1.7945\n",
      " client3 | Train Loss: 1.7916\n",
      " client4 | Train Loss: 1.7916\n",
      " client5 | Train Loss: 1.7695\n",
      " client1 | Error Rate: 73.18 %\n",
      " client2 | Error Rate: 74.01 %\n",
      " client3 | Error Rate: 75.03 %\n",
      " client4 | Error Rate: 74.01 %\n",
      " client5 | Error Rate: 73.36 %\n",
      "\n",
      " Saving checkpoints to ./checkpoint24/RSNA/rsna_fedpl/fedavgclient5sets24seed0False\n",
      " Best Validation Error Rate: 73.92 %, Current Validation Error Rate: 73.92 %\n",
      "\n",
      "============ Train epoch 2 ============\n",
      " client1 | Train Loss: 1.7244\n",
      " client2 | Train Loss: 1.7427\n",
      " client3 | Train Loss: 1.7528\n",
      " client4 | Train Loss: 1.7370\n",
      " client5 | Train Loss: 1.7268\n",
      " client1 | Error Rate: 67.29 %\n",
      " client2 | Error Rate: 67.91 %\n",
      " client3 | Error Rate: 68.64 %\n",
      " client4 | Error Rate: 68.35 %\n",
      " client5 | Error Rate: 67.26 %\n",
      "\n",
      " Saving checkpoints to ./checkpoint24/RSNA/rsna_fedpl/fedavgclient5sets24seed0False\n",
      " Best Validation Error Rate: 67.89 %, Current Validation Error Rate: 67.89 %\n",
      "\n",
      "============ Train epoch 3 ============\n",
      " client1 | Train Loss: 1.7031\n",
      " client2 | Train Loss: 1.7247\n",
      " client3 | Train Loss: 1.7232\n",
      " client4 | Train Loss: 1.7124\n",
      " client5 | Train Loss: 1.7059\n",
      " client1 | Error Rate: 65.13 %\n",
      " client2 | Error Rate: 64.93 %\n",
      " client3 | Error Rate: 66.10 %\n",
      " client4 | Error Rate: 65.01 %\n",
      " client5 | Error Rate: 64.64 %\n",
      "\n",
      " Saving checkpoints to ./checkpoint24/RSNA/rsna_fedpl/fedavgclient5sets24seed0False\n",
      " Best Validation Error Rate: 65.16 %, Current Validation Error Rate: 65.16 %\n",
      "\n",
      "============ Train epoch 4 ============\n",
      " client1 | Train Loss: 1.6818\n",
      " client2 | Train Loss: 1.6932\n",
      " client3 | Train Loss: 1.6975\n",
      " client4 | Train Loss: 1.7003\n",
      " client5 | Train Loss: 1.6925\n",
      " client1 | Error Rate: 62.93 %\n",
      " client2 | Error Rate: 63.68 %\n",
      " client3 | Error Rate: 65.45 %\n",
      " client4 | Error Rate: 65.06 %\n",
      " client5 | Error Rate: 63.32 %\n",
      "\n",
      " Saving checkpoints to ./checkpoint24/RSNA/rsna_fedpl/fedavgclient5sets24seed0False\n",
      " Best Validation Error Rate: 64.09 %, Current Validation Error Rate: 64.09 %\n",
      "\n",
      "============ Train epoch 5 ============\n",
      " client1 | Train Loss: 1.6470\n",
      " client2 | Train Loss: 1.6793\n",
      " client3 | Train Loss: 1.6846\n",
      " client4 | Train Loss: 1.6766\n",
      " client5 | Train Loss: 1.6666\n",
      " client1 | Error Rate: 63.50 %\n",
      " client2 | Error Rate: 63.03 %\n",
      " client3 | Error Rate: 64.85 %\n",
      " client4 | Error Rate: 64.17 %\n",
      " client5 | Error Rate: 63.63 %\n",
      "\n",
      " Saving checkpoints to ./checkpoint24/RSNA/rsna_fedpl/fedavgclient5sets24seed0False\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 63.84 %\n",
      "\n",
      "============ Train epoch 6 ============\n",
      " client1 | Train Loss: 0.3848\n",
      " client2 | Train Loss: 0.4437\n",
      " client3 | Train Loss: 0.4555\n",
      " client4 | Train Loss: 0.4394\n",
      " client5 | Train Loss: 0.4244\n",
      " client1 | Error Rate: 76.38 %\n",
      " client2 | Error Rate: 76.97 %\n",
      " client3 | Error Rate: 77.62 %\n",
      " client4 | Error Rate: 76.45 %\n",
      " client5 | Error Rate: 76.14 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 76.71 %\n",
      "\n",
      "============ Train epoch 7 ============\n",
      " client1 | Train Loss: 0.1483\n",
      " client2 | Train Loss: 0.0987\n",
      " client3 | Train Loss: 0.0690\n",
      " client4 | Train Loss: 0.0786\n",
      " client5 | Train Loss: 0.0616\n",
      " client1 | Error Rate: 76.58 %\n",
      " client2 | Error Rate: 77.23 %\n",
      " client3 | Error Rate: 78.22 %\n",
      " client4 | Error Rate: 76.87 %\n",
      " client5 | Error Rate: 76.45 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.07 %\n",
      "\n",
      "============ Train epoch 8 ============\n",
      " client1 | Train Loss: 0.0436\n",
      " client2 | Train Loss: 0.0050\n",
      " client3 | Train Loss: 0.0941\n",
      " client4 | Train Loss: 0.0192\n",
      " client5 | Train Loss: 0.0479\n",
      " client1 | Error Rate: 76.64 %\n",
      " client2 | Error Rate: 77.28 %\n",
      " client3 | Error Rate: 78.12 %\n",
      " client4 | Error Rate: 76.90 %\n",
      " client5 | Error Rate: 76.45 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.08 %\n",
      "\n",
      "============ Train epoch 9 ============\n",
      " client1 | Train Loss: 0.0156\n",
      " client2 | Train Loss: 0.0312\n",
      " client3 | Train Loss: 0.0285\n",
      " client4 | Train Loss: 0.0454\n",
      " client5 | Train Loss: 0.0036\n",
      " client1 | Error Rate: 76.25 %\n",
      " client2 | Error Rate: 77.02 %\n",
      " client3 | Error Rate: 77.88 %\n",
      " client4 | Error Rate: 76.74 %\n",
      " client5 | Error Rate: 76.19 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 76.82 %\n",
      "\n",
      "============ Train epoch 10 ============\n",
      " client1 | Train Loss: 0.0076\n",
      " client2 | Train Loss: 0.0643\n",
      " client3 | Train Loss: 0.0145\n",
      " client4 | Train Loss: 0.0224\n",
      " client5 | Train Loss: 0.0487\n",
      " client1 | Error Rate: 76.25 %\n",
      " client2 | Error Rate: 77.02 %\n",
      " client3 | Error Rate: 77.88 %\n",
      " client4 | Error Rate: 76.69 %\n",
      " client5 | Error Rate: 76.19 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 76.81 %\n",
      "\n",
      "============ Train epoch 11 ============\n",
      " client1 | Train Loss: 0.0290\n",
      " client2 | Train Loss: 0.0129\n",
      " client3 | Train Loss: 0.0121\n",
      " client4 | Train Loss: 0.0263\n",
      " client5 | Train Loss: 0.0127\n",
      " client1 | Error Rate: 76.32 %\n",
      " client2 | Error Rate: 76.97 %\n",
      " client3 | Error Rate: 78.09 %\n",
      " client4 | Error Rate: 76.95 %\n",
      " client5 | Error Rate: 76.61 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 76.99 %\n",
      "\n",
      "============ Train epoch 12 ============\n",
      " client1 | Train Loss: 0.0216\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0142\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 76.53 %\n",
      " client2 | Error Rate: 77.31 %\n",
      " client3 | Error Rate: 78.25 %\n",
      " client4 | Error Rate: 77.10 %\n",
      " client5 | Error Rate: 76.58 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.15 %\n",
      "\n",
      "============ Train epoch 13 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 76.53 %\n",
      " client2 | Error Rate: 77.31 %\n",
      " client3 | Error Rate: 78.17 %\n",
      " client4 | Error Rate: 77.23 %\n",
      " client5 | Error Rate: 76.69 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.19 %\n",
      "\n",
      "============ Train epoch 14 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0174\n",
      " client3 | Train Loss: 0.0204\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0716\n",
      " client1 | Error Rate: 76.35 %\n",
      " client2 | Error Rate: 77.28 %\n",
      " client3 | Error Rate: 78.27 %\n",
      " client4 | Error Rate: 77.08 %\n",
      " client5 | Error Rate: 76.58 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.11 %\n",
      "\n",
      "============ Train epoch 15 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0191\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 76.25 %\n",
      " client2 | Error Rate: 77.13 %\n",
      " client3 | Error Rate: 78.12 %\n",
      " client4 | Error Rate: 76.92 %\n",
      " client5 | Error Rate: 76.43 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 76.97 %\n",
      "\n",
      "============ Train epoch 16 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0088\n",
      " client3 | Train Loss: 0.0026\n",
      " client4 | Train Loss: 0.0135\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 76.30 %\n",
      " client2 | Error Rate: 77.21 %\n",
      " client3 | Error Rate: 78.14 %\n",
      " client4 | Error Rate: 76.95 %\n",
      " client5 | Error Rate: 76.56 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.03 %\n",
      "\n",
      "============ Train epoch 17 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0071\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 76.56 %\n",
      " client2 | Error Rate: 77.47 %\n",
      " client3 | Error Rate: 78.40 %\n",
      " client4 | Error Rate: 77.23 %\n",
      " client5 | Error Rate: 76.84 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.30 %\n",
      "\n",
      "============ Train epoch 18 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0131\n",
      " client3 | Train Loss: 0.0023\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 76.51 %\n",
      " client2 | Error Rate: 77.52 %\n",
      " client3 | Error Rate: 78.32 %\n",
      " client4 | Error Rate: 77.34 %\n",
      " client5 | Error Rate: 76.92 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.32 %\n",
      "\n",
      "============ Train epoch 19 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0252\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 76.53 %\n",
      " client2 | Error Rate: 77.57 %\n",
      " client3 | Error Rate: 78.45 %\n",
      " client4 | Error Rate: 77.31 %\n",
      " client5 | Error Rate: 76.90 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.35 %\n",
      "\n",
      "============ Train epoch 20 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0200\n",
      " client1 | Error Rate: 76.48 %\n",
      " client2 | Error Rate: 77.41 %\n",
      " client3 | Error Rate: 78.43 %\n",
      " client4 | Error Rate: 77.31 %\n",
      " client5 | Error Rate: 76.97 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.32 %\n",
      "\n",
      "============ Train epoch 21 ============\n",
      " client1 | Train Loss: 0.0024\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0024\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0175\n",
      " client1 | Error Rate: 76.58 %\n",
      " client2 | Error Rate: 77.39 %\n",
      " client3 | Error Rate: 78.50 %\n",
      " client4 | Error Rate: 77.52 %\n",
      " client5 | Error Rate: 77.00 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.40 %\n",
      "\n",
      "============ Train epoch 22 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0323\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 76.53 %\n",
      " client2 | Error Rate: 77.39 %\n",
      " client3 | Error Rate: 78.48 %\n",
      " client4 | Error Rate: 77.47 %\n",
      " client5 | Error Rate: 77.02 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.38 %\n",
      "\n",
      "============ Train epoch 23 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0228\n",
      " client1 | Error Rate: 76.56 %\n",
      " client2 | Error Rate: 77.31 %\n",
      " client3 | Error Rate: 78.48 %\n",
      " client4 | Error Rate: 77.49 %\n",
      " client5 | Error Rate: 76.95 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.36 %\n",
      "\n",
      "============ Train epoch 24 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0221\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0183\n",
      " client1 | Error Rate: 76.53 %\n",
      " client2 | Error Rate: 77.21 %\n",
      " client3 | Error Rate: 78.30 %\n",
      " client4 | Error Rate: 77.23 %\n",
      " client5 | Error Rate: 76.66 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.19 %\n",
      "\n",
      "============ Train epoch 25 ============\n",
      " client1 | Train Loss: 0.0150\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0103\n",
      " client4 | Train Loss: 0.0122\n",
      " client5 | Train Loss: 0.0155\n",
      " client1 | Error Rate: 76.45 %\n",
      " client2 | Error Rate: 76.97 %\n",
      " client3 | Error Rate: 78.17 %\n",
      " client4 | Error Rate: 77.05 %\n",
      " client5 | Error Rate: 76.66 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.06 %\n",
      "\n",
      "============ Train epoch 26 ============\n",
      " client1 | Train Loss: 0.0407\n",
      " client2 | Train Loss: 0.0389\n",
      " client3 | Train Loss: 0.0136\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0170\n",
      " client1 | Error Rate: 76.53 %\n",
      " client2 | Error Rate: 77.10 %\n",
      " client3 | Error Rate: 78.35 %\n",
      " client4 | Error Rate: 77.26 %\n",
      " client5 | Error Rate: 76.61 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.17 %\n",
      "\n",
      "============ Train epoch 27 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0024\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0025\n",
      " client5 | Train Loss: 0.0216\n",
      " client1 | Error Rate: 76.61 %\n",
      " client2 | Error Rate: 77.00 %\n",
      " client3 | Error Rate: 78.30 %\n",
      " client4 | Error Rate: 77.26 %\n",
      " client5 | Error Rate: 76.69 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.17 %\n",
      "\n",
      "============ Train epoch 28 ============\n",
      " client1 | Train Loss: 0.0128\n",
      " client2 | Train Loss: 0.0089\n",
      " client3 | Train Loss: 0.0519\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0096\n",
      " client1 | Error Rate: 76.71 %\n",
      " client2 | Error Rate: 77.13 %\n",
      " client3 | Error Rate: 78.30 %\n",
      " client4 | Error Rate: 77.21 %\n",
      " client5 | Error Rate: 76.77 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.22 %\n",
      "\n",
      "============ Train epoch 29 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0083\n",
      " client1 | Error Rate: 76.71 %\n",
      " client2 | Error Rate: 77.13 %\n",
      " client3 | Error Rate: 78.27 %\n",
      " client4 | Error Rate: 77.21 %\n",
      " client5 | Error Rate: 76.77 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.22 %\n",
      "\n",
      "============ Train epoch 30 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0242\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 76.71 %\n",
      " client2 | Error Rate: 77.23 %\n",
      " client3 | Error Rate: 78.48 %\n",
      " client4 | Error Rate: 77.31 %\n",
      " client5 | Error Rate: 76.95 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.34 %\n",
      "\n",
      "============ Train epoch 31 ============\n",
      " client1 | Train Loss: 0.0359\n",
      " client2 | Train Loss: 0.0443\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0065\n",
      " client1 | Error Rate: 76.92 %\n",
      " client2 | Error Rate: 77.23 %\n",
      " client3 | Error Rate: 78.53 %\n",
      " client4 | Error Rate: 77.41 %\n",
      " client5 | Error Rate: 77.00 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.42 %\n",
      "\n",
      "============ Train epoch 32 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0518\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 76.77 %\n",
      " client2 | Error Rate: 77.23 %\n",
      " client3 | Error Rate: 78.45 %\n",
      " client4 | Error Rate: 77.10 %\n",
      " client5 | Error Rate: 77.05 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.32 %\n",
      "\n",
      "============ Train epoch 33 ============\n",
      " client1 | Train Loss: 0.0155\n",
      " client2 | Train Loss: 0.0214\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0494\n",
      " client1 | Error Rate: 77.08 %\n",
      " client2 | Error Rate: 77.57 %\n",
      " client3 | Error Rate: 78.58 %\n",
      " client4 | Error Rate: 77.49 %\n",
      " client5 | Error Rate: 77.31 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.61 %\n",
      "\n",
      "============ Train epoch 34 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0514\n",
      " client5 | Train Loss: 0.0306\n",
      " client1 | Error Rate: 77.39 %\n",
      " client2 | Error Rate: 77.73 %\n",
      " client3 | Error Rate: 78.76 %\n",
      " client4 | Error Rate: 77.65 %\n",
      " client5 | Error Rate: 77.39 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.78 %\n",
      "\n",
      "============ Train epoch 35 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0212\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.02 %\n",
      " client2 | Error Rate: 77.49 %\n",
      " client3 | Error Rate: 78.66 %\n",
      " client4 | Error Rate: 77.31 %\n",
      " client5 | Error Rate: 77.26 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.55 %\n",
      "\n",
      "============ Train epoch 36 ============\n",
      " client1 | Train Loss: 0.0139\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0114\n",
      " client4 | Train Loss: 0.0204\n",
      " client5 | Train Loss: 0.0147\n",
      " client1 | Error Rate: 77.00 %\n",
      " client2 | Error Rate: 77.15 %\n",
      " client3 | Error Rate: 78.37 %\n",
      " client4 | Error Rate: 77.28 %\n",
      " client5 | Error Rate: 76.90 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.34 %\n",
      "\n",
      "============ Train epoch 37 ============\n",
      " client1 | Train Loss: 0.0209\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0201\n",
      " client5 | Train Loss: 0.0380\n",
      " client1 | Error Rate: 76.90 %\n",
      " client2 | Error Rate: 77.18 %\n",
      " client3 | Error Rate: 78.27 %\n",
      " client4 | Error Rate: 77.23 %\n",
      " client5 | Error Rate: 76.92 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.30 %\n",
      "\n",
      "============ Train epoch 38 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0112\n",
      " client3 | Train Loss: 0.0219\n",
      " client4 | Train Loss: 0.0037\n",
      " client5 | Train Loss: 0.0142\n",
      " client1 | Error Rate: 77.02 %\n",
      " client2 | Error Rate: 77.26 %\n",
      " client3 | Error Rate: 78.48 %\n",
      " client4 | Error Rate: 77.34 %\n",
      " client5 | Error Rate: 76.95 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.41 %\n",
      "\n",
      "============ Train epoch 39 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0041\n",
      " client3 | Train Loss: 0.0047\n",
      " client4 | Train Loss: 0.0289\n",
      " client5 | Train Loss: 0.0344\n",
      " client1 | Error Rate: 77.05 %\n",
      " client2 | Error Rate: 77.21 %\n",
      " client3 | Error Rate: 78.40 %\n",
      " client4 | Error Rate: 77.47 %\n",
      " client5 | Error Rate: 76.92 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.41 %\n",
      "\n",
      "============ Train epoch 40 ============\n",
      " client1 | Train Loss: 0.0040\n",
      " client2 | Train Loss: 0.0326\n",
      " client3 | Train Loss: 0.0674\n",
      " client4 | Train Loss: 0.0179\n",
      " client5 | Train Loss: 0.0387\n",
      " client1 | Error Rate: 76.84 %\n",
      " client2 | Error Rate: 77.23 %\n",
      " client3 | Error Rate: 78.43 %\n",
      " client4 | Error Rate: 77.39 %\n",
      " client5 | Error Rate: 76.97 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.37 %\n",
      "\n",
      "============ Train epoch 41 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0467\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.05 %\n",
      " client2 | Error Rate: 77.44 %\n",
      " client3 | Error Rate: 78.69 %\n",
      " client4 | Error Rate: 77.65 %\n",
      " client5 | Error Rate: 77.31 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.63 %\n",
      "\n",
      "============ Train epoch 42 ============\n",
      " client1 | Train Loss: 0.0386\n",
      " client2 | Train Loss: 0.0050\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0187\n",
      " client1 | Error Rate: 77.13 %\n",
      " client2 | Error Rate: 77.47 %\n",
      " client3 | Error Rate: 78.66 %\n",
      " client4 | Error Rate: 77.49 %\n",
      " client5 | Error Rate: 77.31 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.61 %\n",
      "\n",
      "============ Train epoch 43 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0191\n",
      " client3 | Train Loss: 0.0180\n",
      " client4 | Train Loss: 0.0035\n",
      " client5 | Train Loss: 0.0036\n",
      " client1 | Error Rate: 77.21 %\n",
      " client2 | Error Rate: 77.62 %\n",
      " client3 | Error Rate: 78.79 %\n",
      " client4 | Error Rate: 77.60 %\n",
      " client5 | Error Rate: 77.34 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.71 %\n",
      "\n",
      "============ Train epoch 44 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0080\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.26 %\n",
      " client2 | Error Rate: 77.80 %\n",
      " client3 | Error Rate: 78.92 %\n",
      " client4 | Error Rate: 77.70 %\n",
      " client5 | Error Rate: 77.47 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.83 %\n",
      "\n",
      "============ Train epoch 45 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0250\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.10 %\n",
      " client2 | Error Rate: 77.52 %\n",
      " client3 | Error Rate: 78.79 %\n",
      " client4 | Error Rate: 77.47 %\n",
      " client5 | Error Rate: 77.15 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.61 %\n",
      "\n",
      "============ Train epoch 46 ============\n",
      " client1 | Train Loss: 0.0036\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0025\n",
      " client1 | Error Rate: 77.13 %\n",
      " client2 | Error Rate: 77.70 %\n",
      " client3 | Error Rate: 78.74 %\n",
      " client4 | Error Rate: 77.65 %\n",
      " client5 | Error Rate: 77.39 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.72 %\n",
      "\n",
      "============ Train epoch 47 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0092\n",
      " client4 | Train Loss: 0.0682\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.41 %\n",
      " client2 | Error Rate: 78.17 %\n",
      " client3 | Error Rate: 79.15 %\n",
      " client4 | Error Rate: 78.01 %\n",
      " client5 | Error Rate: 77.65 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.08 %\n",
      "\n",
      "============ Train epoch 48 ============\n",
      " client1 | Train Loss: 0.0198\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.28 %\n",
      " client2 | Error Rate: 77.96 %\n",
      " client3 | Error Rate: 79.08 %\n",
      " client4 | Error Rate: 77.99 %\n",
      " client5 | Error Rate: 77.70 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.00 %\n",
      "\n",
      "============ Train epoch 49 ============\n",
      " client1 | Train Loss: 0.0117\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0314\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0194\n",
      " client1 | Error Rate: 77.36 %\n",
      " client2 | Error Rate: 77.96 %\n",
      " client3 | Error Rate: 79.18 %\n",
      " client4 | Error Rate: 78.14 %\n",
      " client5 | Error Rate: 77.75 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.08 %\n",
      "\n",
      "============ Train epoch 50 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0262\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0209\n",
      " client1 | Error Rate: 77.62 %\n",
      " client2 | Error Rate: 78.27 %\n",
      " client3 | Error Rate: 79.34 %\n",
      " client4 | Error Rate: 78.14 %\n",
      " client5 | Error Rate: 77.83 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.24 %\n",
      "\n",
      "============ Train epoch 51 ============\n",
      " client1 | Train Loss: 0.0149\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0194\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0031\n",
      " client1 | Error Rate: 77.57 %\n",
      " client2 | Error Rate: 78.19 %\n",
      " client3 | Error Rate: 79.28 %\n",
      " client4 | Error Rate: 78.22 %\n",
      " client5 | Error Rate: 77.80 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.21 %\n",
      "\n",
      "============ Train epoch 52 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0169\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0242\n",
      " client1 | Error Rate: 77.57 %\n",
      " client2 | Error Rate: 78.04 %\n",
      " client3 | Error Rate: 79.02 %\n",
      " client4 | Error Rate: 78.06 %\n",
      " client5 | Error Rate: 77.67 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.07 %\n",
      "\n",
      "============ Train epoch 53 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0334\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.21 %\n",
      " client2 | Error Rate: 77.65 %\n",
      " client3 | Error Rate: 78.89 %\n",
      " client4 | Error Rate: 78.04 %\n",
      " client5 | Error Rate: 77.60 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.88 %\n",
      "\n",
      "============ Train epoch 54 ============\n",
      " client1 | Train Loss: 0.0029\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0156\n",
      " client5 | Train Loss: 0.0232\n",
      " client1 | Error Rate: 77.18 %\n",
      " client2 | Error Rate: 77.44 %\n",
      " client3 | Error Rate: 78.84 %\n",
      " client4 | Error Rate: 77.75 %\n",
      " client5 | Error Rate: 77.36 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.72 %\n",
      "\n",
      "============ Train epoch 55 ============\n",
      " client1 | Train Loss: 0.0044\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0568\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0482\n",
      " client1 | Error Rate: 77.23 %\n",
      " client2 | Error Rate: 77.57 %\n",
      " client3 | Error Rate: 79.00 %\n",
      " client4 | Error Rate: 77.99 %\n",
      " client5 | Error Rate: 77.54 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.87 %\n",
      "\n",
      "============ Train epoch 56 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.31 %\n",
      " client2 | Error Rate: 77.57 %\n",
      " client3 | Error Rate: 78.95 %\n",
      " client4 | Error Rate: 78.09 %\n",
      " client5 | Error Rate: 77.52 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.89 %\n",
      "\n",
      "============ Train epoch 57 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0028\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.52 %\n",
      " client2 | Error Rate: 77.75 %\n",
      " client3 | Error Rate: 78.97 %\n",
      " client4 | Error Rate: 78.14 %\n",
      " client5 | Error Rate: 77.65 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.01 %\n",
      "\n",
      "============ Train epoch 58 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.54 %\n",
      " client2 | Error Rate: 77.70 %\n",
      " client3 | Error Rate: 78.97 %\n",
      " client4 | Error Rate: 78.14 %\n",
      " client5 | Error Rate: 77.65 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.00 %\n",
      "\n",
      "============ Train epoch 59 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0526\n",
      " client3 | Train Loss: 0.0413\n",
      " client4 | Train Loss: 0.0019\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.47 %\n",
      " client2 | Error Rate: 77.70 %\n",
      " client3 | Error Rate: 78.97 %\n",
      " client4 | Error Rate: 78.09 %\n",
      " client5 | Error Rate: 77.52 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.95 %\n",
      "\n",
      "============ Train epoch 60 ============\n",
      " client1 | Train Loss: 0.0051\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0616\n",
      " client4 | Train Loss: 0.0139\n",
      " client5 | Train Loss: 0.0020\n",
      " client1 | Error Rate: 77.52 %\n",
      " client2 | Error Rate: 77.41 %\n",
      " client3 | Error Rate: 78.82 %\n",
      " client4 | Error Rate: 77.96 %\n",
      " client5 | Error Rate: 77.23 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.79 %\n",
      "\n",
      "============ Train epoch 61 ============\n",
      " client1 | Train Loss: 0.0316\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0065\n",
      " client5 | Train Loss: 0.0229\n",
      " client1 | Error Rate: 77.39 %\n",
      " client2 | Error Rate: 77.44 %\n",
      " client3 | Error Rate: 78.71 %\n",
      " client4 | Error Rate: 78.04 %\n",
      " client5 | Error Rate: 77.31 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.78 %\n",
      "\n",
      "============ Train epoch 62 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0183\n",
      " client3 | Train Loss: 0.0484\n",
      " client4 | Train Loss: 0.0017\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.36 %\n",
      " client2 | Error Rate: 77.21 %\n",
      " client3 | Error Rate: 78.69 %\n",
      " client4 | Error Rate: 77.99 %\n",
      " client5 | Error Rate: 77.00 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.65 %\n",
      "\n",
      "============ Train epoch 63 ============\n",
      " client1 | Train Loss: 0.0050\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0018\n",
      " client4 | Train Loss: 0.0113\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.26 %\n",
      " client2 | Error Rate: 77.10 %\n",
      " client3 | Error Rate: 78.48 %\n",
      " client4 | Error Rate: 77.80 %\n",
      " client5 | Error Rate: 76.84 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.50 %\n",
      "\n",
      "============ Train epoch 64 ============\n",
      " client1 | Train Loss: 0.0110\n",
      " client2 | Train Loss: 0.0036\n",
      " client3 | Train Loss: 0.0258\n",
      " client4 | Train Loss: 0.0129\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.28 %\n",
      " client2 | Error Rate: 77.08 %\n",
      " client3 | Error Rate: 78.56 %\n",
      " client4 | Error Rate: 77.75 %\n",
      " client5 | Error Rate: 76.95 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.52 %\n",
      "\n",
      "============ Train epoch 65 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0315\n",
      " client5 | Train Loss: 0.0069\n",
      " client1 | Error Rate: 77.34 %\n",
      " client2 | Error Rate: 77.18 %\n",
      " client3 | Error Rate: 78.66 %\n",
      " client4 | Error Rate: 77.75 %\n",
      " client5 | Error Rate: 77.08 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.60 %\n",
      "\n",
      "============ Train epoch 66 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0019\n",
      " client4 | Train Loss: 0.0106\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.31 %\n",
      " client2 | Error Rate: 77.05 %\n",
      " client3 | Error Rate: 78.30 %\n",
      " client4 | Error Rate: 77.60 %\n",
      " client5 | Error Rate: 76.90 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.43 %\n",
      "\n",
      "============ Train epoch 67 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0396\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.21 %\n",
      " client2 | Error Rate: 77.05 %\n",
      " client3 | Error Rate: 78.45 %\n",
      " client4 | Error Rate: 77.73 %\n",
      " client5 | Error Rate: 76.95 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.48 %\n",
      "\n",
      "============ Train epoch 68 ============\n",
      " client1 | Train Loss: 0.0208\n",
      " client2 | Train Loss: 0.0125\n",
      " client3 | Train Loss: 0.0420\n",
      " client4 | Train Loss: 0.0218\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.13 %\n",
      " client2 | Error Rate: 77.08 %\n",
      " client3 | Error Rate: 78.48 %\n",
      " client4 | Error Rate: 77.57 %\n",
      " client5 | Error Rate: 76.97 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.45 %\n",
      "\n",
      "============ Train epoch 69 ============\n",
      " client1 | Train Loss: 0.0402\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0199\n",
      " client1 | Error Rate: 77.26 %\n",
      " client2 | Error Rate: 76.97 %\n",
      " client3 | Error Rate: 78.43 %\n",
      " client4 | Error Rate: 77.70 %\n",
      " client5 | Error Rate: 77.00 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.47 %\n",
      "\n",
      "============ Train epoch 70 ============\n",
      " client1 | Train Loss: 0.0174\n",
      " client2 | Train Loss: 0.0643\n",
      " client3 | Train Loss: 0.0204\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.23 %\n",
      " client2 | Error Rate: 76.97 %\n",
      " client3 | Error Rate: 78.48 %\n",
      " client4 | Error Rate: 77.54 %\n",
      " client5 | Error Rate: 76.82 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.41 %\n",
      "\n",
      "============ Train epoch 71 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0037\n",
      " client3 | Train Loss: 0.0175\n",
      " client4 | Train Loss: 0.0189\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.23 %\n",
      " client2 | Error Rate: 77.08 %\n",
      " client3 | Error Rate: 78.56 %\n",
      " client4 | Error Rate: 77.47 %\n",
      " client5 | Error Rate: 76.82 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.43 %\n",
      "\n",
      "============ Train epoch 72 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0222\n",
      " client4 | Train Loss: 0.0152\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.31 %\n",
      " client2 | Error Rate: 77.21 %\n",
      " client3 | Error Rate: 78.58 %\n",
      " client4 | Error Rate: 77.65 %\n",
      " client5 | Error Rate: 76.90 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.53 %\n",
      "\n",
      "============ Train epoch 73 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0258\n",
      " client1 | Error Rate: 77.28 %\n",
      " client2 | Error Rate: 77.08 %\n",
      " client3 | Error Rate: 78.61 %\n",
      " client4 | Error Rate: 77.65 %\n",
      " client5 | Error Rate: 76.92 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.51 %\n",
      "\n",
      "============ Train epoch 74 ============\n",
      " client1 | Train Loss: 0.0753\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0318\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.26 %\n",
      " client2 | Error Rate: 77.02 %\n",
      " client3 | Error Rate: 78.53 %\n",
      " client4 | Error Rate: 77.70 %\n",
      " client5 | Error Rate: 76.90 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.48 %\n",
      "\n",
      "============ Train epoch 75 ============\n",
      " client1 | Train Loss: 0.0329\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0237\n",
      " client1 | Error Rate: 77.26 %\n",
      " client2 | Error Rate: 76.84 %\n",
      " client3 | Error Rate: 78.32 %\n",
      " client4 | Error Rate: 77.41 %\n",
      " client5 | Error Rate: 76.74 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.32 %\n",
      "\n",
      "============ Train epoch 76 ============\n",
      " client1 | Train Loss: 0.0078\n",
      " client2 | Train Loss: 0.0016\n",
      " client3 | Train Loss: 0.0165\n",
      " client4 | Train Loss: 0.0087\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.26 %\n",
      " client2 | Error Rate: 77.10 %\n",
      " client3 | Error Rate: 78.45 %\n",
      " client4 | Error Rate: 77.54 %\n",
      " client5 | Error Rate: 76.90 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.45 %\n",
      "\n",
      "============ Train epoch 77 ============\n",
      " client1 | Train Loss: 0.0309\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0184\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.31 %\n",
      " client2 | Error Rate: 77.13 %\n",
      " client3 | Error Rate: 78.50 %\n",
      " client4 | Error Rate: 77.57 %\n",
      " client5 | Error Rate: 77.02 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.51 %\n",
      "\n",
      "============ Train epoch 78 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0216\n",
      " client1 | Error Rate: 77.23 %\n",
      " client2 | Error Rate: 77.18 %\n",
      " client3 | Error Rate: 78.48 %\n",
      " client4 | Error Rate: 77.54 %\n",
      " client5 | Error Rate: 77.05 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.50 %\n",
      "\n",
      "============ Train epoch 79 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0135\n",
      " client5 | Train Loss: 0.0243\n",
      " client1 | Error Rate: 77.41 %\n",
      " client2 | Error Rate: 77.52 %\n",
      " client3 | Error Rate: 78.63 %\n",
      " client4 | Error Rate: 77.75 %\n",
      " client5 | Error Rate: 77.31 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.73 %\n",
      "\n",
      "============ Train epoch 80 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0274\n",
      " client3 | Train Loss: 0.0036\n",
      " client4 | Train Loss: 0.0341\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.65 %\n",
      " client2 | Error Rate: 77.86 %\n",
      " client3 | Error Rate: 79.05 %\n",
      " client4 | Error Rate: 78.06 %\n",
      " client5 | Error Rate: 77.65 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.05 %\n",
      "\n",
      "============ Train epoch 81 ============\n",
      " client1 | Train Loss: 0.0145\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0220\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.39 %\n",
      " client2 | Error Rate: 77.65 %\n",
      " client3 | Error Rate: 78.82 %\n",
      " client4 | Error Rate: 77.91 %\n",
      " client5 | Error Rate: 77.44 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 77.84 %\n",
      "\n",
      "============ Train epoch 82 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0134\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0121\n",
      " client1 | Error Rate: 77.78 %\n",
      " client2 | Error Rate: 77.99 %\n",
      " client3 | Error Rate: 79.18 %\n",
      " client4 | Error Rate: 78.19 %\n",
      " client5 | Error Rate: 77.78 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.18 %\n",
      "\n",
      "============ Train epoch 83 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0135\n",
      " client1 | Error Rate: 77.73 %\n",
      " client2 | Error Rate: 78.01 %\n",
      " client3 | Error Rate: 79.26 %\n",
      " client4 | Error Rate: 78.30 %\n",
      " client5 | Error Rate: 77.86 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.23 %\n",
      "\n",
      "============ Train epoch 84 ============\n",
      " client1 | Train Loss: 0.0030\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.88 %\n",
      " client2 | Error Rate: 78.06 %\n",
      " client3 | Error Rate: 79.08 %\n",
      " client4 | Error Rate: 78.22 %\n",
      " client5 | Error Rate: 77.80 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.21 %\n",
      "\n",
      "============ Train epoch 85 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0363\n",
      " client4 | Train Loss: 0.0121\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.96 %\n",
      " client2 | Error Rate: 78.12 %\n",
      " client3 | Error Rate: 79.18 %\n",
      " client4 | Error Rate: 78.45 %\n",
      " client5 | Error Rate: 78.01 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.34 %\n",
      "\n",
      "============ Train epoch 86 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0039\n",
      " client1 | Error Rate: 77.80 %\n",
      " client2 | Error Rate: 78.09 %\n",
      " client3 | Error Rate: 79.13 %\n",
      " client4 | Error Rate: 78.35 %\n",
      " client5 | Error Rate: 77.86 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.25 %\n",
      "\n",
      "============ Train epoch 87 ============\n",
      " client1 | Train Loss: 0.0023\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0260\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0121\n",
      " client1 | Error Rate: 77.78 %\n",
      " client2 | Error Rate: 77.99 %\n",
      " client3 | Error Rate: 79.08 %\n",
      " client4 | Error Rate: 78.27 %\n",
      " client5 | Error Rate: 77.83 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.19 %\n",
      "\n",
      "============ Train epoch 88 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.73 %\n",
      " client2 | Error Rate: 77.88 %\n",
      " client3 | Error Rate: 79.02 %\n",
      " client4 | Error Rate: 78.25 %\n",
      " client5 | Error Rate: 77.75 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.13 %\n",
      "\n",
      "============ Train epoch 89 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.75 %\n",
      " client2 | Error Rate: 77.93 %\n",
      " client3 | Error Rate: 79.02 %\n",
      " client4 | Error Rate: 78.27 %\n",
      " client5 | Error Rate: 77.73 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.14 %\n",
      "\n",
      "============ Train epoch 90 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0071\n",
      " client4 | Train Loss: 0.0072\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.70 %\n",
      " client2 | Error Rate: 77.91 %\n",
      " client3 | Error Rate: 79.05 %\n",
      " client4 | Error Rate: 78.19 %\n",
      " client5 | Error Rate: 77.70 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.11 %\n",
      "\n",
      "============ Train epoch 91 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0032\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0305\n",
      " client1 | Error Rate: 77.75 %\n",
      " client2 | Error Rate: 77.86 %\n",
      " client3 | Error Rate: 79.13 %\n",
      " client4 | Error Rate: 78.25 %\n",
      " client5 | Error Rate: 77.78 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.15 %\n",
      "\n",
      "============ Train epoch 92 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.70 %\n",
      " client2 | Error Rate: 77.88 %\n",
      " client3 | Error Rate: 79.10 %\n",
      " client4 | Error Rate: 78.19 %\n",
      " client5 | Error Rate: 77.75 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.13 %\n",
      "\n",
      "============ Train epoch 93 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0373\n",
      " client5 | Train Loss: 0.0022\n",
      " client1 | Error Rate: 77.91 %\n",
      " client2 | Error Rate: 78.12 %\n",
      " client3 | Error Rate: 79.18 %\n",
      " client4 | Error Rate: 78.22 %\n",
      " client5 | Error Rate: 77.86 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.26 %\n",
      "\n",
      "============ Train epoch 94 ============\n",
      " client1 | Train Loss: 0.0244\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0142\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.70 %\n",
      " client2 | Error Rate: 77.91 %\n",
      " client3 | Error Rate: 79.02 %\n",
      " client4 | Error Rate: 78.12 %\n",
      " client5 | Error Rate: 77.62 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.07 %\n",
      "\n",
      "============ Train epoch 95 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.65 %\n",
      " client2 | Error Rate: 77.80 %\n",
      " client3 | Error Rate: 78.95 %\n",
      " client4 | Error Rate: 78.06 %\n",
      " client5 | Error Rate: 77.57 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.01 %\n",
      "\n",
      "============ Train epoch 96 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0115\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.80 %\n",
      " client2 | Error Rate: 78.01 %\n",
      " client3 | Error Rate: 79.15 %\n",
      " client4 | Error Rate: 78.19 %\n",
      " client5 | Error Rate: 77.75 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.18 %\n",
      "\n",
      "============ Train epoch 97 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0015\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.86 %\n",
      " client2 | Error Rate: 78.09 %\n",
      " client3 | Error Rate: 79.34 %\n",
      " client4 | Error Rate: 78.32 %\n",
      " client5 | Error Rate: 77.83 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.29 %\n",
      "\n",
      "============ Train epoch 98 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0155\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0000\n",
      " client5 | Train Loss: 0.0162\n",
      " client1 | Error Rate: 77.91 %\n",
      " client2 | Error Rate: 78.06 %\n",
      " client3 | Error Rate: 79.39 %\n",
      " client4 | Error Rate: 78.43 %\n",
      " client5 | Error Rate: 77.91 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.34 %\n",
      "\n",
      "============ Train epoch 99 ============\n",
      " client1 | Train Loss: 0.0000\n",
      " client2 | Train Loss: 0.0183\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0132\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.86 %\n",
      " client2 | Error Rate: 77.93 %\n",
      " client3 | Error Rate: 79.36 %\n",
      " client4 | Error Rate: 78.27 %\n",
      " client5 | Error Rate: 77.78 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.24 %\n",
      "\n",
      "============ Train epoch 100 ============\n",
      " client1 | Train Loss: 0.0061\n",
      " client2 | Train Loss: 0.0000\n",
      " client3 | Train Loss: 0.0000\n",
      " client4 | Train Loss: 0.0152\n",
      " client5 | Train Loss: 0.0000\n",
      " client1 | Error Rate: 77.99 %\n",
      " client2 | Error Rate: 78.17 %\n",
      " client3 | Error Rate: 79.34 %\n",
      " client4 | Error Rate: 78.32 %\n",
      " client5 | Error Rate: 77.88 %\n",
      "\n",
      " Best Validation Error Rate: 63.84 %, Current Validation Error Rate: 78.34 %\n",
      "\n",
      " Start final testing\n",
      "\n",
      " client1 | Error Rate: 63.14 %\n",
      " client2 | Error Rate: 63.50 %\n",
      " client3 | Error Rate: 64.17 %\n",
      " client4 | Error Rate: 63.04 %\n",
      " client5 | Error Rate: 64.07 %\n",
      " Best Test Error: 63.58 %\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "    print('Device:', device, '\\n')\n",
    "    parser = argparse.ArgumentParser()\n",
    "    parser.add_argument('--test', action='store_true', help='test the pretrained model')\n",
    "    parser.add_argument('--lr', type=float, default=1e-4, help='learning rate')\n",
    "    parser.add_argument('--wdecay', type=float, default=8e-7, help='learning rate')\n",
    "    parser.add_argument('--batch', type=int, default=128, help='batch size')\n",
    "    parser.add_argument('--iters', type=int, default=100, help='iterations for communication')\n",
    "    parser.add_argument('--wk_iters', type=int, default=1,\n",
    "                        help='optimization iters in local worker between communication')\n",
    "    parser.add_argument('--mode', type=str, default='fedavg', help='fedavg')\n",
    "    parser.add_argument('--save_path', type=str, default='./checkpoint24/RSNA', help='path to save the checkpoint')\n",
    "    parser.add_argument('--clientnum', type=int, default=5, help='client number')\n",
    "    parser.add_argument('--setnum', type=int, default=24, help='set number per client has')\n",
    "\n",
    "    parser.add_argument('--classnum', type=int, default=6, help='class num')\n",
    "    parser.add_argument('--seed', type=int, default=0, help='random seed')\n",
    "\n",
    "    parser.add_argument('--noniid', action='store_true', help='noniid sampling')\n",
    "\n",
    "    args = parser.parse_known_args()[0] # args = parser.parse_args()\n",
    "    print(args)\n",
    "\n",
    "    setup_seed(args.seed)\n",
    "\n",
    "    exp_folder = 'rsna_fedpl'\n",
    "\n",
    "    args.save_path = os.path.join(args.save_path, exp_folder)\n",
    "\n",
    "    if not os.path.exists(args.save_path):\n",
    "        os.makedirs(args.save_path)\n",
    "    SAVE_PATH = os.path.join(args.save_path,\n",
    "                             args.mode + 'client' + str(args.clientnum) + 'sets' + str(args.setnum) + 'seed' + str(\n",
    "                                 args.seed) + str(args.noniid))\n",
    "\n",
    "    # server model and ce loss\n",
    "    server_model = PLRSNAModel(class_num=args.classnum).to(device)\n",
    "    loss_fun = nn.CrossEntropyLoss()\n",
    "\n",
    "    # prepare the data\n",
    "    train_loaders, validation_loaders, test_loaders, prior_test, client_priors_corr, client_Pi = prepare_data(args)\n",
    "\n",
    "    print('\\nData prepared, start training...\\n')\n",
    "\n",
    "    # federated setting\n",
    "    client_num = args.clientnum\n",
    "    clients = ['client' + str(_) for _ in range(1, client_num + 1)]\n",
    "    client_weights = [1 / client_num for i in range(client_num)]\n",
    "    models = [copy.deepcopy(server_model).to(device) for idx in range(client_num)]\n",
    "\n",
    "    if args.test:\n",
    "        checkpoint = torch.load(SAVE_PATH)\n",
    "        server_model.load_state_dict(checkpoint['server_model'])\n",
    "        this_test_error = []\n",
    "        for test_idx, test_loader in enumerate(test_loaders):\n",
    "            test_loss = test(server_model, test_loader, loss_fun, device, classnum=args.classnum)\n",
    "            this_test_error.append(test_loss)\n",
    "            print(' {:<8s}| Error Rate: {:.2f} %'.format(clients[test_idx], test_loss * 100.))\n",
    "        print('Best Test Error: {:.2f} %'.format(100. * sum(this_test_error) / len(this_test_error)))\n",
    "\n",
    "        exit(0)\n",
    "\n",
    "    best_test_error = 1.\n",
    "    training_loss_log = []\n",
    "    error_rate_log = []\n",
    "\n",
    "    # start training\n",
    "    for a_iter in range(args.iters):\n",
    "        # record training loss and test error rate\n",
    "        this_test_error = []\n",
    "        this_train_loss = []\n",
    "\n",
    "        optimizers = [optim.Adam(params=models[idx].parameters(), lr=args.lr, weight_decay=args.wdecay)\n",
    "                      for idx in range(client_num)]\n",
    "\n",
    "        for wi in range(args.wk_iters):\n",
    "            print(\"============ Train epoch {} ============\".format(wi + 1 + a_iter * args.wk_iters))\n",
    "\n",
    "            for client_idx in range(client_num):\n",
    "                model, train_loader, optimizer = models[client_idx], train_loaders[client_idx], optimizers[client_idx]\n",
    "                train_loss = train(model, train_loader, optimizer, loss_fun, client_num, device,\n",
    "                                   client_Pi[client_idx], client_priors_corr[client_idx], prior_test, a_iter)\n",
    "                print(' {:<8s}| Train Loss: {:.4f}'.format(clients[client_idx], train_loss))\n",
    "\n",
    "                this_train_loss.append(train_loss)\n",
    "\n",
    "        # aggregation\n",
    "        server_model, models = communication(args, server_model, models, client_weights)\n",
    "\n",
    "        # start testing\n",
    "        for test_idx, test_loader in enumerate(validation_loaders):\n",
    "            test_loss = test(models[test_idx], test_loader, loss_fun, device, classnum=args.classnum)\n",
    "            this_test_error.append(test_loss)\n",
    "            print(' {:<8s}| Error Rate: {:.2f} %'.format(clients[test_idx], test_loss * 100.))\n",
    "\n",
    "        print()\n",
    "\n",
    "        # error rate after this communication\n",
    "        this_test_error = sum(this_test_error) / len(this_test_error)\n",
    "        if this_test_error < best_test_error:\n",
    "            best_test_error = this_test_error\n",
    "\n",
    "            # Save checkpoint\n",
    "            print(' Saving checkpoints to {}'.format(SAVE_PATH))\n",
    "            torch.save({\n",
    "                'server_model': server_model.state_dict(),\n",
    "                'a_iter': a_iter,\n",
    "            }, SAVE_PATH)\n",
    "\n",
    "        # Best Validation Error Rate\n",
    "        print(' Best Validation Error Rate: {:.2f} %, Current Validation Error Rate: {:.2f} %\\n'.format(\n",
    "            best_test_error * 100.,\n",
    "            this_test_error * 100.\n",
    "        ))\n",
    "\n",
    "        training_loss_log.append(sum(this_train_loss) / len(this_train_loss))\n",
    "        error_rate_log.append(this_test_error)\n",
    "\n",
    "        if not os.path.exists(os.path.join('./logs24/rsna_fedpl', args.mode)):\n",
    "            os.makedirs(os.path.join('./logs24/rsna_fedpl', args.mode))\n",
    "\n",
    "    print(' Start final testing\\n')\n",
    "    checkpoint = torch.load(SAVE_PATH)\n",
    "    server_model.load_state_dict(checkpoint['server_model'])\n",
    "    this_test_error = []\n",
    "    for test_idx, test_loader in enumerate(test_loaders):\n",
    "        test_loss = test(server_model, test_loader, loss_fun, device, classnum=args.classnum)\n",
    "        this_test_error.append(test_loss)\n",
    "        print(' {:<8s}| Error Rate: {:.2f} %'.format(clients[test_idx], test_loss * 100.))\n",
    "    print(' Best Test Error: {:.2f} %'.format(100. * sum(this_test_error) / len(this_test_error)))\n",
    "\n",
    "    error_rate_log.append(sum(this_test_error) / len(this_test_error))\n",
    "    # save record\n",
    "    np.savetxt(os.path.join('./logs24/rsna_fedpl', args.mode, 'client' + str(args.clientnum) +\n",
    "                            'sets' + str(args.setnum) + 'seed' + str(args.seed) + str(args.noniid) + 'train_loss.txt'),\n",
    "               training_loss_log, newline=\"\\r\\n\")\n",
    "    np.savetxt(os.path.join('./logs24/rsna_fedpl', args.mode, 'client' + str(args.clientnum) +\n",
    "                            'sets' + str(args.setnum) + 'seed' + str(args.seed) + str(args.noniid) + 'error_rate.txt'),\n",
    "               error_rate_log, newline=\"\\r\\n\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 18426.651987,
   "end_time": "2022-11-09T21:02:40.070923",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-11-09T15:55:33.418936",
   "version": "2.3.4"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "0523d9f301f64bcc91bc591edf487485": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "1a28a74e430447ccab17d6242ea88fb9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_a96b85e7ec7a460888d8a51d02a9fe1a",
        "IPY_MODEL_51ae376d74ef48ecbc3687afa334acc1",
        "IPY_MODEL_ee1fa34e561f4644b2fc4a8bfc8e9733"
       ],
       "layout": "IPY_MODEL_2d6f5235499b4cf28205a2594797ce5b"
      }
     },
     "2d6f5235499b4cf28205a2594797ce5b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "51ae376d74ef48ecbc3687afa334acc1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_e9ae54bcddfc4facb6e7c57154dfc890",
       "max": 46830571.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_0523d9f301f64bcc91bc591edf487485",
       "value": 46830571.0
      }
     },
     "6670aaba062f4657be5451120f74fead": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "66b2c5a84f5f4fbfa3563c5ffa629af0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "685594b2a0f24e868c1d0a4dce66f839": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "82f5173700d14221884d4a6e540a5e66": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "a96b85e7ec7a460888d8a51d02a9fe1a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_685594b2a0f24e868c1d0a4dce66f839",
       "placeholder": "​",
       "style": "IPY_MODEL_6670aaba062f4657be5451120f74fead",
       "value": "100%"
      }
     },
     "e9ae54bcddfc4facb6e7c57154dfc890": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ee1fa34e561f4644b2fc4a8bfc8e9733": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_82f5173700d14221884d4a6e540a5e66",
       "placeholder": "​",
       "style": "IPY_MODEL_66b2c5a84f5f4fbfa3563c5ffa629af0",
       "value": " 44.7M/44.7M [00:05&lt;00:00, 11.5MB/s]"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}