diff --git a/fairseq/models/wav2vec/wav2vec2_scribblelens.py b/fairseq/models/wav2vec/wav2vec2_scribblelens.py index 88c3798462..b95c736988 100644 --- a/fairseq/models/wav2vec/wav2vec2_scribblelens.py +++ b/fairseq/models/wav2vec/wav2vec2_scribblelens.py @@ -24,6 +24,7 @@ LayerNorm, MultiheadAttention, SamePad, + Smartpool, TransposeLast, ) from fairseq.modules.transformer_sentence_encoder import init_bert_params @@ -298,6 +299,24 @@ def add_args(parser): "--conv-bias", action="store_true", help="include bias in conv encoder" ) + parser.add_argument( + "--smartpooling", action="store_true", help="whether to perform smartpooling" + ) + + parser.add_argument( + "--smartpooling-factor", + type=float, + default=3, + help="factor by which the sequence's length will be reduced in smartpooling" + ) + + parser.add_argument( + "--smartpooling-search-perc", + type=float, + default=0.3, + help="percentage of length of sequence after smartpooling to search for border. Ideally the border is located somewhere in +-search_perc" + ) + def __init__(self, args): super().__init__() self.args = args @@ -312,6 +331,7 @@ def __init__(self, args): conv_bias=args.conv_bias, ) + self.smartpool = Smartpool(args.smartpooling_factor, args.smartpooling_search_perc) if args.smartpooling else None self.post_extract_proj = ( nn.Linear(self.embed, args.encoder_embed_dim) if self.embed != args.encoder_embed_dim and not args.quantize_input @@ -541,8 +561,7 @@ def forward(self, source, padding_mask=None, mask=True, features_only=False): features = features.transpose(1, 2) features = self.layer_norm(features) - unmasked_features = features.clone() - + if padding_mask is not None: assert padding_mask.size(1) == 1 padding_mask = padding_mask.squeeze(1) @@ -552,6 +571,10 @@ def forward(self, source, padding_mask=None, mask=True, features_only=False): padding_mask = padding_mask[:, ::scale] assert np.all(padding_mask.shape == features.shape[:-1]) + if self.smartpool is not None: + features, padding_mask = self.smartpool(features, padding_mask) + unmasked_features = features.clone() + if self.post_extract_proj is not None: features = self.post_extract_proj(features) diff --git a/fairseq/modules/__init__.py b/fairseq/modules/__init__.py index e2326ac6e3..40ceabc001 100644 --- a/fairseq/modules/__init__.py +++ b/fairseq/modules/__init__.py @@ -29,6 +29,7 @@ from .same_pad import SamePad from .scalar_bias import ScalarBias from .sinusoidal_positional_embedding import SinusoidalPositionalEmbedding +from .smartpool import Smartpool from .transformer_sentence_encoder_layer import TransformerSentenceEncoderLayer from .transformer_sentence_encoder import TransformerSentenceEncoder from .transpose_last import TransposeLast @@ -66,6 +67,7 @@ "SamePad", "ScalarBias", "SinusoidalPositionalEmbedding", + "Smartpool", "TransformerSentenceEncoderLayer", "TransformerSentenceEncoder", "TransformerDecoderLayer", diff --git a/fairseq/modules/smartpool.py b/fairseq/modules/smartpool.py new file mode 100644 index 0000000000..ddfd549600 --- /dev/null +++ b/fairseq/modules/smartpool.py @@ -0,0 +1,117 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +class Smartpool(nn.Module): + def __init__( + self, + factor, + search_perc + ): + """Smart pooling algorithm + + Args: + factor: factor by which the sequence's length will be reduced + search_perc: percentage of length of sequence after smartpooling to search for border. Ideally the border is located somewhere in +-search_perc + """ + super().__init__() + + self.search_perc = search_perc + self.factor = factor + self.register_buffer("filters", torch.FloatTensor([[[[-1,1],[1,-1]]]]), persistent=False) + + def warp(self, X, new_lens): + new_lens_cs = new_lens.cumsum(1) + # This really searches for the low boundary of each new pixel + pixel_contributions = new_lens_cs.view(1, -1, 1) - torch.arange(torch.round(new_lens_cs[0, -1]).item(), device=X.device).view(1, 1, -1) + pixel_contributions = pixel_contributions.view(X.size(0), X.size(1), pixel_contributions.size(2)) + # Zero out the negative contributions, i.e. pixels which come before each row + pixel_contributions = torch.max(torch.tensor(0.0, device=X.device), pixel_contributions) + + # # This contains the cumulated pixel lengths for all pixels in each + # pixel_contributions + + pixel_contributions = pixel_contributions.unsqueeze(1) + interp_weights = F.conv2d(pixel_contributions, self.filters, padding=1) + interp_weights = interp_weights[:,:,:-1,1:] # Removing padding + interp_weights = interp_weights.squeeze(1) + + # # Each column corresponds to a new element. Its values are the + # # weights associated with the original data. + # interp_weights + + interp_weights = interp_weights.transpose(1, 2) + Xnew = interp_weights @ X + return Xnew, interp_weights + + def nonzero_interval_length(self, x, dim): + nonz = (x > 0) + _, low = ((nonz.cumsum(dim) == 1) & nonz).max(dim, keepdim=True) + rev_cumsum = nonz.long().flip(dim).cumsum(dim).flip(dim) + _, high = ((rev_cumsum == 1) & nonz).max(dim, keepdim=True) + + return high - low + 1 + + def forward(self, features, padding_mask): + B,T,C = features.size() + + padding_per_batch = (padding_mask > 0).sum(1) + total_T = padding_mask.numel() - padding_per_batch.sum() + features_together = torch.cat([features[i,:T-x] for i,x in enumerate(padding_per_batch)]).unsqueeze(0) + + features_tmp = F.pad(features, (0,0,1,0), value=features_together.mean().item()) + features_tmp = features_tmp.view(1, B * (T+1), C) + + # We have to remove 1 front padding and X_i back paddings from each batch. X_i can be arbitrary + # but we have to append factors zeros so that there is one on the + # border between batches in resulting reduced sequence + # BATCH_1 000 BATCH_2 000 BATCH_3 -> REDUCED_1 0 REDUCED_2 0 REDUCED_3 + new_lens = (features_tmp[:,1:,:] - features_tmp[:,:-1,:]).abs().sum(dim=2).squeeze(0) + new_lens = F.pad(new_lens, (1,0), value=0) + new_lens = torch.cat([torch.cat([new_lens[i*(T+1)+1:(i+1)*(T+1)-x], torch.zeros(3*int(self.factor), device=new_lens.device)]) for i,x in enumerate(padding_per_batch)]).unsqueeze(0) + new_lens = new_lens / new_lens.sum(1, keepdim=True) * ((total_T / self.factor) + B) # Reducing the original length T by some factor + + features = torch.cat([torch.cat([features[i,:T-x], torch.zeros(3*int(self.factor), C, device=new_lens.device)]) for i,x in enumerate(padding_per_batch)]).unsqueeze(0) + features, interp_weights = self.warp(features, new_lens) + + # The idea is to remove B-1 the longest spanning intervals + # which contain several zeros we added earlier + + # Get the indices to remove + lengths_nonzero = self.nonzero_interval_length(interp_weights, 2) + theor_lengths = ((T - padding_per_batch) // int(self.factor) + 1).view(-1) + theor_cumsum = theor_lengths.cumsum(0) + theor_lengths = (theor_lengths.float() * self.search_perc).long() + to_remove = torch.cat( + [torch.argmax( + lengths_nonzero[:, theor_cumsum[i] - theor_lengths[i] : theor_cumsum[i] + theor_lengths[i], :]).view(1) + + theor_cumsum[i] - theor_lengths[i] for i in range(0,B-1)]) + + indices = torch.arange(lengths_nonzero.size(1), device=lengths_nonzero.device) + to_remove = torch.cat([to_remove.view(-1), indices[-1].view(1)]) + + # Remove indices + mask = torch.ones_like(features, dtype=torch.bool, device=features.device).view(1, -1, C) + mask[0, to_remove, :] = False + features = features[mask].view(-1,C) + + # Compute new features with padding + start_idx, _ = torch.sort(to_remove) + start_idx = start_idx - torch.arange(B, device=features.device) + start_idx = F.pad(start_idx, [1,0]) + sizes = start_idx[1:] - start_idx[:-1] + new_T = torch.max(sizes) + sizes = new_T - sizes + + features = torch.cat([torch.cat([features[start_idx[i-1]:start_idx[i]], torch.zeros(sizes[i-1], C, device=features.device)]) for i in range(1,B+1)]) + features = features.view(B, new_T, C) + + # Compute new mask padding mask + if padding_mask is not None: + padding_mask = torch.zeros(B, new_T, dtype=torch.bool, device=features.device) + for i,x in enumerate(sizes): + padding_mask[i, new_T-x:] = True + + return features, padding_mask + + diff --git a/uwr_related/experiments/jdzikowski/Smartpooling toy task.ipynb b/uwr_related/experiments/jdzikowski/Smartpooling toy task.ipynb new file mode 100644 index 0000000000..29f94bc9fa --- /dev/null +++ b/uwr_related/experiments/jdzikowski/Smartpooling toy task.ipynb @@ -0,0 +1,25991 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "h-QZDVJVnfs1" + }, + "source": [ + "### This notebook is optionally accelerated with a GPU runtime.\n", + "### If you would like to use this acceleration, please select the menu option \"Runtime\" -> \"Change runtime type\", select \"Hardware Accelerator\" -> \"GPU\" and click \"SAVE\"\n", + "\n", + "----------------------------------------------------------------------\n", + "\n", + "# vgg-nets\n", + "\n", + "*Author: Pytorch Team*\n", + "\n", + "**Award winning ConvNets from 2014 Imagenet ILSVRC challenge**\n", + "\n", + "\"alt\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cAq2VY_knfs7" + }, + "source": [ + "All pre-trained models expect input images normalized in the same way,\n", + "i.e. mini-batches of 3-channel RGB images of shape `(3 x H x W)`, where `H` and `W` are expected to be at least `224`.\n", + "The images have to be loaded in to a range of `[0, 1]` and then normalized using `mean = [0.485, 0.456, 0.406]`\n", + "and `std = [0.229, 0.224, 0.225]`.\n", + "\n", + "Here's a sample execution." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "APUCl9km1Tah" + }, + "source": [ + "# Models" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": { + "id": "SPGMNCmpLxgQ" + }, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "from PIL import Image\n", + "from torchvision import transforms\n", + "\n", + "import time\n", + "import math\n", + "import copy\n", + "\n", + "\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "GFxPrB8tE7kq" + }, + "outputs": [], + "source": [ + "# model = torch.hub.load('pytorch/vision:v0.6.0', 'vgg11', pretrained=False)\n", + "# or any of these variants\n", + "# model = torch.hub.load('pytorch/vision:v0.6.0', 'vgg11_bn', pretrained=True)\n", + "# model = torch.hub.load('pytorch/vision:v0.6.0', 'vgg13', pretrained=True)\n", + "# model = torch.hub.load('pytorch/vision:v0.6.0', 'vgg13_bn', pretrained=True)\n", + "# model = torch.hub.load('pytorch/vision:v0.6.0', 'vgg16', pretrained=True)\n", + "# model = torch.hub.load('pytorch/vision:v0.6.0', 'vgg16_bn', pretrained=True)\n", + "# model = torch.hub.load('pytorch/vision:v0.6.0', 'vgg19', pretrained=True)\n", + "# model = torch.hub.load('pytorch/vision:v0.6.0', 'vgg19_bn', pretrained=True)\n", + "\n", + "#print(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "id": "3SvcSw131eON" + }, + "outputs": [], + "source": [ + "class Conv(nn.Module):\n", + " def __init__(\n", + " self,\n", + " conv,\n", + " output_C=None\n", + " ):\n", + " super().__init__()\n", + " self.conv = conv\n", + " self.output_C = output_C\n", + "\n", + " def forward(self, x):\n", + " B,T,C = x.shape\n", + " x = x.unsqueeze(1)\n", + " x = self.conv(x)\n", + " if self.output_C is None:\n", + " self.output_C = C\n", + " x = x.reshape(B, -1, self.output_C)\n", + " x = x.prod(-1).sum(-1)\n", + " return x\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "id": "lpLifQs091Yv" + }, + "outputs": [], + "source": [ + "class Smartpool(nn.Module):\n", + " def __init__(\n", + " self,\n", + " factor,\n", + " search_perc,\n", + " mlp2=False\n", + " ):\n", + " \"\"\"Smart pooling algorithm\n", + "\n", + " Args:\n", + " factor: factor by which the sequence's length will be reduced\n", + " search_perc: percentage of length of sequence after smartpooling to search for border. Ideally the border is located somewhere in +-search_perc\n", + " \"\"\"\n", + " super().__init__()\n", + "\n", + " self.search_perc = search_perc\n", + " self.factor = factor\n", + " self.register_buffer(\"filters\", torch.FloatTensor([[[[-1,1],[1,-1]]]]), persistent=False)\n", + " self.mlp = nn.Sequential(\n", + " nn.Linear(2, 256),\n", + " nn.Dropout(0.1),\n", + " nn.GELU(),\n", + " nn.Linear(256,512),\n", + " nn.Dropout(0.1),\n", + " nn.GELU(),\n", + " nn.Linear(512,256),\n", + " nn.Dropout(0.1),\n", + " nn.GELU(),\n", + " nn.Linear(256,1),\n", + " nn.Sigmoid())\n", + " \n", + " if mlp2 == True:\n", + " self.mlp2 = nn.Sequential(\n", + " nn.Linear(2, 256),\n", + " nn.Dropout(0.1),\n", + " nn.GELU(),\n", + " nn.Linear(256,512),\n", + " nn.Dropout(0.1),\n", + " nn.GELU(),\n", + " nn.Linear(512,256),\n", + " nn.Dropout(0.1),\n", + " nn.GELU(),\n", + " nn.Linear(256,1))\n", + " else:\n", + " self.mlp2 = None\n", + "\n", + " def warp(self, X, new_lens):\n", + " new_lens_cs = new_lens.cumsum(1)\n", + " # This really searches for the low boundary of each new pixel\n", + " pixel_contributions = new_lens_cs.view(1, -1, 1) - torch.arange(torch.round(new_lens_cs[0, -1]).item(), device=X.device).view(1, 1, -1)\n", + " pixel_contributions = pixel_contributions.view(X.size(0), X.size(1), pixel_contributions.size(2))\n", + " # Zero out the negative contributions, i.e. pixels which come before each row \n", + " pixel_contributions = torch.max(torch.tensor(0.0, device=X.device), pixel_contributions) \n", + " \n", + " # # This contains the cumulated pixel lengths for all pixels in each \n", + " # pixel_contributions\n", + " \n", + " pixel_contributions = pixel_contributions.unsqueeze(1)\n", + " interp_weights = F.conv2d(pixel_contributions, self.filters, padding=1)\n", + " interp_weights = interp_weights[:,:,:-1,1:] # Removing padding\n", + " interp_weights = interp_weights.squeeze(1)\n", + "\n", + " # # Each column corresponds to a new element. Its values are the \n", + " # # weights associated with the original data.\n", + " # interp_weights\n", + "\n", + " interp_weights = interp_weights.transpose(1, 2)\n", + " Xnew = interp_weights @ X\n", + " return Xnew, interp_weights\n", + "\n", + " def nonzero_interval_length(self, x, dim):\n", + " nonz = (x > 0)\n", + " _, low = ((nonz.cumsum(dim) == 1) & nonz).max(dim, keepdim=True)\n", + " rev_cumsum = nonz.long().flip(dim).cumsum(dim).flip(dim)\n", + " _, high = ((rev_cumsum == 1) & nonz).max(dim, keepdim=True)\n", + " \n", + " return high - low + 1\n", + "\n", + " def forward(self, features):\n", + " B,T,C = features.size()\n", + "\n", + " padding_mask = torch.zeros(B,T, dtype=torch.bool, device=features.device)\n", + " padding_per_batch = (padding_mask > 0).sum(1)\n", + " total_T = padding_mask.numel() - padding_per_batch.sum()\n", + "\n", + " # MLP test\n", + " new_lens = self.mlp(features.view(B*T,C)).view(1,-1)\n", + " new_lens = new_lens / new_lens.sum(1, keepdim=True) * (total_T / self.factor) # Reducing the original length T by some factor\n", + " \n", + " features, interp_weights = self.warp(features, new_lens)\n", + " \n", + " if self.mlp2 is not None:\n", + " features = self.mlp2(features)\n", + "\n", + " return features\n", + " \n", + "\n", + "class DoXTimes(nn.Module):\n", + " def __init__(self, model):\n", + " super().__init__()\n", + " self.model = model\n", + "\n", + " def forward(self, x): \n", + " return torch.cat([self.model(x[i].unsqueeze(0)) for i in range(x.shape[0])]).prod(-1).sum(-1)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ybPAVStr1HuV" + }, + "source": [ + "# Task code" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "SSUEYv2oBYZG" + }, + "outputs": [], + "source": [ + "def get_batch(batch_size, seq_len, divider):\n", + " assert seq_len % divider == 0\n", + " data = torch.empty(batch_size, seq_len, 2)\n", + " data[:,:,0] = torch.zeros_like(data[:,:,0])\n", + " batch_id = torch.arange(batch_size).view(-1,1)\n", + " data[batch_id, torch.multinomial(torch.ones_like(data[:,:,0]), seq_len//divider), 0] = 1\n", + " data[:,:,1].uniform_(0,1)\n", + " return data\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "QEN8XMlE9CDm" + }, + "outputs": [], + "source": [ + "def train(model, epoch, optimizer, scheduler, dataset_len, batch_size, seq_len, divider):\n", + " model.train()\n", + " total_loss = 0.\n", + " start_time = time.time()\n", + "\n", + " for batch, i in enumerate(range(0, dataset_len, batch_size)):\n", + " data = get_batch(batch_size, seq_len, divider).to(device)\n", + " targets = data.prod(-1).sum(-1)\n", + " optimizer.zero_grad()\n", + " output = model(data)\n", + " \n", + " loss = (1 - output / targets).abs().mean()\n", + " loss.backward()\n", + " torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)\n", + " optimizer.step()\n", + "\n", + " total_loss += loss.item()\n", + " log_interval = 200\n", + " if batch % log_interval == 0 and batch > 0:\n", + " cur_loss = total_loss / log_interval\n", + " elapsed = time.time() - start_time\n", + " \"\"\"\n", + " print('| epoch {:3d} | {:5d}/{:5d} batches | '\n", + " 'lr {:02.2f} | ms/batch {:5.2f} | '\n", + " 'loss {:5.2f} | ppl {:8.2f}'.format(\n", + " epoch, batch, dataset_len // batch_size, scheduler.get_last_lr()[0],\n", + " elapsed * 1000 / log_interval,\n", + " cur_loss, math.exp(cur_loss)))\n", + " \"\"\"\n", + " print('| epoch {:3d} | {:5d}/{:5d} batches | '\n", + " 'lr {:02.2f} | ms/batch {:5.2f} | '\n", + " 'loss {:5.2f} |'.format(\n", + " epoch, batch, dataset_len // batch_size, scheduler.get_last_lr()[0],\n", + " elapsed * 1000 / log_interval,\n", + " cur_loss))\n", + " total_loss = 0\n", + " start_time = time.time()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "id": "AEyFhNHW9EJA" + }, + "outputs": [], + "source": [ + "def evaluate(model, dataset_len, batch_size, seq_len, divider):\n", + " model.eval()\n", + " total_loss = 0.\n", + " with torch.no_grad():\n", + " for i in range(0, dataset_len, batch_size):\n", + " data = get_batch(batch_size, seq_len, divider).to(device)\n", + " targets = data.prod(-1).sum(-1)\n", + " output = model(data)\n", + " total_loss += (1 - output / targets).abs().mean().item()\n", + " return total_loss / (dataset_len / batch_size)" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": { + "id": "-TTTvnM0D9Ru" + }, + "outputs": [], + "source": [ + "def train_model(model, epochs, batch_size, eval_batch_size, dataset_len, seq_len, divider, optimizer, scheduler):\n", + " best_val_loss = float(\"inf\")\n", + " best_model = None\n", + "\n", + " for epoch in range(1, epochs + 1):\n", + " epoch_start_time = time.time()\n", + " train(model, epoch, optimizer, scheduler, dataset_len, batch_size, seq_len, divider)\n", + " val_loss = evaluate(model, dataset_len, eval_batch_size, seq_len, divider)\n", + " print('-' * 89)\n", + " print('| end of epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} | '\n", + " 'valid ppl {:8.2f}'.format(epoch, (time.time() - epoch_start_time),\n", + " val_loss, math.exp(val_loss)))\n", + " print('-' * 89)\n", + "\n", + " if val_loss < best_val_loss:\n", + " best_val_loss = val_loss\n", + " best_model = copy.deepcopy(model)\n", + "\n", + " scheduler.step()\n", + "\n", + "\n", + " test_loss = evaluate(best_model, dataset_len, eval_batch_size, seq_len, divider)\n", + " print('=' * 89)\n", + " print('| End of training | test loss {:5.2f} | test ppl {:8.2f}'.format(\n", + " test_loss, math.exp(test_loss)))\n", + " print('=' * 89)\n", + "\n", + " return best_model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-zk_zQPnYFqX" + }, + "source": [ + "# To 2 rows" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YtQivzJ5xy5d" + }, + "source": [ + "## Pooling T/4" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MxS_DruSDYKz" + }, + "source": [ + "### Average pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "3tdL9jwtc8cC", + "outputId": "0f2cee26-3644-4ca2-e2df-9f5089e0ce68" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Downloading: \"https://github.com/pytorch/vision/archive/v0.6.0.zip\" to /home/i273233/.cache/torch/hub/v0.6.0.zip\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Conv(\n", + " (conv): Sequential(\n", + " (0): Conv2d(1, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (1): ReLU(inplace=True)\n", + " (2): AvgPool2d(kernel_size=2, stride=(2, 1), padding=(0, 1))\n", + " (3): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (4): ReLU(inplace=True)\n", + " (5): AvgPool2d(kernel_size=2, stride=(2, 1), padding=0)\n", + " (6): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (7): ReLU(inplace=True)\n", + " (8): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (linear): Conv2d(256, 1, kernel_size=(1, 1), stride=(1, 1))\n", + " )\n", + ")\n" + ] + } + ], + "source": [ + "model = torch.hub.load('pytorch/vision:v0.6.0', 'vgg11', pretrained=False)\n", + "model = model.features[0:9]\n", + "model[0] = nn.Conv2d(1, 64, kernel_size=(3,3), stride=(1,1), padding=(1,1))\n", + "model[2] = nn.AvgPool2d(kernel_size=2, stride=(2,1), padding=(0,1))\n", + "model[5] = nn.AvgPool2d(kernel_size=2, stride=(2,1), padding=0)\n", + "model[8] = nn.Conv2d(256, 256, kernel_size=(3,3), stride=(1,1), padding=(1,1))\n", + "model.add_module(\"linear\", nn.Conv2d(256, 1, kernel_size=1, stride=1, padding=0))\n", + "model = Conv(model)\n", + "model = model.to(device)\n", + "print(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ZUhhG0Tr3VJs", + "outputId": "51cf775a-61a3-47ad-a4ad-1cc292c11e87" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 5000 batches | lr 1.00 | ms/batch 37.01 | loss 0.95 |\n", + "| epoch 1 | 400/ 5000 batches | lr 1.00 | ms/batch 34.65 | loss 0.93 |\n", + "| epoch 1 | 600/ 5000 batches | lr 1.00 | ms/batch 34.81 | loss 0.93 |\n", + "| epoch 1 | 800/ 5000 batches | lr 1.00 | ms/batch 34.85 | loss 0.93 |\n", + "| epoch 1 | 1000/ 5000 batches | lr 1.00 | ms/batch 34.90 | loss 0.93 |\n", + "| epoch 1 | 1200/ 5000 batches | lr 1.00 | ms/batch 34.92 | loss 0.93 |\n", + "| epoch 1 | 1400/ 5000 batches | lr 1.00 | ms/batch 34.96 | loss 0.93 |\n", + "| epoch 1 | 1600/ 5000 batches | lr 1.00 | ms/batch 35.12 | loss 0.93 |\n", + "| epoch 1 | 1800/ 5000 batches | lr 1.00 | ms/batch 35.17 | loss 0.93 |\n", + "| epoch 1 | 2000/ 5000 batches | lr 1.00 | ms/batch 35.18 | loss 0.93 |\n", + "| epoch 1 | 2200/ 5000 batches | lr 1.00 | ms/batch 35.25 | loss 0.93 |\n", + "| epoch 1 | 2400/ 5000 batches | lr 1.00 | ms/batch 35.25 | loss 0.93 |\n", + "| epoch 1 | 2600/ 5000 batches | lr 1.00 | ms/batch 35.28 | loss 0.93 |\n", + "| epoch 1 | 2800/ 5000 batches | lr 1.00 | ms/batch 35.23 | loss 0.93 |\n", + "| epoch 1 | 3000/ 5000 batches | lr 1.00 | ms/batch 35.27 | loss 0.92 |\n", + "| epoch 1 | 3200/ 5000 batches | lr 1.00 | ms/batch 35.34 | loss 0.93 |\n", + "| epoch 1 | 3400/ 5000 batches | lr 1.00 | ms/batch 35.33 | loss 0.93 |\n", + "| epoch 1 | 3600/ 5000 batches | lr 1.00 | ms/batch 35.31 | loss 0.93 |\n", + "| epoch 1 | 3800/ 5000 batches | lr 1.00 | ms/batch 35.34 | loss 0.93 |\n", + "| epoch 1 | 4000/ 5000 batches | lr 1.00 | ms/batch 35.35 | loss 0.93 |\n", + "| epoch 1 | 4200/ 5000 batches | lr 1.00 | ms/batch 35.36 | loss 0.93 |\n", + "| epoch 1 | 4400/ 5000 batches | lr 1.00 | ms/batch 35.38 | loss 0.93 |\n", + "| epoch 1 | 4600/ 5000 batches | lr 1.00 | ms/batch 35.35 | loss 0.93 |\n", + "| epoch 1 | 4800/ 5000 batches | lr 1.00 | ms/batch 35.47 | loss 0.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 254.30s | valid loss 1.77 | valid ppl 5.87\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 5000 batches | lr 0.95 | ms/batch 36.88 | loss 0.57 |\n", + "| epoch 2 | 400/ 5000 batches | lr 0.95 | ms/batch 36.91 | loss 0.56 |\n", + "| epoch 2 | 600/ 5000 batches | lr 0.95 | ms/batch 36.92 | loss 0.56 |\n", + "| epoch 2 | 800/ 5000 batches | lr 0.95 | ms/batch 36.55 | loss 0.56 |\n", + "| epoch 2 | 1000/ 5000 batches | lr 0.95 | ms/batch 36.62 | loss 0.56 |\n", + "| epoch 2 | 1200/ 5000 batches | lr 0.95 | ms/batch 36.95 | loss 0.56 |\n", + "| epoch 2 | 1400/ 5000 batches | lr 0.95 | ms/batch 36.94 | loss 0.56 |\n", + "| epoch 2 | 1600/ 5000 batches | lr 0.95 | ms/batch 36.95 | loss 0.56 |\n", + "| epoch 2 | 1800/ 5000 batches | lr 0.95 | ms/batch 36.94 | loss 0.56 |\n", + "| epoch 2 | 2000/ 5000 batches | lr 0.95 | ms/batch 37.40 | loss 0.56 |\n", + "| epoch 2 | 2200/ 5000 batches | lr 0.95 | ms/batch 37.31 | loss 0.56 |\n", + "| epoch 2 | 2400/ 5000 batches | lr 0.95 | ms/batch 37.31 | loss 0.56 |\n", + "| epoch 2 | 2600/ 5000 batches | lr 0.95 | ms/batch 37.28 | loss 0.56 |\n", + "| epoch 2 | 2800/ 5000 batches | lr 0.95 | ms/batch 37.29 | loss 0.56 |\n", + "| epoch 2 | 3000/ 5000 batches | lr 0.95 | ms/batch 37.30 | loss 0.56 |\n", + "| epoch 2 | 3200/ 5000 batches | lr 0.95 | ms/batch 37.29 | loss 0.56 |\n", + "| epoch 2 | 3400/ 5000 batches | lr 0.95 | ms/batch 37.28 | loss 0.56 |\n", + "| epoch 2 | 3600/ 5000 batches | lr 0.95 | ms/batch 37.31 | loss 0.56 |\n", + "| epoch 2 | 3800/ 5000 batches | lr 0.95 | ms/batch 37.75 | loss 0.56 |\n", + "| epoch 2 | 4000/ 5000 batches | lr 0.95 | ms/batch 37.71 | loss 0.56 |\n", + "| epoch 2 | 4200/ 5000 batches | lr 0.95 | ms/batch 37.72 | loss 0.56 |\n", + "| epoch 2 | 4400/ 5000 batches | lr 0.95 | ms/batch 37.73 | loss 0.56 |\n", + "| epoch 2 | 4600/ 5000 batches | lr 0.95 | ms/batch 37.69 | loss 0.56 |\n", + "| epoch 2 | 4800/ 5000 batches | lr 0.95 | ms/batch 37.67 | loss 0.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 267.48s | valid loss 0.75 | valid ppl 2.12\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 5000 batches | lr 0.90 | ms/batch 37.92 | loss 0.53 |\n", + "| epoch 3 | 400/ 5000 batches | lr 0.90 | ms/batch 37.17 | loss 0.52 |\n", + "| epoch 3 | 600/ 5000 batches | lr 0.90 | ms/batch 37.05 | loss 0.52 |\n", + "| epoch 3 | 800/ 5000 batches | lr 0.90 | ms/batch 37.06 | loss 0.52 |\n", + "| epoch 3 | 1000/ 5000 batches | lr 0.90 | ms/batch 37.06 | loss 0.52 |\n", + "| epoch 3 | 1200/ 5000 batches | lr 0.90 | ms/batch 37.52 | loss 0.52 |\n", + "| epoch 3 | 1400/ 5000 batches | lr 0.90 | ms/batch 38.02 | loss 0.52 |\n", + "| epoch 3 | 1600/ 5000 batches | lr 0.90 | ms/batch 37.03 | loss 0.52 |\n", + "| epoch 3 | 1800/ 5000 batches | lr 0.90 | ms/batch 37.78 | loss 0.52 |\n", + "| epoch 3 | 2000/ 5000 batches | lr 0.90 | ms/batch 37.83 | loss 0.52 |\n", + "| epoch 3 | 2200/ 5000 batches | lr 0.90 | ms/batch 38.30 | loss 0.52 |\n", + "| epoch 3 | 2400/ 5000 batches | lr 0.90 | ms/batch 38.42 | loss 0.52 |\n", + "| epoch 3 | 2600/ 5000 batches | lr 0.90 | ms/batch 38.47 | loss 0.52 |\n", + "| epoch 3 | 2800/ 5000 batches | lr 0.90 | ms/batch 38.43 | loss 0.52 |\n", + "| epoch 3 | 3000/ 5000 batches | lr 0.90 | ms/batch 38.48 | loss 0.52 |\n", + "| epoch 3 | 3200/ 5000 batches | lr 0.90 | ms/batch 38.40 | loss 0.52 |\n", + "| epoch 3 | 3400/ 5000 batches | lr 0.90 | ms/batch 38.48 | loss 0.52 |\n", + "| epoch 3 | 3600/ 5000 batches | lr 0.90 | ms/batch 38.49 | loss 0.52 |\n", + "| epoch 3 | 3800/ 5000 batches | lr 0.90 | ms/batch 38.45 | loss 0.52 |\n", + "| epoch 3 | 4000/ 5000 batches | lr 0.90 | ms/batch 38.48 | loss 0.52 |\n", + "| epoch 3 | 4200/ 5000 batches | lr 0.90 | ms/batch 38.45 | loss 0.52 |\n", + "| epoch 3 | 4400/ 5000 batches | lr 0.90 | ms/batch 38.48 | loss 0.52 |\n", + "| epoch 3 | 4600/ 5000 batches | lr 0.90 | ms/batch 38.35 | loss 0.52 |\n", + "| epoch 3 | 4800/ 5000 batches | lr 0.90 | ms/batch 37.94 | loss 0.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 273.10s | valid loss 0.75 | valid ppl 2.12\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 5000 batches | lr 0.86 | ms/batch 38.24 | loss 0.49 |\n", + "| epoch 4 | 400/ 5000 batches | lr 0.86 | ms/batch 38.04 | loss 0.49 |\n", + "| epoch 4 | 600/ 5000 batches | lr 0.86 | ms/batch 38.07 | loss 0.49 |\n", + "| epoch 4 | 800/ 5000 batches | lr 0.86 | ms/batch 38.05 | loss 0.49 |\n", + "| epoch 4 | 1000/ 5000 batches | lr 0.86 | ms/batch 38.05 | loss 0.49 |\n", + "| epoch 4 | 1200/ 5000 batches | lr 0.86 | ms/batch 38.05 | loss 0.49 |\n", + "| epoch 4 | 1400/ 5000 batches | lr 0.86 | ms/batch 38.07 | loss 0.49 |\n", + "| epoch 4 | 1600/ 5000 batches | lr 0.86 | ms/batch 38.05 | loss 0.49 |\n", + "| epoch 4 | 1800/ 5000 batches | lr 0.86 | ms/batch 38.02 | loss 0.49 |\n", + "| epoch 4 | 2000/ 5000 batches | lr 0.86 | ms/batch 38.03 | loss 0.49 |\n", + "| epoch 4 | 2200/ 5000 batches | lr 0.86 | ms/batch 38.03 | loss 0.49 |\n", + "| epoch 4 | 2400/ 5000 batches | lr 0.86 | ms/batch 38.03 | loss 0.49 |\n", + "| epoch 4 | 2600/ 5000 batches | lr 0.86 | ms/batch 38.01 | loss 0.49 |\n", + "| epoch 4 | 2800/ 5000 batches | lr 0.86 | ms/batch 38.04 | loss 0.49 |\n", + "| epoch 4 | 3000/ 5000 batches | lr 0.86 | ms/batch 38.02 | loss 0.49 |\n", + "| epoch 4 | 3200/ 5000 batches | lr 0.86 | ms/batch 38.01 | loss 0.49 |\n", + "| epoch 4 | 3400/ 5000 batches | lr 0.86 | ms/batch 38.01 | loss 0.49 |\n", + "| epoch 4 | 3600/ 5000 batches | lr 0.86 | ms/batch 38.02 | loss 0.49 |\n", + "| epoch 4 | 3800/ 5000 batches | lr 0.86 | ms/batch 38.02 | loss 0.49 |\n", + "| epoch 4 | 4000/ 5000 batches | lr 0.86 | ms/batch 38.02 | loss 0.49 |\n", + "| epoch 4 | 4200/ 5000 batches | lr 0.86 | ms/batch 38.03 | loss 0.49 |\n", + "| epoch 4 | 4400/ 5000 batches | lr 0.86 | ms/batch 38.05 | loss 0.49 |\n", + "| epoch 4 | 4600/ 5000 batches | lr 0.86 | ms/batch 38.10 | loss 0.49 |\n", + "| epoch 4 | 4800/ 5000 batches | lr 0.86 | ms/batch 38.06 | loss 0.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 273.44s | valid loss 0.75 | valid ppl 2.12\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 5000 batches | lr 0.81 | ms/batch 38.24 | loss 0.46 |\n", + "| epoch 5 | 400/ 5000 batches | lr 0.81 | ms/batch 38.08 | loss 0.45 |\n", + "| epoch 5 | 600/ 5000 batches | lr 0.81 | ms/batch 38.08 | loss 0.45 |\n", + "| epoch 5 | 800/ 5000 batches | lr 0.81 | ms/batch 38.08 | loss 0.45 |\n", + "| epoch 5 | 1000/ 5000 batches | lr 0.81 | ms/batch 38.09 | loss 0.45 |\n", + "| epoch 5 | 1200/ 5000 batches | lr 0.81 | ms/batch 38.11 | loss 0.45 |\n", + "| epoch 5 | 1400/ 5000 batches | lr 0.81 | ms/batch 38.10 | loss 0.45 |\n", + "| epoch 5 | 1600/ 5000 batches | lr 0.81 | ms/batch 38.10 | loss 0.45 |\n", + "| epoch 5 | 1800/ 5000 batches | lr 0.81 | ms/batch 38.08 | loss 0.45 |\n", + "| epoch 5 | 2000/ 5000 batches | lr 0.81 | ms/batch 38.08 | loss 0.45 |\n", + "| epoch 5 | 2200/ 5000 batches | lr 0.81 | ms/batch 38.08 | loss 0.45 |\n", + "| epoch 5 | 2400/ 5000 batches | lr 0.81 | ms/batch 38.05 | loss 0.45 |\n", + "| epoch 5 | 2600/ 5000 batches | lr 0.81 | ms/batch 38.07 | loss 0.45 |\n", + "| epoch 5 | 2800/ 5000 batches | lr 0.81 | ms/batch 38.06 | loss 0.45 |\n", + "| epoch 5 | 3000/ 5000 batches | lr 0.81 | ms/batch 38.10 | loss 0.45 |\n", + "| epoch 5 | 3200/ 5000 batches | lr 0.81 | ms/batch 38.10 | loss 0.45 |\n", + "| epoch 5 | 3400/ 5000 batches | lr 0.81 | ms/batch 38.10 | loss 0.46 |\n", + "| epoch 5 | 3600/ 5000 batches | lr 0.81 | ms/batch 38.10 | loss 0.45 |\n", + "| epoch 5 | 3800/ 5000 batches | lr 0.81 | ms/batch 38.08 | loss 0.45 |\n", + "| epoch 5 | 4000/ 5000 batches | lr 0.81 | ms/batch 38.09 | loss 0.45 |\n", + "| epoch 5 | 4200/ 5000 batches | lr 0.81 | ms/batch 38.08 | loss 0.45 |\n", + "| epoch 5 | 4400/ 5000 batches | lr 0.81 | ms/batch 38.09 | loss 0.45 |\n", + "| epoch 5 | 4600/ 5000 batches | lr 0.81 | ms/batch 38.08 | loss 0.45 |\n", + "| epoch 5 | 4800/ 5000 batches | lr 0.81 | ms/batch 38.06 | loss 0.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 273.66s | valid loss 0.75 | valid ppl 2.12\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 0.75 | test ppl 2.12\n", + "=========================================================================================\n" + ] + } + ], + "source": [ + "batch_size = 20\n", + "eval_batch_size = 10\n", + "seq_len = 1000\n", + "divider = 4\n", + "dataset_len = 100000\n", + "lr = 1.0 # learning rate\n", + "optimizer = torch.optim.SGD(model.parameters(), lr=lr)\n", + "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "epochs = 5 # The number of epochs\n", + "\n", + "model = train_model(model, epochs, batch_size, eval_batch_size, dataset_len, seq_len, divider, optimizer, scheduler)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Wj4jJx_R9VpV", + "outputId": "c96d6bc6-2433-463a-c4ae-2c0278091dee" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "data[0]: tensor([[0.0000e+00, 1.9316e-01],\n", + " [0.0000e+00, 1.2288e-01],\n", + " [1.0000e+00, 8.2184e-01],\n", + " [0.0000e+00, 2.9088e-01],\n", + " [0.0000e+00, 8.8348e-01],\n", + " [0.0000e+00, 4.2186e-01],\n", + " [1.0000e+00, 9.7892e-01],\n", + " [0.0000e+00, 3.2873e-01],\n", + " [0.0000e+00, 9.8628e-04],\n", + " [0.0000e+00, 3.1622e-01],\n", + " [0.0000e+00, 2.9373e-01],\n", + " [0.0000e+00, 7.7972e-01],\n", + " [0.0000e+00, 4.7035e-01],\n", + " [0.0000e+00, 1.2425e-01],\n", + " [0.0000e+00, 3.3145e-01],\n", + " [0.0000e+00, 5.1976e-02],\n", + " [0.0000e+00, 5.1003e-01],\n", + " [0.0000e+00, 1.6871e-01],\n", + " [1.0000e+00, 7.8025e-01],\n", + " [0.0000e+00, 9.3069e-01],\n", + " [1.0000e+00, 8.0811e-01],\n", + " [0.0000e+00, 3.9422e-01],\n", + " [1.0000e+00, 9.8349e-01],\n", + " [1.0000e+00, 7.3919e-01],\n", + " [0.0000e+00, 3.6395e-01],\n", + " [1.0000e+00, 1.2754e-01],\n", + " [0.0000e+00, 7.2988e-01],\n", + " [0.0000e+00, 9.3250e-01],\n", + " [0.0000e+00, 9.5595e-02],\n", + " [0.0000e+00, 1.9342e-01],\n", + " [0.0000e+00, 4.4071e-01],\n", + " [0.0000e+00, 8.9550e-01],\n", + " [0.0000e+00, 2.9614e-01],\n", + " [0.0000e+00, 7.6576e-01],\n", + " [1.0000e+00, 1.7572e-01],\n", + " [0.0000e+00, 8.1042e-01],\n", + " [0.0000e+00, 8.7943e-01],\n", + " [0.0000e+00, 7.7275e-01],\n", + " [0.0000e+00, 9.4483e-01],\n", + " [0.0000e+00, 6.4526e-01],\n", + " [0.0000e+00, 7.1774e-02],\n", + " [0.0000e+00, 4.8297e-02],\n", + " [1.0000e+00, 3.1190e-01],\n", + " [1.0000e+00, 9.0207e-01],\n", + " [1.0000e+00, 2.5033e-01],\n", + " [0.0000e+00, 5.9913e-01],\n", + " [0.0000e+00, 5.2550e-01],\n", + " [0.0000e+00, 9.3465e-01],\n", + " [0.0000e+00, 9.0823e-01],\n", + " [1.0000e+00, 1.9213e-01]], device='cuda:0')\n", + "output: tensor([31.0412, 31.0412, 31.0412, 31.0412, 31.0412, 31.0412, 31.0412, 31.0412,\n", + " 31.0412, 31.0412], device='cuda:0')\n", + "targets: tensor([115.4538, 122.5260, 130.9462, 121.8647, 123.4440, 130.8453, 122.2321,\n", + " 129.7234, 123.2588, 122.7149], device='cuda:0')\n" + ] + } + ], + "source": [ + "model.eval()\n", + "with torch.no_grad():\n", + " data = get_batch(eval_batch_size, seq_len, divider).to(device)\n", + " targets = data.prod(-1).sum(-1).squeeze(-1)\n", + " output = model(data)\n", + " print('data[0]:', data[0,:50,:])\n", + "\n", + " print('output:', output)\n", + " print('targets:', targets)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FRBAvZu7fMqE" + }, + "source": [ + "### Max pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Ez6hLt4efLjM", + "outputId": "6df56109-6233-4f6b-c111-2174ff1cb5dc" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using cache found in /home/i273233/.cache/torch/hub/pytorch_vision_v0.6.0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Conv(\n", + " (conv): Sequential(\n", + " (0): Conv2d(1, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (1): ReLU(inplace=True)\n", + " (2): MaxPool2d(kernel_size=2, stride=(2, 1), padding=(0, 1), dilation=1, ceil_mode=False)\n", + " (3): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (4): ReLU(inplace=True)\n", + " (5): MaxPool2d(kernel_size=2, stride=(2, 1), padding=0, dilation=1, ceil_mode=False)\n", + " (6): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (7): ReLU(inplace=True)\n", + " (8): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (linear): Conv2d(256, 1, kernel_size=(1, 1), stride=(1, 1))\n", + " )\n", + ")\n" + ] + } + ], + "source": [ + "model = torch.hub.load('pytorch/vision:v0.6.0', 'vgg11', pretrained=False)\n", + "model = model.features[0:9]\n", + "model[0] = nn.Conv2d(1, 64, kernel_size=(3,3), stride=(1,1), padding=(1,1))\n", + "model[2] = nn.MaxPool2d(kernel_size=2, stride=(2,1), padding=(0,1))\n", + "model[5] = nn.MaxPool2d(kernel_size=2, stride=(2,1), padding=0)\n", + "model[8] = nn.Conv2d(256, 256, kernel_size=(3,3), stride=(1,1), padding=(1,1))\n", + "model.add_module(\"linear\", nn.Conv2d(256, 1, kernel_size=1, stride=1, padding=0))\n", + "model = Conv(model)\n", + "model = model.to(device)\n", + "print(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "G-B_SfUcfW8Y", + "outputId": "9f4b11e7-a30e-4e56-b5ae-fc37b9dbfd2c" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 5000 batches | lr 1.00 | ms/batch 37.73 | loss 0.67 |\n", + "| epoch 1 | 400/ 5000 batches | lr 1.00 | ms/batch 37.67 | loss 0.48 |\n", + "| epoch 1 | 600/ 5000 batches | lr 1.00 | ms/batch 37.69 | loss 0.48 |\n", + "| epoch 1 | 800/ 5000 batches | lr 1.00 | ms/batch 37.69 | loss 0.48 |\n", + "| epoch 1 | 1000/ 5000 batches | lr 1.00 | ms/batch 37.69 | loss 0.48 |\n", + "| epoch 1 | 1200/ 5000 batches | lr 1.00 | ms/batch 37.68 | loss 0.48 |\n", + "| epoch 1 | 1400/ 5000 batches | lr 1.00 | ms/batch 37.71 | loss 0.48 |\n", + "| epoch 1 | 1600/ 5000 batches | lr 1.00 | ms/batch 37.68 | loss 0.48 |\n", + "| epoch 1 | 1800/ 5000 batches | lr 1.00 | ms/batch 37.67 | loss 0.48 |\n", + "| epoch 1 | 2000/ 5000 batches | lr 1.00 | ms/batch 37.68 | loss 0.48 |\n", + "| epoch 1 | 2200/ 5000 batches | lr 1.00 | ms/batch 37.69 | loss 0.48 |\n", + "| epoch 1 | 2400/ 5000 batches | lr 1.00 | ms/batch 37.69 | loss 0.47 |\n", + "| epoch 1 | 2600/ 5000 batches | lr 1.00 | ms/batch 37.68 | loss 0.48 |\n", + "| epoch 1 | 2800/ 5000 batches | lr 1.00 | ms/batch 37.67 | loss 0.48 |\n", + "| epoch 1 | 3000/ 5000 batches | lr 1.00 | ms/batch 37.68 | loss 0.48 |\n", + "| epoch 1 | 3200/ 5000 batches | lr 1.00 | ms/batch 37.70 | loss 0.48 |\n", + "| epoch 1 | 3400/ 5000 batches | lr 1.00 | ms/batch 37.69 | loss 0.48 |\n", + "| epoch 1 | 3600/ 5000 batches | lr 1.00 | ms/batch 37.67 | loss 0.48 |\n", + "| epoch 1 | 3800/ 5000 batches | lr 1.00 | ms/batch 37.85 | loss 0.48 |\n", + "| epoch 1 | 4000/ 5000 batches | lr 1.00 | ms/batch 37.99 | loss 0.48 |\n", + "| epoch 1 | 4200/ 5000 batches | lr 1.00 | ms/batch 37.98 | loss 0.48 |\n", + "| epoch 1 | 4400/ 5000 batches | lr 1.00 | ms/batch 37.98 | loss 0.48 |\n", + "| epoch 1 | 4600/ 5000 batches | lr 1.00 | ms/batch 37.97 | loss 0.48 |\n", + "| epoch 1 | 4800/ 5000 batches | lr 1.00 | ms/batch 37.98 | loss 0.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 271.28s | valid loss 0.05 | valid ppl 1.05\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 5000 batches | lr 0.95 | ms/batch 38.17 | loss 0.46 |\n", + "| epoch 2 | 400/ 5000 batches | lr 0.95 | ms/batch 37.96 | loss 0.46 |\n", + "| epoch 2 | 600/ 5000 batches | lr 0.95 | ms/batch 37.99 | loss 0.46 |\n", + "| epoch 2 | 800/ 5000 batches | lr 0.95 | ms/batch 37.86 | loss 0.46 |\n", + "| epoch 2 | 1000/ 5000 batches | lr 0.95 | ms/batch 37.88 | loss 0.46 |\n", + "| epoch 2 | 1200/ 5000 batches | lr 0.95 | ms/batch 37.97 | loss 0.46 |\n", + "| epoch 2 | 1400/ 5000 batches | lr 0.95 | ms/batch 37.98 | loss 0.46 |\n", + "| epoch 2 | 1600/ 5000 batches | lr 0.95 | ms/batch 37.98 | loss 0.46 |\n", + "| epoch 2 | 1800/ 5000 batches | lr 0.95 | ms/batch 38.00 | loss 0.46 |\n", + "| epoch 2 | 2000/ 5000 batches | lr 0.95 | ms/batch 37.99 | loss 0.46 |\n", + "| epoch 2 | 2200/ 5000 batches | lr 0.95 | ms/batch 38.01 | loss 0.46 |\n", + "| epoch 2 | 2400/ 5000 batches | lr 0.95 | ms/batch 38.01 | loss 0.46 |\n", + "| epoch 2 | 2600/ 5000 batches | lr 0.95 | ms/batch 37.99 | loss 0.46 |\n", + "| epoch 2 | 2800/ 5000 batches | lr 0.95 | ms/batch 37.99 | loss 0.46 |\n", + "| epoch 2 | 3000/ 5000 batches | lr 0.95 | ms/batch 38.01 | loss 0.46 |\n", + "| epoch 2 | 3200/ 5000 batches | lr 0.95 | ms/batch 37.99 | loss 0.46 |\n", + "| epoch 2 | 3400/ 5000 batches | lr 0.95 | ms/batch 37.99 | loss 0.46 |\n", + "| epoch 2 | 3600/ 5000 batches | lr 0.95 | ms/batch 37.96 | loss 0.46 |\n", + "| epoch 2 | 3800/ 5000 batches | lr 0.95 | ms/batch 37.98 | loss 0.46 |\n", + "| epoch 2 | 4000/ 5000 batches | lr 0.95 | ms/batch 38.02 | loss 0.46 |\n", + "| epoch 2 | 4200/ 5000 batches | lr 0.95 | ms/batch 38.02 | loss 0.46 |\n", + "| epoch 2 | 4400/ 5000 batches | lr 0.95 | ms/batch 38.00 | loss 0.46 |\n", + "| epoch 2 | 4600/ 5000 batches | lr 0.95 | ms/batch 38.00 | loss 0.46 |\n", + "| epoch 2 | 4800/ 5000 batches | lr 0.95 | ms/batch 38.03 | loss 0.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 272.51s | valid loss 0.05 | valid ppl 1.05\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 5000 batches | lr 0.90 | ms/batch 38.19 | loss 0.45 |\n", + "| epoch 3 | 400/ 5000 batches | lr 0.90 | ms/batch 38.03 | loss 0.45 |\n", + "| epoch 3 | 600/ 5000 batches | lr 0.90 | ms/batch 38.02 | loss 0.45 |\n", + "| epoch 3 | 800/ 5000 batches | lr 0.90 | ms/batch 38.03 | loss 0.45 |\n", + "| epoch 3 | 1000/ 5000 batches | lr 0.90 | ms/batch 38.01 | loss 0.45 |\n", + "| epoch 3 | 1200/ 5000 batches | lr 0.90 | ms/batch 38.01 | loss 0.45 |\n", + "| epoch 3 | 1400/ 5000 batches | lr 0.90 | ms/batch 38.06 | loss 0.45 |\n", + "| epoch 3 | 1600/ 5000 batches | lr 0.90 | ms/batch 37.72 | loss 0.45 |\n", + "| epoch 3 | 1800/ 5000 batches | lr 0.90 | ms/batch 37.71 | loss 0.45 |\n", + "| epoch 3 | 2000/ 5000 batches | lr 0.90 | ms/batch 37.71 | loss 0.45 |\n", + "| epoch 3 | 2200/ 5000 batches | lr 0.90 | ms/batch 37.73 | loss 0.45 |\n", + "| epoch 3 | 2400/ 5000 batches | lr 0.90 | ms/batch 37.72 | loss 0.45 |\n", + "| epoch 3 | 2600/ 5000 batches | lr 0.90 | ms/batch 37.73 | loss 0.45 |\n", + "| epoch 3 | 2800/ 5000 batches | lr 0.90 | ms/batch 37.73 | loss 0.45 |\n", + "| epoch 3 | 3000/ 5000 batches | lr 0.90 | ms/batch 37.74 | loss 0.45 |\n", + "| epoch 3 | 3200/ 5000 batches | lr 0.90 | ms/batch 37.73 | loss 0.45 |\n", + "| epoch 3 | 3400/ 5000 batches | lr 0.90 | ms/batch 37.72 | loss 0.45 |\n", + "| epoch 3 | 3600/ 5000 batches | lr 0.90 | ms/batch 37.72 | loss 0.45 |\n", + "| epoch 3 | 3800/ 5000 batches | lr 0.90 | ms/batch 37.71 | loss 0.45 |\n", + "| epoch 3 | 4000/ 5000 batches | lr 0.90 | ms/batch 37.72 | loss 0.45 |\n", + "| epoch 3 | 4200/ 5000 batches | lr 0.90 | ms/batch 37.72 | loss 0.45 |\n", + "| epoch 3 | 4400/ 5000 batches | lr 0.90 | ms/batch 37.70 | loss 0.45 |\n", + "| epoch 3 | 4600/ 5000 batches | lr 0.90 | ms/batch 37.77 | loss 0.45 |\n", + "| epoch 3 | 4800/ 5000 batches | lr 0.90 | ms/batch 38.05 | loss 0.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 271.76s | valid loss 0.05 | valid ppl 1.05\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 5000 batches | lr 0.86 | ms/batch 38.18 | loss 0.44 |\n", + "| epoch 4 | 400/ 5000 batches | lr 0.86 | ms/batch 38.04 | loss 0.44 |\n", + "| epoch 4 | 600/ 5000 batches | lr 0.86 | ms/batch 38.04 | loss 0.44 |\n", + "| epoch 4 | 800/ 5000 batches | lr 0.86 | ms/batch 38.02 | loss 0.44 |\n", + "| epoch 4 | 1000/ 5000 batches | lr 0.86 | ms/batch 38.02 | loss 0.44 |\n", + "| epoch 4 | 1200/ 5000 batches | lr 0.86 | ms/batch 38.16 | loss 0.44 |\n", + "| epoch 4 | 1400/ 5000 batches | lr 0.86 | ms/batch 38.04 | loss 0.44 |\n", + "| epoch 4 | 1600/ 5000 batches | lr 0.86 | ms/batch 38.04 | loss 0.44 |\n", + "| epoch 4 | 1800/ 5000 batches | lr 0.86 | ms/batch 38.06 | loss 0.44 |\n", + "| epoch 4 | 2000/ 5000 batches | lr 0.86 | ms/batch 38.04 | loss 0.44 |\n", + "| epoch 4 | 2200/ 5000 batches | lr 0.86 | ms/batch 38.06 | loss 0.44 |\n", + "| epoch 4 | 2400/ 5000 batches | lr 0.86 | ms/batch 38.03 | loss 0.44 |\n", + "| epoch 4 | 2600/ 5000 batches | lr 0.86 | ms/batch 38.02 | loss 0.44 |\n", + "| epoch 4 | 2800/ 5000 batches | lr 0.86 | ms/batch 38.05 | loss 0.44 |\n", + "| epoch 4 | 3000/ 5000 batches | lr 0.86 | ms/batch 38.03 | loss 0.44 |\n", + "| epoch 4 | 3200/ 5000 batches | lr 0.86 | ms/batch 38.06 | loss 0.44 |\n", + "| epoch 4 | 3400/ 5000 batches | lr 0.86 | ms/batch 38.02 | loss 0.44 |\n", + "| epoch 4 | 3600/ 5000 batches | lr 0.86 | ms/batch 38.02 | loss 0.44 |\n", + "| epoch 4 | 3800/ 5000 batches | lr 0.86 | ms/batch 38.04 | loss 0.44 |\n", + "| epoch 4 | 4000/ 5000 batches | lr 0.86 | ms/batch 37.76 | loss 0.44 |\n", + "| epoch 4 | 4200/ 5000 batches | lr 0.86 | ms/batch 37.74 | loss 0.44 |\n", + "| epoch 4 | 4400/ 5000 batches | lr 0.86 | ms/batch 37.75 | loss 0.44 |\n", + "| epoch 4 | 4600/ 5000 batches | lr 0.86 | ms/batch 37.72 | loss 0.44 |\n", + "| epoch 4 | 4800/ 5000 batches | lr 0.86 | ms/batch 37.73 | loss 0.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 272.51s | valid loss 0.05 | valid ppl 1.05\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 5000 batches | lr 0.81 | ms/batch 37.91 | loss 0.43 |\n", + "| epoch 5 | 400/ 5000 batches | lr 0.81 | ms/batch 37.70 | loss 0.42 |\n", + "| epoch 5 | 600/ 5000 batches | lr 0.81 | ms/batch 37.71 | loss 0.43 |\n", + "| epoch 5 | 800/ 5000 batches | lr 0.81 | ms/batch 37.72 | loss 0.43 |\n", + "| epoch 5 | 1000/ 5000 batches | lr 0.81 | ms/batch 37.71 | loss 0.43 |\n", + "| epoch 5 | 1200/ 5000 batches | lr 0.81 | ms/batch 37.88 | loss 0.43 |\n", + "| epoch 5 | 1400/ 5000 batches | lr 0.81 | ms/batch 38.02 | loss 0.43 |\n", + "| epoch 5 | 1600/ 5000 batches | lr 0.81 | ms/batch 38.01 | loss 0.43 |\n", + "| epoch 5 | 1800/ 5000 batches | lr 0.81 | ms/batch 37.98 | loss 0.43 |\n", + "| epoch 5 | 2000/ 5000 batches | lr 0.81 | ms/batch 38.00 | loss 0.43 |\n", + "| epoch 5 | 2200/ 5000 batches | lr 0.81 | ms/batch 38.00 | loss 0.43 |\n", + "| epoch 5 | 2400/ 5000 batches | lr 0.81 | ms/batch 38.01 | loss 0.43 |\n", + "| epoch 5 | 2600/ 5000 batches | lr 0.81 | ms/batch 38.02 | loss 0.43 |\n", + "| epoch 5 | 2800/ 5000 batches | lr 0.81 | ms/batch 37.98 | loss 0.43 |\n", + "| epoch 5 | 3000/ 5000 batches | lr 0.81 | ms/batch 38.00 | loss 0.43 |\n", + "| epoch 5 | 3200/ 5000 batches | lr 0.81 | ms/batch 38.01 | loss 0.43 |\n", + "| epoch 5 | 3400/ 5000 batches | lr 0.81 | ms/batch 37.99 | loss 0.43 |\n", + "| epoch 5 | 3600/ 5000 batches | lr 0.81 | ms/batch 37.97 | loss 0.43 |\n", + "| epoch 5 | 3800/ 5000 batches | lr 0.81 | ms/batch 38.00 | loss 0.43 |\n", + "| epoch 5 | 4000/ 5000 batches | lr 0.81 | ms/batch 37.97 | loss 0.43 |\n", + "| epoch 5 | 4200/ 5000 batches | lr 0.81 | ms/batch 38.03 | loss 0.43 |\n", + "| epoch 5 | 4400/ 5000 batches | lr 0.81 | ms/batch 38.00 | loss 0.43 |\n", + "| epoch 5 | 4600/ 5000 batches | lr 0.81 | ms/batch 38.00 | loss 0.43 |\n", + "| epoch 5 | 4800/ 5000 batches | lr 0.81 | ms/batch 37.97 | loss 0.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 272.24s | valid loss 0.05 | valid ppl 1.05\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 0.05 | test ppl 1.05\n", + "=========================================================================================\n" + ] + } + ], + "source": [ + "batch_size = 20\n", + "eval_batch_size = 10\n", + "seq_len = 1000\n", + "divider = 4\n", + "dataset_len = 100000\n", + "lr = 1.0 # learning rate\n", + "optimizer = torch.optim.SGD(model.parameters(), lr=lr)\n", + "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "epochs = 5 # The number of epochs\n", + "\n", + "model = train_model(model, epochs, batch_size, eval_batch_size, dataset_len, seq_len, divider, optimizer, scheduler)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "5cnpf7knfdNW", + "outputId": "933ea7d7-bfa5-4a12-9cd9-99ebd337c58c" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "data[0]: tensor([[0.0000, 0.5144],\n", + " [0.0000, 0.4849],\n", + " [0.0000, 0.8429],\n", + " [1.0000, 0.5824],\n", + " [0.0000, 0.7493],\n", + " [0.0000, 0.3822],\n", + " [0.0000, 0.3252],\n", + " [0.0000, 0.6235],\n", + " [1.0000, 0.2795],\n", + " [1.0000, 0.5185],\n", + " [0.0000, 0.1497],\n", + " [1.0000, 0.9238],\n", + " [0.0000, 0.4850],\n", + " [1.0000, 0.8176],\n", + " [1.0000, 0.3368],\n", + " [0.0000, 0.5251],\n", + " [0.0000, 0.0786],\n", + " [0.0000, 0.8424],\n", + " [0.0000, 0.7199],\n", + " [0.0000, 0.9394],\n", + " [0.0000, 0.0350],\n", + " [0.0000, 0.5511],\n", + " [0.0000, 0.0335],\n", + " [0.0000, 0.6633],\n", + " [0.0000, 0.2322],\n", + " [0.0000, 0.6020],\n", + " [1.0000, 0.5486],\n", + " [0.0000, 0.9701],\n", + " [1.0000, 0.8008],\n", + " [0.0000, 0.0352],\n", + " [0.0000, 0.8791],\n", + " [1.0000, 0.2091],\n", + " [0.0000, 0.9137],\n", + " [0.0000, 0.7905],\n", + " [0.0000, 0.7517],\n", + " [1.0000, 0.0346],\n", + " [0.0000, 0.0902],\n", + " [0.0000, 0.9536],\n", + " [0.0000, 0.8477],\n", + " [1.0000, 0.4926],\n", + " [0.0000, 0.4298],\n", + " [0.0000, 0.2160],\n", + " [1.0000, 0.7268],\n", + " [1.0000, 0.4519],\n", + " [1.0000, 0.4687],\n", + " [0.0000, 0.4201],\n", + " [0.0000, 0.8140],\n", + " [1.0000, 0.0196],\n", + " [0.0000, 0.3433],\n", + " [1.0000, 0.2634]], device='cuda:0')\n", + "output: tensor([131.0273, 131.0265, 131.0272, 131.0270, 131.0276, 131.0260, 131.0255,\n", + " 131.0255, 131.0267, 131.0267], device='cuda:0')\n", + "targets: tensor([125.7444, 129.0693, 127.3049, 121.3991, 134.2188, 118.3687, 126.0899,\n", + " 122.0352, 126.7802, 131.4394], device='cuda:0')\n" + ] + } + ], + "source": [ + "model.eval()\n", + "with torch.no_grad():\n", + " data = get_batch(eval_batch_size, seq_len, divider).to(device)\n", + " targets = data.prod(-1).sum(-1).squeeze(-1)\n", + " output = model(data)\n", + " print('data[0]:', data[0,:50,:])\n", + "\n", + " print('output:', output)\n", + " print('targets:', targets)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vywCegytDUbK" + }, + "source": [ + "### Smart pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 674 + }, + "id": "HQ08b1msmZKw", + "outputId": "e7a0898e-cb4e-42dd-fbd3-45ccc8a69e04" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "DoXTimes(\n", + " (model): Smartpool(\n", + " (mlp): Sequential(\n", + " (0): Linear(in_features=2, out_features=256, bias=True)\n", + " (1): Dropout(p=0.1, inplace=False)\n", + " (2): GELU()\n", + " (3): Linear(in_features=256, out_features=512, bias=True)\n", + " (4): Dropout(p=0.1, inplace=False)\n", + " (5): GELU()\n", + " (6): Linear(in_features=512, out_features=256, bias=True)\n", + " (7): Dropout(p=0.1, inplace=False)\n", + " (8): GELU()\n", + " (9): Linear(in_features=256, out_features=1, bias=True)\n", + " (10): Sigmoid()\n", + " )\n", + " )\n", + ")\n", + "| epoch 1 | 200/ 5000 batches | lr 1.00 | ms/batch 72.50 | loss 0.04 |\n", + "| epoch 1 | 400/ 5000 batches | lr 1.00 | ms/batch 73.39 | loss 0.00 |\n", + "| epoch 1 | 600/ 5000 batches | lr 1.00 | ms/batch 73.41 | loss 0.00 |\n", + "| epoch 1 | 800/ 5000 batches | lr 1.00 | ms/batch 73.26 | loss 0.00 |\n", + "| epoch 1 | 1000/ 5000 batches | lr 1.00 | ms/batch 73.10 | loss 0.00 |\n", + "| epoch 1 | 1200/ 5000 batches | lr 1.00 | ms/batch 73.38 | loss 0.00 |\n", + "| epoch 1 | 1400/ 5000 batches | lr 1.00 | ms/batch 73.12 | loss 0.00 |\n", + "| epoch 1 | 1600/ 5000 batches | lr 1.00 | ms/batch 73.06 | loss 0.00 |\n", + "| epoch 1 | 1800/ 5000 batches | lr 1.00 | ms/batch 73.35 | loss 0.00 |\n", + "| epoch 1 | 2000/ 5000 batches | lr 1.00 | ms/batch 73.29 | loss 0.00 |\n", + "| epoch 1 | 2200/ 5000 batches | lr 1.00 | ms/batch 72.92 | loss 0.00 |\n", + "| epoch 1 | 2400/ 5000 batches | lr 1.00 | ms/batch 73.45 | loss 0.00 |\n", + "| epoch 1 | 2600/ 5000 batches | lr 1.00 | ms/batch 72.68 | loss 0.00 |\n", + "| epoch 1 | 2800/ 5000 batches | lr 1.00 | ms/batch 73.20 | loss 0.00 |\n", + "| epoch 1 | 3000/ 5000 batches | lr 1.00 | ms/batch 73.26 | loss 0.00 |\n", + "| epoch 1 | 3200/ 5000 batches | lr 1.00 | ms/batch 73.33 | loss 0.00 |\n", + "| epoch 1 | 3400/ 5000 batches | lr 1.00 | ms/batch 72.86 | loss 0.00 |\n", + "| epoch 1 | 3600/ 5000 batches | lr 1.00 | ms/batch 73.05 | loss 0.00 |\n", + "| epoch 1 | 3800/ 5000 batches | lr 1.00 | ms/batch 72.98 | loss 0.00 |\n", + "| epoch 1 | 4000/ 5000 batches | lr 1.00 | ms/batch 73.47 | loss 0.00 |\n", + "| epoch 1 | 4200/ 5000 batches | lr 1.00 | ms/batch 73.19 | loss 0.00 |\n", + "| epoch 1 | 4400/ 5000 batches | lr 1.00 | ms/batch 73.13 | loss 0.00 |\n", + "| epoch 1 | 4600/ 5000 batches | lr 1.00 | ms/batch 72.83 | loss 0.00 |\n", + "| epoch 1 | 4800/ 5000 batches | lr 1.00 | ms/batch 73.16 | loss 0.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 498.63s | valid loss 0.00 | valid ppl 1.00\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 5000 batches | lr 0.95 | ms/batch 73.47 | loss 0.00 |\n", + "| epoch 2 | 400/ 5000 batches | lr 0.95 | ms/batch 72.93 | loss 0.00 |\n", + "| epoch 2 | 600/ 5000 batches | lr 0.95 | ms/batch 72.96 | loss 0.00 |\n", + "| epoch 2 | 800/ 5000 batches | lr 0.95 | ms/batch 72.94 | loss 0.00 |\n", + "| epoch 2 | 1000/ 5000 batches | lr 0.95 | ms/batch 72.69 | loss 0.00 |\n", + "| epoch 2 | 1200/ 5000 batches | lr 0.95 | ms/batch 72.92 | loss 0.00 |\n", + "| epoch 2 | 1400/ 5000 batches | lr 0.95 | ms/batch 72.87 | loss 0.00 |\n", + "| epoch 2 | 1600/ 5000 batches | lr 0.95 | ms/batch 72.73 | loss 0.00 |\n", + "| epoch 2 | 1800/ 5000 batches | lr 0.95 | ms/batch 68.17 | loss 0.00 |\n", + "| epoch 2 | 2000/ 5000 batches | lr 0.95 | ms/batch 63.07 | loss 0.00 |\n", + "| epoch 2 | 2200/ 5000 batches | lr 0.95 | ms/batch 63.06 | loss 0.00 |\n", + "| epoch 2 | 2400/ 5000 batches | lr 0.95 | ms/batch 63.07 | loss 0.00 |\n", + "| epoch 2 | 2600/ 5000 batches | lr 0.95 | ms/batch 63.05 | loss 0.00 |\n", + "| epoch 2 | 2800/ 5000 batches | lr 0.95 | ms/batch 64.84 | loss 0.00 |\n", + "| epoch 2 | 3000/ 5000 batches | lr 0.95 | ms/batch 72.64 | loss 0.00 |\n", + "| epoch 2 | 3200/ 5000 batches | lr 0.95 | ms/batch 72.81 | loss 0.00 |\n", + "| epoch 2 | 3400/ 5000 batches | lr 0.95 | ms/batch 73.00 | loss 0.00 |\n", + "| epoch 2 | 3600/ 5000 batches | lr 0.95 | ms/batch 72.91 | loss 0.00 |\n", + "| epoch 2 | 3800/ 5000 batches | lr 0.95 | ms/batch 73.11 | loss 0.00 |\n", + "| epoch 2 | 4000/ 5000 batches | lr 0.95 | ms/batch 64.45 | loss 0.00 |\n", + "| epoch 2 | 4200/ 5000 batches | lr 0.95 | ms/batch 63.04 | loss 0.00 |\n", + "| epoch 2 | 4400/ 5000 batches | lr 0.95 | ms/batch 63.06 | loss 0.00 |\n", + "| epoch 2 | 4600/ 5000 batches | lr 0.95 | ms/batch 63.04 | loss 0.00 |\n", + "| epoch 2 | 4800/ 5000 batches | lr 0.95 | ms/batch 63.04 | loss 0.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 475.58s | valid loss 0.00 | valid ppl 1.00\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 5000 batches | lr 0.90 | ms/batch 63.36 | loss 0.00 |\n", + "| epoch 3 | 400/ 5000 batches | lr 0.90 | ms/batch 63.04 | loss 0.00 |\n", + "| epoch 3 | 600/ 5000 batches | lr 0.90 | ms/batch 63.04 | loss 0.00 |\n", + "| epoch 3 | 800/ 5000 batches | lr 0.90 | ms/batch 63.02 | loss 0.00 |\n", + "| epoch 3 | 1000/ 5000 batches | lr 0.90 | ms/batch 63.05 | loss 0.00 |\n", + "| epoch 3 | 1200/ 5000 batches | lr 0.90 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 3 | 1400/ 5000 batches | lr 0.90 | ms/batch 63.04 | loss 0.00 |\n", + "| epoch 3 | 1600/ 5000 batches | lr 0.90 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 3 | 1800/ 5000 batches | lr 0.90 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 3 | 2000/ 5000 batches | lr 0.90 | ms/batch 63.06 | loss 0.00 |\n", + "| epoch 3 | 2200/ 5000 batches | lr 0.90 | ms/batch 63.05 | loss 0.00 |\n", + "| epoch 3 | 2400/ 5000 batches | lr 0.90 | ms/batch 63.04 | loss 0.00 |\n", + "| epoch 3 | 2600/ 5000 batches | lr 0.90 | ms/batch 63.05 | loss 0.00 |\n", + "| epoch 3 | 2800/ 5000 batches | lr 0.90 | ms/batch 63.05 | loss 0.00 |\n", + "| epoch 3 | 3000/ 5000 batches | lr 0.90 | ms/batch 63.05 | loss 0.00 |\n", + "| epoch 3 | 3200/ 5000 batches | lr 0.90 | ms/batch 63.04 | loss 0.00 |\n", + "| epoch 3 | 3400/ 5000 batches | lr 0.90 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 3 | 3600/ 5000 batches | lr 0.90 | ms/batch 63.05 | loss 0.00 |\n", + "| epoch 3 | 3800/ 5000 batches | lr 0.90 | ms/batch 63.33 | loss 0.00 |\n", + "| epoch 3 | 4000/ 5000 batches | lr 0.90 | ms/batch 63.04 | loss 0.00 |\n", + "| epoch 3 | 4200/ 5000 batches | lr 0.90 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 3 | 4400/ 5000 batches | lr 0.90 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 3 | 4600/ 5000 batches | lr 0.90 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 3 | 4800/ 5000 batches | lr 0.90 | ms/batch 63.04 | loss 0.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 448.19s | valid loss 0.00 | valid ppl 1.00\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 5000 batches | lr 0.86 | ms/batch 63.33 | loss 0.00 |\n", + "| epoch 4 | 400/ 5000 batches | lr 0.86 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 4 | 600/ 5000 batches | lr 0.86 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 4 | 800/ 5000 batches | lr 0.86 | ms/batch 63.04 | loss 0.00 |\n", + "| epoch 4 | 1000/ 5000 batches | lr 0.86 | ms/batch 63.01 | loss 0.00 |\n", + "| epoch 4 | 1200/ 5000 batches | lr 0.86 | ms/batch 63.02 | loss 0.00 |\n", + "| epoch 4 | 1400/ 5000 batches | lr 0.86 | ms/batch 63.04 | loss 0.00 |\n", + "| epoch 4 | 1600/ 5000 batches | lr 0.86 | ms/batch 63.02 | loss 0.00 |\n", + "| epoch 4 | 1800/ 5000 batches | lr 0.86 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 4 | 2000/ 5000 batches | lr 0.86 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 4 | 2200/ 5000 batches | lr 0.86 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 4 | 2400/ 5000 batches | lr 0.86 | ms/batch 63.05 | loss 0.00 |\n", + "| epoch 4 | 2600/ 5000 batches | lr 0.86 | ms/batch 63.05 | loss 0.00 |\n", + "| epoch 4 | 2800/ 5000 batches | lr 0.86 | ms/batch 63.02 | loss 0.00 |\n", + "| epoch 4 | 3000/ 5000 batches | lr 0.86 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 4 | 3200/ 5000 batches | lr 0.86 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 4 | 3400/ 5000 batches | lr 0.86 | ms/batch 63.02 | loss 0.00 |\n", + "| epoch 4 | 3600/ 5000 batches | lr 0.86 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 4 | 3800/ 5000 batches | lr 0.86 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 4 | 4000/ 5000 batches | lr 0.86 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 4 | 4200/ 5000 batches | lr 0.86 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 4 | 4400/ 5000 batches | lr 0.86 | ms/batch 63.02 | loss 0.00 |\n", + "| epoch 4 | 4600/ 5000 batches | lr 0.86 | ms/batch 63.01 | loss 0.00 |\n", + "| epoch 4 | 4800/ 5000 batches | lr 0.86 | ms/batch 63.04 | loss 0.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 448.16s | valid loss 0.00 | valid ppl 1.00\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 5000 batches | lr 0.81 | ms/batch 63.34 | loss 0.00 |\n", + "| epoch 5 | 400/ 5000 batches | lr 0.81 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 5 | 600/ 5000 batches | lr 0.81 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 5 | 800/ 5000 batches | lr 0.81 | ms/batch 63.04 | loss 0.00 |\n", + "| epoch 5 | 1000/ 5000 batches | lr 0.81 | ms/batch 63.01 | loss 0.00 |\n", + "| epoch 5 | 1200/ 5000 batches | lr 0.81 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 5 | 1400/ 5000 batches | lr 0.81 | ms/batch 63.02 | loss 0.00 |\n", + "| epoch 5 | 1600/ 5000 batches | lr 0.81 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 5 | 1800/ 5000 batches | lr 0.81 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 5 | 2000/ 5000 batches | lr 0.81 | ms/batch 63.02 | loss 0.00 |\n", + "| epoch 5 | 2200/ 5000 batches | lr 0.81 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 5 | 2400/ 5000 batches | lr 0.81 | ms/batch 63.02 | loss 0.00 |\n", + "| epoch 5 | 2600/ 5000 batches | lr 0.81 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 5 | 2800/ 5000 batches | lr 0.81 | ms/batch 63.02 | loss 0.00 |\n", + "| epoch 5 | 3000/ 5000 batches | lr 0.81 | ms/batch 63.02 | loss 0.00 |\n", + "| epoch 5 | 3200/ 5000 batches | lr 0.81 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 5 | 3400/ 5000 batches | lr 0.81 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 5 | 3600/ 5000 batches | lr 0.81 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 5 | 3800/ 5000 batches | lr 0.81 | ms/batch 63.02 | loss 0.00 |\n", + "| epoch 5 | 4000/ 5000 batches | lr 0.81 | ms/batch 63.04 | loss 0.00 |\n", + "| epoch 5 | 4200/ 5000 batches | lr 0.81 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 5 | 4400/ 5000 batches | lr 0.81 | ms/batch 63.02 | loss 0.00 |\n", + "| epoch 5 | 4600/ 5000 batches | lr 0.81 | ms/batch 63.03 | loss 0.00 |\n", + "| epoch 5 | 4800/ 5000 batches | lr 0.81 | ms/batch 63.03 | loss 0.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 448.04s | valid loss 0.00 | valid ppl 1.00\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 0.00 | test ppl 1.00\n", + "=========================================================================================\n" + ] + } + ], + "source": [ + "batch_size = 20\n", + "eval_batch_size = 10\n", + "seq_len = 1000\n", + "divider = 4\n", + "dataset_len = 100000\n", + "epochs = 5 # The number of epochs\n", + "\n", + "model = DoXTimes(Smartpool(divider, 0.3))\n", + "model = model.to(device)\n", + "print(model)\n", + "\n", + "lr = 1.0 # learning rate\n", + "optimizer = torch.optim.SGD(model.parameters(), lr=lr)\n", + "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "\n", + "model = train_model(model, epochs, batch_size, eval_batch_size, dataset_len, seq_len, divider, optimizer, scheduler)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Vc3RgEAzvtnA", + "outputId": "24228518-297b-4f34-c008-6ddc05397c40" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "data[0]: tensor([[0.0000, 0.8117],\n", + " [0.0000, 0.2613],\n", + " [1.0000, 0.9511],\n", + " ...,\n", + " [1.0000, 0.7560],\n", + " [0.0000, 0.4698],\n", + " [0.0000, 0.4036]], device='cuda:0')\n", + "output: tensor([118.1460, 126.8192, 119.2346, 123.9964, 125.6771, 126.9987, 119.8981,\n", + " 121.9886, 126.0591, 131.1343], device='cuda:0')\n", + "targets: tensor([118.1460, 126.8192, 119.2346, 123.9963, 125.6771, 126.9987, 119.8981,\n", + " 121.9886, 126.0591, 131.1343], device='cuda:0')\n" + ] + } + ], + "source": [ + "model.eval()\n", + "with torch.no_grad():\n", + " data = get_batch(eval_batch_size, seq_len, divider).to(device)\n", + " targets = data.prod(-1).sum(-1)\n", + " output = model(data)\n", + " print('data[0]:', data[0,:,:])\n", + "\n", + " print('output:', output)\n", + " print('targets:', targets)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "v5eKkfBrw6Pd" + }, + "source": [ + "## Pooling T/16" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WCtbVJecx_mc" + }, + "source": [ + "### Average pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "At-q3olSyDSt", + "outputId": "077a2f8d-d16f-4c2f-8880-2d90a3f45fe2" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using cache found in /home/i273233/.cache/torch/hub/pytorch_vision_v0.6.0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Conv(\n", + " (conv): Sequential(\n", + " (0): Conv2d(1, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (1): ReLU(inplace=True)\n", + " (2): AvgPool2d(kernel_size=2, stride=(2, 1), padding=(0, 1))\n", + " (3): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (4): ReLU(inplace=True)\n", + " (5): AvgPool2d(kernel_size=2, stride=(2, 1), padding=(0, 1))\n", + " (6): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (7): ReLU(inplace=True)\n", + " (8): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (9): ReLU(inplace=True)\n", + " (10): AvgPool2d(kernel_size=2, stride=(2, 1), padding=0)\n", + " (11): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (12): ReLU(inplace=True)\n", + " (13): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (14): ReLU(inplace=True)\n", + " (15): AvgPool2d(kernel_size=2, stride=(2, 1), padding=0)\n", + " (16): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (17): ReLU(inplace=True)\n", + " (18): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (linear): Conv2d(512, 1, kernel_size=(1, 1), stride=(1, 1))\n", + " )\n", + ")\n" + ] + } + ], + "source": [ + "\n", + "model = torch.hub.load('pytorch/vision:v0.6.0', 'vgg11', pretrained=False)\n", + "model = model.features[0:19]\n", + "model[0] = nn.Conv2d(1, 64, kernel_size=(3,3), stride=(1,1), padding=(1,1))\n", + "model[2] = nn.AvgPool2d(kernel_size=2, stride=(2,1), padding=(0,1))\n", + "model[5] = nn.AvgPool2d(kernel_size=2, stride=(2,1), padding=(0,1))\n", + "model[10] = nn.AvgPool2d(kernel_size=2, stride=(2,1), padding=0)\n", + "model[15] = nn.AvgPool2d(kernel_size=2, stride=(2,1), padding=0)\n", + "model[18] = nn.Conv2d(512, 512, kernel_size=(3,3), stride=(1,1), padding=(1,1))\n", + "model.add_module(\"linear\", nn.Conv2d(512, 1, kernel_size=1, stride=1, padding=0))\n", + "model = Conv(model)\n", + "model = model.to(device)\n", + "print(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "JThVMqz30f2h", + "outputId": "8aa27cdc-e714-4d56-f32b-47a1cf2e1053" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 5120 batches | lr 1.00 | ms/batch 115.94 | loss 0.55 |\n", + "| epoch 1 | 400/ 5120 batches | lr 1.00 | ms/batch 120.74 | loss 0.49 |\n", + "| epoch 1 | 600/ 5120 batches | lr 1.00 | ms/batch 123.80 | loss 0.49 |\n", + "| epoch 1 | 800/ 5120 batches | lr 1.00 | ms/batch 123.74 | loss 0.49 |\n", + "| epoch 1 | 1000/ 5120 batches | lr 1.00 | ms/batch 123.79 | loss 0.49 |\n", + "| epoch 1 | 1200/ 5120 batches | lr 1.00 | ms/batch 123.74 | loss 0.49 |\n", + "| epoch 1 | 1400/ 5120 batches | lr 1.00 | ms/batch 123.86 | loss 0.49 |\n", + "| epoch 1 | 1600/ 5120 batches | lr 1.00 | ms/batch 123.89 | loss 0.49 |\n", + "| epoch 1 | 1800/ 5120 batches | lr 1.00 | ms/batch 123.79 | loss 0.49 |\n", + "| epoch 1 | 2000/ 5120 batches | lr 1.00 | ms/batch 123.77 | loss 0.49 |\n", + "| epoch 1 | 2200/ 5120 batches | lr 1.00 | ms/batch 123.64 | loss 0.49 |\n", + "| epoch 1 | 2400/ 5120 batches | lr 1.00 | ms/batch 123.64 | loss 0.49 |\n", + "| epoch 1 | 2600/ 5120 batches | lr 1.00 | ms/batch 123.62 | loss 0.49 |\n", + "| epoch 1 | 2800/ 5120 batches | lr 1.00 | ms/batch 123.64 | loss 0.49 |\n", + "| epoch 1 | 3000/ 5120 batches | lr 1.00 | ms/batch 123.61 | loss 0.49 |\n", + "| epoch 1 | 3200/ 5120 batches | lr 1.00 | ms/batch 123.10 | loss 0.49 |\n", + "| epoch 1 | 3400/ 5120 batches | lr 1.00 | ms/batch 123.09 | loss 0.50 |\n", + "| epoch 1 | 3600/ 5120 batches | lr 1.00 | ms/batch 123.14 | loss 0.49 |\n", + "| epoch 1 | 3800/ 5120 batches | lr 1.00 | ms/batch 123.13 | loss 0.50 |\n", + "| epoch 1 | 4000/ 5120 batches | lr 1.00 | ms/batch 123.10 | loss 0.49 |\n", + "| epoch 1 | 4200/ 5120 batches | lr 1.00 | ms/batch 123.03 | loss 0.49 |\n", + "| epoch 1 | 4400/ 5120 batches | lr 1.00 | ms/batch 122.94 | loss 0.49 |\n", + "| epoch 1 | 4600/ 5120 batches | lr 1.00 | ms/batch 122.85 | loss 0.49 |\n", + "| epoch 1 | 4800/ 5120 batches | lr 1.00 | ms/batch 122.92 | loss 0.49 |\n", + "| epoch 1 | 5000/ 5120 batches | lr 1.00 | ms/batch 122.91 | loss 0.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 912.09s | valid loss 0.89 | valid ppl 2.43\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 5120 batches | lr 0.95 | ms/batch 123.72 | loss 0.54 |\n", + "| epoch 2 | 400/ 5120 batches | lr 0.95 | ms/batch 123.19 | loss 0.54 |\n", + "| epoch 2 | 600/ 5120 batches | lr 0.95 | ms/batch 123.22 | loss 0.54 |\n", + "| epoch 2 | 800/ 5120 batches | lr 0.95 | ms/batch 123.21 | loss 0.54 |\n", + "| epoch 2 | 1000/ 5120 batches | lr 0.95 | ms/batch 123.17 | loss 0.54 |\n", + "| epoch 2 | 1200/ 5120 batches | lr 0.95 | ms/batch 123.18 | loss 0.54 |\n", + "| epoch 2 | 1400/ 5120 batches | lr 0.95 | ms/batch 123.11 | loss 0.54 |\n", + "| epoch 2 | 1600/ 5120 batches | lr 0.95 | ms/batch 123.22 | loss 0.54 |\n", + "| epoch 2 | 1800/ 5120 batches | lr 0.95 | ms/batch 123.15 | loss 0.54 |\n", + "| epoch 2 | 2000/ 5120 batches | lr 0.95 | ms/batch 123.26 | loss 0.54 |\n", + "| epoch 2 | 2200/ 5120 batches | lr 0.95 | ms/batch 123.24 | loss 0.54 |\n", + "| epoch 2 | 2400/ 5120 batches | lr 0.95 | ms/batch 123.18 | loss 0.54 |\n", + "| epoch 2 | 2600/ 5120 batches | lr 0.95 | ms/batch 123.07 | loss 0.54 |\n", + "| epoch 2 | 2800/ 5120 batches | lr 0.95 | ms/batch 123.19 | loss 0.54 |\n", + "| epoch 2 | 3000/ 5120 batches | lr 0.95 | ms/batch 123.19 | loss 0.54 |\n", + "| epoch 2 | 3200/ 5120 batches | lr 0.95 | ms/batch 123.18 | loss 0.54 |\n", + "| epoch 2 | 3400/ 5120 batches | lr 0.95 | ms/batch 123.10 | loss 0.54 |\n", + "| epoch 2 | 3600/ 5120 batches | lr 0.95 | ms/batch 123.12 | loss 0.54 |\n", + "| epoch 2 | 3800/ 5120 batches | lr 0.95 | ms/batch 123.06 | loss 0.54 |\n", + "| epoch 2 | 4000/ 5120 batches | lr 0.95 | ms/batch 122.98 | loss 0.54 |\n", + "| epoch 2 | 4200/ 5120 batches | lr 0.95 | ms/batch 123.05 | loss 0.54 |\n", + "| epoch 2 | 4400/ 5120 batches | lr 0.95 | ms/batch 123.00 | loss 0.54 |\n", + "| epoch 2 | 4600/ 5120 batches | lr 0.95 | ms/batch 123.04 | loss 0.54 |\n", + "| epoch 2 | 4800/ 5120 batches | lr 0.95 | ms/batch 123.09 | loss 0.54 |\n", + "| epoch 2 | 5000/ 5120 batches | lr 0.95 | ms/batch 123.00 | loss 0.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 912.89s | valid loss 0.79 | valid ppl 2.20\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 5120 batches | lr 0.90 | ms/batch 123.65 | loss 0.51 |\n", + "| epoch 3 | 400/ 5120 batches | lr 0.90 | ms/batch 123.00 | loss 0.50 |\n", + "| epoch 3 | 600/ 5120 batches | lr 0.90 | ms/batch 123.08 | loss 0.50 |\n", + "| epoch 3 | 800/ 5120 batches | lr 0.90 | ms/batch 123.10 | loss 0.50 |\n", + "| epoch 3 | 1000/ 5120 batches | lr 0.90 | ms/batch 123.05 | loss 0.50 |\n", + "| epoch 3 | 1200/ 5120 batches | lr 0.90 | ms/batch 123.14 | loss 0.50 |\n", + "| epoch 3 | 1400/ 5120 batches | lr 0.90 | ms/batch 123.11 | loss 0.50 |\n", + "| epoch 3 | 1600/ 5120 batches | lr 0.90 | ms/batch 123.07 | loss 0.50 |\n", + "| epoch 3 | 1800/ 5120 batches | lr 0.90 | ms/batch 123.06 | loss 0.50 |\n", + "| epoch 3 | 2000/ 5120 batches | lr 0.90 | ms/batch 123.05 | loss 0.50 |\n", + "| epoch 3 | 2200/ 5120 batches | lr 0.90 | ms/batch 123.18 | loss 0.50 |\n", + "| epoch 3 | 2400/ 5120 batches | lr 0.90 | ms/batch 123.12 | loss 0.50 |\n", + "| epoch 3 | 2600/ 5120 batches | lr 0.90 | ms/batch 123.07 | loss 0.50 |\n", + "| epoch 3 | 2800/ 5120 batches | lr 0.90 | ms/batch 123.17 | loss 0.50 |\n", + "| epoch 3 | 3000/ 5120 batches | lr 0.90 | ms/batch 123.16 | loss 0.50 |\n", + "| epoch 3 | 3200/ 5120 batches | lr 0.90 | ms/batch 123.15 | loss 0.50 |\n", + "| epoch 3 | 3400/ 5120 batches | lr 0.90 | ms/batch 123.15 | loss 0.50 |\n", + "| epoch 3 | 3600/ 5120 batches | lr 0.90 | ms/batch 123.20 | loss 0.50 |\n", + "| epoch 3 | 3800/ 5120 batches | lr 0.90 | ms/batch 123.16 | loss 0.50 |\n", + "| epoch 3 | 4000/ 5120 batches | lr 0.90 | ms/batch 123.10 | loss 0.50 |\n", + "| epoch 3 | 4200/ 5120 batches | lr 0.90 | ms/batch 123.16 | loss 0.50 |\n", + "| epoch 3 | 4400/ 5120 batches | lr 0.90 | ms/batch 123.26 | loss 0.50 |\n", + "| epoch 3 | 4600/ 5120 batches | lr 0.90 | ms/batch 123.21 | loss 0.50 |\n", + "| epoch 3 | 4800/ 5120 batches | lr 0.90 | ms/batch 123.18 | loss 0.50 |\n", + "| epoch 3 | 5000/ 5120 batches | lr 0.90 | ms/batch 123.11 | loss 0.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 913.27s | valid loss 0.79 | valid ppl 2.20\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 5120 batches | lr 0.86 | ms/batch 123.79 | loss 0.47 |\n", + "| epoch 4 | 400/ 5120 batches | lr 0.86 | ms/batch 123.22 | loss 0.47 |\n", + "| epoch 4 | 600/ 5120 batches | lr 0.86 | ms/batch 123.24 | loss 0.47 |\n", + "| epoch 4 | 800/ 5120 batches | lr 0.86 | ms/batch 123.32 | loss 0.47 |\n", + "| epoch 4 | 1000/ 5120 batches | lr 0.86 | ms/batch 123.31 | loss 0.47 |\n", + "| epoch 4 | 1200/ 5120 batches | lr 0.86 | ms/batch 123.23 | loss 0.47 |\n", + "| epoch 4 | 1400/ 5120 batches | lr 0.86 | ms/batch 123.11 | loss 0.47 |\n", + "| epoch 4 | 1600/ 5120 batches | lr 0.86 | ms/batch 123.23 | loss 0.47 |\n", + "| epoch 4 | 1800/ 5120 batches | lr 0.86 | ms/batch 123.31 | loss 0.47 |\n", + "| epoch 4 | 2000/ 5120 batches | lr 0.86 | ms/batch 123.25 | loss 0.47 |\n", + "| epoch 4 | 2200/ 5120 batches | lr 0.86 | ms/batch 123.18 | loss 0.46 |\n", + "| epoch 4 | 2400/ 5120 batches | lr 0.86 | ms/batch 123.13 | loss 0.47 |\n", + "| epoch 4 | 2600/ 5120 batches | lr 0.86 | ms/batch 123.23 | loss 0.47 |\n", + "| epoch 4 | 2800/ 5120 batches | lr 0.86 | ms/batch 123.21 | loss 0.47 |\n", + "| epoch 4 | 3000/ 5120 batches | lr 0.86 | ms/batch 123.20 | loss 0.47 |\n", + "| epoch 4 | 3200/ 5120 batches | lr 0.86 | ms/batch 123.09 | loss 0.47 |\n", + "| epoch 4 | 3400/ 5120 batches | lr 0.86 | ms/batch 123.13 | loss 0.46 |\n", + "| epoch 4 | 3600/ 5120 batches | lr 0.86 | ms/batch 123.07 | loss 0.47 |\n", + "| epoch 4 | 3800/ 5120 batches | lr 0.86 | ms/batch 123.06 | loss 0.47 |\n", + "| epoch 4 | 4000/ 5120 batches | lr 0.86 | ms/batch 123.03 | loss 0.47 |\n", + "| epoch 4 | 4200/ 5120 batches | lr 0.86 | ms/batch 123.02 | loss 0.47 |\n", + "| epoch 4 | 4400/ 5120 batches | lr 0.86 | ms/batch 123.09 | loss 0.46 |\n", + "| epoch 4 | 4600/ 5120 batches | lr 0.86 | ms/batch 123.14 | loss 0.47 |\n", + "| epoch 4 | 4800/ 5120 batches | lr 0.86 | ms/batch 123.19 | loss 0.46 |\n", + "| epoch 4 | 5000/ 5120 batches | lr 0.86 | ms/batch 123.19 | loss 0.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 913.40s | valid loss 0.79 | valid ppl 2.20\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 5120 batches | lr 0.81 | ms/batch 123.63 | loss 0.44 |\n", + "| epoch 5 | 400/ 5120 batches | lr 0.81 | ms/batch 123.08 | loss 0.44 |\n", + "| epoch 5 | 600/ 5120 batches | lr 0.81 | ms/batch 123.19 | loss 0.44 |\n", + "| epoch 5 | 800/ 5120 batches | lr 0.81 | ms/batch 123.11 | loss 0.49 |\n", + "| epoch 5 | 1000/ 5120 batches | lr 0.81 | ms/batch 123.13 | loss 0.50 |\n", + "| epoch 5 | 1200/ 5120 batches | lr 0.81 | ms/batch 123.26 | loss 0.50 |\n", + "| epoch 5 | 1400/ 5120 batches | lr 0.81 | ms/batch 123.33 | loss 0.50 |\n", + "| epoch 5 | 1600/ 5120 batches | lr 0.81 | ms/batch 123.30 | loss 0.50 |\n", + "| epoch 5 | 1800/ 5120 batches | lr 0.81 | ms/batch 123.24 | loss 0.50 |\n", + "| epoch 5 | 2000/ 5120 batches | lr 0.81 | ms/batch 123.13 | loss 0.50 |\n", + "| epoch 5 | 2200/ 5120 batches | lr 0.81 | ms/batch 123.17 | loss 0.50 |\n", + "| epoch 5 | 2400/ 5120 batches | lr 0.81 | ms/batch 123.22 | loss 0.50 |\n", + "| epoch 5 | 2600/ 5120 batches | lr 0.81 | ms/batch 123.20 | loss 0.50 |\n", + "| epoch 5 | 2800/ 5120 batches | lr 0.81 | ms/batch 123.29 | loss 0.50 |\n", + "| epoch 5 | 3000/ 5120 batches | lr 0.81 | ms/batch 123.16 | loss 0.50 |\n", + "| epoch 5 | 3200/ 5120 batches | lr 0.81 | ms/batch 123.26 | loss 0.50 |\n", + "| epoch 5 | 3400/ 5120 batches | lr 0.81 | ms/batch 123.23 | loss 0.50 |\n", + "| epoch 5 | 3600/ 5120 batches | lr 0.81 | ms/batch 123.29 | loss 0.50 |\n", + "| epoch 5 | 3800/ 5120 batches | lr 0.81 | ms/batch 123.33 | loss 0.50 |\n", + "| epoch 5 | 4000/ 5120 batches | lr 0.81 | ms/batch 123.12 | loss 0.50 |\n", + "| epoch 5 | 4200/ 5120 batches | lr 0.81 | ms/batch 123.08 | loss 0.50 |\n", + "| epoch 5 | 4400/ 5120 batches | lr 0.81 | ms/batch 123.09 | loss 0.50 |\n", + "| epoch 5 | 4600/ 5120 batches | lr 0.81 | ms/batch 123.15 | loss 0.50 |\n", + "| epoch 5 | 4800/ 5120 batches | lr 0.81 | ms/batch 123.12 | loss 0.50 |\n", + "| epoch 5 | 5000/ 5120 batches | lr 0.81 | ms/batch 123.15 | loss 0.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 913.66s | valid loss 0.67 | valid ppl 1.95\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 0.67 | test ppl 1.95\n", + "=========================================================================================\n" + ] + } + ], + "source": [ + "batch_size = 20\n", + "eval_batch_size = 10\n", + "seq_len = 1024\n", + "divider = 16\n", + "dataset_len = 102400\n", + "lr = 1.0 # learning rate\n", + "optimizer = torch.optim.SGD(model.parameters(), lr=lr)\n", + "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "epochs = 5 # The number of epochs\n", + "\n", + "model = train_model(model, epochs, batch_size, eval_batch_size, dataset_len, seq_len, divider, optimizer, scheduler)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "84iEiRyL0kkQ", + "outputId": "4beb7bd0-09b1-4ecf-b3ad-a12f9a08b39a" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "data[0]: tensor([[0.0000, 0.3159],\n", + " [0.0000, 0.3881],\n", + " [0.0000, 0.0700],\n", + " [0.0000, 0.3819],\n", + " [0.0000, 0.0777],\n", + " [0.0000, 0.3863],\n", + " [0.0000, 0.2058],\n", + " [0.0000, 0.5430],\n", + " [0.0000, 0.5289],\n", + " [0.0000, 0.0553],\n", + " [0.0000, 0.2238],\n", + " [0.0000, 0.5081],\n", + " [0.0000, 0.3758],\n", + " [0.0000, 0.6520],\n", + " [0.0000, 0.3167],\n", + " [0.0000, 0.8875],\n", + " [0.0000, 0.4066],\n", + " [0.0000, 0.8926],\n", + " [0.0000, 0.8963],\n", + " [0.0000, 0.0858],\n", + " [0.0000, 0.2926],\n", + " [0.0000, 0.7110],\n", + " [0.0000, 0.6208],\n", + " [0.0000, 0.2697],\n", + " [0.0000, 0.9850],\n", + " [0.0000, 0.8698],\n", + " [0.0000, 0.7973],\n", + " [0.0000, 0.1435],\n", + " [0.0000, 0.7069],\n", + " [0.0000, 0.4728],\n", + " [0.0000, 0.0657],\n", + " [0.0000, 0.6978],\n", + " [0.0000, 0.0677],\n", + " [0.0000, 0.6686],\n", + " [0.0000, 0.6546],\n", + " [0.0000, 0.5797],\n", + " [0.0000, 0.9028],\n", + " [0.0000, 0.5751],\n", + " [0.0000, 0.3478],\n", + " [0.0000, 0.6721],\n", + " [0.0000, 0.9224],\n", + " [0.0000, 0.0895],\n", + " [0.0000, 0.4444],\n", + " [0.0000, 0.5574],\n", + " [0.0000, 0.7633],\n", + " [0.0000, 0.8915],\n", + " [0.0000, 0.9144],\n", + " [0.0000, 0.3515],\n", + " [0.0000, 0.8710],\n", + " [0.0000, 0.6082]], device='cuda:0')\n", + "output: tensor([10.6232, 10.6232, 10.6232, 10.6232, 10.6232, 10.6232, 10.6232, 10.6232,\n", + " 10.6232, 10.6232], device='cuda:0')\n", + "targets: tensor([32.2080, 32.9095, 34.1089, 31.3225, 33.6871, 27.5791, 33.6800, 26.6792,\n", + " 29.9740, 36.1351], device='cuda:0')\n" + ] + } + ], + "source": [ + "model.eval()\n", + "with torch.no_grad():\n", + " data = get_batch(eval_batch_size, seq_len, divider).to(device)\n", + " targets = data.prod(-1).sum(-1).squeeze(-1)\n", + " output = model(data)\n", + " print('data[0]:', data[0,:50,:])\n", + "\n", + " print('output:', output)\n", + " print('targets:', targets)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AsrMRkVd1ksb" + }, + "source": [ + "### Max pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "H-B9LVmr1hJ7", + "outputId": "a0c86c4b-85de-400e-ea88-e0e83c52d50f" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using cache found in /home/i273233/.cache/torch/hub/pytorch_vision_v0.6.0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Conv(\n", + " (conv): Sequential(\n", + " (0): Conv2d(1, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (1): ReLU(inplace=True)\n", + " (2): MaxPool2d(kernel_size=2, stride=(2, 1), padding=(0, 1), dilation=1, ceil_mode=False)\n", + " (3): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (4): ReLU(inplace=True)\n", + " (5): MaxPool2d(kernel_size=2, stride=(2, 1), padding=(0, 1), dilation=1, ceil_mode=False)\n", + " (6): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (7): ReLU(inplace=True)\n", + " (8): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (9): ReLU(inplace=True)\n", + " (10): MaxPool2d(kernel_size=2, stride=(2, 1), padding=0, dilation=1, ceil_mode=False)\n", + " (11): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (12): ReLU(inplace=True)\n", + " (13): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (14): ReLU(inplace=True)\n", + " (15): MaxPool2d(kernel_size=2, stride=(2, 1), padding=0, dilation=1, ceil_mode=False)\n", + " (16): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (17): ReLU(inplace=True)\n", + " (18): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (linear): Conv2d(512, 1, kernel_size=(1, 1), stride=(1, 1))\n", + " )\n", + ")\n" + ] + } + ], + "source": [ + "\n", + "model = torch.hub.load('pytorch/vision:v0.6.0', 'vgg11', pretrained=False)\n", + "model = model.features[0:19]\n", + "model[0] = nn.Conv2d(1, 64, kernel_size=(3,3), stride=(1,1), padding=(1,1))\n", + "model[2] = nn.MaxPool2d(kernel_size=2, stride=(2,1), padding=(0,1))\n", + "model[5] = nn.MaxPool2d(kernel_size=2, stride=(2,1), padding=(0,1))\n", + "model[10] = nn.MaxPool2d(kernel_size=2, stride=(2,1), padding=0)\n", + "model[15] = nn.MaxPool2d(kernel_size=2, stride=(2,1), padding=0)\n", + "model[18] = nn.Conv2d(512, 512, kernel_size=(3,3), stride=(1,1), padding=(1,1))\n", + "model.add_module(\"linear\", nn.Conv2d(512, 1, kernel_size=1, stride=1, padding=0))\n", + "model = Conv(model)\n", + "model = model.to(device)\n", + "print(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "5vIgdWg51hRP", + "outputId": "b574cc0a-2788-4508-eb89-ac563544e8c3" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 5120 batches | lr 1.00 | ms/batch 122.65 | loss 0.83 |\n", + "| epoch 1 | 400/ 5120 batches | lr 1.00 | ms/batch 121.98 | loss 0.83 |\n", + "| epoch 1 | 600/ 5120 batches | lr 1.00 | ms/batch 122.04 | loss 0.82 |\n", + "| epoch 1 | 800/ 5120 batches | lr 1.00 | ms/batch 122.01 | loss 0.83 |\n", + "| epoch 1 | 1000/ 5120 batches | lr 1.00 | ms/batch 121.96 | loss 0.83 |\n", + "| epoch 1 | 1200/ 5120 batches | lr 1.00 | ms/batch 121.91 | loss 0.83 |\n", + "| epoch 1 | 1400/ 5120 batches | lr 1.00 | ms/batch 121.94 | loss 0.83 |\n", + "| epoch 1 | 1600/ 5120 batches | lr 1.00 | ms/batch 121.98 | loss 0.82 |\n", + "| epoch 1 | 1800/ 5120 batches | lr 1.00 | ms/batch 121.86 | loss 0.83 |\n", + "| epoch 1 | 2000/ 5120 batches | lr 1.00 | ms/batch 121.88 | loss 0.83 |\n", + "| epoch 1 | 2200/ 5120 batches | lr 1.00 | ms/batch 121.95 | loss 0.83 |\n", + "| epoch 1 | 2400/ 5120 batches | lr 1.00 | ms/batch 122.00 | loss 0.82 |\n", + "| epoch 1 | 2600/ 5120 batches | lr 1.00 | ms/batch 121.84 | loss 0.83 |\n", + "| epoch 1 | 2800/ 5120 batches | lr 1.00 | ms/batch 121.92 | loss 0.83 |\n", + "| epoch 1 | 3000/ 5120 batches | lr 1.00 | ms/batch 121.89 | loss 0.83 |\n", + "| epoch 1 | 3200/ 5120 batches | lr 1.00 | ms/batch 121.89 | loss 0.82 |\n", + "| epoch 1 | 3400/ 5120 batches | lr 1.00 | ms/batch 121.92 | loss 0.82 |\n", + "| epoch 1 | 3600/ 5120 batches | lr 1.00 | ms/batch 121.86 | loss 0.82 |\n", + "| epoch 1 | 3800/ 5120 batches | lr 1.00 | ms/batch 121.95 | loss 0.83 |\n", + "| epoch 1 | 4000/ 5120 batches | lr 1.00 | ms/batch 121.82 | loss 0.82 |\n", + "| epoch 1 | 4200/ 5120 batches | lr 1.00 | ms/batch 121.79 | loss 0.83 |\n", + "| epoch 1 | 4400/ 5120 batches | lr 1.00 | ms/batch 121.86 | loss 0.83 |\n", + "| epoch 1 | 4600/ 5120 batches | lr 1.00 | ms/batch 121.87 | loss 0.83 |\n", + "| epoch 1 | 4800/ 5120 batches | lr 1.00 | ms/batch 121.90 | loss 0.83 |\n", + "| epoch 1 | 5000/ 5120 batches | lr 1.00 | ms/batch 121.80 | loss 0.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 905.45s | valid loss 1.31 | valid ppl 3.70\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 5120 batches | lr 0.95 | ms/batch 122.55 | loss 0.80 |\n", + "| epoch 2 | 400/ 5120 batches | lr 0.95 | ms/batch 121.95 | loss 0.80 |\n", + "| epoch 2 | 600/ 5120 batches | lr 0.95 | ms/batch 122.05 | loss 0.80 |\n", + "| epoch 2 | 800/ 5120 batches | lr 0.95 | ms/batch 122.01 | loss 0.80 |\n", + "| epoch 2 | 1000/ 5120 batches | lr 0.95 | ms/batch 121.95 | loss 0.80 |\n", + "| epoch 2 | 1200/ 5120 batches | lr 0.95 | ms/batch 122.01 | loss 0.79 |\n", + "| epoch 2 | 1400/ 5120 batches | lr 0.95 | ms/batch 121.99 | loss 0.80 |\n", + "| epoch 2 | 1600/ 5120 batches | lr 0.95 | ms/batch 121.95 | loss 0.80 |\n", + "| epoch 2 | 1800/ 5120 batches | lr 0.95 | ms/batch 121.88 | loss 0.79 |\n", + "| epoch 2 | 2000/ 5120 batches | lr 0.95 | ms/batch 121.96 | loss 0.80 |\n", + "| epoch 2 | 2200/ 5120 batches | lr 0.95 | ms/batch 121.91 | loss 0.80 |\n", + "| epoch 2 | 2400/ 5120 batches | lr 0.95 | ms/batch 122.01 | loss 0.79 |\n", + "| epoch 2 | 2600/ 5120 batches | lr 0.95 | ms/batch 122.04 | loss 0.80 |\n", + "| epoch 2 | 2800/ 5120 batches | lr 0.95 | ms/batch 121.94 | loss 0.80 |\n", + "| epoch 2 | 3000/ 5120 batches | lr 0.95 | ms/batch 121.98 | loss 0.80 |\n", + "| epoch 2 | 3200/ 5120 batches | lr 0.95 | ms/batch 122.03 | loss 0.80 |\n", + "| epoch 2 | 3400/ 5120 batches | lr 0.95 | ms/batch 122.07 | loss 0.80 |\n", + "| epoch 2 | 3600/ 5120 batches | lr 0.95 | ms/batch 121.94 | loss 0.80 |\n", + "| epoch 2 | 3800/ 5120 batches | lr 0.95 | ms/batch 122.00 | loss 0.80 |\n", + "| epoch 2 | 4000/ 5120 batches | lr 0.95 | ms/batch 121.94 | loss 0.80 |\n", + "| epoch 2 | 4200/ 5120 batches | lr 0.95 | ms/batch 122.05 | loss 0.79 |\n", + "| epoch 2 | 4400/ 5120 batches | lr 0.95 | ms/batch 122.00 | loss 0.80 |\n", + "| epoch 2 | 4600/ 5120 batches | lr 0.95 | ms/batch 122.02 | loss 0.80 |\n", + "| epoch 2 | 4800/ 5120 batches | lr 0.95 | ms/batch 121.95 | loss 0.80 |\n", + "| epoch 2 | 5000/ 5120 batches | lr 0.95 | ms/batch 122.07 | loss 0.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 905.90s | valid loss 1.31 | valid ppl 3.70\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 5120 batches | lr 0.90 | ms/batch 122.60 | loss 0.77 |\n", + "| epoch 3 | 400/ 5120 batches | lr 0.90 | ms/batch 121.97 | loss 0.77 |\n", + "| epoch 3 | 600/ 5120 batches | lr 0.90 | ms/batch 122.03 | loss 0.77 |\n", + "| epoch 3 | 800/ 5120 batches | lr 0.90 | ms/batch 121.99 | loss 0.77 |\n", + "| epoch 3 | 1000/ 5120 batches | lr 0.90 | ms/batch 122.05 | loss 0.77 |\n", + "| epoch 3 | 1200/ 5120 batches | lr 0.90 | ms/batch 122.02 | loss 0.77 |\n", + "| epoch 3 | 1400/ 5120 batches | lr 0.90 | ms/batch 121.96 | loss 0.77 |\n", + "| epoch 3 | 1600/ 5120 batches | lr 0.90 | ms/batch 121.98 | loss 0.77 |\n", + "| epoch 3 | 1800/ 5120 batches | lr 0.90 | ms/batch 122.00 | loss 0.77 |\n", + "| epoch 3 | 2000/ 5120 batches | lr 0.90 | ms/batch 122.00 | loss 0.77 |\n", + "| epoch 3 | 2200/ 5120 batches | lr 0.90 | ms/batch 122.08 | loss 0.77 |\n", + "| epoch 3 | 2400/ 5120 batches | lr 0.90 | ms/batch 121.99 | loss 0.77 |\n", + "| epoch 3 | 2600/ 5120 batches | lr 0.90 | ms/batch 122.03 | loss 0.77 |\n", + "| epoch 3 | 2800/ 5120 batches | lr 0.90 | ms/batch 122.04 | loss 0.77 |\n", + "| epoch 3 | 3000/ 5120 batches | lr 0.90 | ms/batch 122.06 | loss 0.77 |\n", + "| epoch 3 | 3200/ 5120 batches | lr 0.90 | ms/batch 122.15 | loss 0.77 |\n", + "| epoch 3 | 3400/ 5120 batches | lr 0.90 | ms/batch 122.07 | loss 0.77 |\n", + "| epoch 3 | 3600/ 5120 batches | lr 0.90 | ms/batch 122.07 | loss 0.77 |\n", + "| epoch 3 | 3800/ 5120 batches | lr 0.90 | ms/batch 122.02 | loss 0.76 |\n", + "| epoch 3 | 4000/ 5120 batches | lr 0.90 | ms/batch 122.07 | loss 0.76 |\n", + "| epoch 3 | 4200/ 5120 batches | lr 0.90 | ms/batch 122.06 | loss 0.77 |\n", + "| epoch 3 | 4400/ 5120 batches | lr 0.90 | ms/batch 121.97 | loss 0.77 |\n", + "| epoch 3 | 4600/ 5120 batches | lr 0.90 | ms/batch 121.98 | loss 0.77 |\n", + "| epoch 3 | 4800/ 5120 batches | lr 0.90 | ms/batch 121.98 | loss 0.77 |\n", + "| epoch 3 | 5000/ 5120 batches | lr 0.90 | ms/batch 122.04 | loss 0.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 906.00s | valid loss 1.31 | valid ppl 3.69\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 5120 batches | lr 0.86 | ms/batch 122.59 | loss 0.74 |\n", + "| epoch 4 | 400/ 5120 batches | lr 0.86 | ms/batch 121.99 | loss 0.74 |\n", + "| epoch 4 | 600/ 5120 batches | lr 0.86 | ms/batch 121.97 | loss 0.74 |\n", + "| epoch 4 | 800/ 5120 batches | lr 0.86 | ms/batch 122.06 | loss 0.74 |\n", + "| epoch 4 | 1000/ 5120 batches | lr 0.86 | ms/batch 122.01 | loss 0.74 |\n", + "| epoch 4 | 1200/ 5120 batches | lr 0.86 | ms/batch 121.98 | loss 0.74 |\n", + "| epoch 4 | 1400/ 5120 batches | lr 0.86 | ms/batch 121.91 | loss 0.74 |\n", + "| epoch 4 | 1600/ 5120 batches | lr 0.86 | ms/batch 121.97 | loss 0.74 |\n", + "| epoch 4 | 1800/ 5120 batches | lr 0.86 | ms/batch 121.99 | loss 0.74 |\n", + "| epoch 4 | 2000/ 5120 batches | lr 0.86 | ms/batch 121.93 | loss 0.74 |\n", + "| epoch 4 | 2200/ 5120 batches | lr 0.86 | ms/batch 121.95 | loss 0.74 |\n", + "| epoch 4 | 2400/ 5120 batches | lr 0.86 | ms/batch 121.98 | loss 0.74 |\n", + "| epoch 4 | 2600/ 5120 batches | lr 0.86 | ms/batch 121.93 | loss 0.74 |\n", + "| epoch 4 | 2800/ 5120 batches | lr 0.86 | ms/batch 121.96 | loss 0.74 |\n", + "| epoch 4 | 3000/ 5120 batches | lr 0.86 | ms/batch 121.95 | loss 0.74 |\n", + "| epoch 4 | 3200/ 5120 batches | lr 0.86 | ms/batch 121.97 | loss 0.74 |\n", + "| epoch 4 | 3400/ 5120 batches | lr 0.86 | ms/batch 121.88 | loss 0.74 |\n", + "| epoch 4 | 3600/ 5120 batches | lr 0.86 | ms/batch 121.94 | loss 0.74 |\n", + "| epoch 4 | 3800/ 5120 batches | lr 0.86 | ms/batch 121.90 | loss 0.74 |\n", + "| epoch 4 | 4000/ 5120 batches | lr 0.86 | ms/batch 121.90 | loss 0.73 |\n", + "| epoch 4 | 4200/ 5120 batches | lr 0.86 | ms/batch 121.94 | loss 0.74 |\n", + "| epoch 4 | 4400/ 5120 batches | lr 0.86 | ms/batch 121.93 | loss 0.74 |\n", + "| epoch 4 | 4600/ 5120 batches | lr 0.86 | ms/batch 121.84 | loss 0.74 |\n", + "| epoch 4 | 4800/ 5120 batches | lr 0.86 | ms/batch 121.90 | loss 0.74 |\n", + "| epoch 4 | 5000/ 5120 batches | lr 0.86 | ms/batch 121.83 | loss 0.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 905.42s | valid loss 1.31 | valid ppl 3.69\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 5120 batches | lr 0.81 | ms/batch 122.53 | loss 0.72 |\n", + "| epoch 5 | 400/ 5120 batches | lr 0.81 | ms/batch 121.97 | loss 0.71 |\n", + "| epoch 5 | 600/ 5120 batches | lr 0.81 | ms/batch 122.01 | loss 0.72 |\n", + "| epoch 5 | 800/ 5120 batches | lr 0.81 | ms/batch 121.97 | loss 0.71 |\n", + "| epoch 5 | 1000/ 5120 batches | lr 0.81 | ms/batch 121.86 | loss 0.71 |\n", + "| epoch 5 | 1200/ 5120 batches | lr 0.81 | ms/batch 121.86 | loss 0.71 |\n", + "| epoch 5 | 1400/ 5120 batches | lr 0.81 | ms/batch 121.87 | loss 0.72 |\n", + "| epoch 5 | 1600/ 5120 batches | lr 0.81 | ms/batch 121.94 | loss 0.71 |\n", + "| epoch 5 | 1800/ 5120 batches | lr 0.81 | ms/batch 121.87 | loss 0.71 |\n", + "| epoch 5 | 2000/ 5120 batches | lr 0.81 | ms/batch 121.83 | loss 0.71 |\n", + "| epoch 5 | 2200/ 5120 batches | lr 0.81 | ms/batch 121.77 | loss 0.72 |\n", + "| epoch 5 | 2400/ 5120 batches | lr 0.81 | ms/batch 121.81 | loss 0.71 |\n", + "| epoch 5 | 2600/ 5120 batches | lr 0.81 | ms/batch 121.80 | loss 0.71 |\n", + "| epoch 5 | 2800/ 5120 batches | lr 0.81 | ms/batch 121.90 | loss 0.71 |\n", + "| epoch 5 | 3000/ 5120 batches | lr 0.81 | ms/batch 121.84 | loss 0.71 |\n", + "| epoch 5 | 3200/ 5120 batches | lr 0.81 | ms/batch 121.81 | loss 0.72 |\n", + "| epoch 5 | 3400/ 5120 batches | lr 0.81 | ms/batch 121.78 | loss 0.71 |\n", + "| epoch 5 | 3600/ 5120 batches | lr 0.81 | ms/batch 121.86 | loss 0.71 |\n", + "| epoch 5 | 3800/ 5120 batches | lr 0.81 | ms/batch 121.83 | loss 0.72 |\n", + "| epoch 5 | 4000/ 5120 batches | lr 0.81 | ms/batch 121.72 | loss 0.71 |\n", + "| epoch 5 | 4200/ 5120 batches | lr 0.81 | ms/batch 121.80 | loss 0.71 |\n", + "| epoch 5 | 4400/ 5120 batches | lr 0.81 | ms/batch 121.78 | loss 0.71 |\n", + "| epoch 5 | 4600/ 5120 batches | lr 0.81 | ms/batch 121.81 | loss 0.71 |\n", + "| epoch 5 | 4800/ 5120 batches | lr 0.81 | ms/batch 121.90 | loss 0.71 |\n", + "| epoch 5 | 5000/ 5120 batches | lr 0.81 | ms/batch 121.88 | loss 0.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 904.82s | valid loss 1.30 | valid ppl 3.68\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 1.30 | test ppl 3.69\n", + "=========================================================================================\n" + ] + } + ], + "source": [ + "batch_size = 20\n", + "eval_batch_size = 10\n", + "seq_len = 1024\n", + "divider = 16\n", + "dataset_len = 102400\n", + "lr = 1.0 # learning rate\n", + "optimizer = torch.optim.SGD(model.parameters(), lr=lr)\n", + "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "epochs = 5 # The number of epochs\n", + "\n", + "model = train_model(model, epochs, batch_size, eval_batch_size, dataset_len, seq_len, divider, optimizer, scheduler)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "00ERMl4S1hUY", + "outputId": "d9b77f76-c848-40f3-cb7b-110d44fec1d4" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "data[0]: tensor([[0.0000, 0.1578],\n", + " [0.0000, 0.1991],\n", + " [0.0000, 0.9562],\n", + " [0.0000, 0.0171],\n", + " [0.0000, 0.7283],\n", + " [0.0000, 0.1087],\n", + " [0.0000, 0.2220],\n", + " [0.0000, 0.6999],\n", + " [0.0000, 0.9050],\n", + " [0.0000, 0.4827],\n", + " [0.0000, 0.8929],\n", + " [0.0000, 0.3969],\n", + " [0.0000, 0.1445],\n", + " [0.0000, 0.6179],\n", + " [0.0000, 0.8380],\n", + " [0.0000, 0.4753],\n", + " [0.0000, 0.6989],\n", + " [0.0000, 0.9309],\n", + " [1.0000, 0.9744],\n", + " [0.0000, 0.0209],\n", + " [0.0000, 0.7686],\n", + " [1.0000, 0.5637],\n", + " [0.0000, 0.3032],\n", + " [0.0000, 0.1705],\n", + " [0.0000, 0.7484],\n", + " [0.0000, 0.3289],\n", + " [0.0000, 0.0521],\n", + " [0.0000, 0.4144],\n", + " [0.0000, 0.2647],\n", + " [0.0000, 0.8528],\n", + " [0.0000, 0.2144],\n", + " [0.0000, 0.8776],\n", + " [0.0000, 0.7546],\n", + " [0.0000, 0.7464],\n", + " [1.0000, 0.9732],\n", + " [0.0000, 0.6804],\n", + " [0.0000, 0.1254],\n", + " [0.0000, 0.7739],\n", + " [0.0000, 0.8321],\n", + " [0.0000, 0.4663],\n", + " [0.0000, 0.5382],\n", + " [0.0000, 0.6184],\n", + " [0.0000, 0.6527],\n", + " [0.0000, 0.2243],\n", + " [0.0000, 0.4778],\n", + " [0.0000, 0.4105],\n", + " [0.0000, 0.0087],\n", + " [0.0000, 0.2360],\n", + " [0.0000, 0.4538],\n", + " [1.0000, 0.6617]], device='cuda:0')\n", + "output: tensor([73.3645, 73.3644, 73.3645, 73.3644, 73.3644, 73.3644, 73.3644, 73.3644,\n", + " 73.3644, 73.3644], device='cuda:0')\n", + "targets: tensor([34.7800, 34.7476, 31.4978, 34.0282, 33.8993, 36.2574, 27.0724, 27.0513,\n", + " 33.6134, 30.7847], device='cuda:0')\n" + ] + } + ], + "source": [ + "model.eval()\n", + "with torch.no_grad():\n", + " data = get_batch(eval_batch_size, seq_len, divider).to(device)\n", + " targets = data.prod(-1).sum(-1).squeeze(-1)\n", + " output = model(data)\n", + " print('data[0]:', data[0,:50,:])\n", + "\n", + " print('output:', output)\n", + " print('targets:', targets)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DUotUfNTyEIW" + }, + "source": [ + "### Smart pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "k17jb6go106m", + "outputId": "73bf1ba6-4c04-4503-87e3-a08f242a1396" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "DoXTimes(\n", + " (model): Smartpool(\n", + " (mlp): Sequential(\n", + " (0): Linear(in_features=2, out_features=256, bias=True)\n", + " (1): Dropout(p=0.1, inplace=False)\n", + " (2): GELU()\n", + " (3): Linear(in_features=256, out_features=512, bias=True)\n", + " (4): Dropout(p=0.1, inplace=False)\n", + " (5): GELU()\n", + " (6): Linear(in_features=512, out_features=256, bias=True)\n", + " (7): Dropout(p=0.1, inplace=False)\n", + " (8): GELU()\n", + " (9): Linear(in_features=256, out_features=1, bias=True)\n", + " (10): Sigmoid()\n", + " )\n", + " )\n", + ")\n", + "| epoch 1 | 200/ 5120 batches | lr 1.00 | ms/batch 63.51 | loss 0.15 |\n", + "| epoch 1 | 400/ 5120 batches | lr 1.00 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 1 | 600/ 5120 batches | lr 1.00 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 1 | 800/ 5120 batches | lr 1.00 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 1 | 1000/ 5120 batches | lr 1.00 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 1 | 1200/ 5120 batches | lr 1.00 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 1 | 1400/ 5120 batches | lr 1.00 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 1 | 1600/ 5120 batches | lr 1.00 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 1 | 1800/ 5120 batches | lr 1.00 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 1 | 2000/ 5120 batches | lr 1.00 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 1 | 2200/ 5120 batches | lr 1.00 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 1 | 2400/ 5120 batches | lr 1.00 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 1 | 2600/ 5120 batches | lr 1.00 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 1 | 2800/ 5120 batches | lr 1.00 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 1 | 3000/ 5120 batches | lr 1.00 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 1 | 3200/ 5120 batches | lr 1.00 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 1 | 3400/ 5120 batches | lr 1.00 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 1 | 3600/ 5120 batches | lr 1.00 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 1 | 3800/ 5120 batches | lr 1.00 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 1 | 4000/ 5120 batches | lr 1.00 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 1 | 4200/ 5120 batches | lr 1.00 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 1 | 4400/ 5120 batches | lr 1.00 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 1 | 4600/ 5120 batches | lr 1.00 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 1 | 4800/ 5120 batches | lr 1.00 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 1 | 5000/ 5120 batches | lr 1.00 | ms/batch 63.12 | loss 0.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 458.77s | valid loss 0.00 | valid ppl 1.00\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 5120 batches | lr 0.95 | ms/batch 63.45 | loss 0.00 |\n", + "| epoch 2 | 400/ 5120 batches | lr 0.95 | ms/batch 63.15 | loss 0.00 |\n", + "| epoch 2 | 600/ 5120 batches | lr 0.95 | ms/batch 63.15 | loss 0.00 |\n", + "| epoch 2 | 800/ 5120 batches | lr 0.95 | ms/batch 63.15 | loss 0.00 |\n", + "| epoch 2 | 1000/ 5120 batches | lr 0.95 | ms/batch 63.15 | loss 0.00 |\n", + "| epoch 2 | 1200/ 5120 batches | lr 0.95 | ms/batch 63.15 | loss 0.00 |\n", + "| epoch 2 | 1400/ 5120 batches | lr 0.95 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 2 | 1600/ 5120 batches | lr 0.95 | ms/batch 63.15 | loss 0.00 |\n", + "| epoch 2 | 1800/ 5120 batches | lr 0.95 | ms/batch 63.15 | loss 0.00 |\n", + "| epoch 2 | 2000/ 5120 batches | lr 0.95 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 2 | 2200/ 5120 batches | lr 0.95 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 2 | 2400/ 5120 batches | lr 0.95 | ms/batch 63.15 | loss 0.00 |\n", + "| epoch 2 | 2600/ 5120 batches | lr 0.95 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 2 | 2800/ 5120 batches | lr 0.95 | ms/batch 63.16 | loss 0.00 |\n", + "| epoch 2 | 3000/ 5120 batches | lr 0.95 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 2 | 3200/ 5120 batches | lr 0.95 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 2 | 3400/ 5120 batches | lr 0.95 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 2 | 3600/ 5120 batches | lr 0.95 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 2 | 3800/ 5120 batches | lr 0.95 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 2 | 4000/ 5120 batches | lr 0.95 | ms/batch 63.15 | loss 0.00 |\n", + "| epoch 2 | 4200/ 5120 batches | lr 0.95 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 2 | 4400/ 5120 batches | lr 0.95 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 2 | 4600/ 5120 batches | lr 0.95 | ms/batch 63.15 | loss 0.00 |\n", + "| epoch 2 | 4800/ 5120 batches | lr 0.95 | ms/batch 63.15 | loss 0.00 |\n", + "| epoch 2 | 5000/ 5120 batches | lr 0.95 | ms/batch 63.14 | loss 0.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 458.92s | valid loss 0.00 | valid ppl 1.00\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 5120 batches | lr 0.90 | ms/batch 63.46 | loss 0.00 |\n", + "| epoch 3 | 400/ 5120 batches | lr 0.90 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 3 | 600/ 5120 batches | lr 0.90 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 3 | 800/ 5120 batches | lr 0.90 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 3 | 1000/ 5120 batches | lr 0.90 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 3 | 1200/ 5120 batches | lr 0.90 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 3 | 1400/ 5120 batches | lr 0.90 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 3 | 1600/ 5120 batches | lr 0.90 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 3 | 1800/ 5120 batches | lr 0.90 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 3 | 2000/ 5120 batches | lr 0.90 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 3 | 2200/ 5120 batches | lr 0.90 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 3 | 2400/ 5120 batches | lr 0.90 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 3 | 2600/ 5120 batches | lr 0.90 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 3 | 2800/ 5120 batches | lr 0.90 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 3 | 3000/ 5120 batches | lr 0.90 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 3 | 3200/ 5120 batches | lr 0.90 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 3 | 3400/ 5120 batches | lr 0.90 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 3 | 3600/ 5120 batches | lr 0.90 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 3 | 3800/ 5120 batches | lr 0.90 | ms/batch 63.15 | loss 0.00 |\n", + "| epoch 3 | 4000/ 5120 batches | lr 0.90 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 3 | 4200/ 5120 batches | lr 0.90 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 3 | 4400/ 5120 batches | lr 0.90 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 3 | 4600/ 5120 batches | lr 0.90 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 3 | 4800/ 5120 batches | lr 0.90 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 3 | 5000/ 5120 batches | lr 0.90 | ms/batch 63.13 | loss 0.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 458.75s | valid loss 0.00 | valid ppl 1.00\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 5120 batches | lr 0.86 | ms/batch 63.43 | loss 0.00 |\n", + "| epoch 4 | 400/ 5120 batches | lr 0.86 | ms/batch 63.14 | loss 0.00 |\n", + "| epoch 4 | 600/ 5120 batches | lr 0.86 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 4 | 800/ 5120 batches | lr 0.86 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 4 | 1000/ 5120 batches | lr 0.86 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 4 | 1200/ 5120 batches | lr 0.86 | ms/batch 63.11 | loss 0.00 |\n", + "| epoch 4 | 1400/ 5120 batches | lr 0.86 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 4 | 1600/ 5120 batches | lr 0.86 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 4 | 1800/ 5120 batches | lr 0.86 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 4 | 2000/ 5120 batches | lr 0.86 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 4 | 2200/ 5120 batches | lr 0.86 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 4 | 2400/ 5120 batches | lr 0.86 | ms/batch 63.11 | loss 0.00 |\n", + "| epoch 4 | 2600/ 5120 batches | lr 0.86 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 4 | 2800/ 5120 batches | lr 0.86 | ms/batch 63.11 | loss 0.00 |\n", + "| epoch 4 | 3000/ 5120 batches | lr 0.86 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 4 | 3200/ 5120 batches | lr 0.86 | ms/batch 63.11 | loss 0.00 |\n", + "| epoch 4 | 3400/ 5120 batches | lr 0.86 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 4 | 3600/ 5120 batches | lr 0.86 | ms/batch 63.11 | loss 0.00 |\n", + "| epoch 4 | 3800/ 5120 batches | lr 0.86 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 4 | 4000/ 5120 batches | lr 0.86 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 4 | 4200/ 5120 batches | lr 0.86 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 4 | 4400/ 5120 batches | lr 0.86 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 4 | 4600/ 5120 batches | lr 0.86 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 4 | 4800/ 5120 batches | lr 0.86 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 4 | 5000/ 5120 batches | lr 0.86 | ms/batch 63.12 | loss 0.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 458.77s | valid loss 0.00 | valid ppl 1.00\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 5120 batches | lr 0.81 | ms/batch 63.44 | loss 0.00 |\n", + "| epoch 5 | 400/ 5120 batches | lr 0.81 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 5 | 600/ 5120 batches | lr 0.81 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 5 | 800/ 5120 batches | lr 0.81 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 5 | 1000/ 5120 batches | lr 0.81 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 5 | 1200/ 5120 batches | lr 0.81 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 5 | 1400/ 5120 batches | lr 0.81 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 5 | 1600/ 5120 batches | lr 0.81 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 5 | 1800/ 5120 batches | lr 0.81 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 5 | 2000/ 5120 batches | lr 0.81 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 5 | 2200/ 5120 batches | lr 0.81 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 5 | 2400/ 5120 batches | lr 0.81 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 5 | 2600/ 5120 batches | lr 0.81 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 5 | 2800/ 5120 batches | lr 0.81 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 5 | 3000/ 5120 batches | lr 0.81 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 5 | 3200/ 5120 batches | lr 0.81 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 5 | 3400/ 5120 batches | lr 0.81 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 5 | 3600/ 5120 batches | lr 0.81 | ms/batch 63.13 | loss 0.00 |\n", + "| epoch 5 | 3800/ 5120 batches | lr 0.81 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 5 | 4000/ 5120 batches | lr 0.81 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 5 | 4200/ 5120 batches | lr 0.81 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 5 | 4400/ 5120 batches | lr 0.81 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 5 | 4600/ 5120 batches | lr 0.81 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 5 | 4800/ 5120 batches | lr 0.81 | ms/batch 63.12 | loss 0.00 |\n", + "| epoch 5 | 5000/ 5120 batches | lr 0.81 | ms/batch 63.12 | loss 0.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 458.86s | valid loss 0.00 | valid ppl 1.00\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 0.00 | test ppl 1.00\n", + "=========================================================================================\n" + ] + } + ], + "source": [ + "batch_size = 20\n", + "eval_batch_size = 10\n", + "seq_len = 1024\n", + "divider = 16\n", + "dataset_len = 102400\n", + "epochs = 5 # The number of epochs\n", + "\n", + "model = DoXTimes(Smartpool(divider, 0.3))\n", + "model = model.to(device)\n", + "print(model)\n", + "\n", + "lr = 1.0 # learning rate\n", + "optimizer = torch.optim.SGD(model.parameters(), lr=lr)\n", + "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "\n", + "model = train_model(model, epochs, batch_size, eval_batch_size, dataset_len, seq_len, divider, optimizer, scheduler)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "HOMLGHgA12cP", + "outputId": "f1065bfc-fbfe-4883-ef19-b43a3d0d139a" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "data[0]: tensor([[1.0000, 0.9635],\n", + " [0.0000, 0.6513],\n", + " [0.0000, 0.1477],\n", + " [0.0000, 0.5895],\n", + " [0.0000, 0.7661],\n", + " [0.0000, 0.4402],\n", + " [0.0000, 0.7964],\n", + " [0.0000, 0.4043],\n", + " [0.0000, 0.3817],\n", + " [0.0000, 0.4003],\n", + " [0.0000, 0.7647],\n", + " [0.0000, 0.7788],\n", + " [0.0000, 0.3174],\n", + " [0.0000, 0.5332],\n", + " [0.0000, 0.9874],\n", + " [0.0000, 0.1417],\n", + " [1.0000, 0.2138],\n", + " [0.0000, 0.2606],\n", + " [1.0000, 0.4714],\n", + " [0.0000, 0.2989],\n", + " [0.0000, 0.8235],\n", + " [0.0000, 0.1116],\n", + " [0.0000, 0.7541],\n", + " [0.0000, 0.4020],\n", + " [1.0000, 0.6784],\n", + " [1.0000, 0.7113],\n", + " [0.0000, 0.5481],\n", + " [0.0000, 0.9088],\n", + " [0.0000, 0.2680],\n", + " [0.0000, 0.9701],\n", + " [0.0000, 0.2193],\n", + " [0.0000, 0.8612],\n", + " [0.0000, 0.4654],\n", + " [0.0000, 0.3765],\n", + " [0.0000, 0.5589],\n", + " [0.0000, 0.6168],\n", + " [0.0000, 0.0343],\n", + " [0.0000, 0.0091],\n", + " [0.0000, 0.7646],\n", + " [0.0000, 0.1852],\n", + " [0.0000, 0.8836],\n", + " [0.0000, 0.1186],\n", + " [1.0000, 0.8067],\n", + " [0.0000, 0.3529],\n", + " [0.0000, 0.3338],\n", + " [0.0000, 0.1691],\n", + " [0.0000, 0.4034],\n", + " [0.0000, 0.5362],\n", + " [0.0000, 0.2243],\n", + " [0.0000, 0.8713]], device='cuda:0')\n", + "output: tensor([28.6111, 31.7701, 30.9115, 36.6633, 27.8306, 32.6912, 37.0948, 29.6992,\n", + " 33.2462, 35.4836], device='cuda:0')\n", + "targets: tensor([28.6111, 31.7701, 30.9116, 36.6633, 27.8306, 32.6913, 37.0948, 29.6992,\n", + " 33.2462, 35.4836], device='cuda:0')\n" + ] + } + ], + "source": [ + "model.eval()\n", + "with torch.no_grad():\n", + " data = get_batch(eval_batch_size, seq_len, divider).to(device)\n", + " targets = data.prod(-1).sum(-1).squeeze(-1)\n", + " output = model(data)\n", + " print('data[0]:', data[0,:50,:])\n", + "\n", + " print('output:', output)\n", + " print('targets:', targets)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "9VakAqWwX4jZ" + }, + "source": [ + "# To 1 row" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ra3-YwdbWbP9" + }, + "source": [ + "## Pooling T/4" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "45Da8PVqZZIY" + }, + "source": [ + "### Average pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "XxTnA1yQZZIf", + "outputId": "757b60ad-97ec-4be3-f4ec-b4e8d7f09a0f" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using cache found in /home/i273233/.cache/torch/hub/pytorch_vision_v0.6.0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Conv(\n", + " (conv): Sequential(\n", + " (0): Conv2d(1, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (1): ReLU(inplace=True)\n", + " (2): AvgPool2d(kernel_size=(2, 1), stride=(2, 1), padding=0)\n", + " (3): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (4): ReLU(inplace=True)\n", + " (5): AvgPool2d(kernel_size=2, stride=(2, 1), padding=0)\n", + " (6): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (7): ReLU(inplace=True)\n", + " (8): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (linear): Conv2d(256, 1, kernel_size=(1, 1), stride=(1, 1))\n", + " )\n", + ")\n" + ] + } + ], + "source": [ + "\n", + "model = torch.hub.load('pytorch/vision:v0.6.0', 'vgg11', pretrained=False)\n", + "model = model.features[0:9]\n", + "model[0] = nn.Conv2d(1, 64, kernel_size=(3,3), stride=(1,1), padding=(1,1))\n", + "model[2] = nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=0)\n", + "model[5] = nn.AvgPool2d(kernel_size=2, stride=(2,1), padding=0)\n", + "model[8] = nn.Conv2d(256, 256, kernel_size=(3,3), stride=(1,1), padding=(1,1))\n", + "model.add_module(\"linear\", nn.Conv2d(256, 1, kernel_size=1, stride=1, padding=0))\n", + "model = Conv(model, 1)\n", + "model = model.to(device)\n", + "print(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "3vo-S9WIZZIg", + "outputId": "7ab15729-423d-46e9-d64b-0bc216bce0cd" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 5000 batches | lr 1.00 | ms/batch 31.55 | loss 0.51 |\n", + "| epoch 1 | 400/ 5000 batches | lr 1.00 | ms/batch 31.34 | loss 0.50 |\n", + "| epoch 1 | 600/ 5000 batches | lr 1.00 | ms/batch 31.36 | loss 0.50 |\n", + "| epoch 1 | 800/ 5000 batches | lr 1.00 | ms/batch 31.37 | loss 0.50 |\n", + "| epoch 1 | 1000/ 5000 batches | lr 1.00 | ms/batch 31.37 | loss 0.50 |\n", + "| epoch 1 | 1200/ 5000 batches | lr 1.00 | ms/batch 31.37 | loss 0.50 |\n", + "| epoch 1 | 1400/ 5000 batches | lr 1.00 | ms/batch 31.55 | loss 0.50 |\n", + "| epoch 1 | 1600/ 5000 batches | lr 1.00 | ms/batch 32.08 | loss 0.50 |\n", + "| epoch 1 | 1800/ 5000 batches | lr 1.00 | ms/batch 32.21 | loss 0.50 |\n", + "| epoch 1 | 2000/ 5000 batches | lr 1.00 | ms/batch 32.52 | loss 0.50 |\n", + "| epoch 1 | 2200/ 5000 batches | lr 1.00 | ms/batch 32.81 | loss 0.50 |\n", + "| epoch 1 | 2400/ 5000 batches | lr 1.00 | ms/batch 32.83 | loss 0.50 |\n", + "| epoch 1 | 2600/ 5000 batches | lr 1.00 | ms/batch 33.19 | loss 0.50 |\n", + "| epoch 1 | 2800/ 5000 batches | lr 1.00 | ms/batch 33.28 | loss 0.50 |\n", + "| epoch 1 | 3000/ 5000 batches | lr 1.00 | ms/batch 33.27 | loss 0.50 |\n", + "| epoch 1 | 3200/ 5000 batches | lr 1.00 | ms/batch 33.28 | loss 0.50 |\n", + "| epoch 1 | 3400/ 5000 batches | lr 1.00 | ms/batch 33.28 | loss 0.50 |\n", + "| epoch 1 | 3600/ 5000 batches | lr 1.00 | ms/batch 33.86 | loss 0.50 |\n", + "| epoch 1 | 3800/ 5000 batches | lr 1.00 | ms/batch 33.82 | loss 0.50 |\n", + "| epoch 1 | 4000/ 5000 batches | lr 1.00 | ms/batch 33.82 | loss 0.50 |\n", + "| epoch 1 | 4200/ 5000 batches | lr 1.00 | ms/batch 33.83 | loss 0.50 |\n", + "| epoch 1 | 4400/ 5000 batches | lr 1.00 | ms/batch 33.84 | loss 0.50 |\n", + "| epoch 1 | 4600/ 5000 batches | lr 1.00 | ms/batch 33.84 | loss 0.50 |\n", + "| epoch 1 | 4800/ 5000 batches | lr 1.00 | ms/batch 33.85 | loss 0.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 243.91s | valid loss 0.77 | valid ppl 2.16\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 5000 batches | lr 0.95 | ms/batch 34.72 | loss 0.48 |\n", + "| epoch 2 | 400/ 5000 batches | lr 0.95 | ms/batch 34.56 | loss 0.48 |\n", + "| epoch 2 | 600/ 5000 batches | lr 0.95 | ms/batch 34.52 | loss 0.48 |\n", + "| epoch 2 | 800/ 5000 batches | lr 0.95 | ms/batch 34.54 | loss 0.48 |\n", + "| epoch 2 | 1000/ 5000 batches | lr 0.95 | ms/batch 34.51 | loss 0.48 |\n", + "| epoch 2 | 1200/ 5000 batches | lr 0.95 | ms/batch 34.53 | loss 0.48 |\n", + "| epoch 2 | 1400/ 5000 batches | lr 0.95 | ms/batch 34.54 | loss 0.48 |\n", + "| epoch 2 | 1600/ 5000 batches | lr 0.95 | ms/batch 34.52 | loss 0.48 |\n", + "| epoch 2 | 1800/ 5000 batches | lr 0.95 | ms/batch 34.50 | loss 0.48 |\n", + "| epoch 2 | 2000/ 5000 batches | lr 0.95 | ms/batch 34.52 | loss 0.47 |\n", + "| epoch 2 | 2200/ 5000 batches | lr 0.95 | ms/batch 34.51 | loss 0.47 |\n", + "| epoch 2 | 2400/ 5000 batches | lr 0.95 | ms/batch 34.50 | loss 0.47 |\n", + "| epoch 2 | 2600/ 5000 batches | lr 0.95 | ms/batch 34.49 | loss 0.48 |\n", + "| epoch 2 | 2800/ 5000 batches | lr 0.95 | ms/batch 34.49 | loss 0.48 |\n", + "| epoch 2 | 3000/ 5000 batches | lr 0.95 | ms/batch 34.50 | loss 0.48 |\n", + "| epoch 2 | 3200/ 5000 batches | lr 0.95 | ms/batch 34.52 | loss 0.47 |\n", + "| epoch 2 | 3400/ 5000 batches | lr 0.95 | ms/batch 34.60 | loss 0.48 |\n", + "| epoch 2 | 3600/ 5000 batches | lr 0.95 | ms/batch 34.60 | loss 0.48 |\n", + "| epoch 2 | 3800/ 5000 batches | lr 0.95 | ms/batch 34.63 | loss 0.48 |\n", + "| epoch 2 | 4000/ 5000 batches | lr 0.95 | ms/batch 34.62 | loss 0.48 |\n", + "| epoch 2 | 4200/ 5000 batches | lr 0.95 | ms/batch 34.62 | loss 0.48 |\n", + "| epoch 2 | 4400/ 5000 batches | lr 0.95 | ms/batch 34.62 | loss 0.48 |\n", + "| epoch 2 | 4600/ 5000 batches | lr 0.95 | ms/batch 34.63 | loss 0.48 |\n", + "| epoch 2 | 4800/ 5000 batches | lr 0.95 | ms/batch 34.62 | loss 0.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 254.01s | valid loss 0.77 | valid ppl 2.16\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 5000 batches | lr 0.90 | ms/batch 34.81 | loss 0.45 |\n", + "| epoch 3 | 400/ 5000 batches | lr 0.90 | ms/batch 34.62 | loss 0.45 |\n", + "| epoch 3 | 600/ 5000 batches | lr 0.90 | ms/batch 34.64 | loss 0.45 |\n", + "| epoch 3 | 800/ 5000 batches | lr 0.90 | ms/batch 34.63 | loss 0.45 |\n", + "| epoch 3 | 1000/ 5000 batches | lr 0.90 | ms/batch 34.63 | loss 0.45 |\n", + "| epoch 3 | 1200/ 5000 batches | lr 0.90 | ms/batch 34.65 | loss 0.45 |\n", + "| epoch 3 | 1400/ 5000 batches | lr 0.90 | ms/batch 34.64 | loss 0.45 |\n", + "| epoch 3 | 1600/ 5000 batches | lr 0.90 | ms/batch 34.64 | loss 0.45 |\n", + "| epoch 3 | 1800/ 5000 batches | lr 0.90 | ms/batch 34.62 | loss 0.45 |\n", + "| epoch 3 | 2000/ 5000 batches | lr 0.90 | ms/batch 34.62 | loss 0.45 |\n", + "| epoch 3 | 2200/ 5000 batches | lr 0.90 | ms/batch 34.63 | loss 0.45 |\n", + "| epoch 3 | 2400/ 5000 batches | lr 0.90 | ms/batch 34.63 | loss 0.45 |\n", + "| epoch 3 | 2600/ 5000 batches | lr 0.90 | ms/batch 34.64 | loss 0.45 |\n", + "| epoch 3 | 2800/ 5000 batches | lr 0.90 | ms/batch 34.65 | loss 0.45 |\n", + "| epoch 3 | 3000/ 5000 batches | lr 0.90 | ms/batch 34.65 | loss 0.45 |\n", + "| epoch 3 | 3200/ 5000 batches | lr 0.90 | ms/batch 34.63 | loss 0.45 |\n", + "| epoch 3 | 3400/ 5000 batches | lr 0.90 | ms/batch 34.63 | loss 0.45 |\n", + "| epoch 3 | 3600/ 5000 batches | lr 0.90 | ms/batch 34.64 | loss 0.45 |\n", + "| epoch 3 | 3800/ 5000 batches | lr 0.90 | ms/batch 34.64 | loss 0.45 |\n", + "| epoch 3 | 4000/ 5000 batches | lr 0.90 | ms/batch 34.64 | loss 0.45 |\n", + "| epoch 3 | 4200/ 5000 batches | lr 0.90 | ms/batch 34.62 | loss 0.45 |\n", + "| epoch 3 | 4400/ 5000 batches | lr 0.90 | ms/batch 34.64 | loss 0.45 |\n", + "| epoch 3 | 4600/ 5000 batches | lr 0.90 | ms/batch 34.65 | loss 0.45 |\n", + "| epoch 3 | 4800/ 5000 batches | lr 0.90 | ms/batch 34.64 | loss 0.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 254.47s | valid loss 0.77 | valid ppl 2.16\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 5000 batches | lr 0.86 | ms/batch 34.80 | loss 0.43 |\n", + "| epoch 4 | 400/ 5000 batches | lr 0.86 | ms/batch 34.62 | loss 0.43 |\n", + "| epoch 4 | 600/ 5000 batches | lr 0.86 | ms/batch 34.62 | loss 0.43 |\n", + "| epoch 4 | 800/ 5000 batches | lr 0.86 | ms/batch 34.64 | loss 0.43 |\n", + "| epoch 4 | 1000/ 5000 batches | lr 0.86 | ms/batch 34.64 | loss 0.43 |\n", + "| epoch 4 | 1200/ 5000 batches | lr 0.86 | ms/batch 34.63 | loss 0.43 |\n", + "| epoch 4 | 1400/ 5000 batches | lr 0.86 | ms/batch 34.63 | loss 0.43 |\n", + "| epoch 4 | 1600/ 5000 batches | lr 0.86 | ms/batch 34.63 | loss 0.43 |\n", + "| epoch 4 | 1800/ 5000 batches | lr 0.86 | ms/batch 34.63 | loss 0.43 |\n", + "| epoch 4 | 2000/ 5000 batches | lr 0.86 | ms/batch 34.62 | loss 0.43 |\n", + "| epoch 4 | 2200/ 5000 batches | lr 0.86 | ms/batch 34.63 | loss 0.43 |\n", + "| epoch 4 | 2400/ 5000 batches | lr 0.86 | ms/batch 34.63 | loss 0.43 |\n", + "| epoch 4 | 2600/ 5000 batches | lr 0.86 | ms/batch 34.65 | loss 0.43 |\n", + "| epoch 4 | 2800/ 5000 batches | lr 0.86 | ms/batch 34.64 | loss 0.43 |\n", + "| epoch 4 | 3000/ 5000 batches | lr 0.86 | ms/batch 34.65 | loss 0.43 |\n", + "| epoch 4 | 3200/ 5000 batches | lr 0.86 | ms/batch 34.64 | loss 0.43 |\n", + "| epoch 4 | 3400/ 5000 batches | lr 0.86 | ms/batch 34.65 | loss 0.43 |\n", + "| epoch 4 | 3600/ 5000 batches | lr 0.86 | ms/batch 34.65 | loss 0.43 |\n", + "| epoch 4 | 3800/ 5000 batches | lr 0.86 | ms/batch 34.66 | loss 0.43 |\n", + "| epoch 4 | 4000/ 5000 batches | lr 0.86 | ms/batch 34.64 | loss 0.43 |\n", + "| epoch 4 | 4200/ 5000 batches | lr 0.86 | ms/batch 34.66 | loss 0.43 |\n", + "| epoch 4 | 4400/ 5000 batches | lr 0.86 | ms/batch 34.65 | loss 0.43 |\n", + "| epoch 4 | 4600/ 5000 batches | lr 0.86 | ms/batch 34.64 | loss 0.43 |\n", + "| epoch 4 | 4800/ 5000 batches | lr 0.86 | ms/batch 34.63 | loss 0.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 254.50s | valid loss 0.77 | valid ppl 2.16\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 5000 batches | lr 0.81 | ms/batch 34.84 | loss 0.41 |\n", + "| epoch 5 | 400/ 5000 batches | lr 0.81 | ms/batch 34.66 | loss 0.41 |\n", + "| epoch 5 | 600/ 5000 batches | lr 0.81 | ms/batch 34.67 | loss 0.41 |\n", + "| epoch 5 | 800/ 5000 batches | lr 0.81 | ms/batch 34.66 | loss 0.41 |\n", + "| epoch 5 | 1000/ 5000 batches | lr 0.81 | ms/batch 34.66 | loss 0.41 |\n", + "| epoch 5 | 1200/ 5000 batches | lr 0.81 | ms/batch 34.67 | loss 0.41 |\n", + "| epoch 5 | 1400/ 5000 batches | lr 0.81 | ms/batch 34.67 | loss 0.41 |\n", + "| epoch 5 | 1600/ 5000 batches | lr 0.81 | ms/batch 34.65 | loss 0.41 |\n", + "| epoch 5 | 1800/ 5000 batches | lr 0.81 | ms/batch 34.66 | loss 0.41 |\n", + "| epoch 5 | 2000/ 5000 batches | lr 0.81 | ms/batch 34.65 | loss 0.41 |\n", + "| epoch 5 | 2200/ 5000 batches | lr 0.81 | ms/batch 34.65 | loss 0.41 |\n", + "| epoch 5 | 2400/ 5000 batches | lr 0.81 | ms/batch 34.64 | loss 0.41 |\n", + "| epoch 5 | 2600/ 5000 batches | lr 0.81 | ms/batch 34.65 | loss 0.41 |\n", + "| epoch 5 | 2800/ 5000 batches | lr 0.81 | ms/batch 34.66 | loss 0.41 |\n", + "| epoch 5 | 3000/ 5000 batches | lr 0.81 | ms/batch 34.63 | loss 0.41 |\n", + "| epoch 5 | 3200/ 5000 batches | lr 0.81 | ms/batch 34.64 | loss 0.41 |\n", + "| epoch 5 | 3400/ 5000 batches | lr 0.81 | ms/batch 34.67 | loss 0.41 |\n", + "| epoch 5 | 3600/ 5000 batches | lr 0.81 | ms/batch 34.64 | loss 0.41 |\n", + "| epoch 5 | 3800/ 5000 batches | lr 0.81 | ms/batch 34.65 | loss 0.41 |\n", + "| epoch 5 | 4000/ 5000 batches | lr 0.81 | ms/batch 34.65 | loss 0.41 |\n", + "| epoch 5 | 4200/ 5000 batches | lr 0.81 | ms/batch 34.63 | loss 0.41 |\n", + "| epoch 5 | 4400/ 5000 batches | lr 0.81 | ms/batch 34.63 | loss 0.41 |\n", + "| epoch 5 | 4600/ 5000 batches | lr 0.81 | ms/batch 34.62 | loss 0.41 |\n", + "| epoch 5 | 4800/ 5000 batches | lr 0.81 | ms/batch 34.64 | loss 0.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 254.54s | valid loss 0.72 | valid ppl 2.05\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 0.72 | test ppl 2.05\n", + "=========================================================================================\n" + ] + } + ], + "source": [ + "batch_size = 20\n", + "eval_batch_size = 10\n", + "seq_len = 1000\n", + "divider = 4\n", + "dataset_len = 100000\n", + "lr = 1.0 # learning rate\n", + "optimizer = torch.optim.SGD(model.parameters(), lr=lr)\n", + "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "epochs = 5 # The number of epochs\n", + "\n", + "model = train_model(model, epochs, batch_size, eval_batch_size, dataset_len, seq_len, divider, optimizer, scheduler)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "-PgRcYUzZZIg", + "outputId": "42caac6b-cbfc-4c21-abb5-6f7090dbb86b" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "data[0]: tensor([[0.0000, 0.2617],\n", + " [0.0000, 0.3286],\n", + " [0.0000, 0.5726],\n", + " [0.0000, 0.9218],\n", + " [0.0000, 0.3122],\n", + " [1.0000, 0.6036],\n", + " [0.0000, 0.2931],\n", + " [0.0000, 0.2205],\n", + " [0.0000, 0.9611],\n", + " [0.0000, 0.6294],\n", + " [0.0000, 0.9402],\n", + " [0.0000, 0.4345],\n", + " [0.0000, 0.6048],\n", + " [0.0000, 0.7359],\n", + " [0.0000, 0.6780],\n", + " [0.0000, 0.0507],\n", + " [0.0000, 0.5398],\n", + " [1.0000, 0.0462],\n", + " [0.0000, 0.3863],\n", + " [0.0000, 0.4442],\n", + " [0.0000, 0.2296],\n", + " [1.0000, 0.0960],\n", + " [0.0000, 0.8954],\n", + " [0.0000, 0.1312],\n", + " [0.0000, 0.9982],\n", + " [0.0000, 0.8226],\n", + " [0.0000, 0.3142],\n", + " [0.0000, 0.3401],\n", + " [1.0000, 0.6303],\n", + " [1.0000, 0.5184],\n", + " [0.0000, 0.7984],\n", + " [0.0000, 0.4285],\n", + " [0.0000, 0.7007],\n", + " [0.0000, 0.9994],\n", + " [1.0000, 0.2633],\n", + " [1.0000, 0.9739],\n", + " [1.0000, 0.8579],\n", + " [0.0000, 0.1787],\n", + " [0.0000, 0.1542],\n", + " [0.0000, 0.6181],\n", + " [0.0000, 0.9850],\n", + " [1.0000, 0.7632],\n", + " [0.0000, 0.8211],\n", + " [0.0000, 0.9135],\n", + " [0.0000, 0.1294],\n", + " [0.0000, 0.4572],\n", + " [0.0000, 0.2037],\n", + " [0.0000, 0.1750],\n", + " [1.0000, 0.6225],\n", + " [0.0000, 0.7357]], device='cuda:0')\n", + "output: tensor([35.3504, 35.3503, 35.3503, 35.3504, 35.3504, 35.3504, 35.3504, 35.3504,\n", + " 35.3504, 35.3504], device='cuda:0')\n", + "targets: tensor([130.9722, 122.9335, 128.7154, 137.8438, 122.2178, 119.1000, 121.1519,\n", + " 131.2838, 125.7765, 132.0219], device='cuda:0')\n" + ] + } + ], + "source": [ + "model.eval()\n", + "with torch.no_grad():\n", + " data = get_batch(eval_batch_size, seq_len, divider).to(device)\n", + " targets = data.prod(-1).sum(-1).squeeze(-1)\n", + " output = model(data)\n", + " print('data[0]:', data[0,:50,:])\n", + "\n", + " print('output:', output)\n", + " print('targets:', targets)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tDMMy0rQZOsm" + }, + "source": [ + "### Max pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "W6bz_4tZZOs0", + "outputId": "0e86ade7-7fb2-4cd9-f4fd-ff24853c030a" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using cache found in /home/i273233/.cache/torch/hub/pytorch_vision_v0.6.0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Conv(\n", + " (conv): Sequential(\n", + " (0): Conv2d(1, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (1): ReLU(inplace=True)\n", + " (2): MaxPool2d(kernel_size=(2, 1), stride=(2, 1), padding=0, dilation=1, ceil_mode=False)\n", + " (3): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (4): ReLU(inplace=True)\n", + " (5): MaxPool2d(kernel_size=2, stride=(2, 1), padding=0, dilation=1, ceil_mode=False)\n", + " (6): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (7): ReLU(inplace=True)\n", + " (8): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (linear): Conv2d(256, 1, kernel_size=(1, 1), stride=(1, 1))\n", + " )\n", + ")\n" + ] + } + ], + "source": [ + "model = torch.hub.load('pytorch/vision:v0.6.0', 'vgg11', pretrained=False)\n", + "model = model.features[0:9]\n", + "model[0] = nn.Conv2d(1, 64, kernel_size=(3,3), stride=(1,1), padding=(1,1))\n", + "model[2] = nn.MaxPool2d(kernel_size=(2,1), stride=(2,1), padding=0)\n", + "model[5] = nn.MaxPool2d(kernel_size=2, stride=(2,1), padding=0)\n", + "model[8] = nn.Conv2d(256, 256, kernel_size=(3,3), stride=(1,1), padding=(1,1))\n", + "model.add_module(\"linear\", nn.Conv2d(256, 1, kernel_size=1, stride=1, padding=0))\n", + "model = Conv(model, 1)\n", + "model = model.to(device)\n", + "print(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "1XdECFdvZOs1", + "outputId": "215eb78b-cb2c-47a5-ad42-eaec9e7a32df" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 5000 batches | lr 1.00 | ms/batch 34.37 | loss 0.51 |\n", + "| epoch 1 | 400/ 5000 batches | lr 1.00 | ms/batch 34.32 | loss 0.50 |\n", + "| epoch 1 | 600/ 5000 batches | lr 1.00 | ms/batch 34.32 | loss 0.50 |\n", + "| epoch 1 | 800/ 5000 batches | lr 1.00 | ms/batch 34.33 | loss 0.50 |\n", + "| epoch 1 | 1000/ 5000 batches | lr 1.00 | ms/batch 34.34 | loss 0.50 |\n", + "| epoch 1 | 1200/ 5000 batches | lr 1.00 | ms/batch 34.31 | loss 0.50 |\n", + "| epoch 1 | 1400/ 5000 batches | lr 1.00 | ms/batch 34.33 | loss 0.50 |\n", + "| epoch 1 | 1600/ 5000 batches | lr 1.00 | ms/batch 34.33 | loss 0.50 |\n", + "| epoch 1 | 1800/ 5000 batches | lr 1.00 | ms/batch 34.33 | loss 0.50 |\n", + "| epoch 1 | 2000/ 5000 batches | lr 1.00 | ms/batch 34.33 | loss 0.50 |\n", + "| epoch 1 | 2200/ 5000 batches | lr 1.00 | ms/batch 34.34 | loss 0.50 |\n", + "| epoch 1 | 2400/ 5000 batches | lr 1.00 | ms/batch 34.31 | loss 0.50 |\n", + "| epoch 1 | 2600/ 5000 batches | lr 1.00 | ms/batch 34.31 | loss 0.50 |\n", + "| epoch 1 | 2800/ 5000 batches | lr 1.00 | ms/batch 34.32 | loss 0.50 |\n", + "| epoch 1 | 3000/ 5000 batches | lr 1.00 | ms/batch 34.33 | loss 0.50 |\n", + "| epoch 1 | 3200/ 5000 batches | lr 1.00 | ms/batch 34.32 | loss 0.50 |\n", + "| epoch 1 | 3400/ 5000 batches | lr 1.00 | ms/batch 34.31 | loss 0.50 |\n", + "| epoch 1 | 3600/ 5000 batches | lr 1.00 | ms/batch 34.33 | loss 0.50 |\n", + "| epoch 1 | 3800/ 5000 batches | lr 1.00 | ms/batch 34.34 | loss 0.50 |\n", + "| epoch 1 | 4000/ 5000 batches | lr 1.00 | ms/batch 34.34 | loss 0.50 |\n", + "| epoch 1 | 4200/ 5000 batches | lr 1.00 | ms/batch 34.34 | loss 0.50 |\n", + "| epoch 1 | 4400/ 5000 batches | lr 1.00 | ms/batch 34.33 | loss 0.50 |\n", + "| epoch 1 | 4600/ 5000 batches | lr 1.00 | ms/batch 34.34 | loss 0.50 |\n", + "| epoch 1 | 4800/ 5000 batches | lr 1.00 | ms/batch 34.34 | loss 0.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 252.63s | valid loss 0.65 | valid ppl 1.92\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 5000 batches | lr 0.95 | ms/batch 34.47 | loss 0.48 |\n", + "| epoch 2 | 400/ 5000 batches | lr 0.95 | ms/batch 34.30 | loss 0.48 |\n", + "| epoch 2 | 600/ 5000 batches | lr 0.95 | ms/batch 34.31 | loss 0.48 |\n", + "| epoch 2 | 800/ 5000 batches | lr 0.95 | ms/batch 34.28 | loss 0.48 |\n", + "| epoch 2 | 1000/ 5000 batches | lr 0.95 | ms/batch 34.30 | loss 0.48 |\n", + "| epoch 2 | 1200/ 5000 batches | lr 0.95 | ms/batch 34.30 | loss 0.48 |\n", + "| epoch 2 | 1400/ 5000 batches | lr 0.95 | ms/batch 34.29 | loss 0.48 |\n", + "| epoch 2 | 1600/ 5000 batches | lr 0.95 | ms/batch 34.30 | loss 0.48 |\n", + "| epoch 2 | 1800/ 5000 batches | lr 0.95 | ms/batch 34.29 | loss 0.48 |\n", + "| epoch 2 | 2000/ 5000 batches | lr 0.95 | ms/batch 34.29 | loss 0.48 |\n", + "| epoch 2 | 2200/ 5000 batches | lr 0.95 | ms/batch 34.29 | loss 0.48 |\n", + "| epoch 2 | 2400/ 5000 batches | lr 0.95 | ms/batch 34.31 | loss 0.48 |\n", + "| epoch 2 | 2600/ 5000 batches | lr 0.95 | ms/batch 34.30 | loss 0.48 |\n", + "| epoch 2 | 2800/ 5000 batches | lr 0.95 | ms/batch 34.29 | loss 0.48 |\n", + "| epoch 2 | 3000/ 5000 batches | lr 0.95 | ms/batch 34.29 | loss 0.47 |\n", + "| epoch 2 | 3200/ 5000 batches | lr 0.95 | ms/batch 34.29 | loss 0.48 |\n", + "| epoch 2 | 3400/ 5000 batches | lr 0.95 | ms/batch 34.29 | loss 0.48 |\n", + "| epoch 2 | 3600/ 5000 batches | lr 0.95 | ms/batch 34.30 | loss 0.48 |\n", + "| epoch 2 | 3800/ 5000 batches | lr 0.95 | ms/batch 34.30 | loss 0.47 |\n", + "| epoch 2 | 4000/ 5000 batches | lr 0.95 | ms/batch 34.29 | loss 0.47 |\n", + "| epoch 2 | 4200/ 5000 batches | lr 0.95 | ms/batch 34.28 | loss 0.48 |\n", + "| epoch 2 | 4400/ 5000 batches | lr 0.95 | ms/batch 34.28 | loss 0.47 |\n", + "| epoch 2 | 4600/ 5000 batches | lr 0.95 | ms/batch 34.29 | loss 0.48 |\n", + "| epoch 2 | 4800/ 5000 batches | lr 0.95 | ms/batch 34.27 | loss 0.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 252.38s | valid loss 0.65 | valid ppl 1.92\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 5000 batches | lr 0.90 | ms/batch 34.45 | loss 0.45 |\n", + "| epoch 3 | 400/ 5000 batches | lr 0.90 | ms/batch 34.27 | loss 0.45 |\n", + "| epoch 3 | 600/ 5000 batches | lr 0.90 | ms/batch 34.25 | loss 0.45 |\n", + "| epoch 3 | 800/ 5000 batches | lr 0.90 | ms/batch 34.25 | loss 0.45 |\n", + "| epoch 3 | 1000/ 5000 batches | lr 0.90 | ms/batch 34.25 | loss 0.45 |\n", + "| epoch 3 | 1200/ 5000 batches | lr 0.90 | ms/batch 34.25 | loss 0.45 |\n", + "| epoch 3 | 1400/ 5000 batches | lr 0.90 | ms/batch 34.26 | loss 0.45 |\n", + "| epoch 3 | 1600/ 5000 batches | lr 0.90 | ms/batch 34.27 | loss 0.45 |\n", + "| epoch 3 | 1800/ 5000 batches | lr 0.90 | ms/batch 34.25 | loss 0.45 |\n", + "| epoch 3 | 2000/ 5000 batches | lr 0.90 | ms/batch 34.26 | loss 0.45 |\n", + "| epoch 3 | 2200/ 5000 batches | lr 0.90 | ms/batch 34.25 | loss 0.45 |\n", + "| epoch 3 | 2400/ 5000 batches | lr 0.90 | ms/batch 34.26 | loss 0.45 |\n", + "| epoch 3 | 2600/ 5000 batches | lr 0.90 | ms/batch 34.26 | loss 0.45 |\n", + "| epoch 3 | 2800/ 5000 batches | lr 0.90 | ms/batch 34.26 | loss 0.45 |\n", + "| epoch 3 | 3000/ 5000 batches | lr 0.90 | ms/batch 34.26 | loss 0.45 |\n", + "| epoch 3 | 3200/ 5000 batches | lr 0.90 | ms/batch 34.27 | loss 0.45 |\n", + "| epoch 3 | 3400/ 5000 batches | lr 0.90 | ms/batch 34.28 | loss 0.45 |\n", + "| epoch 3 | 3600/ 5000 batches | lr 0.90 | ms/batch 34.27 | loss 0.45 |\n", + "| epoch 3 | 3800/ 5000 batches | lr 0.90 | ms/batch 34.28 | loss 0.45 |\n", + "| epoch 3 | 4000/ 5000 batches | lr 0.90 | ms/batch 34.25 | loss 0.45 |\n", + "| epoch 3 | 4200/ 5000 batches | lr 0.90 | ms/batch 34.24 | loss 0.45 |\n", + "| epoch 3 | 4400/ 5000 batches | lr 0.90 | ms/batch 34.25 | loss 0.45 |\n", + "| epoch 3 | 4600/ 5000 batches | lr 0.90 | ms/batch 34.26 | loss 0.45 |\n", + "| epoch 3 | 4800/ 5000 batches | lr 0.90 | ms/batch 34.25 | loss 0.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 252.17s | valid loss 0.65 | valid ppl 1.92\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 5000 batches | lr 0.86 | ms/batch 34.42 | loss 0.43 |\n", + "| epoch 4 | 400/ 5000 batches | lr 0.86 | ms/batch 34.27 | loss 0.43 |\n", + "| epoch 4 | 600/ 5000 batches | lr 0.86 | ms/batch 34.26 | loss 0.43 |\n", + "| epoch 4 | 800/ 5000 batches | lr 0.86 | ms/batch 34.30 | loss 0.43 |\n", + "| epoch 4 | 1000/ 5000 batches | lr 0.86 | ms/batch 34.28 | loss 0.43 |\n", + "| epoch 4 | 1200/ 5000 batches | lr 0.86 | ms/batch 34.29 | loss 0.43 |\n", + "| epoch 4 | 1400/ 5000 batches | lr 0.86 | ms/batch 34.28 | loss 0.43 |\n", + "| epoch 4 | 1600/ 5000 batches | lr 0.86 | ms/batch 34.27 | loss 0.43 |\n", + "| epoch 4 | 1800/ 5000 batches | lr 0.86 | ms/batch 34.27 | loss 0.43 |\n", + "| epoch 4 | 2000/ 5000 batches | lr 0.86 | ms/batch 34.27 | loss 0.43 |\n", + "| epoch 4 | 2200/ 5000 batches | lr 0.86 | ms/batch 34.26 | loss 0.43 |\n", + "| epoch 4 | 2400/ 5000 batches | lr 0.86 | ms/batch 34.26 | loss 0.43 |\n", + "| epoch 4 | 2600/ 5000 batches | lr 0.86 | ms/batch 34.27 | loss 0.43 |\n", + "| epoch 4 | 2800/ 5000 batches | lr 0.86 | ms/batch 34.27 | loss 0.43 |\n", + "| epoch 4 | 3000/ 5000 batches | lr 0.86 | ms/batch 34.26 | loss 0.43 |\n", + "| epoch 4 | 3200/ 5000 batches | lr 0.86 | ms/batch 34.27 | loss 0.43 |\n", + "| epoch 4 | 3400/ 5000 batches | lr 0.86 | ms/batch 34.27 | loss 0.43 |\n", + "| epoch 4 | 3600/ 5000 batches | lr 0.86 | ms/batch 34.27 | loss 0.43 |\n", + "| epoch 4 | 3800/ 5000 batches | lr 0.86 | ms/batch 34.27 | loss 0.43 |\n", + "| epoch 4 | 4000/ 5000 batches | lr 0.86 | ms/batch 34.26 | loss 0.43 |\n", + "| epoch 4 | 4200/ 5000 batches | lr 0.86 | ms/batch 34.27 | loss 0.43 |\n", + "| epoch 4 | 4400/ 5000 batches | lr 0.86 | ms/batch 34.26 | loss 0.43 |\n", + "| epoch 4 | 4600/ 5000 batches | lr 0.86 | ms/batch 34.29 | loss 0.43 |\n", + "| epoch 4 | 4800/ 5000 batches | lr 0.86 | ms/batch 34.26 | loss 0.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 252.29s | valid loss 0.65 | valid ppl 1.92\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 5000 batches | lr 0.81 | ms/batch 34.45 | loss 0.41 |\n", + "| epoch 5 | 400/ 5000 batches | lr 0.81 | ms/batch 34.27 | loss 0.41 |\n", + "| epoch 5 | 600/ 5000 batches | lr 0.81 | ms/batch 34.27 | loss 0.41 |\n", + "| epoch 5 | 800/ 5000 batches | lr 0.81 | ms/batch 34.27 | loss 0.41 |\n", + "| epoch 5 | 1000/ 5000 batches | lr 0.81 | ms/batch 34.25 | loss 0.41 |\n", + "| epoch 5 | 1200/ 5000 batches | lr 0.81 | ms/batch 34.27 | loss 0.41 |\n", + "| epoch 5 | 1400/ 5000 batches | lr 0.81 | ms/batch 34.27 | loss 0.41 |\n", + "| epoch 5 | 1600/ 5000 batches | lr 0.81 | ms/batch 34.26 | loss 0.41 |\n", + "| epoch 5 | 1800/ 5000 batches | lr 0.81 | ms/batch 34.27 | loss 0.41 |\n", + "| epoch 5 | 2000/ 5000 batches | lr 0.81 | ms/batch 34.28 | loss 0.41 |\n", + "| epoch 5 | 2200/ 5000 batches | lr 0.81 | ms/batch 34.26 | loss 0.41 |\n", + "| epoch 5 | 2400/ 5000 batches | lr 0.81 | ms/batch 34.23 | loss 0.41 |\n", + "| epoch 5 | 2600/ 5000 batches | lr 0.81 | ms/batch 34.24 | loss 0.41 |\n", + "| epoch 5 | 2800/ 5000 batches | lr 0.81 | ms/batch 34.23 | loss 0.41 |\n", + "| epoch 5 | 3000/ 5000 batches | lr 0.81 | ms/batch 34.24 | loss 0.41 |\n", + "| epoch 5 | 3200/ 5000 batches | lr 0.81 | ms/batch 34.26 | loss 0.41 |\n", + "| epoch 5 | 3400/ 5000 batches | lr 0.81 | ms/batch 34.25 | loss 0.41 |\n", + "| epoch 5 | 3600/ 5000 batches | lr 0.81 | ms/batch 34.25 | loss 0.41 |\n", + "| epoch 5 | 3800/ 5000 batches | lr 0.81 | ms/batch 34.26 | loss 0.41 |\n", + "| epoch 5 | 4000/ 5000 batches | lr 0.81 | ms/batch 34.27 | loss 0.41 |\n", + "| epoch 5 | 4200/ 5000 batches | lr 0.81 | ms/batch 34.26 | loss 0.41 |\n", + "| epoch 5 | 4400/ 5000 batches | lr 0.81 | ms/batch 34.27 | loss 0.41 |\n", + "| epoch 5 | 4600/ 5000 batches | lr 0.81 | ms/batch 34.27 | loss 0.41 |\n", + "| epoch 5 | 4800/ 5000 batches | lr 0.81 | ms/batch 34.28 | loss 0.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 252.22s | valid loss 0.65 | valid ppl 1.92\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 0.65 | test ppl 1.92\n", + "=========================================================================================\n" + ] + } + ], + "source": [ + "batch_size = 20\n", + "eval_batch_size = 10\n", + "seq_len = 1000\n", + "divider = 4\n", + "dataset_len = 100000\n", + "lr = 1.0 # learning rate\n", + "optimizer = torch.optim.SGD(model.parameters(), lr=lr)\n", + "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "epochs = 5 # The number of epochs\n", + "\n", + "model = train_model(model, epochs, batch_size, eval_batch_size, dataset_len, seq_len, divider, optimizer, scheduler)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "dDhuiN10ZOs1", + "outputId": "874f9161-b32e-4e45-eb5d-92badf2e3acc" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "data[0]: tensor([[0.0000, 0.3971],\n", + " [0.0000, 0.7832],\n", + " [1.0000, 0.7652],\n", + " [0.0000, 0.2995],\n", + " [0.0000, 0.5102],\n", + " [0.0000, 0.1984],\n", + " [0.0000, 0.4456],\n", + " [1.0000, 0.4550],\n", + " [0.0000, 0.9313],\n", + " [1.0000, 0.1801],\n", + " [1.0000, 0.1968],\n", + " [0.0000, 0.4223],\n", + " [0.0000, 0.9183],\n", + " [0.0000, 0.0417],\n", + " [0.0000, 0.6817],\n", + " [0.0000, 0.5202],\n", + " [0.0000, 0.8040],\n", + " [1.0000, 0.4950],\n", + " [0.0000, 0.0670],\n", + " [0.0000, 0.5878],\n", + " [1.0000, 0.2242],\n", + " [0.0000, 0.8064],\n", + " [0.0000, 0.7436],\n", + " [1.0000, 0.8182],\n", + " [0.0000, 0.3291],\n", + " [1.0000, 0.3496],\n", + " [0.0000, 0.0335],\n", + " [1.0000, 0.0215],\n", + " [0.0000, 0.1326],\n", + " [0.0000, 0.6483],\n", + " [0.0000, 0.8870],\n", + " [0.0000, 0.6982],\n", + " [0.0000, 0.2311],\n", + " [1.0000, 0.7782],\n", + " [1.0000, 0.2888],\n", + " [0.0000, 0.1660],\n", + " [0.0000, 0.1541],\n", + " [0.0000, 0.9906],\n", + " [0.0000, 0.3401],\n", + " [0.0000, 0.9369],\n", + " [1.0000, 0.6185],\n", + " [0.0000, 0.2701],\n", + " [1.0000, 0.0662],\n", + " [1.0000, 0.3710],\n", + " [0.0000, 0.4112],\n", + " [1.0000, 0.2546],\n", + " [0.0000, 0.8135],\n", + " [0.0000, 0.6596],\n", + " [0.0000, 0.6201],\n", + " [0.0000, 0.8674]], device='cuda:0')\n", + "output: tensor([43.1101, 43.1101, 43.1103, 43.1101, 43.1101, 43.1101, 43.1101, 43.1101,\n", + " 43.1101, 43.1101], device='cuda:0')\n", + "targets: tensor([128.0948, 122.4410, 124.1064, 128.2139, 118.4931, 118.2855, 124.2150,\n", + " 128.9219, 122.3726, 129.6235], device='cuda:0')\n" + ] + } + ], + "source": [ + "model.eval()\n", + "with torch.no_grad():\n", + " data = get_batch(eval_batch_size, seq_len, divider).to(device)\n", + " targets = data.prod(-1).sum(-1).squeeze(-1)\n", + " output = model(data)\n", + " print('data[0]:', data[0,:50,:])\n", + "\n", + " print('output:', output)\n", + " print('targets:', targets)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OJHfbyzxZOs2" + }, + "source": [ + "### Smart pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Lc0QH-OqZOs2", + "outputId": "7dd89e37-4cc4-4c75-d2d5-58f0d5034ec1" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "DoXTimes(\n", + " (model): Smartpool(\n", + " (mlp): Sequential(\n", + " (0): Linear(in_features=2, out_features=256, bias=True)\n", + " (1): Dropout(p=0.1, inplace=False)\n", + " (2): GELU()\n", + " (3): Linear(in_features=256, out_features=512, bias=True)\n", + " (4): Dropout(p=0.1, inplace=False)\n", + " (5): GELU()\n", + " (6): Linear(in_features=512, out_features=256, bias=True)\n", + " (7): Dropout(p=0.1, inplace=False)\n", + " (8): GELU()\n", + " (9): Linear(in_features=256, out_features=1, bias=True)\n", + " (10): Sigmoid()\n", + " )\n", + " (mlp2): Sequential(\n", + " (0): Linear(in_features=2, out_features=256, bias=True)\n", + " (1): Dropout(p=0.1, inplace=False)\n", + " (2): GELU()\n", + " (3): Linear(in_features=256, out_features=512, bias=True)\n", + " (4): Dropout(p=0.1, inplace=False)\n", + " (5): GELU()\n", + " (6): Linear(in_features=512, out_features=256, bias=True)\n", + " (7): Dropout(p=0.1, inplace=False)\n", + " (8): GELU()\n", + " (9): Linear(in_features=256, out_features=1, bias=True)\n", + " )\n", + " )\n", + ")\n", + "| epoch 1 | 200/ 5000 batches | lr 1.00 | ms/batch 96.44 | loss 0.51 |\n", + "| epoch 1 | 400/ 5000 batches | lr 1.00 | ms/batch 95.86 | loss 0.50 |\n", + "| epoch 1 | 600/ 5000 batches | lr 1.00 | ms/batch 95.84 | loss 0.50 |\n", + "| epoch 1 | 800/ 5000 batches | lr 1.00 | ms/batch 95.84 | loss 0.50 |\n", + "| epoch 1 | 1000/ 5000 batches | lr 1.00 | ms/batch 95.85 | loss 0.50 |\n", + "| epoch 1 | 1200/ 5000 batches | lr 1.00 | ms/batch 95.85 | loss 0.50 |\n", + "| epoch 1 | 1400/ 5000 batches | lr 1.00 | ms/batch 99.00 | loss 0.50 |\n", + "| epoch 1 | 1600/ 5000 batches | lr 1.00 | ms/batch 118.93 | loss 0.50 |\n", + "| epoch 1 | 1800/ 5000 batches | lr 1.00 | ms/batch 97.01 | loss 0.50 |\n", + "| epoch 1 | 2000/ 5000 batches | lr 1.00 | ms/batch 96.17 | loss 0.50 |\n", + "| epoch 1 | 2200/ 5000 batches | lr 1.00 | ms/batch 96.17 | loss 0.50 |\n", + "| epoch 1 | 2400/ 5000 batches | lr 1.00 | ms/batch 96.18 | loss 0.50 |\n", + "| epoch 1 | 2600/ 5000 batches | lr 1.00 | ms/batch 96.22 | loss 0.50 |\n", + "| epoch 1 | 2800/ 5000 batches | lr 1.00 | ms/batch 96.16 | loss 0.50 |\n", + "| epoch 1 | 3000/ 5000 batches | lr 1.00 | ms/batch 96.15 | loss 0.50 |\n", + "| epoch 1 | 3200/ 5000 batches | lr 1.00 | ms/batch 96.16 | loss 0.50 |\n", + "| epoch 1 | 3400/ 5000 batches | lr 1.00 | ms/batch 96.13 | loss 0.50 |\n", + "| epoch 1 | 3600/ 5000 batches | lr 1.00 | ms/batch 96.14 | loss 0.50 |\n", + "| epoch 1 | 3800/ 5000 batches | lr 1.00 | ms/batch 96.14 | loss 0.50 |\n", + "| epoch 1 | 4000/ 5000 batches | lr 1.00 | ms/batch 96.19 | loss 0.50 |\n", + "| epoch 1 | 4200/ 5000 batches | lr 1.00 | ms/batch 96.12 | loss 0.50 |\n", + "| epoch 1 | 4400/ 5000 batches | lr 1.00 | ms/batch 96.16 | loss 0.50 |\n", + "| epoch 1 | 4600/ 5000 batches | lr 1.00 | ms/batch 96.13 | loss 0.50 |\n", + "| epoch 1 | 4800/ 5000 batches | lr 1.00 | ms/batch 96.16 | loss 0.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 672.38s | valid loss 0.75 | valid ppl 2.11\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 5000 batches | lr 0.95 | ms/batch 96.66 | loss 0.48 |\n", + "| epoch 2 | 400/ 5000 batches | lr 0.95 | ms/batch 96.14 | loss 0.47 |\n", + "| epoch 2 | 600/ 5000 batches | lr 0.95 | ms/batch 96.16 | loss 0.48 |\n", + "| epoch 2 | 800/ 5000 batches | lr 0.95 | ms/batch 96.18 | loss 0.48 |\n", + "| epoch 2 | 1000/ 5000 batches | lr 0.95 | ms/batch 96.17 | loss 0.48 |\n", + "| epoch 2 | 1200/ 5000 batches | lr 0.95 | ms/batch 96.15 | loss 0.48 |\n", + "| epoch 2 | 1400/ 5000 batches | lr 0.95 | ms/batch 96.16 | loss 0.47 |\n", + "| epoch 2 | 1600/ 5000 batches | lr 0.95 | ms/batch 96.14 | loss 0.48 |\n", + "| epoch 2 | 1800/ 5000 batches | lr 0.95 | ms/batch 96.18 | loss 0.48 |\n", + "| epoch 2 | 2000/ 5000 batches | lr 0.95 | ms/batch 96.22 | loss 0.48 |\n", + "| epoch 2 | 2200/ 5000 batches | lr 0.95 | ms/batch 96.18 | loss 0.47 |\n", + "| epoch 2 | 2400/ 5000 batches | lr 0.95 | ms/batch 96.15 | loss 0.48 |\n", + "| epoch 2 | 2600/ 5000 batches | lr 0.95 | ms/batch 96.18 | loss 0.48 |\n", + "| epoch 2 | 2800/ 5000 batches | lr 0.95 | ms/batch 96.14 | loss 0.48 |\n", + "| epoch 2 | 3000/ 5000 batches | lr 0.95 | ms/batch 96.16 | loss 0.48 |\n", + "| epoch 2 | 3200/ 5000 batches | lr 0.95 | ms/batch 96.22 | loss 0.47 |\n", + "| epoch 2 | 3400/ 5000 batches | lr 0.95 | ms/batch 96.15 | loss 0.48 |\n", + "| epoch 2 | 3600/ 5000 batches | lr 0.95 | ms/batch 96.15 | loss 0.48 |\n", + "| epoch 2 | 3800/ 5000 batches | lr 0.95 | ms/batch 96.17 | loss 0.48 |\n", + "| epoch 2 | 4000/ 5000 batches | lr 0.95 | ms/batch 96.14 | loss 0.48 |\n", + "| epoch 2 | 4200/ 5000 batches | lr 0.95 | ms/batch 96.17 | loss 0.48 |\n", + "| epoch 2 | 4400/ 5000 batches | lr 0.95 | ms/batch 96.18 | loss 0.48 |\n", + "| epoch 2 | 4600/ 5000 batches | lr 0.95 | ms/batch 96.19 | loss 0.48 |\n", + "| epoch 2 | 4800/ 5000 batches | lr 0.95 | ms/batch 96.16 | loss 0.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 667.42s | valid loss 0.75 | valid ppl 2.11\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 5000 batches | lr 0.90 | ms/batch 96.67 | loss 0.46 |\n", + "| epoch 3 | 400/ 5000 batches | lr 0.90 | ms/batch 96.14 | loss 0.45 |\n", + "| epoch 3 | 600/ 5000 batches | lr 0.90 | ms/batch 96.16 | loss 0.45 |\n", + "| epoch 3 | 800/ 5000 batches | lr 0.90 | ms/batch 96.13 | loss 0.45 |\n", + "| epoch 3 | 1000/ 5000 batches | lr 0.90 | ms/batch 96.15 | loss 0.45 |\n", + "| epoch 3 | 1200/ 5000 batches | lr 0.90 | ms/batch 96.12 | loss 0.45 |\n", + "| epoch 3 | 1400/ 5000 batches | lr 0.90 | ms/batch 96.15 | loss 0.45 |\n", + "| epoch 3 | 1600/ 5000 batches | lr 0.90 | ms/batch 96.15 | loss 0.45 |\n", + "| epoch 3 | 1800/ 5000 batches | lr 0.90 | ms/batch 96.15 | loss 0.45 |\n", + "| epoch 3 | 2000/ 5000 batches | lr 0.90 | ms/batch 96.15 | loss 0.45 |\n", + "| epoch 3 | 2200/ 5000 batches | lr 0.90 | ms/batch 96.14 | loss 0.45 |\n", + "| epoch 3 | 2400/ 5000 batches | lr 0.90 | ms/batch 96.15 | loss 0.45 |\n", + "| epoch 3 | 2600/ 5000 batches | lr 0.90 | ms/batch 96.17 | loss 0.45 |\n", + "| epoch 3 | 2800/ 5000 batches | lr 0.90 | ms/batch 96.15 | loss 0.45 |\n", + "| epoch 3 | 3000/ 5000 batches | lr 0.90 | ms/batch 96.12 | loss 0.45 |\n", + "| epoch 3 | 3200/ 5000 batches | lr 0.90 | ms/batch 96.14 | loss 0.45 |\n", + "| epoch 3 | 3400/ 5000 batches | lr 0.90 | ms/batch 96.14 | loss 0.45 |\n", + "| epoch 3 | 3600/ 5000 batches | lr 0.90 | ms/batch 96.15 | loss 0.45 |\n", + "| epoch 3 | 3800/ 5000 batches | lr 0.90 | ms/batch 96.17 | loss 0.45 |\n", + "| epoch 3 | 4000/ 5000 batches | lr 0.90 | ms/batch 98.25 | loss 0.45 |\n", + "| epoch 3 | 4200/ 5000 batches | lr 0.90 | ms/batch 96.15 | loss 0.45 |\n", + "| epoch 3 | 4400/ 5000 batches | lr 0.90 | ms/batch 96.15 | loss 0.45 |\n", + "| epoch 3 | 4600/ 5000 batches | lr 0.90 | ms/batch 96.16 | loss 0.45 |\n", + "| epoch 3 | 4800/ 5000 batches | lr 0.90 | ms/batch 96.15 | loss 0.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 667.73s | valid loss 0.75 | valid ppl 2.11\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 5000 batches | lr 0.86 | ms/batch 96.64 | loss 0.43 |\n", + "| epoch 4 | 400/ 5000 batches | lr 0.86 | ms/batch 96.15 | loss 0.43 |\n", + "| epoch 4 | 600/ 5000 batches | lr 0.86 | ms/batch 96.15 | loss 0.43 |\n", + "| epoch 4 | 800/ 5000 batches | lr 0.86 | ms/batch 96.28 | loss 0.43 |\n", + "| epoch 4 | 1000/ 5000 batches | lr 0.86 | ms/batch 96.16 | loss 0.43 |\n", + "| epoch 4 | 1200/ 5000 batches | lr 0.86 | ms/batch 96.15 | loss 0.43 |\n", + "| epoch 4 | 1400/ 5000 batches | lr 0.86 | ms/batch 96.15 | loss 0.43 |\n", + "| epoch 4 | 1600/ 5000 batches | lr 0.86 | ms/batch 96.15 | loss 0.43 |\n", + "| epoch 4 | 1800/ 5000 batches | lr 0.86 | ms/batch 96.16 | loss 0.43 |\n", + "| epoch 4 | 2000/ 5000 batches | lr 0.86 | ms/batch 96.19 | loss 0.43 |\n", + "| epoch 4 | 2200/ 5000 batches | lr 0.86 | ms/batch 96.16 | loss 0.43 |\n", + "| epoch 4 | 2400/ 5000 batches | lr 0.86 | ms/batch 96.16 | loss 0.43 |\n", + "| epoch 4 | 2600/ 5000 batches | lr 0.86 | ms/batch 96.15 | loss 0.43 |\n", + "| epoch 4 | 2800/ 5000 batches | lr 0.86 | ms/batch 96.15 | loss 0.43 |\n", + "| epoch 4 | 3000/ 5000 batches | lr 0.86 | ms/batch 96.18 | loss 0.43 |\n", + "| epoch 4 | 3200/ 5000 batches | lr 0.86 | ms/batch 96.28 | loss 0.43 |\n", + "| epoch 4 | 3400/ 5000 batches | lr 0.86 | ms/batch 96.17 | loss 0.43 |\n", + "| epoch 4 | 3600/ 5000 batches | lr 0.86 | ms/batch 96.16 | loss 0.43 |\n", + "| epoch 4 | 3800/ 5000 batches | lr 0.86 | ms/batch 96.16 | loss 0.43 |\n", + "| epoch 4 | 4000/ 5000 batches | lr 0.86 | ms/batch 96.16 | loss 0.43 |\n", + "| epoch 4 | 4200/ 5000 batches | lr 0.86 | ms/batch 96.16 | loss 0.43 |\n", + "| epoch 4 | 4400/ 5000 batches | lr 0.86 | ms/batch 96.19 | loss 0.43 |\n", + "| epoch 4 | 4600/ 5000 batches | lr 0.86 | ms/batch 96.15 | loss 0.43 |\n", + "| epoch 4 | 4800/ 5000 batches | lr 0.86 | ms/batch 96.14 | loss 0.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 667.20s | valid loss 0.75 | valid ppl 2.11\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 5000 batches | lr 0.81 | ms/batch 96.65 | loss 0.41 |\n", + "| epoch 5 | 400/ 5000 batches | lr 0.81 | ms/batch 96.18 | loss 0.41 |\n", + "| epoch 5 | 600/ 5000 batches | lr 0.81 | ms/batch 96.15 | loss 0.41 |\n", + "| epoch 5 | 800/ 5000 batches | lr 0.81 | ms/batch 96.16 | loss 0.41 |\n", + "| epoch 5 | 1000/ 5000 batches | lr 0.81 | ms/batch 96.17 | loss 0.41 |\n", + "| epoch 5 | 1200/ 5000 batches | lr 0.81 | ms/batch 96.13 | loss 0.41 |\n", + "| epoch 5 | 1400/ 5000 batches | lr 0.81 | ms/batch 96.19 | loss 0.41 |\n", + "| epoch 5 | 1600/ 5000 batches | lr 0.81 | ms/batch 96.13 | loss 0.41 |\n", + "| epoch 5 | 1800/ 5000 batches | lr 0.81 | ms/batch 96.13 | loss 0.41 |\n", + "| epoch 5 | 2000/ 5000 batches | lr 0.81 | ms/batch 96.14 | loss 0.41 |\n", + "| epoch 5 | 2200/ 5000 batches | lr 0.81 | ms/batch 96.16 | loss 0.41 |\n", + "| epoch 5 | 2400/ 5000 batches | lr 0.81 | ms/batch 96.16 | loss 0.41 |\n", + "| epoch 5 | 2600/ 5000 batches | lr 0.81 | ms/batch 96.20 | loss 0.41 |\n", + "| epoch 5 | 2800/ 5000 batches | lr 0.81 | ms/batch 96.16 | loss 0.41 |\n", + "| epoch 5 | 3000/ 5000 batches | lr 0.81 | ms/batch 96.15 | loss 0.41 |\n", + "| epoch 5 | 3200/ 5000 batches | lr 0.81 | ms/batch 96.14 | loss 0.41 |\n", + "| epoch 5 | 3400/ 5000 batches | lr 0.81 | ms/batch 96.15 | loss 0.41 |\n", + "| epoch 5 | 3600/ 5000 batches | lr 0.81 | ms/batch 96.14 | loss 0.41 |\n", + "| epoch 5 | 3800/ 5000 batches | lr 0.81 | ms/batch 96.17 | loss 0.41 |\n", + "| epoch 5 | 4000/ 5000 batches | lr 0.81 | ms/batch 96.15 | loss 0.41 |\n", + "| epoch 5 | 4200/ 5000 batches | lr 0.81 | ms/batch 96.15 | loss 0.41 |\n", + "| epoch 5 | 4400/ 5000 batches | lr 0.81 | ms/batch 96.15 | loss 0.41 |\n", + "| epoch 5 | 4600/ 5000 batches | lr 0.81 | ms/batch 96.17 | loss 0.41 |\n", + "| epoch 5 | 4800/ 5000 batches | lr 0.81 | ms/batch 96.18 | loss 0.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 667.23s | valid loss 0.74 | valid ppl 2.11\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 0.74 | test ppl 2.11\n", + "=========================================================================================\n" + ] + } + ], + "source": [ + "batch_size = 20\n", + "eval_batch_size = 10\n", + "seq_len = 1000\n", + "divider = 4\n", + "dataset_len = 100000\n", + "epochs = 5 # The number of epochs\n", + "\n", + "model = DoXTimes(Smartpool(divider, 0.3, mlp2=True))\n", + "model = model.to(device)\n", + "print(model)\n", + "\n", + "lr = 1.0 # learning rate\n", + "optimizer = torch.optim.SGD(model.parameters(), lr=lr)\n", + "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "\n", + "model = train_model(model, epochs, batch_size, eval_batch_size, dataset_len, seq_len, divider, optimizer, scheduler)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "n_7radp_ZOs2", + "outputId": "abef3b35-dc08-49f1-fdc7-ea1773f932e0" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "data[0]: tensor([[1.0000, 0.6150],\n", + " [0.0000, 0.0595],\n", + " [0.0000, 0.5246],\n", + " ...,\n", + " [0.0000, 0.6626],\n", + " [0.0000, 0.9729],\n", + " [0.0000, 0.3818]], device='cuda:0')\n", + "output: tensor([31.8880, 31.8882, 31.8881, 31.8885, 31.8886, 31.8884, 31.8888, 31.8887,\n", + " 31.8880, 31.8886], device='cuda:0')\n", + "targets: tensor([127.4935, 130.1996, 131.9557, 127.1092, 122.5810, 124.3430, 120.3016,\n", + " 120.7829, 123.7960, 131.3994], device='cuda:0')\n" + ] + } + ], + "source": [ + "model.eval()\n", + "with torch.no_grad():\n", + " data = get_batch(eval_batch_size, seq_len, divider).to(device)\n", + " targets = data.prod(-1).sum(-1).squeeze(-1)\n", + " output = model(data)\n", + " print('data[0]:', data[0,:,:])\n", + "\n", + " print('output:', output)\n", + " print('targets:', targets)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GJXTDtqYZhOX" + }, + "source": [ + "## Pooling T/16" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6qEjuca5ZhOb" + }, + "source": [ + "### Average pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "BGPv7w9xZhOb", + "outputId": "f95cd1a3-65f7-493d-b8d4-213854f57e83" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using cache found in /home/i273233/.cache/torch/hub/pytorch_vision_v0.6.0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Conv(\n", + " (conv): Sequential(\n", + " (0): Conv2d(1, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (1): ReLU(inplace=True)\n", + " (2): AvgPool2d(kernel_size=(2, 1), stride=(2, 1), padding=0)\n", + " (3): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (4): ReLU(inplace=True)\n", + " (5): AvgPool2d(kernel_size=(2, 1), stride=(2, 1), padding=0)\n", + " (6): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (7): ReLU(inplace=True)\n", + " (8): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (9): ReLU(inplace=True)\n", + " (10): AvgPool2d(kernel_size=(2, 1), stride=(2, 1), padding=0)\n", + " (11): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (12): ReLU(inplace=True)\n", + " (13): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (14): ReLU(inplace=True)\n", + " (15): AvgPool2d(kernel_size=2, stride=(2, 1), padding=0)\n", + " (16): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (17): ReLU(inplace=True)\n", + " (18): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (linear): Conv2d(512, 1, kernel_size=(1, 1), stride=(1, 1))\n", + " )\n", + ")\n" + ] + } + ], + "source": [ + "model = torch.hub.load('pytorch/vision:v0.6.0', 'vgg11', pretrained=False)\n", + "model = model.features[0:19]\n", + "model[0] = nn.Conv2d(1, 64, kernel_size=(3,3), stride=(1,1), padding=(1,1))\n", + "model[2] = nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=0)\n", + "model[5] = nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=0)\n", + "model[10] = nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=0)\n", + "model[15] = nn.AvgPool2d(kernel_size=2, stride=(2,1), padding=0)\n", + "model[18] = nn.Conv2d(512, 512, kernel_size=(3,3), stride=(1,1), padding=(1,1))\n", + "model.add_module(\"linear\", nn.Conv2d(512, 1, kernel_size=1, stride=1, padding=0))\n", + "model = Conv(model, 1)\n", + "model = model.to(device)\n", + "print(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "677pIwmSZhOd", + "outputId": "7e4e2b95-431d-485e-a4d0-5871e89c4916" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 5120 batches | lr 1.00 | ms/batch 107.09 | loss 0.50 |\n", + "| epoch 1 | 400/ 5120 batches | lr 1.00 | ms/batch 107.01 | loss 0.50 |\n", + "| epoch 1 | 600/ 5120 batches | lr 1.00 | ms/batch 107.42 | loss 0.50 |\n", + "| epoch 1 | 800/ 5120 batches | lr 1.00 | ms/batch 107.66 | loss 0.50 |\n", + "| epoch 1 | 1000/ 5120 batches | lr 1.00 | ms/batch 108.56 | loss 0.50 |\n", + "| epoch 1 | 1200/ 5120 batches | lr 1.00 | ms/batch 108.75 | loss 0.50 |\n", + "| epoch 1 | 1400/ 5120 batches | lr 1.00 | ms/batch 108.91 | loss 0.50 |\n", + "| epoch 1 | 1600/ 5120 batches | lr 1.00 | ms/batch 108.88 | loss 0.50 |\n", + "| epoch 1 | 1800/ 5120 batches | lr 1.00 | ms/batch 109.49 | loss 0.50 |\n", + "| epoch 1 | 2000/ 5120 batches | lr 1.00 | ms/batch 110.21 | loss 0.50 |\n", + "| epoch 1 | 2200/ 5120 batches | lr 1.00 | ms/batch 112.75 | loss 0.50 |\n", + "| epoch 1 | 2400/ 5120 batches | lr 1.00 | ms/batch 112.83 | loss 0.50 |\n", + "| epoch 1 | 2600/ 5120 batches | lr 1.00 | ms/batch 112.87 | loss 0.50 |\n", + "| epoch 1 | 2800/ 5120 batches | lr 1.00 | ms/batch 114.30 | loss 0.50 |\n", + "| epoch 1 | 3000/ 5120 batches | lr 1.00 | ms/batch 115.47 | loss 0.51 |\n", + "| epoch 1 | 3200/ 5120 batches | lr 1.00 | ms/batch 115.28 | loss 0.50 |\n", + "| epoch 1 | 3400/ 5120 batches | lr 1.00 | ms/batch 117.82 | loss 0.50 |\n", + "| epoch 1 | 3600/ 5120 batches | lr 1.00 | ms/batch 116.51 | loss 0.50 |\n", + "| epoch 1 | 3800/ 5120 batches | lr 1.00 | ms/batch 116.82 | loss 0.50 |\n", + "| epoch 1 | 4000/ 5120 batches | lr 1.00 | ms/batch 117.91 | loss 0.50 |\n", + "| epoch 1 | 4200/ 5120 batches | lr 1.00 | ms/batch 118.22 | loss 0.50 |\n", + "| epoch 1 | 4400/ 5120 batches | lr 1.00 | ms/batch 118.22 | loss 0.50 |\n", + "| epoch 1 | 4600/ 5120 batches | lr 1.00 | ms/batch 118.04 | loss 0.50 |\n", + "| epoch 1 | 4800/ 5120 batches | lr 1.00 | ms/batch 118.16 | loss 0.50 |\n", + "| epoch 1 | 5000/ 5120 batches | lr 1.00 | ms/batch 118.26 | loss 0.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 857.10s | valid loss 0.51 | valid ppl 1.66\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 5120 batches | lr 0.95 | ms/batch 119.23 | loss 0.48 |\n", + "| epoch 2 | 400/ 5120 batches | lr 0.95 | ms/batch 118.77 | loss 0.48 |\n", + "| epoch 2 | 600/ 5120 batches | lr 0.95 | ms/batch 118.79 | loss 0.48 |\n", + "| epoch 2 | 800/ 5120 batches | lr 0.95 | ms/batch 118.61 | loss 0.48 |\n", + "| epoch 2 | 1000/ 5120 batches | lr 0.95 | ms/batch 118.79 | loss 0.48 |\n", + "| epoch 2 | 1200/ 5120 batches | lr 0.95 | ms/batch 118.53 | loss 0.48 |\n", + "| epoch 2 | 1400/ 5120 batches | lr 0.95 | ms/batch 118.51 | loss 0.48 |\n", + "| epoch 2 | 1600/ 5120 batches | lr 0.95 | ms/batch 118.43 | loss 0.47 |\n", + "| epoch 2 | 1800/ 5120 batches | lr 0.95 | ms/batch 118.51 | loss 0.48 |\n", + "| epoch 2 | 2000/ 5120 batches | lr 0.95 | ms/batch 118.52 | loss 0.48 |\n", + "| epoch 2 | 2200/ 5120 batches | lr 0.95 | ms/batch 118.71 | loss 0.48 |\n", + "| epoch 2 | 2400/ 5120 batches | lr 0.95 | ms/batch 118.67 | loss 0.48 |\n", + "| epoch 2 | 2600/ 5120 batches | lr 0.95 | ms/batch 118.88 | loss 0.48 |\n", + "| epoch 2 | 2800/ 5120 batches | lr 0.95 | ms/batch 118.72 | loss 0.48 |\n", + "| epoch 2 | 3000/ 5120 batches | lr 0.95 | ms/batch 118.53 | loss 0.48 |\n", + "| epoch 2 | 3200/ 5120 batches | lr 0.95 | ms/batch 118.48 | loss 0.48 |\n", + "| epoch 2 | 3400/ 5120 batches | lr 0.95 | ms/batch 118.22 | loss 0.48 |\n", + "| epoch 2 | 3600/ 5120 batches | lr 0.95 | ms/batch 118.43 | loss 0.48 |\n", + "| epoch 2 | 3800/ 5120 batches | lr 0.95 | ms/batch 118.34 | loss 0.48 |\n", + "| epoch 2 | 4000/ 5120 batches | lr 0.95 | ms/batch 118.53 | loss 0.48 |\n", + "| epoch 2 | 4200/ 5120 batches | lr 0.95 | ms/batch 118.51 | loss 0.48 |\n", + "| epoch 2 | 4400/ 5120 batches | lr 0.95 | ms/batch 118.36 | loss 0.48 |\n", + "| epoch 2 | 4600/ 5120 batches | lr 0.95 | ms/batch 118.65 | loss 0.48 |\n", + "| epoch 2 | 4800/ 5120 batches | lr 0.95 | ms/batch 118.71 | loss 0.48 |\n", + "| epoch 2 | 5000/ 5120 batches | lr 0.95 | ms/batch 118.56 | loss 0.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 885.47s | valid loss 0.51 | valid ppl 1.66\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 5120 batches | lr 0.90 | ms/batch 119.37 | loss 0.46 |\n", + "| epoch 3 | 400/ 5120 batches | lr 0.90 | ms/batch 118.78 | loss 0.46 |\n", + "| epoch 3 | 600/ 5120 batches | lr 0.90 | ms/batch 118.66 | loss 0.45 |\n", + "| epoch 3 | 800/ 5120 batches | lr 0.90 | ms/batch 118.61 | loss 0.45 |\n", + "| epoch 3 | 1000/ 5120 batches | lr 0.90 | ms/batch 118.54 | loss 0.45 |\n", + "| epoch 3 | 1200/ 5120 batches | lr 0.90 | ms/batch 118.54 | loss 0.45 |\n", + "| epoch 3 | 1400/ 5120 batches | lr 0.90 | ms/batch 118.66 | loss 0.45 |\n", + "| epoch 3 | 1600/ 5120 batches | lr 0.90 | ms/batch 118.53 | loss 0.45 |\n", + "| epoch 3 | 1800/ 5120 batches | lr 0.90 | ms/batch 118.50 | loss 0.45 |\n", + "| epoch 3 | 2000/ 5120 batches | lr 0.90 | ms/batch 118.69 | loss 0.46 |\n", + "| epoch 3 | 2200/ 5120 batches | lr 0.90 | ms/batch 118.52 | loss 0.45 |\n", + "| epoch 3 | 2400/ 5120 batches | lr 0.90 | ms/batch 118.32 | loss 0.45 |\n", + "| epoch 3 | 2600/ 5120 batches | lr 0.90 | ms/batch 118.62 | loss 0.45 |\n", + "| epoch 3 | 2800/ 5120 batches | lr 0.90 | ms/batch 118.60 | loss 0.45 |\n", + "| epoch 3 | 3000/ 5120 batches | lr 0.90 | ms/batch 118.65 | loss 0.45 |\n", + "| epoch 3 | 3200/ 5120 batches | lr 0.90 | ms/batch 118.38 | loss 0.45 |\n", + "| epoch 3 | 3400/ 5120 batches | lr 0.90 | ms/batch 118.34 | loss 0.45 |\n", + "| epoch 3 | 3600/ 5120 batches | lr 0.90 | ms/batch 118.42 | loss 0.45 |\n", + "| epoch 3 | 3800/ 5120 batches | lr 0.90 | ms/batch 118.50 | loss 0.45 |\n", + "| epoch 3 | 4000/ 5120 batches | lr 0.90 | ms/batch 118.24 | loss 0.45 |\n", + "| epoch 3 | 4200/ 5120 batches | lr 0.90 | ms/batch 118.38 | loss 0.45 |\n", + "| epoch 3 | 4400/ 5120 batches | lr 0.90 | ms/batch 118.58 | loss 0.45 |\n", + "| epoch 3 | 4600/ 5120 batches | lr 0.90 | ms/batch 118.69 | loss 0.46 |\n", + "| epoch 3 | 4800/ 5120 batches | lr 0.90 | ms/batch 118.53 | loss 0.45 |\n", + "| epoch 3 | 5000/ 5120 batches | lr 0.90 | ms/batch 118.48 | loss 0.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 884.80s | valid loss 0.51 | valid ppl 1.66\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 5120 batches | lr 0.86 | ms/batch 119.03 | loss 0.43 |\n", + "| epoch 4 | 400/ 5120 batches | lr 0.86 | ms/batch 118.45 | loss 0.43 |\n", + "| epoch 4 | 600/ 5120 batches | lr 0.86 | ms/batch 118.46 | loss 0.43 |\n", + "| epoch 4 | 800/ 5120 batches | lr 0.86 | ms/batch 118.38 | loss 0.43 |\n", + "| epoch 4 | 1000/ 5120 batches | lr 0.86 | ms/batch 118.46 | loss 0.43 |\n", + "| epoch 4 | 1200/ 5120 batches | lr 0.86 | ms/batch 118.58 | loss 0.43 |\n", + "| epoch 4 | 1400/ 5120 batches | lr 0.86 | ms/batch 118.53 | loss 0.43 |\n", + "| epoch 4 | 1600/ 5120 batches | lr 0.86 | ms/batch 118.57 | loss 0.43 |\n", + "| epoch 4 | 1800/ 5120 batches | lr 0.86 | ms/batch 118.67 | loss 0.43 |\n", + "| epoch 4 | 2000/ 5120 batches | lr 0.86 | ms/batch 118.74 | loss 0.43 |\n", + "| epoch 4 | 2200/ 5120 batches | lr 0.86 | ms/batch 118.78 | loss 0.43 |\n", + "| epoch 4 | 2400/ 5120 batches | lr 0.86 | ms/batch 118.47 | loss 0.43 |\n", + "| epoch 4 | 2600/ 5120 batches | lr 0.86 | ms/batch 118.41 | loss 0.43 |\n", + "| epoch 4 | 2800/ 5120 batches | lr 0.86 | ms/batch 118.51 | loss 0.43 |\n", + "| epoch 4 | 3000/ 5120 batches | lr 0.86 | ms/batch 118.56 | loss 0.43 |\n", + "| epoch 4 | 3200/ 5120 batches | lr 0.86 | ms/batch 118.53 | loss 0.43 |\n", + "| epoch 4 | 3400/ 5120 batches | lr 0.86 | ms/batch 118.70 | loss 0.43 |\n", + "| epoch 4 | 3600/ 5120 batches | lr 0.86 | ms/batch 118.64 | loss 0.43 |\n", + "| epoch 4 | 3800/ 5120 batches | lr 0.86 | ms/batch 118.43 | loss 0.43 |\n", + "| epoch 4 | 4000/ 5120 batches | lr 0.86 | ms/batch 118.41 | loss 0.43 |\n", + "| epoch 4 | 4200/ 5120 batches | lr 0.86 | ms/batch 118.64 | loss 0.43 |\n", + "| epoch 4 | 4400/ 5120 batches | lr 0.86 | ms/batch 118.80 | loss 0.43 |\n", + "| epoch 4 | 4600/ 5120 batches | lr 0.86 | ms/batch 118.51 | loss 0.43 |\n", + "| epoch 4 | 4800/ 5120 batches | lr 0.86 | ms/batch 118.74 | loss 0.43 |\n", + "| epoch 4 | 5000/ 5120 batches | lr 0.86 | ms/batch 118.73 | loss 0.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 886.10s | valid loss 0.51 | valid ppl 1.66\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 5120 batches | lr 0.81 | ms/batch 119.42 | loss 0.41 |\n", + "| epoch 5 | 400/ 5120 batches | lr 0.81 | ms/batch 118.77 | loss 0.41 |\n", + "| epoch 5 | 600/ 5120 batches | lr 0.81 | ms/batch 118.63 | loss 0.41 |\n", + "| epoch 5 | 800/ 5120 batches | lr 0.81 | ms/batch 118.69 | loss 0.41 |\n", + "| epoch 5 | 1000/ 5120 batches | lr 0.81 | ms/batch 118.79 | loss 0.41 |\n", + "| epoch 5 | 1200/ 5120 batches | lr 0.81 | ms/batch 118.79 | loss 0.41 |\n", + "| epoch 5 | 1400/ 5120 batches | lr 0.81 | ms/batch 118.71 | loss 0.41 |\n", + "| epoch 5 | 1600/ 5120 batches | lr 0.81 | ms/batch 118.76 | loss 0.41 |\n", + "| epoch 5 | 1800/ 5120 batches | lr 0.81 | ms/batch 118.62 | loss 0.41 |\n", + "| epoch 5 | 2000/ 5120 batches | lr 0.81 | ms/batch 118.55 | loss 0.41 |\n", + "| epoch 5 | 2200/ 5120 batches | lr 0.81 | ms/batch 118.81 | loss 0.41 |\n", + "| epoch 5 | 2400/ 5120 batches | lr 0.81 | ms/batch 118.89 | loss 0.41 |\n", + "| epoch 5 | 2600/ 5120 batches | lr 0.81 | ms/batch 118.74 | loss 0.41 |\n", + "| epoch 5 | 2800/ 5120 batches | lr 0.81 | ms/batch 118.62 | loss 0.41 |\n", + "| epoch 5 | 3000/ 5120 batches | lr 0.81 | ms/batch 118.91 | loss 0.41 |\n", + "| epoch 5 | 3200/ 5120 batches | lr 0.81 | ms/batch 118.75 | loss 0.41 |\n", + "| epoch 5 | 3400/ 5120 batches | lr 0.81 | ms/batch 118.80 | loss 0.41 |\n", + "| epoch 5 | 3600/ 5120 batches | lr 0.81 | ms/batch 118.76 | loss 0.41 |\n", + "| epoch 5 | 3800/ 5120 batches | lr 0.81 | ms/batch 118.76 | loss 0.41 |\n", + "| epoch 5 | 4000/ 5120 batches | lr 0.81 | ms/batch 118.58 | loss 0.41 |\n", + "| epoch 5 | 4200/ 5120 batches | lr 0.81 | ms/batch 118.62 | loss 0.41 |\n", + "| epoch 5 | 4400/ 5120 batches | lr 0.81 | ms/batch 118.66 | loss 0.41 |\n", + "| epoch 5 | 4600/ 5120 batches | lr 0.81 | ms/batch 118.81 | loss 0.41 |\n", + "| epoch 5 | 4800/ 5120 batches | lr 0.81 | ms/batch 118.73 | loss 0.41 |\n", + "| epoch 5 | 5000/ 5120 batches | lr 0.81 | ms/batch 119.04 | loss 0.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 887.23s | valid loss 0.51 | valid ppl 1.66\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 0.51 | test ppl 1.66\n", + "=========================================================================================\n" + ] + } + ], + "source": [ + "batch_size = 20\n", + "eval_batch_size = 10\n", + "seq_len = 1024\n", + "divider = 16\n", + "dataset_len = 102400\n", + "lr = 1.0 # learning rate\n", + "optimizer = torch.optim.SGD(model.parameters(), lr=lr)\n", + "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "epochs = 5 # The number of epochs\n", + "\n", + "model = train_model(model, epochs, batch_size, eval_batch_size, dataset_len, seq_len, divider, optimizer, scheduler)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 231 + }, + "id": "SatDK8bcZhOd", + "outputId": "60567777-3faf-4a53-a229-6ff92fcaf974" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "data[0]: tensor([[0.0000, 0.5810],\n", + " [0.0000, 0.6137],\n", + " [0.0000, 0.3804],\n", + " [0.0000, 0.7532],\n", + " [0.0000, 0.5587],\n", + " [0.0000, 0.4855],\n", + " [0.0000, 0.1757],\n", + " [0.0000, 0.1030],\n", + " [0.0000, 0.4494],\n", + " [0.0000, 0.7263],\n", + " [0.0000, 0.5921],\n", + " [0.0000, 0.9249],\n", + " [1.0000, 0.6204],\n", + " [0.0000, 0.8577],\n", + " [0.0000, 0.8268],\n", + " [0.0000, 0.3762],\n", + " [1.0000, 0.6489],\n", + " [0.0000, 0.9748],\n", + " [0.0000, 0.6496],\n", + " [0.0000, 0.5912],\n", + " [0.0000, 0.3919],\n", + " [0.0000, 0.8667],\n", + " [0.0000, 0.4704],\n", + " [0.0000, 0.1632],\n", + " [0.0000, 0.5355],\n", + " [0.0000, 0.1074],\n", + " [0.0000, 0.3211],\n", + " [0.0000, 0.4606],\n", + " [0.0000, 0.3553],\n", + " [0.0000, 0.0128],\n", + " [0.0000, 0.1509],\n", + " [0.0000, 0.0864],\n", + " [0.0000, 0.8225],\n", + " [0.0000, 0.0753],\n", + " [0.0000, 0.4122],\n", + " [0.0000, 0.6764],\n", + " [0.0000, 0.6808],\n", + " [0.0000, 0.7612],\n", + " [0.0000, 0.2584],\n", + " [0.0000, 0.4378],\n", + " [1.0000, 0.3192],\n", + " [0.0000, 0.0829],\n", + " [0.0000, 0.2837],\n", + " [1.0000, 0.2408],\n", + " [0.0000, 0.2520],\n", + " [0.0000, 0.7281],\n", + " [0.0000, 0.4087],\n", + " [0.0000, 0.4585],\n", + " [0.0000, 0.4028],\n", + " [0.0000, 0.5639]], device='cuda:0')\n", + "output: tensor([15.6888, 15.6888, 15.6888, 15.6888, 15.6888, 15.6888, 15.6888, 15.6888,\n", + " 15.6888, 15.6888], device='cuda:0')\n", + "targets: tensor([28.6718, 32.1147, 32.6677, 30.7145, 33.6836, 33.2020, 28.7610, 32.8572,\n", + " 34.0891, 30.2188], device='cuda:0')\n" + ] + } + ], + "source": [ + "model.eval()\n", + "with torch.no_grad():\n", + " data = get_batch(eval_batch_size, seq_len, divider).to(device)\n", + " targets = data.prod(-1).sum(-1).squeeze(-1)\n", + " output = model(data)\n", + " print('data[0]:', data[0,:50,:])\n", + "\n", + " print('output:', output)\n", + " print('targets:', targets)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UmTVSH-5ZhOd" + }, + "source": [ + "### Max pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "R1nOR_FoZhOd", + "outputId": "5921fab3-2859-4e86-e49b-b9f3c6a4fe69" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using cache found in /home/i273233/.cache/torch/hub/pytorch_vision_v0.6.0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Conv(\n", + " (conv): Sequential(\n", + " (0): Conv2d(1, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (1): ReLU(inplace=True)\n", + " (2): MaxPool2d(kernel_size=(2, 1), stride=(2, 1), padding=0, dilation=1, ceil_mode=False)\n", + " (3): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (4): ReLU(inplace=True)\n", + " (5): MaxPool2d(kernel_size=(2, 1), stride=(2, 1), padding=0, dilation=1, ceil_mode=False)\n", + " (6): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (7): ReLU(inplace=True)\n", + " (8): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (9): ReLU(inplace=True)\n", + " (10): MaxPool2d(kernel_size=(2, 1), stride=(2, 1), padding=0, dilation=1, ceil_mode=False)\n", + " (11): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (12): ReLU(inplace=True)\n", + " (13): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (14): ReLU(inplace=True)\n", + " (15): MaxPool2d(kernel_size=2, stride=(2, 1), padding=0, dilation=1, ceil_mode=False)\n", + " (16): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (17): ReLU(inplace=True)\n", + " (18): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n", + " (linear): Conv2d(512, 1, kernel_size=(1, 1), stride=(1, 1))\n", + " )\n", + ")\n" + ] + } + ], + "source": [ + "\n", + "model = torch.hub.load('pytorch/vision:v0.6.0', 'vgg11', pretrained=False)\n", + "model = model.features[0:19]\n", + "model[0] = nn.Conv2d(1, 64, kernel_size=(3,3), stride=(1,1), padding=(1,1))\n", + "model[2] = nn.MaxPool2d(kernel_size=(2,1), stride=(2,1), padding=0)\n", + "model[5] = nn.MaxPool2d(kernel_size=(2,1), stride=(2,1), padding=0)\n", + "model[10] = nn.MaxPool2d(kernel_size=(2,1), stride=(2,1), padding=0)\n", + "model[15] = nn.MaxPool2d(kernel_size=2, stride=(2,1), padding=0)\n", + "model[18] = nn.Conv2d(512, 512, kernel_size=(3,3), stride=(1,1), padding=(1,1))\n", + "model.add_module(\"linear\", nn.Conv2d(512, 1, kernel_size=1, stride=1, padding=0))\n", + "model = Conv(model, 1)\n", + "model = model.to(device)\n", + "print(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "icdEBc6sZhOe", + "outputId": "d612c4e3-7f0f-4bb9-92c5-3204b4a52391" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 5120 batches | lr 1.00 | ms/batch 119.06 | loss 0.51 |\n", + "| epoch 1 | 400/ 5120 batches | lr 1.00 | ms/batch 118.54 | loss 0.50 |\n", + "| epoch 1 | 600/ 5120 batches | lr 1.00 | ms/batch 118.50 | loss 0.50 |\n", + "| epoch 1 | 800/ 5120 batches | lr 1.00 | ms/batch 118.57 | loss 0.50 |\n", + "| epoch 1 | 1000/ 5120 batches | lr 1.00 | ms/batch 118.42 | loss 0.50 |\n", + "| epoch 1 | 1200/ 5120 batches | lr 1.00 | ms/batch 118.27 | loss 0.50 |\n", + "| epoch 1 | 1400/ 5120 batches | lr 1.00 | ms/batch 118.39 | loss 0.50 |\n", + "| epoch 1 | 1600/ 5120 batches | lr 1.00 | ms/batch 118.31 | loss 0.50 |\n", + "| epoch 1 | 1800/ 5120 batches | lr 1.00 | ms/batch 118.18 | loss 0.50 |\n", + "| epoch 1 | 2000/ 5120 batches | lr 1.00 | ms/batch 118.23 | loss 0.50 |\n", + "| epoch 1 | 2200/ 5120 batches | lr 1.00 | ms/batch 118.17 | loss 0.50 |\n", + "| epoch 1 | 2400/ 5120 batches | lr 1.00 | ms/batch 117.90 | loss 0.50 |\n", + "| epoch 1 | 2600/ 5120 batches | lr 1.00 | ms/batch 118.12 | loss 0.50 |\n", + "| epoch 1 | 2800/ 5120 batches | lr 1.00 | ms/batch 118.17 | loss 0.50 |\n", + "| epoch 1 | 3000/ 5120 batches | lr 1.00 | ms/batch 118.09 | loss 0.50 |\n", + "| epoch 1 | 3200/ 5120 batches | lr 1.00 | ms/batch 118.21 | loss 0.50 |\n", + "| epoch 1 | 3400/ 5120 batches | lr 1.00 | ms/batch 118.14 | loss 0.50 |\n", + "| epoch 1 | 3600/ 5120 batches | lr 1.00 | ms/batch 118.11 | loss 0.50 |\n", + "| epoch 1 | 3800/ 5120 batches | lr 1.00 | ms/batch 118.27 | loss 0.51 |\n", + "| epoch 1 | 4000/ 5120 batches | lr 1.00 | ms/batch 118.28 | loss 0.50 |\n", + "| epoch 1 | 4200/ 5120 batches | lr 1.00 | ms/batch 117.93 | loss 0.50 |\n", + "| epoch 1 | 4400/ 5120 batches | lr 1.00 | ms/batch 118.03 | loss 0.50 |\n", + "| epoch 1 | 4600/ 5120 batches | lr 1.00 | ms/batch 118.11 | loss 0.50 |\n", + "| epoch 1 | 4800/ 5120 batches | lr 1.00 | ms/batch 118.26 | loss 0.50 |\n", + "| epoch 1 | 5000/ 5120 batches | lr 1.00 | ms/batch 118.22 | loss 0.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 883.64s | valid loss 0.46 | valid ppl 1.58\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 5120 batches | lr 0.95 | ms/batch 119.12 | loss 0.48 |\n", + "| epoch 2 | 400/ 5120 batches | lr 0.95 | ms/batch 118.27 | loss 0.48 |\n", + "| epoch 2 | 600/ 5120 batches | lr 0.95 | ms/batch 118.02 | loss 0.48 |\n", + "| epoch 2 | 800/ 5120 batches | lr 0.95 | ms/batch 118.28 | loss 0.48 |\n", + "| epoch 2 | 1000/ 5120 batches | lr 0.95 | ms/batch 118.25 | loss 0.48 |\n", + "| epoch 2 | 1200/ 5120 batches | lr 0.95 | ms/batch 118.09 | loss 0.48 |\n", + "| epoch 2 | 1400/ 5120 batches | lr 0.95 | ms/batch 118.16 | loss 0.48 |\n", + "| epoch 2 | 1600/ 5120 batches | lr 0.95 | ms/batch 117.99 | loss 0.48 |\n", + "| epoch 2 | 1800/ 5120 batches | lr 0.95 | ms/batch 118.07 | loss 0.48 |\n", + "| epoch 2 | 2000/ 5120 batches | lr 0.95 | ms/batch 117.87 | loss 0.48 |\n", + "| epoch 2 | 2200/ 5120 batches | lr 0.95 | ms/batch 118.04 | loss 0.48 |\n", + "| epoch 2 | 2400/ 5120 batches | lr 0.95 | ms/batch 117.97 | loss 0.48 |\n", + "| epoch 2 | 2600/ 5120 batches | lr 0.95 | ms/batch 117.87 | loss 0.48 |\n", + "| epoch 2 | 2800/ 5120 batches | lr 0.95 | ms/batch 117.95 | loss 0.48 |\n", + "| epoch 2 | 3000/ 5120 batches | lr 0.95 | ms/batch 118.09 | loss 0.48 |\n", + "| epoch 2 | 3200/ 5120 batches | lr 0.95 | ms/batch 118.07 | loss 0.48 |\n", + "| epoch 2 | 3400/ 5120 batches | lr 0.95 | ms/batch 118.23 | loss 0.48 |\n", + "| epoch 2 | 3600/ 5120 batches | lr 0.95 | ms/batch 118.08 | loss 0.48 |\n", + "| epoch 2 | 3800/ 5120 batches | lr 0.95 | ms/batch 117.86 | loss 0.48 |\n", + "| epoch 2 | 4000/ 5120 batches | lr 0.95 | ms/batch 117.96 | loss 0.48 |\n", + "| epoch 2 | 4200/ 5120 batches | lr 0.95 | ms/batch 118.09 | loss 0.48 |\n", + "| epoch 2 | 4400/ 5120 batches | lr 0.95 | ms/batch 118.18 | loss 0.48 |\n", + "| epoch 2 | 4600/ 5120 batches | lr 0.95 | ms/batch 118.27 | loss 0.48 |\n", + "| epoch 2 | 4800/ 5120 batches | lr 0.95 | ms/batch 118.21 | loss 0.48 |\n", + "| epoch 2 | 5000/ 5120 batches | lr 0.95 | ms/batch 118.15 | loss 0.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 882.96s | valid loss 0.46 | valid ppl 1.58\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 5120 batches | lr 0.90 | ms/batch 118.66 | loss 0.46 |\n", + "| epoch 3 | 400/ 5120 batches | lr 0.90 | ms/batch 118.41 | loss 0.45 |\n", + "| epoch 3 | 600/ 5120 batches | lr 0.90 | ms/batch 118.19 | loss 0.45 |\n", + "| epoch 3 | 800/ 5120 batches | lr 0.90 | ms/batch 118.07 | loss 0.45 |\n", + "| epoch 3 | 1000/ 5120 batches | lr 0.90 | ms/batch 118.13 | loss 0.45 |\n", + "| epoch 3 | 1200/ 5120 batches | lr 0.90 | ms/batch 118.17 | loss 0.45 |\n", + "| epoch 3 | 1400/ 5120 batches | lr 0.90 | ms/batch 118.07 | loss 0.45 |\n", + "| epoch 3 | 1600/ 5120 batches | lr 0.90 | ms/batch 118.04 | loss 0.45 |\n", + "| epoch 3 | 1800/ 5120 batches | lr 0.90 | ms/batch 118.16 | loss 0.45 |\n", + "| epoch 3 | 2000/ 5120 batches | lr 0.90 | ms/batch 118.39 | loss 0.45 |\n", + "| epoch 3 | 2200/ 5120 batches | lr 0.90 | ms/batch 118.23 | loss 0.45 |\n", + "| epoch 3 | 2400/ 5120 batches | lr 0.90 | ms/batch 118.00 | loss 0.45 |\n", + "| epoch 3 | 2600/ 5120 batches | lr 0.90 | ms/batch 118.03 | loss 0.45 |\n", + "| epoch 3 | 2800/ 5120 batches | lr 0.90 | ms/batch 117.93 | loss 0.45 |\n", + "| epoch 3 | 3000/ 5120 batches | lr 0.90 | ms/batch 118.14 | loss 0.45 |\n", + "| epoch 3 | 3200/ 5120 batches | lr 0.90 | ms/batch 118.14 | loss 0.45 |\n", + "| epoch 3 | 3400/ 5120 batches | lr 0.90 | ms/batch 118.00 | loss 0.45 |\n", + "| epoch 3 | 3600/ 5120 batches | lr 0.90 | ms/batch 118.06 | loss 0.45 |\n", + "| epoch 3 | 3800/ 5120 batches | lr 0.90 | ms/batch 118.18 | loss 0.46 |\n", + "| epoch 3 | 4000/ 5120 batches | lr 0.90 | ms/batch 118.16 | loss 0.45 |\n", + "| epoch 3 | 4200/ 5120 batches | lr 0.90 | ms/batch 118.07 | loss 0.45 |\n", + "| epoch 3 | 4400/ 5120 batches | lr 0.90 | ms/batch 118.23 | loss 0.45 |\n", + "| epoch 3 | 4600/ 5120 batches | lr 0.90 | ms/batch 118.18 | loss 0.45 |\n", + "| epoch 3 | 4800/ 5120 batches | lr 0.90 | ms/batch 117.97 | loss 0.45 |\n", + "| epoch 3 | 5000/ 5120 batches | lr 0.90 | ms/batch 118.21 | loss 0.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 883.32s | valid loss 0.46 | valid ppl 1.58\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 5120 batches | lr 0.86 | ms/batch 118.61 | loss 0.43 |\n", + "| epoch 4 | 400/ 5120 batches | lr 0.86 | ms/batch 117.97 | loss 0.43 |\n", + "| epoch 4 | 600/ 5120 batches | lr 0.86 | ms/batch 118.21 | loss 0.43 |\n", + "| epoch 4 | 800/ 5120 batches | lr 0.86 | ms/batch 118.08 | loss 0.43 |\n", + "| epoch 4 | 1000/ 5120 batches | lr 0.86 | ms/batch 118.13 | loss 0.43 |\n", + "| epoch 4 | 1200/ 5120 batches | lr 0.86 | ms/batch 118.19 | loss 0.43 |\n", + "| epoch 4 | 1400/ 5120 batches | lr 0.86 | ms/batch 118.21 | loss 0.43 |\n", + "| epoch 4 | 1600/ 5120 batches | lr 0.86 | ms/batch 118.33 | loss 0.43 |\n", + "| epoch 4 | 1800/ 5120 batches | lr 0.86 | ms/batch 118.03 | loss 0.43 |\n", + "| epoch 4 | 2000/ 5120 batches | lr 0.86 | ms/batch 118.27 | loss 0.43 |\n", + "| epoch 4 | 2200/ 5120 batches | lr 0.86 | ms/batch 118.20 | loss 0.43 |\n", + "| epoch 4 | 2400/ 5120 batches | lr 0.86 | ms/batch 118.11 | loss 0.43 |\n", + "| epoch 4 | 2600/ 5120 batches | lr 0.86 | ms/batch 118.27 | loss 0.43 |\n", + "| epoch 4 | 2800/ 5120 batches | lr 0.86 | ms/batch 118.42 | loss 0.43 |\n", + "| epoch 4 | 3000/ 5120 batches | lr 0.86 | ms/batch 118.37 | loss 0.43 |\n", + "| epoch 4 | 3200/ 5120 batches | lr 0.86 | ms/batch 118.32 | loss 0.43 |\n", + "| epoch 4 | 3400/ 5120 batches | lr 0.86 | ms/batch 118.25 | loss 0.43 |\n", + "| epoch 4 | 3600/ 5120 batches | lr 0.86 | ms/batch 118.37 | loss 0.43 |\n", + "| epoch 4 | 3800/ 5120 batches | lr 0.86 | ms/batch 117.96 | loss 0.43 |\n", + "| epoch 4 | 4000/ 5120 batches | lr 0.86 | ms/batch 118.10 | loss 0.43 |\n", + "| epoch 4 | 4200/ 5120 batches | lr 0.86 | ms/batch 118.12 | loss 0.43 |\n", + "| epoch 4 | 4400/ 5120 batches | lr 0.86 | ms/batch 118.26 | loss 0.43 |\n", + "| epoch 4 | 4600/ 5120 batches | lr 0.86 | ms/batch 118.14 | loss 0.43 |\n", + "| epoch 4 | 4800/ 5120 batches | lr 0.86 | ms/batch 118.21 | loss 0.43 |\n", + "| epoch 4 | 5000/ 5120 batches | lr 0.86 | ms/batch 118.29 | loss 0.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 883.62s | valid loss 0.46 | valid ppl 1.58\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 5120 batches | lr 0.81 | ms/batch 118.84 | loss 0.41 |\n", + "| epoch 5 | 400/ 5120 batches | lr 0.81 | ms/batch 118.39 | loss 0.41 |\n", + "| epoch 5 | 600/ 5120 batches | lr 0.81 | ms/batch 118.46 | loss 0.41 |\n", + "| epoch 5 | 800/ 5120 batches | lr 0.81 | ms/batch 118.38 | loss 0.41 |\n", + "| epoch 5 | 1000/ 5120 batches | lr 0.81 | ms/batch 118.58 | loss 0.41 |\n", + "| epoch 5 | 1200/ 5120 batches | lr 0.81 | ms/batch 118.28 | loss 0.41 |\n", + "| epoch 5 | 1400/ 5120 batches | lr 0.81 | ms/batch 118.28 | loss 0.41 |\n", + "| epoch 5 | 1600/ 5120 batches | lr 0.81 | ms/batch 118.21 | loss 0.41 |\n", + "| epoch 5 | 1800/ 5120 batches | lr 0.81 | ms/batch 118.27 | loss 0.41 |\n", + "| epoch 5 | 2000/ 5120 batches | lr 0.81 | ms/batch 118.37 | loss 0.41 |\n", + "| epoch 5 | 2200/ 5120 batches | lr 0.81 | ms/batch 118.43 | loss 0.41 |\n", + "| epoch 5 | 2400/ 5120 batches | lr 0.81 | ms/batch 118.16 | loss 0.41 |\n", + "| epoch 5 | 2600/ 5120 batches | lr 0.81 | ms/batch 118.45 | loss 0.41 |\n", + "| epoch 5 | 2800/ 5120 batches | lr 0.81 | ms/batch 118.10 | loss 0.41 |\n", + "| epoch 5 | 3000/ 5120 batches | lr 0.81 | ms/batch 118.38 | loss 0.41 |\n", + "| epoch 5 | 3200/ 5120 batches | lr 0.81 | ms/batch 118.38 | loss 0.41 |\n", + "| epoch 5 | 3400/ 5120 batches | lr 0.81 | ms/batch 118.47 | loss 0.41 |\n", + "| epoch 5 | 3600/ 5120 batches | lr 0.81 | ms/batch 118.34 | loss 0.41 |\n", + "| epoch 5 | 3800/ 5120 batches | lr 0.81 | ms/batch 117.95 | loss 0.41 |\n", + "| epoch 5 | 4000/ 5120 batches | lr 0.81 | ms/batch 118.30 | loss 0.41 |\n", + "| epoch 5 | 4200/ 5120 batches | lr 0.81 | ms/batch 118.27 | loss 0.41 |\n", + "| epoch 5 | 4400/ 5120 batches | lr 0.81 | ms/batch 118.38 | loss 0.41 |\n", + "| epoch 5 | 4600/ 5120 batches | lr 0.81 | ms/batch 118.43 | loss 0.41 |\n", + "| epoch 5 | 4800/ 5120 batches | lr 0.81 | ms/batch 118.35 | loss 0.41 |\n", + "| epoch 5 | 5000/ 5120 batches | lr 0.81 | ms/batch 118.43 | loss 0.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 884.63s | valid loss 0.46 | valid ppl 1.58\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 0.46 | test ppl 1.58\n", + "=========================================================================================\n" + ] + } + ], + "source": [ + "batch_size = 20\n", + "eval_batch_size = 10\n", + "seq_len = 1024\n", + "divider = 16\n", + "dataset_len = 102400\n", + "lr = 1.0 # learning rate\n", + "optimizer = torch.optim.SGD(model.parameters(), lr=lr)\n", + "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "epochs = 5 # The number of epochs\n", + "\n", + "model = train_model(model, epochs, batch_size, eval_batch_size, dataset_len, seq_len, divider, optimizer, scheduler)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "oPNViB4vZhOe", + "outputId": "97a978ec-9b4c-4e24-9840-1b8f6e943993" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "data[0]: tensor([[0.0000, 0.7545],\n", + " [0.0000, 0.5093],\n", + " [0.0000, 0.5365],\n", + " [0.0000, 0.0089],\n", + " [0.0000, 0.7281],\n", + " [0.0000, 0.7589],\n", + " [0.0000, 0.5609],\n", + " [0.0000, 0.8734],\n", + " [0.0000, 0.8879],\n", + " [0.0000, 0.9234],\n", + " [0.0000, 0.7817],\n", + " [0.0000, 0.3907],\n", + " [0.0000, 0.0874],\n", + " [0.0000, 0.9121],\n", + " [0.0000, 0.8126],\n", + " [0.0000, 0.0738],\n", + " [0.0000, 0.5736],\n", + " [0.0000, 0.9918],\n", + " [0.0000, 0.4867],\n", + " [0.0000, 0.6933],\n", + " [0.0000, 0.3557],\n", + " [0.0000, 0.4842],\n", + " [1.0000, 0.1213],\n", + " [0.0000, 0.7083],\n", + " [0.0000, 0.1244],\n", + " [0.0000, 0.0733],\n", + " [0.0000, 0.6778],\n", + " [0.0000, 0.9451],\n", + " [0.0000, 0.4623],\n", + " [0.0000, 0.6713],\n", + " [0.0000, 0.4773],\n", + " [0.0000, 0.9635],\n", + " [0.0000, 0.8699],\n", + " [0.0000, 0.2970],\n", + " [0.0000, 0.8436],\n", + " [0.0000, 0.2610],\n", + " [0.0000, 0.7576],\n", + " [0.0000, 0.2387],\n", + " [1.0000, 0.1822],\n", + " [1.0000, 0.6135],\n", + " [0.0000, 0.5607],\n", + " [0.0000, 0.3519],\n", + " [0.0000, 0.6554],\n", + " [0.0000, 0.0392],\n", + " [0.0000, 0.3510],\n", + " [0.0000, 0.1371],\n", + " [0.0000, 0.3131],\n", + " [0.0000, 0.2484],\n", + " [0.0000, 0.1667],\n", + " [1.0000, 0.8486]], device='cuda:0')\n", + "output: tensor([17.2102, 17.2102, 17.2102, 17.2102, 17.2102, 17.2102, 17.2102, 17.2102,\n", + " 17.2102, 17.2102], device='cuda:0')\n", + "targets: tensor([31.9716, 32.4535, 29.2805, 32.6871, 31.3211, 34.2413, 32.7659, 30.8108,\n", + " 29.9659, 33.2249], device='cuda:0')\n" + ] + } + ], + "source": [ + "model.eval()\n", + "with torch.no_grad():\n", + " data = get_batch(eval_batch_size, seq_len, divider).to(device)\n", + " targets = data.prod(-1).sum(-1).squeeze(-1)\n", + " output = model(data)\n", + " print('data[0]:', data[0,:50,:])\n", + "\n", + " print('output:', output)\n", + " print('targets:', targets)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UJOjI2NaZhOe" + }, + "source": [ + "### Smart pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "lFx86pizZhOe", + "outputId": "8dd123f9-b26c-4964-ffa6-ee33bcf17f20" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "DoXTimes(\n", + " (model): Smartpool(\n", + " (mlp): Sequential(\n", + " (0): Linear(in_features=2, out_features=256, bias=True)\n", + " (1): Dropout(p=0.1, inplace=False)\n", + " (2): GELU()\n", + " (3): Linear(in_features=256, out_features=512, bias=True)\n", + " (4): Dropout(p=0.1, inplace=False)\n", + " (5): GELU()\n", + " (6): Linear(in_features=512, out_features=256, bias=True)\n", + " (7): Dropout(p=0.1, inplace=False)\n", + " (8): GELU()\n", + " (9): Linear(in_features=256, out_features=1, bias=True)\n", + " (10): Sigmoid()\n", + " )\n", + " (mlp2): Sequential(\n", + " (0): Linear(in_features=2, out_features=256, bias=True)\n", + " (1): Dropout(p=0.1, inplace=False)\n", + " (2): GELU()\n", + " (3): Linear(in_features=256, out_features=512, bias=True)\n", + " (4): Dropout(p=0.1, inplace=False)\n", + " (5): GELU()\n", + " (6): Linear(in_features=512, out_features=256, bias=True)\n", + " (7): Dropout(p=0.1, inplace=False)\n", + " (8): GELU()\n", + " (9): Linear(in_features=256, out_features=1, bias=True)\n", + " )\n", + " )\n", + ")\n", + "| epoch 1 | 200/ 5120 batches | lr 1.00 | ms/batch 96.45 | loss 0.51 |\n", + "| epoch 1 | 400/ 5120 batches | lr 1.00 | ms/batch 96.00 | loss 0.50 |\n", + "| epoch 1 | 600/ 5120 batches | lr 1.00 | ms/batch 95.97 | loss 0.50 |\n", + "| epoch 1 | 800/ 5120 batches | lr 1.00 | ms/batch 95.95 | loss 0.50 |\n", + "| epoch 1 | 1000/ 5120 batches | lr 1.00 | ms/batch 95.94 | loss 0.50 |\n", + "| epoch 1 | 1200/ 5120 batches | lr 1.00 | ms/batch 95.98 | loss 0.50 |\n", + "| epoch 1 | 1400/ 5120 batches | lr 1.00 | ms/batch 95.97 | loss 0.50 |\n", + "| epoch 1 | 1600/ 5120 batches | lr 1.00 | ms/batch 95.98 | loss 0.50 |\n", + "| epoch 1 | 1800/ 5120 batches | lr 1.00 | ms/batch 95.97 | loss 0.50 |\n", + "| epoch 1 | 2000/ 5120 batches | lr 1.00 | ms/batch 95.96 | loss 0.50 |\n", + "| epoch 1 | 2200/ 5120 batches | lr 1.00 | ms/batch 95.97 | loss 0.50 |\n", + "| epoch 1 | 2400/ 5120 batches | lr 1.00 | ms/batch 95.96 | loss 0.50 |\n", + "| epoch 1 | 2600/ 5120 batches | lr 1.00 | ms/batch 95.96 | loss 0.50 |\n", + "| epoch 1 | 2800/ 5120 batches | lr 1.00 | ms/batch 95.96 | loss 0.50 |\n", + "| epoch 1 | 3000/ 5120 batches | lr 1.00 | ms/batch 95.96 | loss 0.50 |\n", + "| epoch 1 | 3200/ 5120 batches | lr 1.00 | ms/batch 95.94 | loss 0.50 |\n", + "| epoch 1 | 3400/ 5120 batches | lr 1.00 | ms/batch 95.95 | loss 0.50 |\n", + "| epoch 1 | 3600/ 5120 batches | lr 1.00 | ms/batch 98.30 | loss 0.50 |\n", + "| epoch 1 | 3800/ 5120 batches | lr 1.00 | ms/batch 96.29 | loss 0.50 |\n", + "| epoch 1 | 4000/ 5120 batches | lr 1.00 | ms/batch 96.27 | loss 0.50 |\n", + "| epoch 1 | 4200/ 5120 batches | lr 1.00 | ms/batch 96.30 | loss 0.50 |\n", + "| epoch 1 | 4400/ 5120 batches | lr 1.00 | ms/batch 97.40 | loss 0.50 |\n", + "| epoch 1 | 4600/ 5120 batches | lr 1.00 | ms/batch 94.25 | loss 0.50 |\n", + "| epoch 1 | 4800/ 5120 batches | lr 1.00 | ms/batch 94.28 | loss 0.50 |\n", + "| epoch 1 | 5000/ 5120 batches | lr 1.00 | ms/batch 94.29 | loss 0.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 677.20s | valid loss 0.78 | valid ppl 2.18\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 5120 batches | lr 0.95 | ms/batch 94.73 | loss 0.48 |\n", + "| epoch 2 | 400/ 5120 batches | lr 0.95 | ms/batch 94.26 | loss 0.48 |\n", + "| epoch 2 | 600/ 5120 batches | lr 0.95 | ms/batch 94.28 | loss 0.48 |\n", + "| epoch 2 | 800/ 5120 batches | lr 0.95 | ms/batch 94.27 | loss 0.48 |\n", + "| epoch 2 | 1000/ 5120 batches | lr 0.95 | ms/batch 94.28 | loss 0.48 |\n", + "| epoch 2 | 1200/ 5120 batches | lr 0.95 | ms/batch 94.26 | loss 0.48 |\n", + "| epoch 2 | 1400/ 5120 batches | lr 0.95 | ms/batch 94.27 | loss 0.48 |\n", + "| epoch 2 | 1600/ 5120 batches | lr 0.95 | ms/batch 94.25 | loss 0.48 |\n", + "| epoch 2 | 1800/ 5120 batches | lr 0.95 | ms/batch 94.26 | loss 0.48 |\n", + "| epoch 2 | 2000/ 5120 batches | lr 0.95 | ms/batch 94.26 | loss 0.48 |\n", + "| epoch 2 | 2200/ 5120 batches | lr 0.95 | ms/batch 94.27 | loss 0.48 |\n", + "| epoch 2 | 2400/ 5120 batches | lr 0.95 | ms/batch 94.27 | loss 0.48 |\n", + "| epoch 2 | 2600/ 5120 batches | lr 0.95 | ms/batch 94.26 | loss 0.48 |\n", + "| epoch 2 | 2800/ 5120 batches | lr 0.95 | ms/batch 94.26 | loss 0.48 |\n", + "| epoch 2 | 3000/ 5120 batches | lr 0.95 | ms/batch 94.27 | loss 0.48 |\n", + "| epoch 2 | 3200/ 5120 batches | lr 0.95 | ms/batch 94.26 | loss 0.48 |\n", + "| epoch 2 | 3400/ 5120 batches | lr 0.95 | ms/batch 94.27 | loss 0.48 |\n", + "| epoch 2 | 3600/ 5120 batches | lr 0.95 | ms/batch 94.28 | loss 0.48 |\n", + "| epoch 2 | 3800/ 5120 batches | lr 0.95 | ms/batch 94.27 | loss 0.48 |\n", + "| epoch 2 | 4000/ 5120 batches | lr 0.95 | ms/batch 94.25 | loss 0.48 |\n", + "| epoch 2 | 4200/ 5120 batches | lr 0.95 | ms/batch 94.25 | loss 0.48 |\n", + "| epoch 2 | 4400/ 5120 batches | lr 0.95 | ms/batch 94.25 | loss 0.48 |\n", + "| epoch 2 | 4600/ 5120 batches | lr 0.95 | ms/batch 94.29 | loss 0.48 |\n", + "| epoch 2 | 4800/ 5120 batches | lr 0.95 | ms/batch 94.26 | loss 0.48 |\n", + "| epoch 2 | 5000/ 5120 batches | lr 0.95 | ms/batch 94.27 | loss 0.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 668.86s | valid loss 0.78 | valid ppl 2.18\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 5120 batches | lr 0.90 | ms/batch 94.74 | loss 0.46 |\n", + "| epoch 3 | 400/ 5120 batches | lr 0.90 | ms/batch 94.27 | loss 0.46 |\n", + "| epoch 3 | 600/ 5120 batches | lr 0.90 | ms/batch 94.26 | loss 0.46 |\n", + "| epoch 3 | 800/ 5120 batches | lr 0.90 | ms/batch 94.26 | loss 0.45 |\n", + "| epoch 3 | 1000/ 5120 batches | lr 0.90 | ms/batch 94.26 | loss 0.46 |\n", + "| epoch 3 | 1200/ 5120 batches | lr 0.90 | ms/batch 94.26 | loss 0.46 |\n", + "| epoch 3 | 1400/ 5120 batches | lr 0.90 | ms/batch 94.25 | loss 0.46 |\n", + "| epoch 3 | 1600/ 5120 batches | lr 0.90 | ms/batch 94.25 | loss 0.46 |\n", + "| epoch 3 | 1800/ 5120 batches | lr 0.90 | ms/batch 94.25 | loss 0.45 |\n", + "| epoch 3 | 2000/ 5120 batches | lr 0.90 | ms/batch 94.28 | loss 0.45 |\n", + "| epoch 3 | 2200/ 5120 batches | lr 0.90 | ms/batch 94.25 | loss 0.45 |\n", + "| epoch 3 | 2400/ 5120 batches | lr 0.90 | ms/batch 94.23 | loss 0.46 |\n", + "| epoch 3 | 2600/ 5120 batches | lr 0.90 | ms/batch 94.22 | loss 0.46 |\n", + "| epoch 3 | 2800/ 5120 batches | lr 0.90 | ms/batch 94.24 | loss 0.45 |\n", + "| epoch 3 | 3000/ 5120 batches | lr 0.90 | ms/batch 94.22 | loss 0.45 |\n", + "| epoch 3 | 3200/ 5120 batches | lr 0.90 | ms/batch 94.25 | loss 0.46 |\n", + "| epoch 3 | 3400/ 5120 batches | lr 0.90 | ms/batch 94.24 | loss 0.45 |\n", + "| epoch 3 | 3600/ 5120 batches | lr 0.90 | ms/batch 94.22 | loss 0.45 |\n", + "| epoch 3 | 3800/ 5120 batches | lr 0.90 | ms/batch 94.20 | loss 0.46 |\n", + "| epoch 3 | 4000/ 5120 batches | lr 0.90 | ms/batch 94.24 | loss 0.46 |\n", + "| epoch 3 | 4200/ 5120 batches | lr 0.90 | ms/batch 94.24 | loss 0.45 |\n", + "| epoch 3 | 4400/ 5120 batches | lr 0.90 | ms/batch 94.24 | loss 0.46 |\n", + "| epoch 3 | 4600/ 5120 batches | lr 0.90 | ms/batch 94.25 | loss 0.45 |\n", + "| epoch 3 | 4800/ 5120 batches | lr 0.90 | ms/batch 94.23 | loss 0.45 |\n", + "| epoch 3 | 5000/ 5120 batches | lr 0.90 | ms/batch 94.24 | loss 0.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 668.51s | valid loss 0.77 | valid ppl 2.17\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 5120 batches | lr 0.86 | ms/batch 94.69 | loss 0.44 |\n", + "| epoch 4 | 400/ 5120 batches | lr 0.86 | ms/batch 94.19 | loss 0.44 |\n", + "| epoch 4 | 600/ 5120 batches | lr 0.86 | ms/batch 94.23 | loss 0.44 |\n", + "| epoch 4 | 800/ 5120 batches | lr 0.86 | ms/batch 94.23 | loss 0.43 |\n", + "| epoch 4 | 1000/ 5120 batches | lr 0.86 | ms/batch 94.20 | loss 0.44 |\n", + "| epoch 4 | 1200/ 5120 batches | lr 0.86 | ms/batch 94.21 | loss 0.44 |\n", + "| epoch 4 | 1400/ 5120 batches | lr 0.86 | ms/batch 94.24 | loss 0.43 |\n", + "| epoch 4 | 1600/ 5120 batches | lr 0.86 | ms/batch 94.23 | loss 0.43 |\n", + "| epoch 4 | 1800/ 5120 batches | lr 0.86 | ms/batch 94.23 | loss 0.43 |\n", + "| epoch 4 | 2000/ 5120 batches | lr 0.86 | ms/batch 94.21 | loss 0.43 |\n", + "| epoch 4 | 2200/ 5120 batches | lr 0.86 | ms/batch 94.23 | loss 0.43 |\n", + "| epoch 4 | 2400/ 5120 batches | lr 0.86 | ms/batch 94.21 | loss 0.43 |\n", + "| epoch 4 | 2600/ 5120 batches | lr 0.86 | ms/batch 94.24 | loss 0.44 |\n", + "| epoch 4 | 2800/ 5120 batches | lr 0.86 | ms/batch 94.21 | loss 0.44 |\n", + "| epoch 4 | 3000/ 5120 batches | lr 0.86 | ms/batch 94.23 | loss 0.43 |\n", + "| epoch 4 | 3200/ 5120 batches | lr 0.86 | ms/batch 94.24 | loss 0.43 |\n", + "| epoch 4 | 3400/ 5120 batches | lr 0.86 | ms/batch 94.23 | loss 0.43 |\n", + "| epoch 4 | 3600/ 5120 batches | lr 0.86 | ms/batch 94.23 | loss 0.43 |\n", + "| epoch 4 | 3800/ 5120 batches | lr 0.86 | ms/batch 94.28 | loss 0.43 |\n", + "| epoch 4 | 4000/ 5120 batches | lr 0.86 | ms/batch 94.23 | loss 0.43 |\n", + "| epoch 4 | 4200/ 5120 batches | lr 0.86 | ms/batch 94.25 | loss 0.43 |\n", + "| epoch 4 | 4400/ 5120 batches | lr 0.86 | ms/batch 94.26 | loss 0.43 |\n", + "| epoch 4 | 4600/ 5120 batches | lr 0.86 | ms/batch 94.24 | loss 0.43 |\n", + "| epoch 4 | 4800/ 5120 batches | lr 0.86 | ms/batch 94.31 | loss 0.44 |\n", + "| epoch 4 | 5000/ 5120 batches | lr 0.86 | ms/batch 94.23 | loss 0.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 668.65s | valid loss 0.76 | valid ppl 2.14\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 5120 batches | lr 0.81 | ms/batch 94.70 | loss 0.41 |\n", + "| epoch 5 | 400/ 5120 batches | lr 0.81 | ms/batch 94.23 | loss 0.41 |\n", + "| epoch 5 | 600/ 5120 batches | lr 0.81 | ms/batch 94.22 | loss 0.41 |\n", + "| epoch 5 | 800/ 5120 batches | lr 0.81 | ms/batch 94.22 | loss 0.41 |\n", + "| epoch 5 | 1000/ 5120 batches | lr 0.81 | ms/batch 94.22 | loss 0.41 |\n", + "| epoch 5 | 1200/ 5120 batches | lr 0.81 | ms/batch 94.22 | loss 0.41 |\n", + "| epoch 5 | 1400/ 5120 batches | lr 0.81 | ms/batch 94.24 | loss 0.41 |\n", + "| epoch 5 | 1600/ 5120 batches | lr 0.81 | ms/batch 94.22 | loss 0.41 |\n", + "| epoch 5 | 1800/ 5120 batches | lr 0.81 | ms/batch 94.22 | loss 0.41 |\n", + "| epoch 5 | 2000/ 5120 batches | lr 0.81 | ms/batch 94.23 | loss 0.41 |\n", + "| epoch 5 | 2200/ 5120 batches | lr 0.81 | ms/batch 94.23 | loss 0.41 |\n", + "| epoch 5 | 2400/ 5120 batches | lr 0.81 | ms/batch 94.22 | loss 0.41 |\n", + "| epoch 5 | 2600/ 5120 batches | lr 0.81 | ms/batch 94.21 | loss 0.41 |\n", + "| epoch 5 | 2800/ 5120 batches | lr 0.81 | ms/batch 94.22 | loss 0.41 |\n", + "| epoch 5 | 3000/ 5120 batches | lr 0.81 | ms/batch 94.23 | loss 0.41 |\n", + "| epoch 5 | 3200/ 5120 batches | lr 0.81 | ms/batch 94.22 | loss 0.41 |\n", + "| epoch 5 | 3400/ 5120 batches | lr 0.81 | ms/batch 94.20 | loss 0.41 |\n", + "| epoch 5 | 3600/ 5120 batches | lr 0.81 | ms/batch 94.21 | loss 0.41 |\n", + "| epoch 5 | 3800/ 5120 batches | lr 0.81 | ms/batch 94.22 | loss 0.41 |\n", + "| epoch 5 | 4000/ 5120 batches | lr 0.81 | ms/batch 94.21 | loss 0.41 |\n", + "| epoch 5 | 4200/ 5120 batches | lr 0.81 | ms/batch 94.22 | loss 0.41 |\n", + "| epoch 5 | 4400/ 5120 batches | lr 0.81 | ms/batch 94.25 | loss 0.41 |\n", + "| epoch 5 | 4600/ 5120 batches | lr 0.81 | ms/batch 94.25 | loss 0.41 |\n", + "| epoch 5 | 4800/ 5120 batches | lr 0.81 | ms/batch 94.28 | loss 0.41 |\n", + "| epoch 5 | 5000/ 5120 batches | lr 0.81 | ms/batch 94.26 | loss 0.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 668.31s | valid loss 0.15 | valid ppl 1.16\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 0.15 | test ppl 1.16\n", + "=========================================================================================\n" + ] + } + ], + "source": [ + "batch_size = 20\n", + "eval_batch_size = 10\n", + "seq_len = 1024\n", + "divider = 16\n", + "dataset_len = 102400\n", + "epochs = 5 # The number of epochs\n", + "\n", + "model = DoXTimes(Smartpool(divider, 0.3, mlp2=True))\n", + "model = model.to(device)\n", + "print(model)\n", + "\n", + "lr = 1.0 # learning rate\n", + "optimizer = torch.optim.SGD(model.parameters(), lr=lr)\n", + "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "\n", + "model = train_model(model, epochs, batch_size, eval_batch_size, dataset_len, seq_len, divider, optimizer, scheduler)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "shoKXM8zZhOf", + "outputId": "7e1f1599-be37-4c4f-f8e0-f7b724d652ea" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "data[0]: tensor([[0.0000, 0.5378],\n", + " [1.0000, 0.5299],\n", + " [0.0000, 0.9057],\n", + " [0.0000, 0.8706],\n", + " [0.0000, 0.1930],\n", + " [0.0000, 0.8721],\n", + " [0.0000, 0.3009],\n", + " [0.0000, 0.1162],\n", + " [1.0000, 0.0651],\n", + " [0.0000, 0.7684],\n", + " [0.0000, 0.4840],\n", + " [0.0000, 0.4395],\n", + " [0.0000, 0.1227],\n", + " [0.0000, 0.9553],\n", + " [0.0000, 0.4086],\n", + " [0.0000, 0.1031],\n", + " [0.0000, 0.8250],\n", + " [0.0000, 0.7717],\n", + " [0.0000, 0.1606],\n", + " [0.0000, 0.0623],\n", + " [0.0000, 0.9686],\n", + " [0.0000, 0.6312],\n", + " [0.0000, 0.4187],\n", + " [0.0000, 0.3914],\n", + " [0.0000, 0.6275],\n", + " [0.0000, 0.7836],\n", + " [0.0000, 0.9253],\n", + " [1.0000, 0.8567],\n", + " [1.0000, 0.4542],\n", + " [0.0000, 0.1590],\n", + " [0.0000, 0.6848],\n", + " [0.0000, 0.5018],\n", + " [0.0000, 0.2396],\n", + " [0.0000, 0.4275],\n", + " [0.0000, 0.4752],\n", + " [0.0000, 0.5145],\n", + " [0.0000, 0.8207],\n", + " [0.0000, 0.0278],\n", + " [1.0000, 0.7614],\n", + " [0.0000, 0.4437],\n", + " [0.0000, 0.1933],\n", + " [0.0000, 0.1328],\n", + " [0.0000, 0.6986],\n", + " [0.0000, 0.9801],\n", + " [0.0000, 0.1887],\n", + " [0.0000, 0.2022],\n", + " [0.0000, 0.4904],\n", + " [0.0000, 0.3834],\n", + " [0.0000, 0.4684],\n", + " [0.0000, 0.9203]], device='cuda:0')\n", + "output: tensor([27.0208, 27.0208, 27.0208, 27.0208, 27.0208, 27.0207, 27.0209, 27.0208,\n", + " 27.0208, 27.0209], device='cuda:0')\n", + "targets: tensor([32.8675, 30.0858, 31.7577, 31.3642, 30.4849, 36.7939, 27.4491, 34.0155,\n", + " 28.9980, 30.1141], device='cuda:0')\n" + ] + } + ], + "source": [ + "model.eval()\n", + "with torch.no_grad():\n", + " data = get_batch(eval_batch_size, seq_len, divider).to(device)\n", + " targets = data.prod(-1).sum(-1).squeeze(-1)\n", + " output = model(data)\n", + " print('data[0]:', data[0,:50,:])\n", + "\n", + " print('output:', output)\n", + " print('targets:', targets)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zUFnk6G-JHnu", + "papermill": { + "duration": 0.034695, + "end_time": "2021-01-21T10:12:56.107344", + "exception": false, + "start_time": "2021-01-21T10:12:56.072649", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# MNIST - 2 digits" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-21T10:12:56.247292Z", + "iopub.status.busy": "2021-01-21T10:12:56.246667Z", + "iopub.status.idle": "2021-01-21T10:12:58.456145Z", + "shell.execute_reply": "2021-01-21T10:12:58.456827Z" + }, + "id": "G7IV5OZuJHrM", + "papermill": { + "duration": 2.279969, + "end_time": "2021-01-21T10:12:58.457234", + "exception": false, + "start_time": "2021-01-21T10:12:56.177265", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "from PIL import Image\n", + "from torchvision import transforms, datasets\n", + "\n", + "import time\n", + "import math\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import copy\n", + "\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-21T10:12:58.626870Z", + "iopub.status.busy": "2021-01-21T10:12:58.626028Z", + "iopub.status.idle": "2021-01-21T10:12:58.628188Z", + "shell.execute_reply": "2021-01-21T10:12:58.628868Z" + }, + "id": "3SvcSw131eON", + "papermill": { + "duration": 0.124179, + "end_time": "2021-01-21T10:12:58.629076", + "exception": false, + "start_time": "2021-01-21T10:12:58.504897", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "class ConvMNIST(nn.Module):\n", + " def __init__(\n", + " self,\n", + " features,\n", + " conv,\n", + " classifier\n", + " ):\n", + " super().__init__()\n", + " self.features = features\n", + " self.conv = conv\n", + " self.classifier = classifier\n", + "\n", + " def forward(self, x):\n", + " x = x.unsqueeze(1)\n", + " #print('before features:', x.shape)\n", + " x = self.features(x)\n", + " x = x.squeeze(2)\n", + " #print('after features:', x.shape)\n", + " x = self.conv(x)\n", + " x = x.squeeze(2).transpose(1,2)\n", + " #print('after sq and tr', x.shape)\n", + " x = self.classifier(x)\n", + " #print('after classification:', x.shape)\n", + " x = x.view(x.shape[0] * x.shape[1], -1)\n", + " #print('after x view', x.shape)\n", + " return x\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-21T10:12:58.744019Z", + "iopub.status.busy": "2021-01-21T10:12:58.733738Z", + "iopub.status.idle": "2021-01-21T10:12:58.763315Z", + "shell.execute_reply": "2021-01-21T10:12:58.763968Z" + }, + "id": "lpLifQs091Yv", + "papermill": { + "duration": 0.088504, + "end_time": "2021-01-21T10:12:58.764183", + "exception": false, + "start_time": "2021-01-21T10:12:58.675679", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "class Smartpool(nn.Module):\n", + " def __init__(\n", + " self,\n", + " factor,\n", + " search_perc,\n", + " mlp2=False\n", + " ):\n", + " \"\"\"Smart pooling algorithm\n", + "\n", + " Args:\n", + " factor: factor by which the sequence's length will be reduced\n", + " search_perc: percentage of length of sequence after smartpooling to search for border. Ideally the border is located somewhere in +-search_perc\n", + " \"\"\"\n", + " super().__init__()\n", + "\n", + " self.search_perc = search_perc\n", + " self.factor = factor\n", + " self.register_buffer(\"filters\", torch.FloatTensor([[[[-1,1],[1,-1]]]]), persistent=False)\n", + " self.mlp = nn.Sequential(\n", + " nn.Linear(512, 2048),\n", + " nn.Dropout(0.1),\n", + " nn.GELU(),\n", + " nn.Linear(2048, 2048),\n", + " nn.Dropout(0.1),\n", + " nn.GELU(),\n", + " nn.Linear(2048, 1),\n", + " nn.Sigmoid())\n", + " \n", + " if mlp2 == True:\n", + " self.mlp2 = nn.Sequential(\n", + " nn.Linear(2, 256),\n", + " nn.Dropout(0.1),\n", + " nn.GELU(),\n", + " nn.Linear(256,512),\n", + " nn.Dropout(0.1),\n", + " nn.GELU(),\n", + " nn.Linear(512,256),\n", + " nn.Dropout(0.1),\n", + " nn.GELU(),\n", + " nn.Linear(256,1))\n", + " else:\n", + " self.mlp2 = None\n", + " \n", + " self.visualization = False\n", + "\n", + " def warp(self, X, new_lens):\n", + " new_lens_cs = new_lens.cumsum(1)\n", + " # This really searches for the low boundary of each new pixel\n", + " pixel_contributions = new_lens_cs.view(1, -1, 1) - torch.arange(torch.round(new_lens_cs[0, -1]).item(), device=X.device).view(1, 1, -1)\n", + " pixel_contributions = pixel_contributions.view(X.size(0), X.size(1), pixel_contributions.size(2))\n", + " # Zero out the negative contributions, i.e. pixels which come before each row \n", + " pixel_contributions = torch.max(torch.tensor(0.0, device=X.device), pixel_contributions) \n", + " \n", + " # # This contains the cumulated pixel lengths for all pixels in each \n", + " # pixel_contributions\n", + " \n", + " pixel_contributions = pixel_contributions.unsqueeze(1)\n", + " interp_weights = F.conv2d(pixel_contributions, self.filters, padding=1)\n", + " interp_weights = interp_weights[:,:,:-1,1:] # Removing padding\n", + " interp_weights = interp_weights.squeeze(1)\n", + "\n", + " # # Each column corresponds to a new element. Its values are the \n", + " # # weights associated with the original data.\n", + " # interp_weights\n", + "\n", + " interp_weights = interp_weights.transpose(1, 2)\n", + " Xnew = interp_weights @ X\n", + " return Xnew, interp_weights\n", + "\n", + " def nonzero_interval_length(self, x, dim):\n", + " nonz = (x > 0)\n", + " _, low = ((nonz.cumsum(dim) == 1) & nonz).max(dim, keepdim=True)\n", + " rev_cumsum = nonz.long().flip(dim).cumsum(dim).flip(dim)\n", + " _, high = ((rev_cumsum == 1) & nonz).max(dim, keepdim=True)\n", + " \n", + " return high - low + 1\n", + "\n", + " def forward(self, features):\n", + " #print('features shape', features.shape)\n", + " B,T,C = features.size()\n", + "\n", + " padding_mask = torch.zeros(B,T, dtype=torch.bool, device=features.device)\n", + " padding_per_batch = (padding_mask > 0).sum(1)\n", + " total_T = padding_mask.numel() - padding_per_batch.sum()\n", + "\n", + " # MLP test\n", + " new_lens = self.mlp(features.view(B*T,C)).view(1,-1)\n", + " new_lens = new_lens / new_lens.sum(1, keepdim=True) * (total_T / self.factor) # Reducing the original length T by some factor\n", + " \n", + " if self.visualization:\n", + " return new_lens\n", + " \n", + " features, interp_weights = self.warp(features, new_lens)\n", + " \n", + " if self.mlp2 is not None:\n", + " features = self.mlp2(features)\n", + "\n", + " return features\n", + " \n", + " def set_visualization(self, value):\n", + " self.visualization = value\n", + " \n", + "\n", + "class DoXTimes(nn.Module):\n", + " def __init__(self, model, classifier, features=None):\n", + " super().__init__()\n", + " self.model = model\n", + " self.classifier = classifier\n", + " self.features = features\n", + " \n", + " def forward(self, x):\n", + " #print('1', x.shape)\n", + " \n", + " #print('2', x.shape)\n", + " if self.features is not None:\n", + "\n", + " x = x.unsqueeze(1)\n", + " x = self.features(x)\n", + " x = x.squeeze(2)\n", + " \n", + " #print('3', x.shape)\n", + " x = x.transpose(1,2)\n", + " B = x.shape[0]\n", + " x = torch.cat([self.model(x[i].unsqueeze(0)) for i in range(B)])\n", + " #print('4', x.shape)\n", + " x = self.classifier(x)\n", + " x = x.view(B * x.shape[1], -1)\n", + " return x\n", + " \n", + " def visualize(self, x):\n", + " self.model.set_visualization(True)\n", + " if self.features is not None:\n", + " x = x.unsqueeze(1)\n", + " x = self.features(x)\n", + " x = x.squeeze(2)\n", + " \n", + " #print('3', x.shape)\n", + " x = x.transpose(1,2)\n", + " B = x.shape[0]\n", + " x = torch.cat([self.model(x[i].unsqueeze(0)) for i in range(B)])\n", + " \n", + " x = x.squeeze(1)\n", + " self.model.set_visualization(False)\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-21T10:12:58.862522Z", + "iopub.status.busy": "2021-01-21T10:12:58.861687Z", + "iopub.status.idle": "2021-01-21T10:12:58.893262Z", + "shell.execute_reply": "2021-01-21T10:12:58.893939Z" + }, + "papermill": { + "duration": 0.083492, + "end_time": "2021-01-21T10:12:58.894166", + "exception": false, + "start_time": "2021-01-21T10:12:58.810674", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "seq_len = 128\n", + "digits_per_batch = 2\n", + "divider = seq_len // digits_per_batch\n", + "\n", + "dataset_root = \".\"\n", + "mnist_mean = 0.1307\n", + "mnist_std = 0.3081\n", + "batch_size_train = 32\n", + "batch_size_test = 64\n", + "\n", + "train_loader = torch.utils.data.DataLoader(\n", + " datasets.MNIST(dataset_root, train=True, download=True,\n", + " transform=transforms.Compose([\n", + " transforms.ToTensor(),\n", + " transforms.Normalize(\n", + " (mnist_mean,), (mnist_std,))\n", + " ])),\n", + " batch_size=digits_per_batch * batch_size_train, shuffle=True)\n", + "\n", + "test_loader = torch.utils.data.DataLoader(\n", + " datasets.MNIST(dataset_root, train=False, download=True,\n", + " transform=transforms.Compose([\n", + " transforms.ToTensor(),\n", + " transforms.Normalize(\n", + " (mnist_mean,), (mnist_std,))\n", + " ])),\n", + " batch_size=digits_per_batch * batch_size_test, shuffle=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-21T10:12:58.989865Z", + "iopub.status.busy": "2021-01-21T10:12:58.989048Z", + "iopub.status.idle": "2021-01-21T10:12:58.992011Z", + "shell.execute_reply": "2021-01-21T10:12:58.991298Z" + }, + "id": "QEN8XMlE9CDm", + "papermill": { + "duration": 0.051266, + "end_time": "2021-01-21T10:12:58.992204", + "exception": false, + "start_time": "2021-01-21T10:12:58.940938", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def train(model, epoch, optimizer, scheduler, loader, seq_len, digits_per_batch):\n", + " model.train()\n", + " total_loss = 0.\n", + " start_time = time.time()\n", + "\n", + " for batch, data in enumerate(loader):\n", + " data, targets = get_batch(data, seq_len, digits_per_batch)\n", + " data = data.to(device)\n", + " targets = targets.to(device)\n", + " optimizer.zero_grad()\n", + " output = model(data)\n", + " \n", + " loss = F.cross_entropy(output, targets, reduction=\"sum\")\n", + " loss.backward()\n", + " torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)\n", + " optimizer.step()\n", + "\n", + " total_loss += loss.item()\n", + " log_interval = 200\n", + " if batch % log_interval == 0 and batch > 0:\n", + " cur_loss = total_loss / log_interval\n", + " elapsed = time.time() - start_time\n", + " print('| epoch {:3d} | {:5d}/{:5d} batches | '\n", + " 'lr {:02.5f} | ms/batch {:5.2f} | '\n", + " 'loss {:5.2f} |'.format(\n", + " epoch, batch, len(loader), scheduler.get_last_lr()[0],\n", + " elapsed * 1000 / log_interval,\n", + " cur_loss))\n", + " total_loss = 0\n", + " start_time = time.time()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-21T10:12:59.092445Z", + "iopub.status.busy": "2021-01-21T10:12:59.091597Z", + "iopub.status.idle": "2021-01-21T10:12:59.093784Z", + "shell.execute_reply": "2021-01-21T10:12:59.094433Z" + }, + "id": "AEyFhNHW9EJA", + "papermill": { + "duration": 0.054936, + "end_time": "2021-01-21T10:12:59.094630", + "exception": false, + "start_time": "2021-01-21T10:12:59.039694", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def evaluate(model, loader, seq_len, digits_per_batch):\n", + " model.eval()\n", + " total_loss = 0.\n", + " seed = torch.seed()\n", + " torch.manual_seed(0)\n", + " with torch.no_grad():\n", + " for data in loader:\n", + " data, targets = get_batch(data, seq_len, digits_per_batch)\n", + " data = data.to(device)\n", + " targets = targets.to(device)\n", + " output = model(data)\n", + " total_loss += F.cross_entropy(output, targets, reduction=\"sum\").item()\n", + " torch.manual_seed(seed)\n", + " return total_loss / len(loader)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-21T10:12:59.192510Z", + "iopub.status.busy": "2021-01-21T10:12:59.191970Z", + "iopub.status.idle": "2021-01-21T10:12:59.194147Z", + "shell.execute_reply": "2021-01-21T10:12:59.194566Z" + }, + "id": "-TTTvnM0D9Ru", + "papermill": { + "duration": 0.052094, + "end_time": "2021-01-21T10:12:59.194702", + "exception": false, + "start_time": "2021-01-21T10:12:59.142608", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def train_model(model, epochs, train_loader, test_loader, seq_len, digits_per_batch, optimizer, scheduler):\n", + " best_val_loss = float(\"inf\")\n", + " best_model = None\n", + " patience_expansion = 1.5\n", + " \n", + " epoch = 1\n", + " while epoch <= epochs:\n", + " epoch_start_time = time.time()\n", + " train(model, epoch, optimizer, scheduler, train_loader, seq_len, digits_per_batch)\n", + " val_loss = evaluate(model, test_loader, seq_len, digits_per_batch)\n", + " print('-' * 89)\n", + " print('| end of epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} |'.format(\n", + " epoch, (time.time() - epoch_start_time),val_loss))\n", + " print('-' * 89)\n", + "\n", + " if val_loss < best_val_loss:\n", + " best_val_loss = val_loss\n", + " best_model = copy.deepcopy(model)\n", + " epochs = int(np.maximum(epochs, epoch * patience_expansion + 1))\n", + "\n", + " scheduler.step()\n", + " epoch += 1\n", + "\n", + "\n", + " test_loss = evaluate(best_model, test_loader, seq_len, digits_per_batch)\n", + " print('=' * 89)\n", + " print('| End of training | test loss {:5.2f} |'.format(\n", + " test_loss))\n", + " print('=' * 89)\n", + "\n", + " return best_model, test_loss" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.049243, + "end_time": "2021-01-21T10:12:59.271906", + "exception": false, + "start_time": "2021-01-21T10:12:59.222663", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## No gaussian noise" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-21T10:12:59.339392Z", + "iopub.status.busy": "2021-01-21T10:12:59.338544Z", + "iopub.status.idle": "2021-01-21T10:12:59.341712Z", + "shell.execute_reply": "2021-01-21T10:12:59.341008Z" + }, + "id": "SSUEYv2oBYZG", + "papermill": { + "duration": 0.042199, + "end_time": "2021-01-21T10:12:59.341892", + "exception": false, + "start_time": "2021-01-21T10:12:59.299693", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def get_batch(batch, seq_len, digits_per_batch):\n", + " batch_size = batch[0].shape[0] // digits_per_batch\n", + " width = batch[0].shape[-1]\n", + " data = (torch.zeros(batch_size, batch[0].shape[2], seq_len) - mnist_mean) / mnist_std\n", + " choices = torch.multinomial(torch.ones(batch_size, seq_len - (width - 1) * digits_per_batch), digits_per_batch)\n", + " choices = choices.sort()[0] + torch.arange(digits_per_batch) * (width - 1)\n", + "\n", + " a = batch[0][torch.arange(batch[0].shape[0]),:,:].view(-1)\n", + " b = torch.arange(batch_size).repeat_interleave(digits_per_batch * width * width)\n", + " c = torch.arange(width).repeat_interleave(width).repeat(digits_per_batch * batch_size)\n", + " d = (torch.arange(width).repeat(digits_per_batch * batch_size * width).view(digits_per_batch * batch_size, width, width) + choices.view(digits_per_batch * batch_size, 1, 1)).view(-1)\n", + " data[b,c,d] = a\n", + " batch[0] = data\n", + " \n", + " return data, batch[1]" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-21T10:12:59.444250Z", + "iopub.status.busy": "2021-01-21T10:12:59.443398Z", + "iopub.status.idle": "2021-01-21T10:12:59.879313Z", + "shell.execute_reply": "2021-01-21T10:12:59.879992Z" + }, + "papermill": { + "duration": 0.490095, + "end_time": "2021-01-21T10:12:59.880226", + "exception": false, + "start_time": "2021-01-21T10:12:59.390131", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAKoAAAOfCAYAAABYKVgBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAACiFElEQVR4nOydd3yN1x/H3+femy2bhAwjMszaIShaqmZRHWhVUWpWW9Xq1P66i5YaNdtqq7S1aneoltbeWwYhCRIjEdl3PL8/btwIQcidcd5eed3nOc95zvlGPvfs8z1CURQkEntHZWsDJJLSIIUqcQikUCUOgRSqxCGQQpU4BFKoEofA5kIVQnQSQhwXQsQLIcbfxfuhQoiNQoijQojDQogxheF+Qog/hBBxhZ++d5iuWgixVwixuqzpCSF8hBBLhBDHCu2MKWN6LxX+roeEEIuEEK53kp4Q4mshRJoQ4tA1YTd9XwjxeuHf57gQ4uFSpjex8Pc9IIRYLoTwKW16JaIois1+ADWQAIQBzsB+oM4dplEFaFx47QnEAnWAz4DxheHjgU/vMN2XgR+B1YX3d50esAB4rvDaGfC52/SAYOAk4FZ4/zPw7J2kB7QBGgOHrgkr8f3C/8v9gAtQo/DvpS5Feh0BTeH1p3eSXok221ioMcBv19y/DrxexjR/BR4CjgNVrhHz8TtIIwTYADx4jVDvKj3Aq1BY4rrwu00vGEgC/AANsLpQFHeUHlD9OmGV+P71fxPgNyDmduld96wXsPBO0rv+x9ZV/9X/9KskF4bdFUKI6kAjYDsQqCjKWYDCz4A7SGoK8CpguCbsbtMLA84D3xQ2JeYJITzuNj1FUVKAScBp4CxwWVGU38tg31Vu9r45/kaDgHVlSc/WQhUlhN3VnK4QogKwFHhRUZTMuzZIiG5AmqIou+82jevQYKwWv1IUpRGQjbFqvVv7fIEeGKvNIMBDCPG0OQy9WZYlhJX6bySEeBPQAQvLkp6thZoMhF5zHwKcudNEhBBOGEW6UFGUZYXBqUKIKoXPqwBppUyuFfCIECIRWAw8KIT4oQzpJQPJiqJsL7xfglG4d5teB+CkoijnFUXRAsuAlmVI7yo3e/+u/0ZCiAFAN+AppbCev9v0bC3UnUCEEKKGEMIZ6AOsvJMEhBACmA8cVRTl82serQQGFF4PwNh2vS2KoryuKEqIoijVC+35S1GUp8uQ3jkgSQgRVRjUHjhyt+lhrPJbCCHcC3/39sDRMqR3lZu9vxLoI4RwEULUACKAHbdLTAjRCXgNeERRlJzr8rnj9GzamSr8knXB2FNPAN68i/dbY6w6DgD7Cn+6AP4YO0RxhZ9+d5F2O4o6U3edHtAQ2FVo4wrAt4zpvQccAw4B32PsQZc6PWARxvatFmMJN/hW7wNvFv59jgOdS5lePMa26NW/yazSplfSjygqkSUS+8XWVb9EUiqkUCUOgRSqxCGQQpU4BFKoEofAYkK901VRQoihZs5fpmcHaZktPQuNjd7xqihgl5ltkOnZQVrmSs9SJWo0EK8oyglFUQowTkX2sFBeknsAjYXSLWmFTPNrIxRWB0MB1KibuOKOl/Az2+yDTM8+0ipNeldIv6AoSqVbpWEpod52hYyiKHOAOQBewk9pLtpbyBSJvfOnsuTU7eJYquo3y6ooieQqlhJqmVdFSSTXYpGqX1EUnRBiFMZtBmrga0VRDlsiL4n1ObO8Dv4expV7F7I8CH7U8n9aS7VRURRlLbDWUulbgtye0WjdVfhtTkaXlHzD88x+Ldg0cQYbct354uknYdsBG1hpOzTBQVz52oV/687HCTUAh7SCdyOeRB93wqJ5y5mpaxjw8Uo2fPYl6a1CbhpHq+hp43qF2OecrWiZ7dFUr0ra7Aqsr/uTSaRDT3fkqa1DSHyissXzl0ItgTc/+BZ1RNgN4VlBxv+u5dlVqPPuWWubZVOy6wbyT8MfTPdDT3fkwshgwvrtI/TDLRbPXwq1BB5wzaTjij0Y7m9ULHzPS9MAuKJ3RZecYgvTbEaej9p0/d75aNL7eaHstV63Qwr1Joz0SSCljZutzbAbNn82w3R9qEtldImnrZq/FOo1LH2kJXMuR+Ik1DgJNfm1csntEQ2AtkMTU7hKyO071kYK9Rr0cSeYs7gLWkWPVtFz8IHZfDllGhefi8H5jXOmcINSNPEW+00TYr9uiibkrv1mlCtq7nSl6nYPqm73QF3R32zpWmx4ylGpPiuOoZ078k21DQBEOgmyH86ilmv2DXHVEWHEdpyDVtHTefkoXMtxu/WhocP4Y84sADr9eZTVdY0+1PQPNEbnriZ9SBa7mv2Ak1CjVfTGl/YXvX9Uq+WFkaNxj714V0NZUqjXoT9/nn0rW8LoDaaw3THzC6+MHYpaLmdJersHzz253hTndFeIXGVNS63PVQHWd01i+oddAJjb9yuaOhcUPi8eD+DTi41QoTDOfx9rZ8+gx/HeOA0Lu2Ox2sV2aXtclJIwMYYDfacWCytWWlwX1rvrAAz7j1rTRKuiqVKZhKmV2NNy/i3jOQk1rfb1QSyuCIBXYh6oYPS8n+jglgHAyOT2nGlxxfTOn8qS3YqiNL1l/mW0v9xSc9xW6lYcTmzHOaYwJ6G+IV6vuG6c/7o6Pvu3WtM8q6M7ew4lrobRedAteOi5YXjl6tFkXSkWnqF3BzIAmBGygV5E31H+Uqi3IHSFmloZIzn4+JemsGtL1HobhhExYA8+3BuD/36HFBpsGsrG1tPxU5U8M/fHvFk31DrX0+XIE7iQeEd5y17/LXD7dQcR4/fSeP6LnNUXmMKXZ1ehd9cB1Bodb0PrrI/Xom2E9dtHh29eves0Wu/pj8eQWwu5JGQbtZSkjG/J3tHT0Cp6vsuswfI6t1yQXq5Rubqi1K5J4usq5jT93hReyykbJwQHte5EaLI4oXNHX1gWHssPYlm/B1CfvYDuXGqx9ErTRpVCvQNWp+xGq+g5qy+gx1evEvyJ5ee4HYm4GcbdRhEjt3PikxgiJh5Hf/HSbd8rjVBl1X8HtBk3EiehpqrGjYoPnkETevNVVvciESO3EzHS6AY2bPzWUom0tEih3iFXZ6fW1vn5lssBJeZFCvUOcMo28F1mDbIULWBcDiixDlKod4DbrztYXqcSq7JqAsblgBLrIIV6F8yY2svWJtxzyF6/xObIXr+k3CCFKnEIpFAlDoEUqsQhkEKVOARSqBKHQArVzhHN6nNmeR0+T9xq0w2Eie/HkL4mgimJW4j96s4WPZsDKVQ7RV3Rn9TRLfnsl7nsi/6BWk4uxI2sanU70ka2pMOhKxwZNIPnwv7j+/QWrO48lQtDY6xqh1zhb2eoAwM4OqE6e7pPwUv1B8YjEGDC+QbUePP2Z9uak9PvtGT30Cm8kdqSjv2H4LT5ICgG9jq3p2LeDtR1o7jU0BfvhdssbossUe2F6PqcXxnF/J3LiO8xi4NadwDS9Dk0e38kex4LB8Odr4wvC5/2/5Y3UlsS160imr92o2gLUHQ6DDk5pI5qzgu/ruB853yr2CJLVDvg4nMx/DFhMl4qV8an3s+GOS1wvqLw32cz+TjtASrN2op1JWrEQ5XPuvXNqH62aOOiytOTE6/VY++zU3AR1pOPFKoN0VSpTLWVGXxfeRKrsqvxxZdPUOWHQ1RWH+OJrYfZlg/xfUMBy/oevRmfJXbGo37xxc/HP6lDXM8ZgIbe8Z2JGnPKKl8iWfXbkOxGoXwRtJk4rRuzJjxGwMwt6DMzyWwfSX/Pc/T/9zmLO8i9FQl7QtnY+Fv0DzQGjJ6mD/SYSqYhj6i/B1PwmMGsq/hvhRSqDXFZu5MuR3vTwgXe+fAbYmdFkzq6JSEvxZGgyyVwnTPaDk1QR4WjqWJ5Z7nXEzn7HOuyg1DevMDpd1uytdnX5Cl6Hvz0FWo+tRf9+fNWs0Uu87Mxai8vEsbV5flevzHGt+Tt17HaPC4ZXNmTW4Mpv3cm6oNYq5Vk58a0ZM+r0wFIN+QSs/AVwsab19mG3IXqQKi9vDBEVuXkOBVHWn9L5C8j8DsgSK9T9PcxuChs6jGZF0/15Mr9F6xjV90oxq/6iVYuBsLXDSXyuV1mz0O69HEg9JmZqDNzmdh4HXMvhxL5xgEMOTn4XRev945x/PjBREbQ2uI2qSNrMmzFKlq5GABQZdlOLrKNakfk1PSjq3sWs+Pux5CTU2KcAs+SDkW0DEdf9qerexYRy4dT659BvN1pGZrq1p8dAylUu0LnYfxzaH71LfG50Gio8ngi753pYnFb0ka0ZF+3qbQ9+BhRrx5EnHbjfrcTKO6uFs+7JKRQ7QivkUnMulyNij+VfH6VKjKMVZGrSfiijsVtyWmbxbrsIMScShhycqi+MofqGne0/h4Wz7skpFDtBHV4DaaE/cLpfH8M2SV4t/b1pfb38STqcvDZdNKitqhcXfkxeh7vLeqL+zKj55P0Wu4WzfO2Ntk0d4mJ7NqVqKkp+RQWtb8f2iUV+KTyTh5aOfYGJ2PmJvaThkQ5GQj5s6idfKGZHh16hN5g0bxvhuz12zmaKpWJe6EGh2tNJ2rZCCJGb7d4nqJSPi+ltEf17z4Asp5oweauk+gd9zhiy/5bv2whpFDtDCehR+Xpg6qiH8k9g+nw9DZWVF7D4NMPWEWkAKokV1o0TiDx/s7oPDQ88vYGUvXO5L4fhMZGToulUO2ECnuS+eBCPT4IOIj+WFH1GqvNo97XY6n+tvVcr9dYkU2PpxJ4dvF81EJFvqIl+rMXqbzBdm425cyUHZHzaHPyBqXTPPAUv8XVxnuDGwEr4606p36VhMkt2PLEZM7p1Qx+7yX8vrHcF0VOoUocAunSR1JukEKVOARSqBKHQApV4hBIoUocAilUiUMgB/wltye6PgkvGs+BfanhBoZ6JzIsqS3/nAjHcNGFWm8eRZ9p2fMMpFAlt+TEZzFs6DORKmo3DBhQocKAgVmh/2AI3YgKFdHHRxMw3bKzVlKodoba1xclNNB0f76ZcRF1doggr1oBBztOZ/LFxlRxzuCz1T2oOc5yM0ZnV9TmWLMZGHBDhQBUTEhrxMrFrRk+YBVDvRNRIbgSk0vAdIuZAUih2hXqSpU49nYYx3vPLBauQmDg6gyiE29UPAjAoj+1ljMmuj6rG3+FAWNJGrV0FGHLtbjEpRKcvIU1X9Xgx5+bMTnyFzy3lrw80ZxIodoRSc9GcLz3tJs+r/vdKHb2/5xDBS689upwKvy1F0tNgOsqOFNFXVSSRrxgXLmlK3yuz8ykQqdMJtCEACy/WEUK1Y4ImbGPJobR5FYqkl/FAwrex6/Q/8f1rO43ia6H+uP2oTcem7dbTKQALrvimJFRk5E+CRgwQHR92HHwtu+dXVEbl5U++H1t3iaJXJRi55x9uSWjhqxgoFcSMe+Mwn++9Zb7XRwcw/b/zcCAwrCktiS+EYXmr90lxlX7eJO52J9/6i/BgEK34CalzkcuSnFg1IEBZK6ryV8vTeSzPQ/TcdDzBPxy2Ko2+M/fSpuDj5l6+f+bNxeXfyqX6Pn6TP+6bKj/EwYUYwlsZqRQ7Qjh4sL5YTEMjj2J4VIGmZsC6d99CDWf2ovzb7ssPlZZEhU6naDVOy+wt8BAtIvC8vC1rNi+koQfG6J70FhqakJDGDJ8FSpUqBDUXjrK7HbIqt9OEE7OJC2OYE6j76mmyWFQ36I/ttOp8+iSU2xoHahrR3D5Cz1/1//FNJ66I1/w9F9D6dtkB+8F7MWAgQcPPon3Exfu6EslF047EAk/NuRo2/klPvsuM5jJhztQYZUnQYNOkLCqJkGTbLMtJLdHND4vn+atqqto5KwyDZ1d/ezW69lSdbquRQrVgVBHhJHRJACtu8B/bybHR7nhddCZBk8e4puqf18zjgrphjz+ygnh+wdj0KWcsYm9p99tyb4hU00zVVc/6y18gbBX76zDJ4VaTkiY3IIPuv3Em+ueZFGPaUS7OKFXDDzSsge6U0lWt+fa0YDrS9Srn3cyQiF7/eWEmmO38U1UNcJf3MbAOWPQKvpiJaw1uTg4hmUTJpp6920OPkbnp4dw/9gRDE1qZwpfNmEilwaZ74gfKVQHI3TSLnrEdrdJ3prQEAaOXW2asXrw4JNUeMcD9cY9eC7expkWV6izcBSp+nyC1e48M3YtmtAQs+QthepgqIMC6RhwhHRDHljZvc6pflUZ6p2IAQM78xW8X1Lf0HEKe3Urvd8dhwGFoT7xqH/Q3SS1O0MK1cFouvIEI30SaL7uRZsMWakQqFCxO686iOt8tUbXR10nEq/EApyEGhUq3qq6CnXtiDLnK+f6HYiTH8Ww3H8qU9JrU/u1eJucPXW1DTrUO5GGq0+x5nJD07Nh/l9xRudGIxcDWsU4CmBQzFMWSqE6Aio1acOaM6/PTE7q9Hy78GFC0q0/jhq88QqqF4yrqVQIol0UWgTsu6a370awWmBAhZNQMy09jNUD28DROxtXLQkpVDuj4OGmuKZkYTh0DABNSDCX7g9lx5vTOFyg44XBowixlQ+oHQe5f+wIol48zJzQvwvn9FWmzxkZNZm+7wGcYt0I3pSH8+54yCy7SEEK1e7IqOnM69PW8eayfvgehYxOORxtM5NW+5/EZ4Ibmp0lr16yFp6Lt3FmMXSj5NVRNdlrujZn00QK1c6osugo8/f04MjSor0dkb+MJOqdIzZZlGIvSKHaGfr0dNiWXmw9ZzjbbNJxsifk8JTEIZBClTgEUqgSh0AKVeIQSKFKHAIpVBuRNqola1P2cH5llE3teCH+GKtTdpt+Tv0vBrVvyUdc2hIpVAtyflgMDx7MLvbjsakSAFdqGDCgcPmE7UQRt6AxrV3TjXPyhf8ODp7OpS62/fKUhBxHtQBCoyFpXDTdntjCy37Hij07672X7iuGsLPJ5+QrKjySbFNWnJ7QkuMdprE735nnDzyN31ceZIzIYkfThYQMiydnvR/6i5dsYltJyBLVApx8rxn7Rk3jo4A9prAxZ1pxUpdHFbUbu5r9QAWVC19eakCVybaZt88LNvqtenrrc1TueRTn33aRkeoJwMKwdeQ2DbOJXTdDCtXMJExqweZnJhULa/DVaBK7efHEpHHFwn+d+KA1TSuRyHcv3xD2/vnGuMddsIE1N0dW/WZCUy2UEwNC2d1nMu7CeKb9aV0OryX1oPris+hT0wh6tOi/e0BiByr+cQLzrH+/cypt0dBqyyh8467ZgCfASaj5Lbk2fidibWRZyUihmgnd1woHak0DnE1hT732Cp6LtwEXARgassn0LPbbWvifs54fqevx/bZ43movL8LDzqFV9GQc8sfPRnbdDFn1m4GESS34OeoX0/1RrZb7XxtZKFJQubqSsqwuHd2MnZMPLtxHpcWHbGJrSaj9/UiYW53VtZYBEDHfsses3w1SqGVANK1H1e0ebHlyMu7CWJJOSY9k0Acv4f3DNlO84582ZG/z73ASav7Lc2Ln47UwXLliK7NvIP3hSA60NnppabyjP8o565+9ejtuK1QhxNdCiDQhxKFrwvyEEH8IIeIKP32vefa6ECJeCHFcCPGwpQy3B86+pWdmyCZ8Va6msI3tquM/r6haVTWsw5sdV5juB60fgj42wZpm3hJD20ZM+N83ALQ/+CRBvY7Y1ZfoKqUpUb8FOl0XNh7YoChKBLCh8B4hRB2gD1C38J2ZQgi12ay1I/K6R7On2cLCXZmCZF0ubceORH/hoimO0rIBx0Z48KzXGVQI6i4YRcTI7Ta0uji6B5swZcFM2rvlcEpXgGpWRVubdFNuK1RFUTYB14/89gAWFF4vAHpeE75YUZR8RVFOAvFAtHlMtS98XzlVuCPT+PPI9FdNbdKrxD3rTGzXWRhQWJNTgYjZyTay9kYSJrfgy6+nE+6kYXc+PPvaWNxW7LC1WTflbnv9gYqinAVQFOWsECKgMDwYuPavlVwYVq5IHxDD59U+52oPf1RKa0KXny22Cl/l6Ul819kYgL0FBqYNehLVqb0lJWdVlFYNSRgumNHiGyKdnHkjtSmbJzfH+7ovmb1h7uEpUUJYiU6ShBBDgaEArrib2QzLsvWjGRgKRTrmTCuSunuiTz1peq6pFkrNpedM989PHEPAZuvPQKnc3dE3jCDjzRwqvqwQ/64HP7SYSwNnOFpgYF2OL0d6V8X7pH2LFO5eqKlCiCqFpWkVIK0wPBkIvSZeCFCiX0RFUeYAc8Doze8u7bA567c3ICK1qN2Z2bcF9720n8lVtgGCAwV6PM/YZlj/2KR6HOsxAwDVX1e3NRtF+sqg4ag37gFO2cS2O+VuhboSGAB8Uvj56zXhPwohPgeCgAjAfhs+ZkYdGMAPn0yiqsZ47lLX493JmxiE2zrr/xdceD6G2B7TudoNcRJqtIXFwaP/DCNio223Xd8ppRmeWgRsBaKEEMlCiMEYBfqQECIOeKjwHkVRDgM/A0eA9cBIRVHK9QbKLs33ETejOXEzmhO+Jt0kUoC4lABc1u20iV2ep3XszgcDBt4735Bme57grD4XAwb2dphO3LelP7XEHpCOfO+C387sQ3uL799RrZan9w0k+KnTGLKzrWhZcZSYBmi9nHCPu4DuRCJnX25Jn2c3MNbfOCQ+7mxLdk5ugkuGHo8DKTbzXi09TluIE5/EsKnfRCqqbzxacUp6JD9N6Vhs0N+e0AQHkbdAw4iqf/OIR7qp3frSmfvZlRZK1tZKoED1JWlwNs0qTi+kUC3IiU9jOPJ0kTeTqD+G4nbchapf7MGQl2dDy0rP6XdbAjCh7yJ6VTD2hwefeojdv9cBoNqvl1H2Wv5sKylUiUMgffhLyg1SqBKHQApV4hBIoUocAilUiUMghSpxCKRQHZCkJfVYm7KHjP7mOxnP3pG7UB0Mlbs73zf5mjR9AT7f2+fslyWQQnUwjs+szX3Om6m7eQQ1OGC2dKN2ORUuTYShSe3Y+lv94hGEQsjGfJLbuVJ95WVyQj1Ia6wGReB+TqHSV5b90kihOhBqH2/ebL7GImmvOVaPiVWMi7vnhP6N4bm/ij1XoeLoM1qinNTsfwZ8VAXU0LhiwMDG3ApMPtQP1WbL7WCQbVQHIuXZujzjZTxW0vs3D7OmHTXmFJ0GDaf28lGMOdOKy4aCG+M4GfdpNnCGahrjDoe9+SpG7ehrUZGCLFEdiuFDf0WF4O88JwI2ppjVHZD+4iWcf7tExG+QADzy9Fh0bsadRXmdM3Fd58XMN76kQZEjGNblePLJ289Q0wr7raRQHQR1ZE3qu+7BgML4o73xSyzuG0p1Xy0S+vjidl5QZdoOFF3ZZHytAw3mQm7PaHxUBVzd0Nj+4JN4dDqBJ9bZbyWF6iAcH1aJaBfjSrfM/cV9QxU83JTZc6ZSQ+OKCkHXzc/ALvO5DMrtGc2mmXPQKkZHG12O9Sz0AXDCbHncDilUB2Hr45MBV07q8gifW1Ttq+6rxS/zpuJ9jbcWc5LZrwULPp6MVjF2nOpveo6aQ0+gupJkkfxuhuxMOQCiSV38VcbdBL2nj0OXeNr0LOF1F4uJNKN/DFM/nGbqONXf9BzhI07bxOWPFKoDkPC4F2phPHrcLa1oobtoVJdfY74yuRVSIWh78DEUM1T7+Z2bMe1/xTtPofM0EOCP2surzOnfKVKodo4mOIgPH/0RvWIgRZ+D7/GizYLHR7gT7uRSzLVQ5h+VzZLvpeeziokUYO13s1j+1yLyl/mQ29O6npqkUO0cfZA/vTyMrr967H0Othlno9R1Ivmi3aIb4oesNY/LyI/rLUd1zT8noTZdr6+9nA0zvrLqUT+yM+VAVJxY1BY98WRFurtnFu4hNRL15xAiju658cW74IO3n+Wdfpe4nOlh8vN/KToAbZ9LvF1rLZ3d0zk4eDq1wwdTs1+6WfK8FVKoDoQ6T4eCsTRdPmASULwTVfUn83n49Fq0DRZBRTA5f/OOOwEL4csOT+I66wfau+XQMDSZHH/LH/Ujq34HQIVALVRcaGQ8Xiepa0UinVxNHSwVglb7+uCy1jpeWZz+3M3wv54BrHfUjxSqvaMzkGnIQ68Y+OWtiZz8KIYGvY5gQEGvGMhXdLyd1pCKfe3P7745kUK1c5S9h3lwzyAAQjRuHB4wnW+qbQBgX4GO+98fw+5GKqt4NLlK6uiWxHadhZNQ03rvUzj/tsvieco2qgNQuV8S4Z8/z84uU4oN7i+/3ITKv59BiaxZ5nMBsh9rTq6fsdzyO56H6p/iq6FEk7rEDqjAlM7f09DlXwy4MCujKlm7KuKH5c+kkp5SHAh9u8bo3NVcGprF7mY/oEKwOU/D6FnDCJp4946Csx9rzteTPzfNQP2b58rQrc8QulBD0lM6OkcdoY33MXp4FJ3mtySrMgt7tUd/pOwilS59JKXiwtAYtkz40nSvosjp71V+y/FGqxgr4FfX9iNiwRWz+aUqjVBl1S+h0u5Maq0bzrHOX93w7GiBgb4LXqLqu0UldjjbSvZ3b0FkiSqxOdJJmqTcIIUqcQikUCUOgRSqxCGQQpU4BFKoEoeg3Ag16e2W1N6tYW3KHn47s4+FSf8R/0ULhEYOFZcHHF6oOb2aE7TNk13PT2FyZeMJeXrFgK/KjdgnZnL6tXJ5uPU9h8MWN+qIMBKeCWRF/8lEOrmSpei5f8/TeM3wJrWZE+0e2cO0oC2sGvoZo5c8i/54vPWME4JLz7ag3vOH+GdHHVOw21k1VafuuyG6ITcX7GDixZ5xWKF2/XUnw7xPAa7U/n4kNZZlUXHHQQBC18Px/5rA91uornHH4O5iVds0QVXY9oHxsFxC/yn+cNSN8WtvGojHZg8qL9hv05P+7BmHFGr6szEM9Z5Or/iu5Lc9Rw2Kuzw8+3JL9o41HlZWb8sAqh9PsOrctC7lDF2CG5vuNcFBJD5TveS4DbM42uYbaAMRLZ4j4hnz7HkqbzikUH2/3UqvZ7tx+cuquHOu2LMLQ2P45yXjfqI1ORWo3j/O5ifp6VLOEPLxjeeMqr28uLAoEIDTuhwqr3G+IY7EiEMKFUD7YBoV3DI5O7olWS1yAYiunsjyalNxEa6c1uXwVe9+GPKO2djSklH7+nL++0psa7gYgAfXvkzkT9ZxOOaIOKxQMejx/d2Z1dWnX/fA+Cv13DuEygeOWt+uUqD29SX1u0rsaGQUaesDj1P79eOU6/Piy4hDD0+19zMKUS1u/DVWNJqLOrKmtU26LWp/P84tCGBHY6NID2sL8H1ZhT7jso0ts28ct0QFlsZE8eW3D5CV6E2FRBUhS0+jS0pG92dVfq+9gsGr/2DefXVs3ka9lrN9a7GribEWOKwtYNhrL+J5VFb5t8OhharPuEzlnkUl0VVXjBdXhkBt6OmRwTwnJ7ATocZNb86Kbp8Dzhws0DJi/It4ynZpqXDoqv9mVNqfa7oWzk42tKSIzL4tWNFtKnWdjD37wZ9Ikd4J5VKomVWLthQbwoJtaIkRVYPazP54ikmkjXc+ReDP9jkaYa/YvVBVHh4kLalH2wO5pe4cXelR5GhWHEu0kGWlQ+3rS+an+SaRTjjfgOCB59CnW96xWHnC/tuo4VXZ2mIuFYQLTst0bHy04S2dLWQ90YIdLaYAznQ82hNNdorVTC2Jo5+FE19/NgDPJbUlbmIdPNK329QmR8TuS1TD/qPc//lY1uRU4GXfODov34UmrHqJcUXTevzv47m4CWcSdLm4DlbAYLvRycx+Lfiz4xcAHCzQkjwuHI+lUqR3g/2XqECVz7fwFs/iOmIuI3xO4rJGy2d7HsblsBuV9moBOPNMATvvn0UF4UKKPoen3xmH7ynbnRWq9vKi3piDVNe4A/DE9iHUsPChYeUZh9rXL5rWo/23W3nZN65YuFqo0CtGzx5p+hwe+HYc1d6x7YG2sbOiie8+C4C5l0P5tVl1uTLqJpQ7TynKrkNsvL8qs998CADn6llMa7SI9m56JpxvwE/rW1N9dS7V/rWtSDVVKrOz6xeA8SSTibs7UqvSRagRiuGQ7O3fDQ5VojoMQnDyx/uMy/cAAwoX9Ln0ev0VvBfKsdPrkZ5SbIWiED4hC7VQoRYqdufDgL6j8F5kHY/Q5RGHqvodCX1sAg8HNTTdC/bZzJbygCxRJQ6BFKrEIZBClTgEUqgSh0AKVeIQyF6/xIS6TiQpD1fEu9NZNtZbilqoWJFdgbkdHkB3Ksmmtkmh3ktE18dtYhoJa2oSMn0f2uhaXAl1ocKAFDyd8vms+jfU0BjX8hoAg6Knq/tlPugYgv9cKVSJFRBN66F8nM4v4WthDKwb4kl9598J0bhdE6towXmOUsBxrYYnfx9BnfVJpm0+tkIKtZSo/f04+2QtVJ0vmHaQqoWKWRnBLBvyEJpL2eiPxt0mFdugDq9B7dlH+KRy0cxYZ/crXF2LMCsjjLzCo3lmH7gf1wPuVPs2Ad25VCLZaXORghRqqVD7+3Hs3QiOPzoNgD9y3ZmY+DBhnheZGbKJwT/PZ0ZGTX5vF4H+/HkbW3sdKjVHX6nEr5WXmIJ+z/XgpSUDCdxhXHHmsWovirYAgJqFM2j2IM5rkUItBUc/DCe2+0ySdbmM7DIYkZmNJuk08Q83ha83ATDcJ441tR5AZUdCFU7O1N+uZXVg0flRy7P9mDegJzW2Fq0ws/2ypNsjhVoK1J5aTuryGPTyWDwOl7xCv84vownfbF8ro3K6NOS1Sl9wte055kwrEprlIdhvW8PuAjmOejtUauqFnGHg0f633Ebic1RY0ajS4ZGQySld0Xbxv0+F29CasiGFehtUzk7GnvL1CMG55kXe9wxO9idUw6Fj9Fs4hpM6owOO36NncX5lFLS4D5Wnp42tuzOkUG+DoijsLTAwJmwDmsqBpvDk8TEceH6a6X7qyzNR146whYm3pPpbW3kxuhe1F4/EQ6jY3uRHVi/9hnZbznFuTEsKHm6KcLJ/d5dSqLdByc+n35ah9PK4RGZMdQCSX2/JlpGTmZFRk8ZfjCbLkE+Mi57U+yva1tiboE9No+bYbfQeOJpux3oA8LLfMXa9Oo3fv55N4x15xH/RAk2Vyja29ObIrSil4NxLLdk0djJXDHqSdW7Udi7gxeSOJL5bC+f1O3nzxD5iXPQ03tGfoEeP2rU/fqHRUNCuAYndnVj2yFTqOhf1p0/rcnn2pbG4L7Pulm65FcVMVP5iC62mj8VZCBanN6f1zsEkjwvHeb1xAP3tMUMB2BP9PSmvxdjS1Nui6HQ4/bmbiDHbeLO1sUkwIrkNu/OhqsaNaq8ct7WJJSJL1DvA0Lohqn/33RAuXFyo8Kcni8J+4/dcD74Mr2V9464jr3s0OjcVFX4u3ZBZfpdmTJ4xgzCNjlazXiH0wy0WtrAIWaKamZJECsZ27KG/IgHo4HaFjP62L1WTOqg4015f6o6StoKaes6CCioX7utif1u6pVDNRM15p1me7YcKFRc7597+BQtTa+YFlnecTuqQpqgjwm4bv+CZS6gK5RBVIRW1j7elTbwjpFDNhC4pmQ+PdAGgabXTNrYG9Mfj6fXHKHa8OY3kiS6ow2vcNG7ULie+q/et6f6yzg1Fa1+z/XIK1Ywom32hGXxdbT29ogdD4QFttiJy2B5GbG3DkkbzCPpbzRvn7mfV3oam516VstjU9GvchTNQdGjcv181wz/btt5mrkcK1QI4CTWKWoXN56oMek43z6Z//1f495PpTK6yjclVru9cFW/D9j3xMBXO2FdpClKoZsXvmJYcpaCwhLIf/Jbup41uJGld82lULYmhVf7hAbcbzzWovWgkNV/diYvhgg2svDVyeMrMuPxTmaXhaxxi8N9ekMNTNiDhN2MPe0/09whn+ypZHRlZ9ZuZavPjeaDN46TtC6RGgX2tT3VkpFDNjD41DY9OadTghK1NKVfIql/iEEihShyC2wpVCBEqhNgohDgqhDgshBhTGO4nhPhDCBFX+Ol7zTuvCyHihRDHhRAPW/IXkNwblKZE1QFjFUWpDbQARgoh6gDjgQ2KokQAGwrvKXzWB6gLdAJmCiHUljD+Vmg7NOGjkztw+cd+FwNLSs9thaooyllFUfYUXl8BjgLBQA9gQWG0BUDPwusewGJFUfIVRTkJxAPRZrb7logmdan18WHqOQvmhS0ldpZVsy/3qCtVQlOlcrGtOZbmjnr9QojqQCNgOxCoKMpZMIpZCBFQGC0YuHZcJrkw7Pq0hgJDAVxxv2PDb4bSsgEfLZxLPWfjBKa3yhkn73yzpX8vo+3QhOTBOpbEzKa2kxM5SgEN175ArZnZiBPJ6DMzLZZ3qYUqhKgALAVeVBQlU4ibzmSX9OCG6RlFUeYAc8A4M1VaO27FxSExrH9nEt4qVwwYvYAszwog7JmjDuFkwV4RLi7EfdKIOY/MpY1rAWDcgu0unIntOgu6wojkNmzY2Zyo1w5Z5DytUglVCOGEUaQLFUVZVhicKoSoUliaVgHSCsOTgdBrXg8BzpjL4Juh7diUSeNn46kyzgbFa3U8PncsoRuyQHvA0tmXa/IevI9jT8y4ZZyZIZsgZBO1fZ8jfNARlHzz1mKl6fULYD5wVFGUz695tBIYUHg9APj1mvA+QggXIUQNIALYYT6TS7DRyZmACSeIcTX+5yTrchn85kvG7RTbpEjLgtBocH/1xoOP51yuzn2zR/PQ4d7Fwo+2m0fim43Nbkdpev2tgP7Ag0KIfYU/XYBPgIeEEHHAQ4X3KIpyGPgZOAKsB0YqimLRk3NTl4SxoPpvAFwxFNB/7Fh58JiZSP45il8jVxULa7rzadb0aEbV/23BtesZOvUfypqcoh0BKwdMMrsdt636FUX5l5LbnQAlLnlSFOVD4MMy2FVqtB2bsqrRFK4u/G0/eRyVl1hvY1p5RWg0PHrwDAO9dpvCmu/uh36DP5WnbUdfeGq3oi1A89duZvXtifvin3jALc/kDNicOPTMlKZyIH5vJ1JJXbQ6/b2R3+H7nx/e//rj/a8/CRNtv9HO0VC5u3N6cS0Gehm9TGsVPVPSIwl8S1B56pYSj5YXR06wNdtynmIcelFK4sCa7AmbWiyss3s6Xav/Yer1nwpdSZ+EV6g0y762VtgrKk9PTsyrwaGYb0xhMzJq82c9T+Dmu1OPTanLyorG07T3FhjMb5fZU7QSolFd9o2ahqqEf05CzdY8F/bmq6ihcWX729PJeqKFrU12CI59VptDrYtE2nx3P/7q1+y279WPKvLxP37IMLPb5bAlauygCqZS81pGJbfjwIz7qLgpBcXNhZ7L/mOA1yne+OhbZmy83/48QtsROb2as6TTNMA44700qyKBb6swHDh6y/cy+7VgdrUpXB1fdTuUbHaP1Q5bol7PsKQHaffKaJLb6vH5biu6xNPoj8bxyZ/dAaNjCOHqcptU7l3UlSox44up3OdsFOnPWQF826cLhv23FumVJ1vw9UefU9vJKNJW+/qgv3DR7PaVG6GeyfbGPVWLoiv6Lqs8PVnT/QsAWu55Cl3yjeOBEiOx42uaxLY824/v+3RC2Xv4lu/k9GrOzxMnEe5kLAC+yQyl4isU+xuYi3Ij1JW1lrP2+1nkrytaVpA8rD5hhf/5l856y412t6BTm72m6zeX97ulSIWLCyc/iWH1tCkEqo0nq1zQ5zJlYU+LnQxTboR6lfV1jKd/5PaIJrdRjim8+lIp0lsxPbjI1WTl7Tfvted3bYbXhgoc7j/dtC18b4GBJ0e/TOgHlhu/dtjOVO3PUvikTQPGV7zx4IR3Tuwhwuk/vAvn/ZdnBeCWmIFFp8ccHL1SJM6MmmoqeHqirxeG4mwsy3IrOZPzTAYz682iSWFTf1OeM8/9MZjaX2bgdsSis+SOva/f0Loh83+cTqC6eCdJhco0InBCq+XlRwZhOGB/HursifQ1EfzXcLHpfsyZVrwd+CcV1W4lxl+T482sJ3rcth1bGsr9vn7Vv/vo8emrXDYUlPh8QOLDPPrtK1KkpcDvLWd6xXUz3U8N+q9EkQ5I7ECtn0Yy55HOZhFpaXHoEvUqqga1OTbagy/aLqazezq11g8neK0azzX7MeTd6LpGUjLCxQXn331ZGr6mWHiqPpd2/44i4FdXvFcfNPt609KUqOVCqBLzofb1RbgU9/CiKAr61LSbvFF2SiNUh+1MSSyDPj3d1iaUiEO3USX3DlKoEodAClXiEEihShwCKVSJQyCFKnEIpFDtGLWvL7m/1WB1ym5Wp+zm3Est0dSoZmuzbIIUqj0TFMD6uj+hVfRoFT27XpmGy4Ic1IEBt3+3nCGFaqcY2jbi6EueACzJCuKjC00A+CFsDV7LdDYRq2hUl9ivm3J2RW0Gx55kbcoe00/cVMvuSZMzU3ZKWmM3DneaAsB3wx/BJfYc0b3u55uxX/B1tfU8/UtX8gdUQ3fylMVtUQcGMHjzNpq7/mtaKK1CYLjGo5dHVcs5SDPmJ7FLovsY19k+dKgPzvtOoks5Q8D0LQyc/BJgLFkrLzL/3qTrUVf058X/NvKIRzqBajfW5HhT++/nCF85jMsG44KfrflqXFdZ9uxUKVQ7JGlJPeaG/oeTUJN6OKDY/HvA9C3U/+UFnISaWaH/WNQOlYcHVVbn094tn2RdLvfNHc2c6Gb4/eHKiPs3cErnxIECPUN+HE6rkTs5PaGlxWyRVb+dYWjbiCkNf0Cr6NmQ607k/Es37ExwzlCRZcjHRVj2z1fQvBazQucAguf7jKTq1i0o7u649j3H+jHt+HODsQ3tOQQmVt5ObLsA9O9ZxhZZotoZeX7OtHY1rvcc/tcz6I/E3hCn6ntb+C3HuInx0kDLuSw609oFFQK1UOGUmGoKrzASNBuMPqlEo7o899JKVAgS0ipazBYpVDtlWFJb6nyUett4gQMSLTYCELhTiwEFvWLgTC/jiYSGnBz08SdNcWLHuDLY+zTdY7sRNjDeInaAFKrdkfHMlVLFe/vXPgAsj1hNVovqFrHF/b/jvHe+IQBrx3/GuRdbovLwMD2PnRXN7vbT2JqvRnnJB0NOzk1SKjtSqHbGu3VXG31nJVdHl3j6pvHCXt1K022DcBJqRnz2i0Vs0WdmsnVsNOmGXCqq3dg1bhpN/stEadmAC6si2ddtKhNS2/K/wYMw7DtiERuuIjtTdoZWUaNV9Hi556Hy8Ljl/iRFEWgVPXoLljeaDbvp228UD8zYwjj/I0yotA9+2Vc4jurMvvcb4bbRslulQZaodsfrm42uxjc3+InsDnVtbI0R1ea9/NclnKg/hqJCmDpYewsMFHhaR0JSqHZGyJqis+O6frARdd0oG1pThC45Bdd4Fwwopg5WI2cVf34yhROfWt5ZshSqneG17xyjkx8EYLTvYWIH+ZYYTzSrzyv1/rCKTWofb2LnNOO/542++Rvv6M/jCQ9zVKvFRTix96kvSHrLcoP9IIVqd+hOniK5RRZOQo2TUHO0zwzSRt0ogvgnKvCMVwpOQo26BD+x5kTzqxuxXWdRQeXCsKS2hA67QHab84zv8gx9TzyMi3Ci66NbUbmb72C765FCtVO25elNy/u+GfsFlbb4oK4bha59Eypt8eHvJyeanluyMwWwPHwtYDz07GwPd9Mef/2RWOKWRgLwUeAuEr4Ot5gNstdvp7w98Dnin9ZwuPMMIp0Ec6r+jtPvxhEBI863fN+cqIUKFAPJz1fDkFp8GKrylC1E1h1GfJfZ9Io6wH4XF7MfhgayRLVbVP/sJWrEfprMedG0FrUkNuU589Ubj1nUFr1iKLak73rcTjthQOGjgD2o/UpuU5cVKVQ7RtEWUPV/W9jbK4zRyQ9S55/BN8R5/4VBeCzdXsLb5qPW9yPJV7RcCfdEExoCgKZGNUSjulwcHMPwPkZfVd9mBmHIMv85qCB9T0lKyYkfG7K3zSx25buzIqMx/fy20si5qJxLN+TRc9xYPBff+YmJ0veUxGyE9dtH9Fsvo26SQYugUzRxVrMzX2HI/v7kxntT49dcPP+13LGeskSV2Jxy78hXcu8ghSpxCKRQJQ6BFKrEIZBClTgEcnhKctfkd27G3/PnolX0JocULf43isCNaehjE8yalyxRJXdF+poIXvxykWlhjAEFraJn2zvTqb7Q/GfOSqFK7oqtDX+iq/tlVAichLrY59Sg/8yenxSq5K64WoJe/Qxf/Xyx+1P/M++qfylUyV3Ral8f/sz1QYWgzXtjiHx+J42/HmMqWTt22YU6sqbZ8pNCldwVvl3j+GL0UzR/dyQB/xgXUld7Z6upRJ1cZRtpbcznGOOe6vWrfbwhKBCA/MqenHpOT9uweBp7nWKodyJOQk34xoHUfGrvbVKSADiv34k/FPONdW1bFWG+vO4poR57vxbfd/0KNQqNXAymU6ivfmoVWBAzn//R2NamOixXS1QnoeYWa63vmHItVLWPN/mNwzn1nPE7/0DYYaJdlMIxPxUqBLvzVTRzMX6uvNyIVYn1CMKyXj/KG/mdm3ElVEPMkD2yRL1jouuT+b9sNtSfZSoxG84dQ+d/IotFcz53hYLKnjifu4L+aJwU6R2SviaCNyIX0dX9cmGvXy1L1Dvh7Bs69tZfYio5QUXVtVdgx8Fi8fSA+ig3+CCV3Jz8zs3YMG82BhRU7Cn8vHE89WIzHf7zzJNnuej1a0JDSHmt+N73btUOF3r1MJg+rxep5M65OiN1/ThqSZ+Hu84w23hquRDq8U8rsveFacXCdj/fgEZfjqbJl2MYltQWFaobxCy5M/I7N7thRqrVvj50fW4kkWuH3VCiOgk1hwfPIHZ2szLnXS6q/rZh8RgwcHZFbar0PGoM3HGQ4KtO5npVMJaokjKRPfKyqcR85WwbdsxuhP+8rQDEzt9tapte/3m46wx6EV2mvMtFiTondBMqVOxttpCs9WGmLb3qOpFkrw9jTugm9uarCP50i40tdVwuPB/DlkaLTCVlQrM8/OdtRR0VTsROF1P477kedH1uJA89N8w0c+Uk1OR3LlupWi6EOjOjhqkt+lf9n2iw8jQu/1TmndWL2FD/J3bmK7w1eIitzXRsFExtz8hVw03BYT8k8VmVzaaSdtKo/jiv34nLup28f7ybKfzlLxdS0OnuxVouhDp/VldUhb17FSreD9jH0vA1hGhyeS+tCRPCmqDeuMfWZjo8V9uesd2/4sLQGCJ2ujA1aGuxktR5/U5TfL9usaYStat7Fr/Pn3XXeZcLoQYtOET0x6OL9fJnZNSk17vj2N2oXPyKNifw7zTGnGlVtO50wvQSS9LruX404G5HAeS+fskdkb4mAkURCKGYPj2m+ZQoUjCOFGSPvMy2RotN463N3x2J/9ytpjjSU4rE7Ph2jbuj+C7rduKyDgwpZVsDIIUqsQrdgos8Evqz9RYxS0Y24CQOgRSqxCGQQpU4BFKoEodAClXiEEihShwCKVSJQyCFKnEIHFKo6vAanP6lPmtT9pAwuYWtzZFYAYcUqjbYh50xczGg0LRFLEJjfxNs9XarePCgZY6yuRdxSKGq/tlLs7kvA/B99T+I/+SW6xlsQn+/rTzjLR1ZmAuHFCpA5e1a03XNRsk2tKRkBh4YYGsTyhUOK1R74sGD2TxzPAmVp6cpLCPVkwC1O7Hz7a+0d0SkUMuIytUVX002fSqcR7gYD9JVeXjwxQOLAKhw3HqH65ZnyoVQz66parO8E76NZLBXMiNSWmG4fAWAi4/dR3f3TACcL9t+YXp5wGGFmti76Nr/cIFNbIid04yj938LwJ6vGqJoi9uhQ49ruhSqOXBYoUbVPGPT/DWhIfzT6QsAjmnzqbTyuOmZ3sX4eUmfT4WfLXc+6L2E/Q1AlgK1vx/hXmk2tSGnbhWC1e4A5Clq9BcvmZ65PZpqK7PKLQ4pVHy9aeBh9Lp3oECPy6V8czqOuy2qerX4eOYsrvpVHPzFi1Rsl0d6pAsVzuhZX+9LwBknIdC3M/padd6XgD7jshWtLF84pFCVCm7UcjFW/SszG6FOSkNnxfxzq3rSzKXI+efu16ZfF8PY0/dVufHbwnkk6nIYMuRFnH7fZUUryxcO2UY17DvC+4ndAXir4gFyGoRaNX+XC3k8nvDwLeMk6nJoOHUUzd8Zyag6D0uRlhGHLFFtzo6DZLdT0+z5kWQHFQVX3qHnTGs1x56ewa9X7iPoM6OvK+merezcVqhCCFdgE+BSGH+JoigThBB+wE9AdSAReEJRlPTCd14HBmP0j/uCoii/WcR6W2LQU+mrrVS6LlgVbd7zlSRGSlP15wMPKorSAGgIdBJCtADGAxsURYkANhTeI4SoA/QB6gKdgJlCCLUFbLdL+vfYaGsTyiW3FapiJKvw1qnwRwF6AAsKwxcAPQuvewCLFUXJVxTlJBAPZXSO6WCohUM2/e2aUv2PCiHUQoh9QBrwh6Io24FARVHOAhR+Xj39KhhIuub15MKw69McKoTYJYTYpSW/DL8C5I7JKNP75uTrf9qiV2Sr1NyUSqiKougVRWkIhADRQoh6t4he0qEtNwxzKooyR1GUpoqiNHXCpVTG3oyLB69vKdoOl4vGVk7nCodQ3VfLxtaUH+6ojlIUJQP4G2PbM1UIUQWg8PPqVFEycO14UQhg9vnOEzuMC1F+zgogYr5tZ6lKorazO/kBHrY2o9xwW6EKISoJIXwKr92ADsAxYCVwdXXwAODXwuuVQB8hhIsQogYQAezAzNQYv5VuwU34LioUfWyCuZO/a6ovuUTfkw9xuCAXl3NyK4q5KM04ahVgQWHPXQX8rCjKaiHEVuBnIcRg4DTwOICiKIeFED8DRwAdMFJRlHvmGCfDoWNcbg0vE4Px+ywxB9KRr8TmlMaRrxxHkTgEUqgSh0AKVeIQSKFKHAIpVIlDIIUqcQikUO9BhJMzya+3xLAhlNUpu/ntzD4MG0JJfr0l6sCA2ydgA6RQLYjax5u4ac1Zm7KHbofTWZuyh9Upu0n4sSFX+tjOC2HcZ43ZM2oqq2stw4CBn7O8ifA6z/JhE3lk42Gb2XUr5IC/pYiuT7dvNjHUJx4VKgwYbvh8L60JqxLroWzzwf+IDtdVZp9pLpHVKbsxYKD2huep9qMa99jz6E4kcmlQDDx6Eb9usVax4yqlGfCXQrUQmtAQVm5baTpWcWZGDc4WeAPwfsA+U/i1n05CzbT0asyf1ZWgBYfQZ2ZaxLarQu00eMRNj4a8Faf+F0PD9sfpF7CNz092xKVjYpnskUdM2pAjEyqbDhFud/BJvJ+4YBLeww88B0Dic8ZC4mi7eRgwoFVgqE88w8ZPI1oZTcD0LRaxLVmXS4jGDc24c4gNzjd4eLke4eRM0itNCeqQxNpaK1Cxh3RDLiNOPcK5rUFUI9Eidl6LbKNaALWPNyJXbTqWfVP9JXTeeorU0S3RPdgERSVQhKDmU3up+dReugU34ZFegxiW1BYVKpyEmoZPH7SYff1fHosBA6trLSPus8a3jKt7sAnK+oBibdre8Z3pNeZlLre+SLUJlvkyXY+s+i2A2suLyz9X5O/6v5TYNr22jfrHtFb4fW08G1RdO4LLX+iZHPkLwz+3XImqcncndnYURx6cA0CPR55F2V28E5XfuRnaMRdZXvd7fFWujDnTivVbGlJtnf6umgu3QrZRbczFwTEsmzCRYLU7BhR6x3flYFwI8Z3nmNqmbQ4+hs+QAnRJ1zgjjq4POyxXooJRrPHzIjnUdi5/5noy7cneJrHm9oym/Xv/Mr7ifgC6HXsUTc8MDFeuWMQWuXrKxvjP30rvd8eZ2qrTayxBc9HJdG9A4a/6P3Gq33VuMy0sUgBDTg41J2v5M9eTTm45JIx1Qh0YQPqzMWycMYs3Kh5kVHI7Hho5ElX7JIuJtLTIEtUKZK8PY2P9X27o5asQRCwdTsQL221mm2hSl1Urv8OAgW8uV2egdyJXDAXELHyFyGmn0KVY3muiLFHthDNxlUwl6NXPGRk1afrxKGq9ddSmtim7DxO96ykABnonAtDnmdGEjd9qFZGWFjk8ZWmi6zO2/VpUqFAhoPBz9agHCdi4BXvYo1M/oLgg1Rv32MiSmyNLVAty4rMY5i75iqHeicVK1J35Cs7nbNvmA+PoRPqzMXxT9W/2F8ChAgUVKnJ72p+/EClUC6CuE0nctOYce2oGVdRuqBDMyQin/rzR7M1X0cxFEPjNOVubyckX67H5wy+ZdCmKdx/uw9CPx2DAQNC4eFubdgNSqOYmuj5Vvj7D0Uenm0rQiKXDWRdTjbDJh+i3egQGFJ4L+IeLg23nUO3i4Bh+HTwRgE096hbbct4vwP7cucs2qhlJHd2SWS9No5mLwICKNTnefPFyPyJWbUePcf6/XoNTqBBEuyj4H8y6bZqWQvXoBappnKn791BqnrD/EwZliWpGsmJyaORS1Bad1a9XsRVRuqRk6nmdMT23xnhpSagDA3gu7D8WZFYj8j3j+gOVuzt5/sYp37cO97CJXbdCCtWMtA2LN83Vz8kIJ/YZd3QPNjE9j5vWnPcC9qJC0OLjMbYzNDePfVlVGeB1ikdWbCN9TQSev7uxZ9RUYxv1ffuThf1Z5MAkvhHFjnyBVtEz1Cee44/OZO33s2iy10CTvQaOPzoTAwai/h5MlQ3nbWanPjOTQx/ex/4C49jpfw0X832N9azL8aX+ghdumPe3B+TMlAW4ODiGjqP+46PAA2gVvWkmKnLd80Q+Zz++/DXVQjnfLoTzrXSErFfhtT3JJoP8clGKjbnSpwXpUSpcL4L/4Txc4lLRJafY2iy7Qy6ctjGei7fhec29NY8YKm/INqrEIZBClTgEUqgSh0AKVeIQSKHaKZpqoVwaGMOlgTFcXhvO6wkH0FQOtLVZNkP2+u2IC0NjiBhwnOlVV6FG4KVyBYznqj59ZAB+mjwbW2g7yq1Qz4xryQNP7uSLKttJ0OWSp6h5eeAIu1wUDMYt1hPGLaCrexbgBsDoMy05+HEDPDfF43Uh4Z4e3ip3QhUaDbHzGhD30HRS9DmEr3+R2i/FcnRSFEEVVVSwtYE34egX4XR138j/LtRnZ+9IAPQnTuNu2G4XuwBsTbkTatqQZsQ9NIOIP4ZQ+93zRCXvQ29QqBuZTPofVW+fgA3QVK/KlvZTAXd+XNeGsPittjbJ7ihfnano+nz72hd0ONKLiGd3o0s8jaLToaofya8Ra/A+cNHWFpZIRnQQAWp3LhpyiZhkXQdljkK5Eqrh4wx8VDrchhf/tU487sP41Cboj9vfFgvh5Ey1MccBeOLoU+gv2OeXydaUG6FeGhjDz5E/0Xnn8+jjT5rC1f5+jO39K2uW2m7bx60QdcP5vvoGdOjJXFYFfbvGJL/REk1oCJoqldFUqYzK0/P2CZVzyk0b9XIEOAkVoR8UPyE4s10EcIwa8+2z15zUyQeAPEVH+NOxLKrxh/HByKI4b6Q15lDnQHTnUq1voJ1QLoSqcnXljUeX0nz7YEL2Fi36VVeqxJP/W8/H/3Yl8px5HXuZA3WdSJYNnwi4UUG48G31dTTb3R/NMj8ALjZUeLnjGj4K2EPE28OJGCmF6tDkt6lHf88tzF5qrCKVmAac7uSBS4N0olz+ptaYwxhsbGNJnO5ekZoaNy4acul24FkqjdZS6cRx03NfYGmHjgxbMJd32y9lISG2M9bGlAuhXsV3ZxoGJ2fSxudzuOk3ACzOqoRwdoIcGxtXAgYnOKvP4eGvXiXk4y23bJoYyk934q4oV799wrOBPHogmV1Nf2RznoZOTw7iv8wIEkfWtbVpJVL1kx0MbdKLkI+t4wzXkSkXJarbgSQ+uFCPI8/OQIee0Wfu59STlVGd2MemJPsUKYCi06E/f+tNfqeetcdGi/UpFyWqIeMyG1MjOawtoNU7L5DQLA/diUQA3qq7Fvdztt8XdjNU7u6IZvVLfKb28WZhy3kAfHrwYWuaZXeUixI1/bGG/Fd3Js0nvIT//KLpR7W/HxHOO6i4LxN7k6o6vAbnpzgxpOa/POm5gfkZRSX/posRJKyqydMD/qCZiyDTkEe1Twx29ztYk3IhVM3TaSzN9qXSD3uL9e4vdonCR7USddpluxtDPdOlCrsbTSdRl0PzLcPJT3cluNpFpkctYoxvPLxUFLfNtFcI2n1vt2PLRdU/vMY/fHy0M4a84us1L9WDMScfL+4f3064XE8LGO2r8ZGeOu8m4TLRl49SuqAWRX+W1gceJ+Qr27j+sSccvkTN6dWcpzz3MW9m8QV8hraNmPXYHD7t/zSCszay7uYEblKT1SWfXyPWwJriz/QKpOlz2J5fGd9nr6C3sf98e8DhS1SDBvSKAYQwheV3bsas76YxYtFQxJb9NrTu5nj/sI0nHh1K1N+DOVigNYWvyPYhfPXzDAxrx1cR4ehT02xopf3g8CWq94GL5Cs6At9OICEohqzOWfzTYgoxS14h/B3rnC161+w4SM2n4DWaFwuOZOc93XEqCYcXqv54PNHTX2TbqM9xe/9P5l4Opf3UcYRPurc7H+UN6XtKYnPk8T2ScoMUqsQhkEKVOARSqBKHQApV4hBIoUocAilUiUMghSpxCKRQJQ6Bw0+hSiyDum4Up7v5Q/PL9I/YwSt+xt2xj0R3s8kRP1KoJXB2bEt2vzyNyLXDbhnvZNe5aBWjr71G00ZTde4x9BcvWcNEi5L6Qks+fOFrOrnlkKbPYcrF1tT6wegRIzzzkE1skkItAc+HzmHAQGyXWcYzSwtRoSp2r1WK7nePnkoDtzFUm+DAi2GEIOnNGPYOn2r03LL6BSK/yYdtBwhjK+o6kRydWJs67ydbvVSVbdQS8Bmcx33/DuabzNBbxvvoQn3W5fia7rt2s7/jw++E0+/EcHD4dDSoaTljLJHP74RtBwDjJsTgb1KI7z6Liw9Y332nXD11C9SBAeBd5KDsbMdAMupr8TjpRNVlqXAxHeHqSuzEAA61NTYDOox/Ee8fHE+w6ogw3vntF5q5CCI2PEfkoAMoOuNOM021UPTfGFgTtQqAqB9HUnOc+Xy4ypP7yog+NQ2uWWEfEJtAwNVn18Tz+qcaqrYq3IQanZvAETk32YlmLoJ0Qy61/peOXle0HTJ2ZAjHomaY7j1PWv93lFW/OVDAgAFH3tD8df3vAGi25qVibjvTn43hv76TisXNaKjF2kihSlBaNSRIo+O0LofaX2YAcO7FlnhursjfH0xFryhE/28kbQ4+BoBHRes78pJVv4TLYW74q9yoIHT0XroJgEcrTOKKYqDu+hep/cUVKh7eSpqmJdSHBY2+4Q2irWqjFKoZSG+VjwoV8dp8Ku6xP68st8PrdB7phlx8VW4862Ucdvr0YiPWvteOyKVFp7I4Z9ruN5NCLSNqfz+eb7wZAwbmXWyNsvvw7V+yM1T/7GVAdG+SnwwDoMq/mSi7DuHB9hviqoUKtQ2+ilKoZSQnuiYv+v0GwJrVLaiGYw74686eo/KUcwA3laFztoEcQwF6ZK9fYse4L9tOrFbBT1WAuk6kVfOWQjUDqsJ/CEdrnd4dVTXuXGrsZ9U8ZdVvBkzz/4pjDvY7ArJEldwxOvSorDzmL4VqRgL22psXVstwqEDB8yfrrmeQVX8ZKfBSG9ungNsKO3fK5sDIErWM6J65WDjPf+8cChGoLkA0su4hHrJELSMXEvygoa2tsB4DDwzASaOn0vkMq7qbl0ItI7VmZ9ClTm+a+p+2tSlWIaDHMQCrn4kghVpG9IePo+kA+2xtSDlHtlElDoEUqsTiFPxRjdUpuyno1Oyu05BClVgETXAQZ15tyci4WH6vs4yORx7FJfXuF1xLodoJJz+K4bcz+0gfEGNrU8qMJjiIgu/U7BozlSsGNzoOHoZr91SUvXe/BLLUQhVCqIUQe4UQqwvv/YQQfwgh4go/fa+J+7oQIl4IcVwIcW8f4lkK1L6+TH9iHlpFzwMvmm93p7VRBwaQOrol1VZcYnWtZeQpOj6Z+yTO63fecFjdnXInJeoY4Og19+OBDYqiRAAbCu8RQtQB+gB1gU7ATCGEukxWlnOEizMPuBn/kI9477GxNXeHOjCAhuvPsXP8NL4I2gxAwz9GEWSm02lKJVQhRAjQFZh3TXAPYEHh9QKg5zXhixVFyVcU5SQQD1beYCOxKurwGrT84zTvBezFSaip+88Qej30FJEDd5stj9KWqFOAV6HYPGGgoihnAQo/r255DwaSromXXBgmKYeoo8KJG1KZcf4HMWCg7vQRhD9zCP2RWLPmc1uhCiG6AWmKopT261HSoswbVhQLIYYKIXYJIXZpyS9l0uWThBFhtjbhrlCH16D2jyc49PSXACzJqkzIR1tMHlbMSWlmploBjwghugCugJcQ4gcgVQhRRVGUs0KIKsBVlyLJwLVOm0KAGzxqKYoyB5gDRpc+ZfgdHB6du2P++sdHBrK88s8A/JAZyi/92pM62hvnTuep8IUXTn9asepXFOV1RVFCFEWpjrGT9JeiKE8DK4EBhdEGAL8WXq8E+gghXIQQNYAIQK5/uwWV6p43XU9OdoxBkvRnYzj2RJGbn68+74Wy9zA7x09jc8MfeXXWDyRMbmG2/MoyjvoJ8JAQIg54qPAeRVEOAz8DR4D1wEhFUfQ3TUXC+cOVTNdjQ36zoSWlQx0Vzn8fTsdJGNfi1l4+Cv+5xmG1q2Ht3XL4/JHvzJbnHS1KURTlb+DvwuuLQIku+BRF+RD4sIy2SeyU+Hc9MGBAW+hzq9qqonLovfN1GF/RePS8VjHfmic5MyW5I9T+fjQMTTbd781X4ZRZtIFq0Yp2FslXCtXO6L9qhK1NuCXHp1bj+xrrTfcjD/dFbN1fYtzGLme4/LR52qlSqHaGR5J9/0k6Rx0pdm9YV9F0nf5sDNEdi3z8XzY4USGlwCz5yoXTdkbo9/HYc89TJQymzYxOQk1mhIHXY0/Sy+MSsBsnoUarGJ+f1nmj3mieKWH7/vreI9Rqesp0famDfQ/+GwoP2DB2pvQceWIaPTwuFAu7ev3m7GfNlq8Uqh3wVtXVpusKZ8xTVdqaTkceo+qiRLOlJ4VqZzgfSb59JBuyaVETzuhunPLem69idbY/jXf0p9W7L+D+rNasR/zINqrkjqjy+Rb6pr3C5XAVB4ZMA4xz/N/36YSy9zBBGDtb5p7tl0KV3DHeP2zDG+j2bpNrQi3rwFhW/XbApBTj/P6cy9VRcnJtbI19IktUO+DK/RfoxtXS6YpNbbFXZIkqcQikUCUOgRSqxCGQQpU4BFKoEodAClXiEEihlndUak58FsPalD2sTdlDfte7d1RmS6RQbYQmNISEyS2oudOVUz/Xx3B/I8vkUzWYF7utxoCCAYVz0U4WycfSyAF/K6GuG0VSV39qdk3g3aor8Vb9S1WNu/Fh0BYaOT9Flc3mz/diqyA2XMhlqHci8dp8gv9xTB8KUqhWYuiKNXR3zyy8cwKcWJxVibf+fhRVtpoaK82/vE+4uOD33Gl+qrmey4Y8Rjw/Bue/dpk9H2sghWolfFRG36A78xWe/X40Qf8V4LzxAJHanRbJT7i4EPtZI45HGvfeP36sH86/OaZIQbZRrcbATQMBeGPY81SbsAWn33ehaC24SLpeOMcfM4r0gj4X5YuA27xg30ihWglhxWNSVa6uHB/sYbrvd/wpXNZapuS2FlKoVuLr+7+xWl4n3m5EbI+vABiV0hr3oVbL2mLINqqV8FHlAk6khztjqNOSiN6xdKt0AIAMvTvfzO1C5SnmcXqr1Mw2Xe/5qiF+J4t7sdZUqUxOgyI/dq5/HyyzR2hLI4VqBfK7NCPcyXjI7a43p5cYZ/S46bROHYHXorIdhqtv15gvmy4EYGlWRQL+Si62LaTg4abcP/FfLutSOX4lkKXha4hcO4yoUQdQ8u136EpW/VYg30eNm3A23f+S5U/4qmG0fmUErV8ZQe0FIwG41PPuTw0BY9tUeesC7d1yOKvP5ZMZfdGdKvKprAkOwvONZF7zP4xKKMyusQSA2C6zOPV6k5slaxfIEtUK+O5MI0Wfw/z05vz7cgs0G3YTeY0nTr9NQTAAZjZdyGTfNujT0+8qH5W/H+trLwdgfXYkgV8Wb0qcmenFjnBjadve8zBz0qN5o+JB47t2vktblqhWQB93guHRvdnR2g/Nhps7t23vpke4utx1PtkNizzQT17ao9iztJEtWdaw6AiGYesHYUCwPNuPDsOGEzrRvsdYZYlqJXTnUm/6TPHxBGDg6fvRn79w13mcetTouXpDrjs1pyWYXAMZ7m/Eslc/I0TjZop7vNdMAI5qtVTYm4LOkmO6ZkCWqBZE5eFB7NxmqNzdbxpHaDQ4z8wA4NjUunft/17t483q9sZ99pkGV/SpaWgqB2Jo24gTj7oUE+lVFl0J5MlvX0afev6GZ/aGLFEtiMrHm/gus4nKGEnNcSUfdJbZuylrwo2lW1l6/PrMLB7d/jyHWn9DF/dU1LEKPuqDBKmvEO50Y3Oi1s8jqTXpNFVTttx4EogdIktUC6JLOUOPuK7MfnQOal/fG56rAwNo+Mo+AFrtf6JsmRn0uP1bAQAX4cQjHum0cS24QaR9TzxMz9aPEvHqHrO63LE0UqgWJmFjDdq5ajm3IACVR9G0Zm6PaN7dtoZpQVv4NjMIn15Jt0ildATO2E6PDn2IXDus2M/SrCIfpnOrr+T048GWXWdgAYSi2L7g9xJ+SnNR4nEA5QKPTZX4IWwN3Y89RuLBIAD+fHQSVTXuTLoUxaYuUeiSLOcc7cRnMRx5qmiiYUZGTf7oVM+ied4JfypLdiuK0vRWcWQb1QpktzlPw0VD2Hf/XFxqX/0vd2dVjhcbB7VASTpo0fzDxu+gybnRTBj2A494pDP7ly5UTTLPdK21kCWqFUl6qyXde2/ho4A9hK8aRu0vM8x+FKMjUpoSVQpVYnNKI1TZmZI4BFKoEodAClXiEEihShwCKVSJQyCFKnEIpFBthBLTgLgFjXnvxG7q7VZx4tMYRNN6oFLb2jS7RI6jWhlD20aceaGALc3nUUHcuKqp82MDb3oIriOgqVKZIx+E4BeQyTf1vyNR68fUYX1vuWBcjqPaGUlvtWTOd9M40OJ7KggX8hUdWUo+df59FkPhYrse8/+ysZV3z+kJLam39hzxneawo/Fi6jo509U9i4t17n7XwlXkXL+VyHq8Ob8P/YwqancSdLl8l96CXQPvQ9l7mOocoOXqvmxrtJiazmmAv63NvTNUamKnN+Fkz5noFQMp+hw6fvMqmlzYP3o6Fbqeg2llzMI8lkpuhapeLeZMnEIVtXGlf98DA9nZUI2y17KHiFkDodEQN60p8T1moVcMfHyxDkMfHUa1CVvQehpriaerbi9zPrJEtQLHxnhSq3ABc6YhD9fvii+iVvt481i1vQBkG8peTVqT2K/vI6690StL8wkjCVx/GiX5EJn9WjCzzxxGpLQiqX8wEF+mfKRQrYDINVZcOvQ8/PZYfH8pvi3l7FN1ecVvIwCvL32KGpS8bcUeqfVpNl2CH+H8klAqzduKDqMnwewqKtq5avngcgDOx8smUpBCtQq1ZlwkukYfXL/zvUGk6rpRvDT6Z9N9zcUZGKxtYBnQHz4O7aESxkXYKldXkkc1Zt9L05mfGYLbE5mm3bBlQQrVCuiPx1Oxe8nPjo2rwFOeaQDUmzuKqgccpzQtCe8N7qyuPp1vM4NYEROBPuPunGlcj+xM2ZD8rs34pa2xfTc/M4Sw2SfADsa175a4Gc1ZWP1P9hXoWDS8C/qMy2ZLW5aoNkLt78fk6TNo6Kwh3ZDLvE964HvWcUrT/K7NKPA0zqIZ1FB9RCxx1b/i8YSHyXotCPXWPWbNTwrVBqg8PMhd7ElDZ+N/f+f9A/Fb4Bgi1XZowotfLaKD2w5cRHH59D35EHnd8hGZ5p9Zk0K1MmpfXzIX+fJPHaMnvRZ7+1DxsdN234FSV6pEwvQqrG8xtfA0lxulk/ZRGC6ZlvFsLduoVuboxxH8U98o0s/TI6j0dKrdO9FVV/SnyqpcjrT+tujIoUJWZPuYrk8/qS9xUY1wceHSoBjivmxO3ILGd2WDLFGtyKVBMRzuNhXQcNGQy+/D26DK2Gtrs25L7JehrAr9ulhYlpLPwwf64/dMOq98HEZ8l9nEdZjHtINhTPu9E0In8DsE6bVh7CMrGeK9lQRdLoOO9r8rG6RQrcTFITGsfWcSLsLorKz56peI3LzjNm/ZB96exR0MP57wMBc/q4H36h3ogcjnMwifO5QtD01htM8JRj8x0xQ3VylgSVZVIpYPJ2rWZTwOHbsrG6RQrcClQTGseHsi/qqiatP5ohrRpC7K7hvn+5VWDRH/7bOihbfm+fB/OVig5YltQ6m4wg3vtYdxvXLNl8ygJ3LwLh4e+ypZDYzNmMprnHHKMaDO1eP0524i2F6mdrhcj2phTr3XkoPPTUPFjef36NCjVfTc99cIlHxj2257pyl4q5zRKno6vzAG92VlX9Bh70iXPnaAolJKFCmABjUaoSau/bxrQo1NA41Qk+uv4uaeVe8tpFAtTKV9BhrvfApP13xSkvyJmmVs7xlcnYgfYixFd7efhpfKFYAUfQ7t/hpDtcUq/H8r2wkp5QlZ9UtsjtyKIik3SKFKHAIpVIlDIIUqcQikUCUOgRSqxCGQQpU4BFKoNkDboQlnXmmJ099VWJuyB6e/qxA3tYWtzbJrpFCtiDqyJmm/1mLpN9PY99J0lkesJk2fwzNBWzj+2Aw6HLqCunaEzew7PyyGzoczOLn4PpvZcDPkFKq1iK6P35Qkfq22gZM6hTHJ7YidUQe/PZfgYgbvDwhn35jpuC7Rsb5DbXRnz1nVPE1YdQaOXssIn5MENU5nPjWsmv/tkEK1AqJJXYYtXE5X98vEagsYMfolXFftwJttpj3vQZ+lER42jD1dp7DWPwasLNTcMH9G+Jy0ap53ghSqFaj21Qm6ul/m04t1+ePNNriuKnnBdOSwHTzWfjSaQzd30WgJNGHVaTLRMnudzIVso1qYC0NjmB2ylVhtAX+Ov/+mIr3KrfyIWooTTwfxfsA+q+d7J8gS1cIE9j2FXjEQooFzz+aR/3gTXE67UGVLAU6ZWps77dXUqMaIJ9fY1IbSIIVqQdSBAcyq+TPgjrtw5mCrbzmq1XJe70H1Zy/jLmDGpRh+Wt+ayNln0J08ZVX7NGHVqfbTuRvapm/t7kkN7MvrtRSqBTn7WDhV1MYV+4/HdyFhVU1C115AZGZj8PfidFdfPhz4HRP672P3EzAhrIlV7UsYEMTKoGU03dUPfvNj15vGE6g/aLJC9vrvJSrtzQZgQ64LBT0LCErfUuTZLjmFkP0wa10Puq7+jiYukN27OR5LrbNH6tLqSP687zMOFjhRcZIbepd8q+R7t8jOlAURW/bzwAsj+KJnb/TpJXu1U/YeZvy5ZqgQ+Iw+bXGbNDWqofuzKsvv+5oqanee2D4E1ebivgWWnbduyV4apFAtjMfS7Rhus5d9+aGGGFCo7nEJ4eRsUXt8F17m99orqKJ2p9X+JwgfffaGOMcX1bKoDXeDFKod4Bpv3Ng3NWgr6op+Fs3rbI4Xb6Q1pvHEUfgNuIz+/HmL5mcuZBvVgqjc3THk5Nw+YiGZhjyL+0fVdDjNPqAyW27wBK0WKvSKfbprkyWqhcjv2oyQjSXv578ZT8U/hu5cqoUsuj32KlKQQrUYSR3UfBz0O5n9br18T1OjGt8OmooKQUJaRStZ53hIoVqIyG8vc9mgsOrTyST82BBNjWolxjv5VDCNnFUYUAgbWPbTQ8qCWtivHGQb1UIY9h+l5/RX6fHUZo63/ZptG+DpP55Hk66h5k+Z5FTzILmnnt3tJwEuNN7Rn6CcI7Y2224plVCFEInAFUAP6BRFaSqE8AN+AqoDicATiqKkF8Z/HRhcGP8FRVF+M7vlDkDQxC3sXVSVt1cJ3qq0i9hus4wP+oMKgQGFfEXFgMQOhA67YJZjbsqCqY360CWYbltbrudOyvoHFEVpeI3rlfHABkVRIoANhfcIIeoAfYC6QCdgphDinj3bW5ecwu5GKlp/MIbGO/rzRqrxv29DrrEUjf7qZS62SkefmmZTO91OXGRmhnHa9PXa62xqS0mUpervAbQrvF4A/A28Vhi+WFGUfOCkECIeiAYHOo7OAlSatRVmwQGgG8aZnyDsp6rXnUhk5k9dGfG8nRWlhZS2RFWA34UQu4UQQwvDAhVFOQtQ+BlQGB4MJF3zbnJhWDGEEEOFELuEELu02Pc8871C1f9toUtwY+ZH2teCFCh9idpKUZQzQogA4A8hxK3mBEsaPLxhFFtRlDnAHDB68yulHZJ7lFKVqIqinCn8TAOWY6zKU4UQVQAKP682spKB0GteDwHOmMtgyb3JbYUqhPAQQnhevQY6AoeAlcCAwmgDgF8Lr1cCfYQQLkKIGkAE4BinKkjsltJU/YHAciHE1fg/KoqyXgixE/hZCDEYOA08DqAoymEhxM/AEUAHjFQUxdYjLxIHR3qcltgc6XFaUm6QQpWYheTXW7I6ZTerU3aT2zPa7OlLoUrKTNqoluwbNQ2tYjw369EPfzd7HlKokjLTtL/lt1ZLoUrKhKZaKJWcrxQL+z2tjtnzuaeFmjaiJdNO/cfy5B23XeAsKZkj46vwVqVdxcJEP/PvFCiXQlU1qH3ThcpXSR3dkh1vTqOmxg0XoSE1xvbDdOWBuutHYLicafZ0y6VQI+bH89Pmn6i0xYekN1uitGqIJjjI9Dzp7ZbsHF/yQbqS0pP9WHN+6jTDdJ+qL6DKn5o72tBYWsrlCv+9Hzbmmw/SWFDtLxjxF4yABF0uP182Lq97wnsiqsLDca8SuFWK9k4p8FBRz6moJmq/eiwRiy1zfmu5FKr78u2sXVeFHx/tTHqUish2JxgVvIHRfnupIFy4eoLzpEtR/BDfDK8fPfFesefGJV6Sm6L28Ubdx3qLvculUAEMeXl4/bgNLyAfmCzqUXuXmsmVd5BpyKPjhLH4f7eTIJ1x8bIU6Z0hKlRgQ/1FVsuvXLZRS0L3QGMmVzYu4mo1+xX8vt6KotPZ2CrHxkmoi/0ownJf93Jbol7PyWeMn0uzfQn9yDoe88ozJwZVQ3vdojihWK6dX/5LVCHQVA5kcVvjDtC3lvQDg1x1WFZG9VlV7H5GRk0iFpi/t3+Vci/UnF7RrNy9jibOxo2w6gKB2tfXxlaVPw5nBcOOgxZLv9wL1Wt7Eoe1Bab7Q0OmM3bXJhI/iEEdWdOGljkued2jiXAufrzQPxste4hauW+j6i9cZE9eVWo7naXr44NQ7TiMPqY+NbbuRK+XTYC74exTebR2zS4WVmO8ZXfDl/sSVeXuTn/PcwxI7IDYsh9Fp0O1ea+xx28HuxsckYjxGSzJCioWpg4MuEls81DuhZrd2ni26L61tW1sSflBl3iaK4biM3vBK7NvEts8lHuhnu5lYFs+VF/mGJ6VHYUZi7oXu+/pv9uiK9DKvVBnt1nAn1fqoT8aZ2tTyhVVP9tNowVjTPcTR/fHd98li+VX7jtTAL993AZPLLNY4l5Fyc+n+ptb6fWmcX+UCzst6o2wXJeohvsbEaq5jKeFVvRIrEe5FuqlOq50X/Kyrc2QmIFyXfXXHnCU9L6eyKUnjk+5Fur5lhlAho2tkJiDcl31S8oPUqgSh0AKVeIQlOs2qiXQVKmM4uPJ0THGpYLtGh7lwDf1cM5U8Fq8Xa4fsBBSqKVE7e9H6mNR/PP2F7iJ4idAf/tiHN0rJNCy2ygiP8pBf/i41e1TeXoinJ144J9TjPNLAKBLrTboM82/x94WSKGWAnVgAPofndlRawZw4zHlz3qdAdw43m4+/8WoeHfYc7juPWW1k5s1lQPxXKplQXXjcV755bBQl23UUpD+YBhra60sVdxWLgb++GYOp56LsLBVRVz82tMk0vKKFOptyOseTcQLJZ8HVX/WKJpPGMnYczf6A103/DMuDI2xtHmo60QSVOHyDeGTL9ZDKUcLw2XVfxt0Iy/wTdW/TfdvpDXmlwONAQj/Kwfx3z5il3jTPfAJav94gs8qGx2GBavdce6VVnhAkeU48a4L+2uuLRb2xrnmHOtaCUO27Y5UNzdSqLdCCNTX7FVv+uEoPNL0RCwpvt1an3EZMi6zYlMLPnpiOxqMGwmfrLqbRf074/O9+bdpiKb1uBxZgTqVE4qFj0h6gHP9KqI7l2j2PG2JrPpvgaF1QzbWW2q6r7IhDY8lN/cJEP7SNlpNeMF0P9rnBJc651rEtthnPfh74jQWXVeanh4Xge5EokXytCVSqLcgI9zVdJ2gy0UUaG/7TuBfZ0nQWUacVzHc34iI2inFwnrHPUL9zYNxSrtyk7ccGynUW+CaUeSQ9o3TPTCk3n64SXcikT77B5nuJzZbYlY/Aoa2jeDdC6ystdwU9sixXujGVaRGnwPoj8ebLS97Qgr1Jqgr+vPJ5K9M9z+F/Y4IDbrFG0U4/1wkzO7umQiXG8de7wbD/Y3gnQusrrWsWPjFRaEoOy3n/MEekJ2pmyCcnGjhcnfveiZZ5rTs1Gg3dl4n0vv39aPivivl3huhFOpN0KWep9HOp9jbbKGtTSmRhw8/Tto/QYT+cQVlV/kuTUFW/TfHoEdsLN62rP3jidu2N9WBAXSY/q/pPnLjYHSpZXd4q+3YlOZPFB2Tc2Z3FUI/3GJRf0/2hBTqLQj+MY4PLtQz3X9WeRf1/0onu3fzEuNrQkM4/VUlXvEzLkpJ0+cQ9VG2WVZUXa7uxPSQv3nkWC9aTXiBsKXls3d/M2TVfwv058/z15ut8f40h9E+JwD4KGAPw171IPFCIwA06bkYXJ0wuGloM/1fk0gBHj08AK8jsWW2Q9uhCd1GbqJ33COI8b7479xa7tuk1yOFehtcV+/g++DOPPrmRILV7gDMCtkMP24GYGe+QpAm1/TsWgpWBAAJN4SXFtGkLqEzE6ntsZFtGTXQv+iLsu/eqOqvR1b9paDi7K10/HZcic+auYgbRBqrzSP6fyMJ/KnkxSylQV07gp7f/830kL85nBVE1lB/DPvuPj1HRwq1lIRNPU6PuK63jZeiz2Hway9TcfZW4xqAu0BTLZTBv/7GQO9EAJKzfdCboQnhyAjFDrZOeAk/pblob2szbovKw4NLj97H+fYFxD00F7VQoVcMqIWKsN8HU/vNsygF2jItmFb7+vLm7g00dSlaonfZUECrzSMJ67fPDL+F/fGnsmS3oihNbxVHCtXeEAJV/ShWrPu+WHDLPU9RsXv5LFVLI1TZmbI3FAXDgWM8EtysWHBFyqdIS4tso0ocAilUiUMghSpxCKRQJQ6BFKrEIZBClTgEUqgSh0AKVWJ24r5szm9n9hE760bHHHfLPTvgf3FIDFkPZdEj4iAfBewBoMuxRzh10RefXz1wzdDj+ucBlHzLbCspjwgXFxL+15gdvSahVVxv/8IdcE8KVXVfLVa9PZEqmgrolaKdpmtrrTTO37cyhjXd1Q/NCl/8vrHsOZ/lBXVQZY48PR0wr0jhHhWqyNOSqnei5YbBaM47E9zwrOnZhrpFm+d2Nf0RmkJD/1EETdpiC1MdistfqU3XPWK7EzV6n9kWeN+TQtXHJtB7xRg0+YKw605FfiS4G5nNQ0npoeWXNrNo6Kxh4rD5TPvpIXRJyTay2P7J79qMqVEzADXjzzVD9Bco1xw/X1bu2c5U+EvbbhApgC7lDO7LthMxYA+P/TYKgIfcctEH+ljZQsfi4nPZ3OdsLFHjsgLQJafc5o07454Vamno3nQvAAuvBKA6VX4841mCGQ1+NF0fOlDN7OlLod4Elbs7g/yN2563XQm3mvdoRydWW0CtqWXfHn49Uqg34fjM2tR3dgLg38WNbWyNfZP1RAuqa7IAeGTFS+jjT5o9DynUEsjs24Ld7acBMD61CVUmyx7/zVB7eVHwzCWqqN3YkS+Ies8yB21IoV6HpnpVur++ES+VcSzw199a2Ngi++boxCi2NFoEgFbRoE9Pt0g+UqjXof9az2v+RwGovflZarwuB/tvRaPaiabroTv6WywfKdRr0HZsypqoVQB8nh5BxBvl44wmS5HRP4YZ1Y1+WiPXPU/NoScslpcUaiHqyJp8PnsGAOmGXNa9/EC5dDFuTvL8BRXVbgDUeT8VwxXL+cOSQsV4Kp/7/MumXv5Dn47D6fddNrbKvsnp1Zz1L38GwLCktiiZWRbNTwoVSHg5ip/Cfgdg0qUoKi+4N/073QnJnQym0nTTpvoW60Rd5Z6c67+WS4NiOPKsscrflg9/1fcA7i2XjmVhY64rNVbkWDyfe7pEFS4uPDT6PwD2Feh495lBt3lDAsazV5vVNXac3nt9EGLL/tu8UXbuaaHGfdKI9wP2AfD4P8NR/bvPpvY4CvqQSiys8TsHCvS4XLbOMZb3tFA1QddUWZn3fCvojhl2+GmrdTrv6b+OctIDWhuvq9U+Z1tjHAhl1yG6BTfBz4r+sKQ3P4nNKY03v3u66pc4DlKoEodAClXiEEihShwCKVSJQyCFeg+T2yOatSl76HvsjFmParcEUqjXoPb3I3ZOM9am7DH6TprdDHWlSrY2y2JUHHsSAwp9PVMQFTxsbc4tkUItRNuxKR3+SSS+62wA9IqB+G6z+WTnKhJ/us/uS5y7obXf3Z8qaG2kUAH9A42ZMmc6Y3zj0aGn/tZnaPrRKP7LV1HXyZkjrb/l3IIAW5tpVtSBAQQ5WXZpnjmRQlWpyXstg7pOzuwr0NHm9RcIfewQATO28El0e55LagvA1/W/w9C2kY2NNR+JQ8LpVcH8++8txT0vVE1wFf6pvwSAx34bhc93RZv5DFeyOPd8CPsKdNR3duLCy7m2MtPs+B/Wo0KFCsGiK8EoWdm2NumWlEqoQggfIcQSIcQxIcRRIUSMEMJPCPGHECKu8NP3mvivCyHihRDHhRAPW878sqNkZfNHrnGlOoXHOqojwkh6uyVnfwrjzIO+PLZhBAAR/uXHW4rnlpMszapY7jpTU4H1iqLUAhoAR4HxwAZFUSKADYX3CCHqAH2AukAnYKYQQl1iqnaAPj2dkasGAvBF659Qubpysl9llg6exJ5mC+k+YDPvtV4BwO7Eqja01LzoU9M4o3WcDuJthSqE8ALaAPMBFEUpUBQlA+gBLCiMtgDoWXjdA1isKEq+oigngXjAfD6yLUDNn41Venf3TDz+qECNL48y5NjTALxXaT9PeRrbcp473Gxm471OaUrUMOA88I0QYq8QYp4QwgMIVBTlLEDh59VucTCQdM37yYVhxRBCDBVC7BJC7NJiW/fjqj3HiPzZWL3/FPY7wesLTO3WX7L8yVLyWZXjRdB3h21ppsX4ITO0XLRRNUBj4CtFURoB2RRW8zdBlBB2w6JXRVHmKIrSVFGUpk64lMpYS6Hk5xP5+j5a7O0DwKyQzQCMPRfNggZRdHnpRWY/1h19xmVbmmlWLg6OYbRvHE5CzayENhbfRVpWSiPUZCBZUZTthfdLMAo3VQhRBaDwM+2a+KHXvB8CnDGPuZbDkJeHz6fFOxRbvmyGIS+PCr9sx7D/qI0sswyqRy9gwIBWsc6ep7JyW6EqinIOSBJCRBUGtQeOACuBAYVhA4BfC69XAn2EEC5CiBpABLDDrFZbCEVTvDJQ7LYLaF4y9/nb2oTbUto9U6OBhUIIZ+AEMBCjyH8WQgwGTgOPAyiKclgI8TNGMeuAkYpi/19bdWAAgR/FFwu7b+hBkr+2kUFWJHCX3f95SidURVH2ASXtaSlxo5OiKB8CH969WdZF5epK8mx/VlVdSJo+h+35lenunkkzr5Oc8aiOIdu+OxplJeUxLeErbG3FrbnnZ6YA0ns3ZG8zo0hbrn6Z6UOfAGCIdxLUCL3N246HcHKmcoUrqFDhJNT4+dj/F/GeF6ra34+XJxgd0T56eACRI3agztUBsCrHC3Gu/MxGXUUdFMgv4atMnamLJ+1/4P+eF2rca1H09kjnYIEW3xcU1D7eJL5kfDYrqS36Cxdta6CF2ZEvqDUrw9Zm3JZ7WqjCxYUBnTYC8GtmI861D6T2hiscaf0tAGdXmf8YGntje044+sOW8btvTu5poarc3elQwTjb9FbFQ+x8ewafVd5Foi6HOt+OpPLU7bdJwTHRn0mlwbwxHCpQWHjyln4f7IZ73lOK0rIByvuXWF/rV1L0ObRb/go1f8mTDtOsSGk8pdzTvqcAxJb9iPbQBeNZUhFss7FFkpK4p6t+ieMghSpxCKRQJQ6BFKrEIZBClTgEUqgSh8Chh6c01UI52zmE9IbGZWr3NzjG5v21UHloOfbgPFO8N1KbcqCx7ceLJXePwwr10qAYlr87kUC1GwYMRQ+qbkCFqljYB4E7iH5xDJWnyOPMHRWHrPp1DzZh4YRJVFKXfq/V1NGzLGjRvYvK3Z0Tn8TgsakSUbucCNzqxYlPYkgb0dKs+ThkiXqmjQvVNM62NuOeR+3ri/tKNUfCpqNCYLi6h7P/RvIVLV3OvIDbCvPsQnI4ocYtaMy2BycCJQu19p/P47vFBRQYN3axQ/lXciTUlSrBz04sCltlCpuVEcaBrBBmhmzCRTgRMPYEV1aYJz+HEurZl1tyvMM0wNUU5iTUaAu/yE0+H03EZGM7VLi4sHZgfXpXMC7jO6fztra55RZNlcroftCwOvJXcpQCuh56igpve6A+fxltsB+7v99EExcIcc/AXHt3HaqNunvsNAzX/dMqegwYeOpkR0IXFvn7vPJIQ+ZW3WCK9/FXfW1oeflBU6Uyed87s7rWr6Qb8mg5cywenU6g7DyI4uJM3NPONCnsOoyp+DdnXjFPW9WhhHoztuc7kTm6MrpzqYCxgV/lheJOamWP3zwk9wljfe3lXNDn0m72OEI+Kvp/bf7zEWIf+cp0H6JxY+jANWbJ1+GFOiGtES9+NhzV6bOmMFVgJRaGrTPdN/hPnhptLjr2N7rlbLNoHKEfGEWqqRZK8ustedF/N99lBjMrI4wLeqM/ry4VDpPTq3mZ83UooUZtGFLs/o1zzfn3vRZUmrUV/cVLpnDla63p+sv0WtQcl2EtE8s12b2b87ivsRfvc8wYpqlelXZrjnBg1HS253mxrHM0q+v68vmF+wGoqnHjSmjZPXk4lFAjntlD7T+fZ2ueC9GfjeFQE8MNwx+JH8SwNmotKlRcMRTww9yH0Z1KukmKkjuh7/traeSsYnm2HxV3p6OpXpU2q47yom8sibocpjRrjS7xtEXydqheP0DEgD18zH1U5sY256VBMfTu+p+pg3X/3HFUnSrbpuZmy5UIzrbz46mhO3nRN5Zecd3QtjsLWM7RmsMJ9Vb0eHEj4/wPAir6n+xE2PxEdLY2qpxw7sWWDPOeDggmV94Brxlrso5He6HpcGMpqhIKKgRqoUIpyb/jHeJQVX9JpL7QkvMro7j/QB7P+uwC4II+l7SPwtCl2L0TQcdBAQOK6edggZaIZcNx7lbyhIpBMc5UaRU9wgzrgRyyRE38IIbZfY3nQbVy3X3NAhTjAJ63ypl3p81nxfuN2fZ5U7wXGjfsaaqFEj8kBICak46UK3+nlib4m0MsHF6FIE06L+57Ev+FHkQs337tcqASidUW4JOgvU2s2+NQQhUaDSf+14zDA6abwq6dmbqKi0pNK1ctrSpvh8+2w2dX4+02xWmRNIqKs7ciKR36zEz+t607VX9SEbpuZ6nf+yu7Fi5rSh//ZjiUUDNWVudggy+LfYu1CsWX+ZUy7FIjPRUtaWw5JOLZ3bePdB0rzjZAQ9lHAhyqjbq5wU9lTmNvvorH47sTvqjs1ZGkZDRVKhPsYhwByFx4w/ENd5emWVKxMSOSHmBbSjV8f6yA+7l8FAFCgYTHXXkw5iDTQ/42xf0rqw76JxVU5/bazuByTlrnGoz0WcM3maEE/HPWLCMvDiXUqA1DONp+tum+9p/PU3vCBQznLxKSXXRiydXRkPD/INnTk7Y9RrP50xkAjPU/xPxJLQl/OtWapt+THMkJQnci0SxpOZRQI57ZwyM0K7pnz22/rYYrV/D+YRvdfmhiCgtHlqbWYM2fzQjDPB1Wh2qjShyHhVeqELHAfL5lpVAlZsfv660sqhWE/kis2dKUQpU4BFKoEodAClXiEEihShwCKVRJqRBOzpwfHsPY+MP8nLyVrMfLvr3kTnCocVSJbdBUC+XIO4HEd5pRGOJCnp+KCla0QZaokluiCQ4iY7YT8Z3mABCrzeO+6aMIXGq+oafSIIVqJQr+qMarCQdRe3nZ2pRSo3J15eRUPzbVXwLAq+eaMmLYGEI+3mL1g+KkUK3ES9X/oI1rAWlP1r3hmSY4iFM/1yejf4wNLCsZ4eSM2++eHIz5DoBOx3pwtKMPzuvLvrb0bpBCtQLqqHAqay6Tqs/FM+nG1QnnulZjf6uvaf2ifRzApgmrTsrP4fxS8zdS9DlEbHgOTc+MYlvSrY0UqhU49oI/DZxhT35AiSVStafjAWjrdczapt2AJqw656Y6sy/6B9L0OTz0/TgintmD4coV29pl09zvAVTu7vRrtQUVKpK0/iXGCXHPQIUKtbjdDiTL47Ygix1hv2NAof3cV6n+vn1sN5dCtSAqd3di50SxotIcNuRWYFX/NoBx3aw6vAY5EcbNMMEuf2PAgF6xbQWX9GZLdtT4HANOtHhvFKFziotU5emJCAo03mRkok+1nktPKVQLcqVzfY48YNyIODHxYU6+4EKXOsadsm28/6KXx6Ub9nbZCk2NarzzzCLchDNtDz5GxTlF60g11UI5Oi6Idzosp7/nPwDU3/oM1QZkY8jOto59VsnlHkT3YBMWfD6Zq1u419Zeiqq26jph2k8XIaVbMI9XMA45eT+Xb1qQnvFMDLPfm0J9Z6di8Q/GfEfTgaMImG6dpoEUqgVQe3nBm6nUdKqAVjGe2HJal8v67NpM+6U7AGFfHjP1oj02VeKnmutt1kZVe3kxeJjRPWSthSMJSzH6Qch+rDnfvz+Jmho3jmnz+fhMZw6mVWFBg2+p7+xEdqtsmH6rlM2HFKoFSO1Tly21vkSrGEvQtvv74vmxJ6rNe6lW6DNLf038Uz+EY5hgwFOVhyY4yOoeXgwRoUS4bDd6P/ksFr2ikDaqJf+Nn4KLcGPW5Wp8O6kbfl9vpTIZvFnjCf731y+81WgNi33ro0+3nM+pq9hP3VOO8EzSkarPZ0RyGzoNHoFf/0uoNt9+n1aMi57ULtWsYOGNPOSWy+acSJScXIRGw6SXZuMiNIxIacXKZ9vh93VRm1WXeJpRx/rylGcaSrUqVrFPlqgWwGXdToa3exrlcibOF3cWKz1vxWldLr7H8ixqW0nkhHoA8N3J5vjlxJLbM5p2rjtosL0/wY8eBg4WRRaC2LlNiL9vDgm6XFSZOVbpDsoS1ULoTiSWfiZHgAoV1TXupNdyvX18M+Ny0eiMo2GlFAAu19CwLR+qDj9fLJ4mNISTH7YgvrNxgcqvV+7DcN46c/6yRLUD8jpmYsDAZUMBbhet36FS5etI1OXwdpXf6PjzcDzXKQzZ8ww1nNJRWjXkwn1uZDQvYFnbmabe/+KsSvz8eUf8rljHf5cUqh3wQFXjFOqWvEq4L7P+fL/mdBprs+oywuckh1stgFaQqMvh4mYXmjhf69bcKNLusd3geTf8Yq3nZE5W/RJ051JZ+krHYmHVNe7XiRR2F+i5b/oolK4Z6GOLnzpjaWSJageohAEVKl5dNIBqZvIscqe4/rGX7g/1IfVTuD/oBBXU+bxXaT9qoeKxhA7s3RVOxMIsQnZtsclcmhSqjRFN6vK0/7fGP745fIjfJYpOh/5ILBW7U3janpouNC58eoFwLmDLg+Rl1W9j8gLcaVB4rKt3vC2lYN/IEtXGuP1zmPfSmrBycWuCv7ePJXX2iBSqjTHk5LC7kYrgEo4jkhQhq36JQyCFKnEIpFAlDoEUqsQhkEKVOASO3etXqVHViwDg+GAf/A4IKu1Mh9hEDHnWXy4nsRwOJ1SVqytXujUgPVJNaIdTrIxaWPTsMeP5mwNPtWffuZqFaykl5QGHE+qJNxtxaNCtN+p8U20D+VW11J/1AnU+PovuVJKVrDMvuj+r8lr1dYydM4Sgz+7tcVaHaKOqI2vS4dAV5p7+lx0DPzeFn9XnsjVfbfrZlg9b89XkK1pchBPzHpqPLtDHdoaXgtPvtkT3Z1X0DzS+4VnXyodo75bPtOdnITQOV6aYFYf47eMHBfCrbyzgBsB3mcFMPtyB4CkaVP/uuyF+7KxoYrt/RRvXAjJ++JX5LZtZ3ftcafmq/yzud9XRtPYoAjYWf/Z7Wm3G+MZzv6uO8X2a4v3DNtsYaQc4RIka2jTFdJ1uyOP7l7sT+tihEkUKUOfjs7Tc2xeARzzSyW5R0xpm3jGqhnWopskEoMovcTc871l5n+la52q7lVX2gEOUqE6dztC290h8t6WA3oBL8q1dH+pOJXH5UAw0Mt67J2fZiT+SItQV/YmcH0tVjTujz7TEkJl5y/juj5+DeVYyzg5xiBJV0enw/GkbulNJ6JJTbv8C4JFUVAJlhXlayrS7Rri5MbnyDgDerbwBVQWPG+JMWfKI6TopoZLVbLNHHEKod0N2SNHaTr2T/VWbhvMXGHPG6Lj3tZROGDKzbogTNuOa7R5O9/Za1XIr1D+fmmi69t1WulLYmghvL4ZX+huAwxcro2gLboijjSw6695nv9MNz+8lyqVQE9+PIVjtjgpB7b+fs8txVH1qGiNj+94QrgkO4uzYllTf4cZ3C6eZwj27n7WmeXaHQ3SmSoO6dgSZdf0AqNQsFQMK8dp8ak693UHptue/houp99NAdFo1+9t9hZso3JuCu03tsiccWqjCxYUT7zVmSLffaey2jHauWgzXbEHbkx+KJjUDe5Wq2yuuPDenLfNC/+FI628LQ53JNOSxILMOo31O2NI8u8IhhSpcXIidW5dN7b4kUL2Fy4Y8jmg9UAsVBqXI09MTFdI4uSqOLX3uM+uR3ObCcOAYqYMi4Y+isPaHH8VjYAG6lDO4HglliLex2ZJy3odwG9lpDzicUNW1Izg23pPj7eeiVZzpdqwHeVOCKPBU8e/EmTfEf83/MBfW7+K5+/vaZVvVkHCKuvNGUa31aVJ+r0rwZ1vRKcZa4avYNgxpZlx04/eX9X1S2RMO15lKvb8ixzvMBaDV7megfTKuq3ZwJbT4r9Jyb19WZvsCUFHtRr/ft6Br38Tq9t4OJT+fahO2QPtkgj/dAkpR0yX4+aJpX7cnz9nCPLvB4YSq0hZdXz7tzYlPY2i2T8+O0VMA4xRrg69G49c9jrn9HqH9occAYzMgZvIOsp5oYQOr745Tg4oq+zOHAm1oie1xOKFWXHKI8eeaAXD80ZkceXo6Eyrtw0momXs5lGc7PkvoB8aSSdl1CI9XXVmaZTx9ZEKlfWT2vfVUpT2h8ygqXSve3g9wucbhhGq4coWDo+rT7ViPYuF9TzzMyl4x6I8WX9xh2H+U7x7tSNQfQ/kvz4mKFaxzioc5CJ9nfxMVtkIoiu2n5ryEn9JctL+jd9S+vijBRdWhEncSJT//pvFVrq6oKgegSzx913ZaG7WXF8N276KrexaRfw0mvH/5LFb/VJbsVhSl6a3iOFyv/yr69HS4g0MODHl5GBxIpAD6zExe3vkEXdt+TXBAhq3NsSm3rfqFEFFCiH3X/GQKIV4UQvgJIf4QQsQVfvpe887rQoh4IcRxIcTDlv0Vyjdh0xQa73zK1mbYnDuq+oUQaiAFaA6MBC4pivKJEGI84KsoymtCiDrAIiAaCAL+BCIVRbnpmQt3U/VLyg+lqfrvtDPVHkhQFOUU0ANYUBi+AOhZeN0DWKwoSr6iKCeBeIyilUjumjsVah+MpSVAoKIoZwEKPwMKw4OBa6eAkgvDiiGEGCqE2CWE2KXl5p0giQTuQKhCCGfgEeCX20UtIeyG9oWiKHMURWmqKEpTp8LzQiWSm3EnJWpnYI+iKKmF96lCiCoAhZ9Xz8ROBkKveS8EsO6ZiZJyx50ItS9F1T7ASmBA4fUA4NdrwvsIIVyEEDWACGBHWQ2V3NuUahxVCOEOPAQ8f03wJ8DPQojBwGngcQBFUQ4LIX4GjgA6YOStevwSSWkolVAVRckB/K8Lu4hxFKCk+B8CH5bZurtA7eNNfuNwTj2np22Y8aCxIQH/oBIGBiwYQ9V3Hcc1TtbjzXn5w0X09MgoFr4i24eeHhnMzKjBugej0KemlZxAOcJhZ6ZKJLo+mf/LZkP9WagwHkF+7ee+IVNplfQC/vNtc5bTnfLT5MkEqt0AwbZ8eOX441T40BN1tpZHVn9HD8/DrPNsDKm3TcrhKVdCTXnAk/31F2BARao+l5kXW7JodzQecc7k+yts6DORgWNXs+aXGuhv4/DBHgjRVGBKenXWDW4D2w7ghXH7dJVtnqgQtF39MpHx90bzv1wJFcCAggEDz/UejrLzIJHsMj1rr4zjhW5rbWjdnbEtT8+aYQ+g2la0GCVhUgvWhs5kWz7UnnG51EesOzoOt8zvdqgQqFCh7Dx4w7OwpVmsruuLPjOTU++1JGRbBc6uqG0DK0vHqy8NR7W5SKRZjzfnrycmkaXkM/iHkegPH7ehddal3JaoJbLjICmvtSSsywkOhk/DgIHWc16wroF3gNuvRdV6bs9oU5u1/vxRVHvHMdrZ5qJclajBn25hb4Gx43RxcEyxZ7oHmxCyrQL7X5jO0vA1rMnxpvXbL+D3tWP8wZN66KmidqfhtmfuOZFCORMqwIAFYzBgYNmEiYhm9dGEhhA7rykfzpvDzNCNGFCIWjKSuQ+0dRiRXkvV922/0N0WlLuqv+raK6QOyidY7c6qFd+iwujXX4VgZkY4Cz7vQsT8rXbrlOJm1HknlRWtfKj0VTKpMbePX94odyUqOw7SftE4ZmbUwIDB1GaNWDqcdTHVHGYM9Xp0Scm8vuwp5lfdyMUh955Sy51Q1T7eBDZIZYTPSVSoWJPjzaPNexLxwnaHGDu9FREzkojVFtBi6B5bm2J1yp1Qz/Svy4b6P5lK0h1ZYaV2/usoNKpwytYmWJ1yJVRNaAhDhq8ylaSp+nx6+uxG96D9eUi5G04/WZVaTi78drGurU2xOuVKqKf6VWWodyIGDMx9oC3tF42jkbOKU4PtzYP/naMJDeHdoT9gQOHEt5G2NsfqlCuh5vsbe/fDkh5El5yCd6xxpsrRUbm6Ir7X09Mjg7fSmuA/zzE7hGWhXAn18Yf+w4DC5gSjzyb/+VsxoHB/zXgbW3b3aEJDUK/3ZXnEalL0Ofz3YXNbm2QTypVQV5+qi5NQm+4vDo5BhTAJ15HQhIaQsqwus/9bzK8Razirz+XR98fhsWS7rU2zCeVqwD/3uA/aZnoWxMzn+ddG8d7gH4p5oHYUEia1YGbP+bR3y8eAG9MywljxykP4r7v3qvyrlCuheseCk1AT7aJn/wvTMaCwJscb3w327QRX5epKwruNCG9+isHB/9LTwzhO+v2Vykyf1Bv/eVtx4daHwJV3ylXV7z9/K2FLnjfNSLU58IRDzOkLby/87jtPx4AjPOKRzopsH2p/N5KfW9S+JztOJeGw3vwk5QdLuPSRSGyCFKrEIZBClTgEUqgSh0AK1QaovbxgQwizTv2L4f5GtjbHIZBCtQGpfeqystZyQjRupNey7zFee0EK1Qa4Pmp0bZKqz8UzydE2xRiJn9KC1Sm7jbWDFZBCtTLq8BqMCdsAQLufX8F5vWPOOFWte/bm29ItQLmaQnUELn2pppfHJUBFzVe22doch0GWqFZE5e5OVa90DBjYkOtua3PuGqHR4OlsXXf2UqhW5Ern+nxfYz3rcnx5bdpgW5tz14g64fwSvsqqeUqhWpEFn08G4MPjXag81XH8tF5PQj/f20cyM1KoVkLt70dNpwqoysF/ubaS8YjvJtufRX/lilXydPz/NQfhxAu10Cp6DBjQ/l7R1uaUiQp+OQC4/OYFVlp9J4VqBdRR4Qzu/RsAky/WI2jhsRLjXRgaQ3Zv+98TtSf6ewD8D+ZYLU8pVCuQ2rYSL/gaxfnTN+3RX7xU7HnC5BaMjItl24TpLJvyOSPjYkmY1MIWpt4WfbvGAIxKbod6j/X8s0qhWgMBKlQ4CTVOWcWrypOfxHC0zww6u19BhcBX5Upn9ysc7TuDjP7252PqudnLcRJqLmtdMeTlWS1fKVRroGBsm153ipHSqiGTei/AgIG30prQ/N2RxLw7isg1wzBgYOJ7X6EJDrKR0Tei7diUlm5JaBU9B3+PsmrecmbKhrz23fe0ds2j9obniRp6BP884/6owLVBjLjvAWaF/kNql2r4z7X9wYeaKpWp+v/2zj0+x/r/48/Pfdh5DpthbNjsxOY4pyEqIhRCKhXJoXIukerb6dvpW5RzCdE3SQglX79SkmTOh2FjG2ZzGGPWxs73fV2/P+65Z20Ydp+2z/Px8LDrc32u63p7eO39Ob/f7x6ljtaZgcf7EDD/mFXzB0iPagW8juaRbCgA4NEJv5mb9C4uBWzJ9SDsP1dLNKOGs+dI7eNkE1tvRGFgXeb5/QFA4u+BpfrZlkZ6VCug2XaACQNG89LKVUzyimPSf+JMeQ/RcJ/rVZYsTOdUZjB9/Q/T3DWFnm6ZAGQqBbim20fcrFPjivvWDd6x/mKFFKqVUPfFMmPoEBZ/fIZlAT+XuHft+lryNoDeRwci3q+F2xbbR0bRuLjQL/QQAK+cjwIbxOuWQrUiIjqGKz3c6P7gOM70VEh4aEGJ+8OTu7H9cDDuSXr8ZuxGNaTYyNKSpD/eivfqzAFg58w2VMf6u76kUK2MkpOD29pdhKyFh/hn3Na/CSmKiGL7aAvFZBSl4vorzwXvv87ZJP+BHExJys3o9aMwnLKNl5eRUiQ2R0ZKkVQapFAlDoEUqsQhkEKVOARSqBKHQArVAmjc3ckYFkW9nZ5kDItCF9jI1iY5PHLC3wIkLQ0kppNpJYcPfqfJb88RPOyUTW1ydKRHrUA0Li6c/LYlhzt9ZS67ohRQfU/ljC+l8fTk3aQ9vHnS8rlZpVArkOSXW3Ok6yLz9bqrtbl/1hRqz3Pco9E3I3V4M1o5aTBaQUay6a8gcvu1482nV5ivv8nyZ03/zvjGV06RArinGslU8vDWaNAGBWA8nmSxb0mPWgFofXxo/OpRBnpcQoOG7Xl6lo/vgzHecTMGlgeP1buIK3QnTO/MuV6+Fv2W9KgVwNlFtVjn/7U5tt3HD/ZHn7jPpjZZkz35Kr5LYiwa20961Lvk3NSObI5cDJgGTm1mTsSYeNLGVlkPLQpGBEp2tkW/Iz3qXbJ/4lwUTOeb7ts7inozKm+ftCysMZAC6VHvirQxHc0/j0h+AP9RF2xojW14ZtuzVvmOFOpdMGzsRjQINGjYsTMM46V0W5tkdZzdCq3yHSnUO0Tp2op+HkdQUFFQCFxXHNhW6+PDpdFR6P/wxSe6Bmdf6XiTNzk2X0Qus8p3ZB/1DsnzcqKezhmAORlh6A+fRHVx4UqfFnz2yWxC9driyhN+pWmz0QQ9dcBG1loGjacnLqKQQlV768p3+y2Lf6EKsOpUJGpePqemtWbznHk00etL1dnQeT50aG4D6yyHNVempFAriONLwjg4arb5utvhx2i5aKL5OkjvTG7dyrXmf7WBzIriEGjQoEEQ3WqFuQSg3fvj8V2dSPBPF80RpvXC8s2jtXm2xxarfUt61DvE9XwexwvzzYMpBYUd+Vo6/mscGqOKyxpY4P+7+d6gE93x2HHK1mZbhGfXPW/xb0iPeoeIHTEMWvgyB8fONZc11Wcz6435tHNWSyQLO1Kgkt07H8VK8e6thVYo6IUWbZ6w+LekR70L/D4ouQrlqXGijXNxMMYrSgFtdg/jjba9Kp1IAYyqxhTz1QqhIaRQ75Km34wrVRZbYKDJb8/xxJBx1HskrkouBFQ0MlKK5I7peiiXKd5xzM0I5peIO0/eW55IKbKPKrljtvdqzJ/Lgjh+wJ/GFo7wJ4UquWMMZ8/B/dCYsxb/luyjShwCKVSJQyCFKnEIpFAlDoEUqsQhkEKVOARSqBKHQM6jSsxomodxbEzJFabB7XfzXm1TjILej41A89dBG1gmhSopIvPJDmz7eH6JXV9QnKRtxZX6aLMLbZZWSArVUmi0nB/fnqljVvK4x8VSt0ee7srWvU3RZ2oIXJ2JwdMZ7Y7DqAbrZ3HSBgXQc+o2AD5Jj8BLd5WNF5sBcDnXDXVJbWpsT0E9G2t1264hhWohTr/anpgxpr2qZR3YWOy/FcX/D9PFM6a/Qta/QNN3kjGct258gGMv1mZdrVVEfjoR/+UnQK/HcPoMAO4AnLRJErTrkYMpC6CtU5vJT6297ecS+n7O01t3I9o2s4BVZaMND2VBzyUA+H4SjeH8BbNI7QnpUS2AcHdjaLUbb9T4KD0co6qhp+dhIp1L3hvocQm+2cBXT/ZB3XvEwpaCy+eXuc81D9Dwy7mDpo3Q1zEi5T6SPwjD5afdFrflZkihVjAiMhzPOak3vN/u/fHUWbQXtbCA6A7DSXxBx5AWe3jL56C5zkCPS0x/q4Daj7qg5OVZ1N6YFD8213NjV3ZjtELBqBY3su3dT/Blgy3EzvmVqemjEdExFrXlZkihVjCaPAPJ80LY9G40PVxLRrj7KD0c35XHMBYWmAp2HiJ4J+xrEcG+dQdLeNe6E/IxWFikAEFPHWBOeH+MsfGl7u0K78+bHytsa/ktJ8cJGtsw/pvso1Ywxth4qq3YybQvSgcPe8U7lsSpoaXKlZijvJfykDXMK5OyRHqt3Pt1Pb2PDmRjp/lcHh5lZcuKkR7VAmirVaPDoJLNZLcjg9BN9yIk/rTNR9C3g3IwDn0fZ/69pQ85D2XhtdQ2dpTLowohXhRCxAohjgghVgghXIQQXkKIX4UQiUV/17yu/qtCiONCiHghRE/LmW+fiFpefOb3p/n6yaQeuPU9h/63feUaUTf7chzG05bfNV9e1Px8cgx63F0KEDrb+LZbClUIUR+YALRRVTUC0AKPA9OAzaqqBgObi64RQjQtuh8OPAh8JkQlDBNyE+Km+ZS4zipwQc3PL7OuzrcuCUva8F3QOnNZwNpMm0z83wxF1fB50+VofWrZ5Pvl7aPqAFchhA5wA84B/YD/Ft3/L9C/6Od+wHeqquarqpoEHAfaVZjFDoBwUoriphb9ETdeeEx5OpCkBxfjKpzQIHjnYks05+3zePXQ/cMxpJ63ybdvKVRVVc8CM4AUIBXIVFV1E1BHVdXUojqpQO2iR+oDp697xZmishIIIUYLIfYKIfYWUra3cVRUlaJQP0V/1LIjiWiDAhgxbCOFqhEFlXQll90vtLb6ypQjUJ6mvyYmLxkA1APchRBP3eyRMspKuRRVVReqqtpGVdU2epzLeKTycC6rjDPvGi0pH7sxtsYJc1H3fSMROyp+rlJbozoJn7cj8atIMp/scNvPq1EtWBh4+yttFUl5esbdgSRVVS8CCCHWAh2BC0IIX1VVU4UQvkBaUf0zgP91z/th6ipUWWou9ixVlvt/DTgQ8bX5etip7viPy7LIjICoXo2VD86nlZMGHgA+hrBVYwmbk4py4SJKTs4Nn9V6e9Fj8VY0gN902w01ytNHTQE6CCHchBAC6AYcBdYDw4rqDAN+LPp5PfC4EMJZCBEABAO2XX+zMVkNiv2BNjiQpA+j2BKxxlx2xpBL6r8bYzhjmZG+Ifk070Q9xAOxA7lgzEVB4djg+az963tOfBmMtknwjZ8Na8ALNRK5Z9EU2HnIIvaVh1t6VFVVdwkhvgf2AwbgALAQ8ABWCSFGYBLzo0X1Y4UQq4C4ovpjVfUfC8hVjDqDklE/A21IYzp/f4QfvVdzrYeUasxlwEdTqf2LZZd9DOcv4NwDer08lYCHT7Im6H8AHO6ymC0/ebDg7L1cznPDsKQOABlNNITed4IhvutINhRQf5vlV8luhow9ZQESlkaS0GOh+fq9S83Z8FkXHhm7hVe8TXs6NQjeutiCjQs647Ngh9VtVDq3RHn7MhubFHv2a5ukr+eyMZ9+b06h5leWs7E8saekUC1Bh+ZsWHPzJZy30lrx5/tRuH+/y0pGlUbr40Neq4YkDTD1ABMe/ryEUF87357oT9pR7VvLxpWSQrURQqcjbW1jdkV+W+b9sN9HEjrhFMaMDCtbZp+UR6hyU4oFUA0GfCflE7p2DCuumPp8C/4OJHTtGDq+MY6goQelSG8T6VElNkd6VEmlQQpV4hBIoUocAilUiUMghSpxCKRQJQ6BPDNVxTnzakdym+QxMOIA3arFMfnLEdT/o/j0rC4r74aH/6yJFCqgcXMj/qNm/Nz3U0L07nyVVZtVj3TFeDTR1qZZDF1DfzK/0HOw2dwSy6b7x81GM654zT++0MiYlyfadKkXZNOPaBOB3xZB/IDPCNC5EFuQy2CPM2TNrJwbvjTu7hyf2YEPt65mc7OV5KuFhG8dRfh34wn/bjwbsr3JVwt5OulBjKpKqF5L+1f32Nrsqr0yZbyvNS8vWk431xxWXa3Nv1cNJuizJOInN+L7gbN5rW3vSpceMum75hy+50vzdfMlE2j0RvHOKE2LJmSGVcdz5U7Org1nX/uvAOhbv63FbJKZ+25CziPteXfGIu5xMfBVVn3W9O9Mo/gdph32vr48snksHsOdqDdjh+kQVCXg8vAofomaDkVHfyKWTSDwjZLb95SYo3gWnYbxdLXtHtTrqZJNv2gTwYrZn9DJpZAvs/xY07cjxvjj5vseu1xJ6PUF+yfNJfmd2z9jZK8MnrwJP50rGjTcf/gxAqeVvcdU4+5OxrAo1ob/Fw0aRqXYfh9GlROqtk5tpq38ljpaV77Oqs8PfTtgTDwJgK5+PdJ+DGPXK7PN9X2jKsdxL03LpkyqmYCCwiVjLk6zvEvV0TX059LoKCK25bDtgzl4aZ1RUNjzS4QNLP6HbbY2wKoIQZONl+jkUkiqMZevp/bFJdF0nEsTEUa75Yf4V60jKNf9/qZG16chybayuMJI6V3D/PPQxCdw+tk0QBJtIkgc4kFwy9N8GLiaUL22aKc/nDPk023DZELe32uzkOjXqFJCTRsbxX/qzuWCMZf+H07B56cdGO9rjfG1dEY2+JXBHmn887S395HKMfqvfV/xwcHT6TVwHeePvtdFtrf873XTU8WnTO85OISa77oSvHOXzUUKVUyof4cb0CDw1bqR4wunv4/gcNRi8/2P0pvy65R7yJ6QyY6WK1mTXZMa0Y4V1OxGPOx72PxzTKcl0OnaVXHr8VuuJ+O3DyH4cwO1jiVj/DvTukbehColVO+9WpS+Jv9weMS8olLB3L8DWfhtbxrOisFFjWPGgj9Q0PL6D0MIPGv9g3cVibZpCFdnFjK+5vfohZbC69xjbIGB+Wn3Ev1jCxrMPYxy5QrB7AfA3tqRqiXUJTuZNqYtT3vtIFvVMynucYz/86b2oj34G6JRgNSXOtLcaRuZSh4B624cmMERODelI6+PXMEjHmkoQKEKCgpR+57CY0l13JKzUQ/E4kd0mQkx7IkqJVRUlSORKq/QHgAvEkzF11XpO9SUxuaZE4MsEl7HWuga+ptF+k98x2ZjOB1vF33P8lLlpqduRvqIKN7xiSFTySNrlv+tH7BjTozw4xGPNK4oBUTuesbW5tw1UqhFCJ2OwOEmD/vA/hG4/uDYUYi+e3oWAG03TaTmMg/bGlMBSKEWkfFEW5YHbCJXLaDei/azdHinNHPSo0GDJtPUu9OgQS+0nCwshMJCG1t3+0ihYoq57zvStDrVYssLGE6esq1BFcCyK3VRUHBtdAUwDaIKVSNDjzzjkPFXpVCBrAeasCbo/9AgCP60wNbmVAjLn+vDFaWA3e2WMn9m8ZJwdp6TDa26c6RQgeApcSiotNz9FOoB2yWmrUg0Ww9w/76RaIUgVF+84lTrGzcbWnXnVHmhGu9tzXy/zaQac/Fa4viDjuvxG5lG76efp9vhxwBTCiGPP2x/rOROqFrzqGWQ0tMZvdAy8NCzeNk432dFY7yUju73dHS/Q1/a4kqS3a04lZcqvcNfYh/I2FOSSoMUqsQhkEKVOARSqBKHQApV4hBIoUoo6NmGxPntGXz0PBvP7idjWJStTSpFlZ9HrYpo3N0xNg/i5EBXXuz9P4ZVm4ez0KNBoKCSEQ41b/0aqyKFaiVEZDjxY11J6PkFQNGxECPhX48j4FXrHndJeK8ZCYM/QzFvndaXuL9g4EKmHRuN1xL7OYYjm34rICLDGbL8F471NOVxuraTSUHhr6dmcP/hbERkuFVsMXSLZO+gT0uVr8v24lihKct3F5cCHpq41Sr2lBcpVAuT8lZHnl2xgcc8UwFYnBlIs/9OoMeI54mcO5HFGa2Z5BVH8sPVrWJPWmtnPDTF2bzXZXvRau54vurYhmf+/ZK53EuXXdbjNkM2/RZE6J0whObwiPtlFGBKakeOD2lAQIKpSa3/M2yK6cLkxUesZlO96dF0Pj+OtCgjNeJ01J4XTX2iMQJeT55GUxTXYP+VBsAVq9l1K6RHtSApU9twpOsiAJquHM+JwfUxJpywsVXgtfYQ7ik6Ro75ib5x6ZyY0YGkFS3YGLbeNJhScomfaZ2uSHmRHtVC5PdqS8yYuVB0BCT08zTzyYHc/u3Qjr3ApqZrgX2AhsgecVx82/J2ab290K11IiZonnkwNfqJ+UV3Td607a8TCVlp2fynt4sUqgW5Fion6I8RBCUdQVe/HilPNmLN2OkE6FxKRHret6kpDbBsKnSA+DdCOBY0n3+GLrpG2OqxhE07aHfn/KVQrUA1j1x0v/lwj3cik7x+BJzYkutCYkFdRlY/aVVbFOebSzD4m6soefZ3uFEK1QrsivzW7D0P5Gt4YutogpYo6C9kof1BYXj1U1azxe20jiRDHiF6d1BLilYr7HfIIoVqIdzjL9J01XjiBs81l31/tS7LHn+QkAP7AFN8pyPZflCBQhXOziQsDKd5o7Nc/rQhrj+WPLXg90E0wxNfIr2ZhkbrTaP6j79fTBO9vpRw7QkpVAthOHmKoBdP0ffFtuT2b4fH9iSMFy8CJQ8PaoSCpgInX/K6Nye++wIAFnwQyIYfSy+Geqzehcfq4lBGeaq2VB17w359fSXC9YfdRSItjaJqSgyqKpLq2mw0np43raP19sJF2P9JKinUSswTnhc4N7LZTeuceDHU1OwD2/M1aNPtZ5L/eqRQbUh+r7YMrfVXhb7TPfYC2/NMO6E0CPa/PI/UlzqW8qz5fdpyfFkr4obPN9d9LWEAhiT7DAMv+6g25Iq/jhYVHLjEcCqFv7JD6ORS3Bc+MHke3z3nwxs7+6E76wwBORzt8gUKqt3Nl94IKVQbktEpv9RAKrd/u7uOJPjn8+25NNeD6XWL00IO9khjcPdF19UqOeE/+vS9qF/6ANad1y0vsum3IbM6fmfe9rfymU+ZcPwYbmfuPsq1iI4hvpOODvufuGm9fLWQBX8H0rfnk6R2LcBjtW3znd4M6VHthFC9lvMGI9oLf1dIcgslL4/aQ1J54J7nOTc0n3UdviBE78S08235KTECwyVXGq03ot+0FzhWAV+0LDJSig1JnN2Bo4NMCwIttj+L/1wdmm0HbGyV9ZGRUuyckK+vsO5qbY4WKPh9rq+SIi0vsum3Ieq+WJaGNgQaoi1KmyMpG+lRJQ6BFKrEIZBClTgEUqgSh0AKVeIQVLlR/5XHOpD6YCEtAs8AkLSuMXVnWf6s0h3RoTlX377K9uZrMaoKWqEhYP1oQp6vXCHcy0PVEKoQaIMCSHi7Ooe6zjElrt3zLDlXnVGaFVDX1vaVwfFZHdg6YAZ1tK70PNqPM39XJy/XiT19ZvLog5Nw+nmPrU20KpVeqNpa3hyd3oiEHgsB+Ci9OX8Ni6Senabp0daozrH3wzjUbxaPxD+B+p4P2q0x+ClnEHonLiYKCial4/SzrS21LpW6j6qt5Y1ujZ6EHgtJKCyg3Qfj2d6uhl3nkoqfG0h8/89ovnYi+oFZaLfsB8W0A18tNCVr02scZXNexVEpParQO5HTuyUfzlxAC6cC2kx/Cb91p6md7AB56VNcCFs9ltDXDmHMKbmTSlutGlph+70ZtqBSCvXc+DbsmzyX/+VUp/PHQ6k7J7pCdiRZg0avm+JSlfULlbzUnwCdCynnvAnmlFXtsjWVTqhaby8ef2YzOWoBi3p0o06SnY7obxOdX33eiNjI7nxByNw8qppfrXR91KPvBzHW6yBdpk++o/M/Gjc3dHXroKtbB6Gzn9/juNfqM9DjEsNXjkXdZ799bEthP/8TFcSrXTcw4Nhj1J19e55UGxpE/PO1eLjLXqbX3QiY4jAFTbJ+sDBtjeokTSgZTS+27yxAS9A3lx02TeTdUOmEOqLaGT7c+hAhpJT7GU3LpoxevZ4+bpkszfKn1a6hfNR8Dccenc9DkyItaG1psge1Z9knn+Cn+/0fd0xBIi619aJm1XOola/pB/DdUv5/1sXno3hu9XpmJXWn11Oj+KF9EPUHxDInubsFLSwbbY3qPPDGNvx0rszNCC5x797DjwLw1r+Wkjq5o9VtszWVUqjlRevjw5uTl7EqrS0ufc6h3bIfY1aW+f6WXBfrGuTjzWu1DgMwvmYisQUGhid3o1/XQXj0SiLq7XHoMfLK6JVow0Ota5uNqZRCvTywfPHnhZsLfdwySZ4dap5MB0h5syOzGq/ixS9HWcrEMlHPnid03RhSjbmMO9uZySPHcLHj3xiPJ4Gq4r1oB89tGs5gjzRSPyg7vmllpdIJ9awxhwjfVIT+1pEdDClniHpzHDV2nwNAuacVbPZjz+hPGf76S/h9ZN3jw0pODsHjdvF8q76kdFHRbd5Xqk7YgiwuGXP5vNlyq9pmayqdULsvn8KKwF9ImNnq1pVVFe8vd5Ab5MOJ6VFMWfoNG8J+5IGpk6i+fKd56dLaGNMv3zCYrnLoGJ3/HE+oPp/8Pm2tbJntqHSj/sbvxtCkwUgO9Z9N+7MvUT1JwfO7klNM2YPakxGsRWl9hVFNtvNEtdnkqfDAqikErs2h2k47DsQQ0JDhzXdw0qDD9c9jdr8kXFFUznP9Gi2F3VshpqYxzC+axk5pRDiZkn0dLXAioaAOTkWhFl/f3Z+wtzIwnj5Xop9qjxi6RZIx8Sq7Ir8l5McXCBlTOfalludcf6XzqAAoRRFANsFy/NC07EFGRDUAahy7irq3OK9TEAfsYh+A0Dtx8p1IXhuwhpWPd0ecOmu6Ud+0WzZ+VE3+GjADF6EhZP0kQicerFLLqJXTozogOQPas2HOLNzEjQeB3Y4MwuWdaojoGCtaZnmqrkd1QNzW7qJLw8lcbZ2LzsnIkc5L+fRyGAsPdsYp0RW/33PwiE3GmJFka1NtghSqHVF3ZvH+hIcwLd0GURzmpyqu8V+j0k1PSSonUqgSh0AKVeIQ2MWoXwhxEcgGLlXga2vJ99nFu8rzvoaqqvrc7AV2IVQAIcTeW01RyPdZ5332aJts+iUOgRSqxCGwJ6EulO+zm/fZnW1200eVSG6GPXlUieSGSKFKHAIpVIlDIIUqcQikUCUOwf8DxlqkFntmJbkAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "for batch in train_loader:\n", + " data, targets = get_batch(batch, seq_len, digits_per_batch)\n", + "\n", + " plt.matshow((data.view(-1, data.shape[-1]) * mnist_std + mnist_mean).numpy())\n", + " plt.show()\n", + " break" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.057399, + "end_time": "2021-01-20T09:44:04.103436", + "exception": false, + "start_time": "2021-01-20T09:44:04.046037", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "### Average pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-20T09:44:04.277197Z", + "iopub.status.busy": "2021-01-20T09:44:04.276636Z", + "iopub.status.idle": "2021-01-20T13:16:09.396466Z", + "shell.execute_reply": "2021-01-20T13:16:09.397430Z" + }, + "papermill": { + "duration": 12725.214396, + "end_time": "2021-01-20T13:16:09.397659", + "exception": false, + "start_time": "2021-01-20T09:44:04.183263", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 82.52 | loss 64.37 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 79.99 | loss 26.51 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 78.62 | loss 21.16 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 73.52 | loss 16.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 77.94s | valid loss 18.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 75.53 | loss 12.05 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 76.08 | loss 11.87 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 76.41 | loss 9.91 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 74.87 | loss 10.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 94.39s | valid loss 27.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 81.17 | loss 9.38 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 80.76 | loss 8.75 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 80.57 | loss 7.40 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 80.08 | loss 8.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 81.48s | valid loss 12.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 81.39 | loss 7.21 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 80.60 | loss 6.76 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 80.49 | loss 7.13 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 80.02 | loss 7.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 81.30s | valid loss 11.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 80.99 | loss 6.32 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 80.82 | loss 6.53 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 79.37 | loss 7.61 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 80.15 | loss 6.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 81.45s | valid loss 9.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 79.88 | loss 6.53 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 80.22 | loss 5.90 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 79.90 | loss 5.83 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 80.19 | loss 5.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 81.10s | valid loss 7.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 79.94 | loss 5.20 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 79.63 | loss 5.82 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 80.01 | loss 5.44 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 79.93 | loss 5.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 80.92s | valid loss 10.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 80.03 | loss 5.10 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 80.23 | loss 6.11 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 79.61 | loss 4.79 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 80.44 | loss 5.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 80.86s | valid loss 15.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 79.60 | loss 5.00 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 79.01 | loss 5.22 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 79.78 | loss 4.17 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 79.93 | loss 5.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 80.38s | valid loss 7.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 77.07 | loss 4.09 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 75.22 | loss 4.43 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 75.92 | loss 4.71 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 75.10 | loss 4.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 76.52s | valid loss 8.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 76.47 | loss 3.46 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 124.86 | loss 4.06 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 115.85 | loss 4.10 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 78.96 | loss 4.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 95.97s | valid loss 9.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 78.94 | loss 2.88 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 79.10 | loss 3.14 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 79.36 | loss 3.88 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 79.00 | loss 3.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 80.09s | valid loss 9.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 79.57 | loss 2.56 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 79.39 | loss 4.12 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 78.06 | loss 2.59 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 77.86 | loss 3.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 79.69s | valid loss 10.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 78.76 | loss 2.99 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 78.64 | loss 2.83 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 78.76 | loss 2.75 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 78.51 | loss 2.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 79.73s | valid loss 8.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 79.40 | loss 2.08 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 78.59 | loss 2.39 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 77.94 | loss 2.93 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 75.29 | loss 3.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 86.58s | valid loss 7.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 121.80 | loss 2.22 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 78.00 | loss 2.81 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 78.41 | loss 2.26 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 78.64 | loss 3.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 88.13s | valid loss 8.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 78.95 | loss 2.30 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 78.53 | loss 1.99 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 79.26 | loss 2.70 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 79.11 | loss 2.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 80.01s | valid loss 9.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 78.81 | loss 2.62 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 78.93 | loss 2.24 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 77.78 | loss 2.17 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 78.52 | loss 2.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 79.63s | valid loss 6.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 79.75 | loss 1.91 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 77.87 | loss 1.86 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 77.80 | loss 1.44 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 77.51 | loss 2.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 78.66s | valid loss 8.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 74.85 | loss 2.29 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 74.39 | loss 1.96 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 156.99 | loss 1.23 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 101.88 | loss 1.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 98.82s | valid loss 9.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 80.03 | loss 1.72 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 79.35 | loss 1.08 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 79.84 | loss 1.32 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 80.11 | loss 2.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 81.04s | valid loss 8.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 79.42 | loss 0.70 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 80.11 | loss 1.28 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 80.13 | loss 2.13 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 79.85 | loss 1.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 81.10s | valid loss 9.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 80.40 | loss 1.14 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 78.79 | loss 1.36 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 80.21 | loss 1.49 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 79.36 | loss 1.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 80.96s | valid loss 8.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 79.37 | loss 0.98 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 79.57 | loss 0.99 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 79.86 | loss 1.75 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 79.58 | loss 1.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 80.65s | valid loss 8.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 78.15 | loss 0.66 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 79.48 | loss 0.46 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 79.25 | loss 1.19 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 79.90 | loss 0.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 80.44s | valid loss 9.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 77.91 | loss 0.82 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 76.01 | loss 0.43 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 74.84 | loss 1.48 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 73.69 | loss 1.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 76.40s | valid loss 8.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 76.02 | loss 0.73 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 181.43 | loss 0.52 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 79.09 | loss 0.84 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 79.78 | loss 0.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 100.40s | valid loss 8.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 80.35 | loss 0.41 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 77.39 | loss 0.78 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 78.46 | loss 0.94 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 79.49 | loss 1.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 79.99s | valid loss 8.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 78.86 | loss 0.49 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 79.64 | loss 0.51 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 78.88 | loss 0.40 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 79.69 | loss 0.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 80.55s | valid loss 10.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 79.09 | loss 0.72 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 79.74 | loss 0.49 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 79.91 | loss 0.56 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 79.77 | loss 0.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 80.72s | valid loss 9.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.62 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 81.50 | loss 74.06 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 80.98 | loss 32.75 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 81.49 | loss 19.13 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 80.73 | loss 15.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 82.23s | valid loss 31.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 81.80 | loss 12.90 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 81.13 | loss 11.46 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 82.17 | loss 10.28 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 81.24 | loss 10.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 82.45s | valid loss 34.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 79.21 | loss 8.85 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 77.17 | loss 9.05 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 74.62 | loss 8.68 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 76.77 | loss 8.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 77.24s | valid loss 8.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 76.82 | loss 7.90 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 77.04 | loss 7.13 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 79.69 | loss 6.79 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 167.55 | loss 6.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 97.51s | valid loss 9.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 81.17 | loss 6.16 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 80.72 | loss 6.32 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 80.74 | loss 7.11 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 80.41 | loss 7.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 81.62s | valid loss 13.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 79.92 | loss 5.98 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 79.40 | loss 6.25 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 80.30 | loss 6.01 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 80.65 | loss 7.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 81.29s | valid loss 11.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 81.03 | loss 5.48 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 80.35 | loss 6.05 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 80.19 | loss 5.69 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 80.96 | loss 5.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 81.46s | valid loss 14.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 81.35 | loss 5.39 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 80.57 | loss 4.68 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 80.19 | loss 5.57 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 79.65 | loss 5.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 81.58s | valid loss 7.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 79.74 | loss 5.56 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 81.18 | loss 4.36 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 80.32 | loss 4.68 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 80.80 | loss 4.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 81.41s | valid loss 7.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 81.01 | loss 4.32 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 79.58 | loss 4.20 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 79.61 | loss 4.70 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 80.03 | loss 4.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 80.98s | valid loss 8.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 80.51 | loss 4.00 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 79.78 | loss 3.96 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 79.90 | loss 4.27 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 80.50 | loss 4.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 81.12s | valid loss 9.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 79.56 | loss 3.12 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 80.07 | loss 3.63 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 78.99 | loss 3.86 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 79.72 | loss 3.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 80.65s | valid loss 6.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 80.06 | loss 3.08 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 79.86 | loss 3.84 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 79.27 | loss 3.29 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 80.23 | loss 3.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 80.51s | valid loss 8.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 76.19 | loss 2.35 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 74.79 | loss 3.57 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 74.95 | loss 2.90 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 75.58 | loss 3.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 76.01s | valid loss 10.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 136.63 | loss 2.52 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 109.92 | loss 2.66 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 78.97 | loss 3.30 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 79.00 | loss 2.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 97.91s | valid loss 7.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 79.62 | loss 2.48 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 79.23 | loss 2.43 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 78.82 | loss 2.83 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 79.85 | loss 2.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 80.11s | valid loss 7.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 79.96 | loss 2.16 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 78.32 | loss 2.45 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 77.96 | loss 2.02 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 78.83 | loss 2.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 79.92s | valid loss 7.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 79.79 | loss 2.32 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 78.72 | loss 2.33 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 78.59 | loss 1.27 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 78.74 | loss 2.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 80.14s | valid loss 8.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 80.00 | loss 2.29 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 78.67 | loss 1.66 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 78.72 | loss 1.36 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 78.28 | loss 2.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 79.98s | valid loss 8.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 79.25 | loss 1.64 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 79.03 | loss 1.80 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 79.03 | loss 2.14 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 78.86 | loss 1.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 80.15s | valid loss 8.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 79.18 | loss 0.93 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 79.26 | loss 1.95 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 78.95 | loss 2.00 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 78.00 | loss 1.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 79.64s | valid loss 8.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 78.81 | loss 0.87 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 79.21 | loss 1.40 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 78.99 | loss 1.32 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 79.37 | loss 1.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 80.18s | valid loss 6.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 79.44 | loss 1.29 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 78.81 | loss 0.93 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 78.82 | loss 1.07 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 78.16 | loss 0.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 79.84s | valid loss 9.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 78.87 | loss 1.21 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 78.80 | loss 0.82 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 78.67 | loss 1.04 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 79.15 | loss 1.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 80.02s | valid loss 10.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 79.06 | loss 0.88 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 78.26 | loss 0.99 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 77.92 | loss 0.92 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 75.36 | loss 1.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 77.40s | valid loss 10.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 74.48 | loss 0.88 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 73.61 | loss 0.98 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 74.12 | loss 1.62 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 133.60 | loss 0.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 94.73s | valid loss 7.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 79.08 | loss 0.76 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 78.57 | loss 0.81 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 78.10 | loss 0.53 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 77.50 | loss 1.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 79.40s | valid loss 8.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 77.46 | loss 0.15 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 77.98 | loss 0.94 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 77.52 | loss 0.93 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 78.19 | loss 0.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 78.93s | valid loss 10.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 78.78 | loss 0.44 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 78.63 | loss 0.61 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 78.30 | loss 0.93 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 78.63 | loss 0.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 79.46s | valid loss 10.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 79.38 | loss 1.01 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 78.17 | loss 0.47 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 77.81 | loss 0.82 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 78.27 | loss 0.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 79.62s | valid loss 7.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 31 | 200/ 938 batches | lr 0.00021 | ms/batch 76.85 | loss 0.15 |\n", + "| epoch 31 | 400/ 938 batches | lr 0.00021 | ms/batch 78.13 | loss 0.58 |\n", + "| epoch 31 | 600/ 938 batches | lr 0.00021 | ms/batch 77.04 | loss 0.86 |\n", + "| epoch 31 | 800/ 938 batches | lr 0.00021 | ms/batch 78.03 | loss 0.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 31 | time: 78.68s | valid loss 8.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 32 | 200/ 938 batches | lr 0.00020 | ms/batch 78.81 | loss 0.67 |\n", + "| epoch 32 | 400/ 938 batches | lr 0.00020 | ms/batch 77.69 | loss 0.39 |\n", + "| epoch 32 | 600/ 938 batches | lr 0.00020 | ms/batch 76.97 | loss 0.69 |\n", + "| epoch 32 | 800/ 938 batches | lr 0.00020 | ms/batch 77.74 | loss 0.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 32 | time: 78.70s | valid loss 8.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 33 | 200/ 938 batches | lr 0.00019 | ms/batch 78.54 | loss 0.61 |\n", + "| epoch 33 | 400/ 938 batches | lr 0.00019 | ms/batch 77.55 | loss 0.54 |\n", + "| epoch 33 | 600/ 938 batches | lr 0.00019 | ms/batch 76.38 | loss 0.26 |\n", + "| epoch 33 | 800/ 938 batches | lr 0.00019 | ms/batch 75.63 | loss 0.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 33 | time: 77.70s | valid loss 9.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 34 | 200/ 938 batches | lr 0.00018 | ms/batch 76.31 | loss 0.29 |\n", + "| epoch 34 | 400/ 938 batches | lr 0.00018 | ms/batch 74.22 | loss 0.52 |\n", + "| epoch 34 | 600/ 938 batches | lr 0.00018 | ms/batch 73.88 | loss 0.47 |\n", + "| epoch 34 | 800/ 938 batches | lr 0.00018 | ms/batch 72.67 | loss 0.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 34 | time: 75.14s | valid loss 7.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.38 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 119.60 | loss 80.45 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 81.42 | loss 31.42 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 79.18 | loss 20.80 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 79.15 | loss 15.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 88.71s | valid loss 19.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 79.96 | loss 12.60 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 80.03 | loss 11.39 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 79.58 | loss 10.65 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 79.31 | loss 11.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 80.51s | valid loss 10.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 81.46 | loss 8.77 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 80.45 | loss 8.33 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 79.63 | loss 8.10 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 80.62 | loss 8.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 81.50s | valid loss 13.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 79.74 | loss 7.61 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 80.58 | loss 8.39 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 80.27 | loss 6.93 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 79.04 | loss 7.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 80.84s | valid loss 10.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 80.34 | loss 6.31 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 79.23 | loss 6.67 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 80.46 | loss 6.89 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 80.79 | loss 6.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 80.90s | valid loss 9.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 80.79 | loss 6.31 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 80.13 | loss 6.13 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 78.98 | loss 6.70 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 79.90 | loss 6.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 81.31s | valid loss 19.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 80.46 | loss 5.63 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 79.51 | loss 4.82 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 79.04 | loss 5.94 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 78.38 | loss 5.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 80.14s | valid loss 8.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 80.38 | loss 5.00 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 78.97 | loss 5.03 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 80.37 | loss 5.00 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 80.74 | loss 5.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 81.07s | valid loss 8.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 79.38 | loss 4.94 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 80.33 | loss 4.31 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 79.97 | loss 5.40 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 79.56 | loss 4.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 80.91s | valid loss 8.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 79.95 | loss 3.98 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 78.96 | loss 5.83 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 79.61 | loss 3.87 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 79.93 | loss 3.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 80.71s | valid loss 9.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 78.57 | loss 3.62 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 79.36 | loss 4.00 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 78.84 | loss 3.90 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 79.22 | loss 4.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 79.40s | valid loss 9.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 77.23 | loss 4.29 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 76.64 | loss 3.08 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 75.47 | loss 3.89 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 75.42 | loss 3.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 87.84s | valid loss 9.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 118.04 | loss 3.21 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 79.18 | loss 3.58 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 80.75 | loss 3.37 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 79.45 | loss 3.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 88.41s | valid loss 9.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 80.00 | loss 3.07 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 79.81 | loss 2.93 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 79.31 | loss 3.47 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 80.36 | loss 1.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 80.81s | valid loss 10.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 81.01 | loss 2.56 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 79.32 | loss 2.52 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 80.34 | loss 3.03 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 79.01 | loss 2.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 80.99s | valid loss 10.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 80.05 | loss 2.99 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 79.28 | loss 1.85 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 79.10 | loss 2.54 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 80.44 | loss 2.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 80.91s | valid loss 8.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 79.84 | loss 2.10 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 78.97 | loss 1.63 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 79.30 | loss 1.92 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 79.19 | loss 2.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 80.66s | valid loss 8.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 79.30 | loss 1.84 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 78.12 | loss 2.34 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 75.20 | loss 2.42 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 75.18 | loss 1.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 90.51s | valid loss 8.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 108.77 | loss 1.58 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 78.99 | loss 1.14 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 79.20 | loss 2.19 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 78.45 | loss 2.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 85.88s | valid loss 12.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 79.74 | loss 1.46 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 79.17 | loss 1.80 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 79.15 | loss 1.58 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 79.62 | loss 2.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 80.21s | valid loss 10.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 80.41 | loss 1.71 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 78.24 | loss 1.26 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 79.24 | loss 1.64 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 79.37 | loss 1.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 80.06s | valid loss 9.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 80.09 | loss 1.74 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 77.14 | loss 1.34 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 79.61 | loss 1.21 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 78.90 | loss 1.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 79.95s | valid loss 10.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 79.67 | loss 0.87 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 79.31 | loss 1.38 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 78.87 | loss 1.07 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 79.19 | loss 1.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 80.34s | valid loss 8.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 79.24 | loss 1.35 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 78.15 | loss 1.20 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 74.79 | loss 1.67 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 116.26 | loss 0.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 102.68s | valid loss 11.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 81.78 | loss 0.70 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 81.18 | loss 1.21 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 80.68 | loss 0.61 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 80.61 | loss 1.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 82.16s | valid loss 10.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 80.25 | loss 0.59 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 79.72 | loss 1.07 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 79.40 | loss 0.80 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 82.02 | loss 0.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 81.43s | valid loss 11.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 82.02 | loss 0.55 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 80.55 | loss 0.64 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 82.02 | loss 1.41 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 80.89 | loss 1.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 82.45s | valid loss 12.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 81.70 | loss 1.14 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 81.74 | loss 0.42 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 81.10 | loss 1.30 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 80.58 | loss 0.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 82.41s | valid loss 11.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 80.80 | loss 0.95 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 80.47 | loss 0.66 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 80.80 | loss 0.79 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 80.65 | loss 0.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 81.58s | valid loss 13.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 81.10 | loss 0.47 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 81.45 | loss 0.64 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 81.04 | loss 0.95 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 80.77 | loss 0.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 82.20s | valid loss 11.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 8.01 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 83.71 | loss 73.29 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 83.53 | loss 32.14 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 83.47 | loss 19.54 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 83.48 | loss 16.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 84.42s | valid loss 35.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 83.70 | loss 12.40 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 83.22 | loss 11.98 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 82.91 | loss 10.91 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 83.44 | loss 9.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 84.28s | valid loss 17.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 83.39 | loss 9.58 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 83.23 | loss 8.64 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 83.09 | loss 8.05 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 82.93 | loss 7.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 84.09s | valid loss 13.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 82.68 | loss 7.79 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 83.45 | loss 7.69 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 83.35 | loss 7.36 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 84.25 | loss 6.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 84.19s | valid loss 12.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 83.94 | loss 6.52 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 82.86 | loss 6.15 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 84.72 | loss 5.73 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 82.11 | loss 6.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 84.44s | valid loss 10.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 83.71 | loss 6.54 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 82.07 | loss 6.17 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 82.61 | loss 6.03 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 82.96 | loss 6.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 83.84s | valid loss 8.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 82.61 | loss 6.24 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 82.44 | loss 5.34 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 81.69 | loss 4.76 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 83.51 | loss 6.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 83.61s | valid loss 12.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 83.50 | loss 4.62 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 80.41 | loss 5.53 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 81.97 | loss 5.09 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 82.67 | loss 5.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 83.12s | valid loss 7.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 83.45 | loss 5.17 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 80.93 | loss 4.66 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 82.08 | loss 4.76 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 80.92 | loss 4.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 83.00s | valid loss 9.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 82.09 | loss 4.47 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 81.67 | loss 4.46 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 82.44 | loss 4.38 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 83.29 | loss 3.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 83.40s | valid loss 8.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 82.24 | loss 3.16 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 82.44 | loss 3.92 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 82.81 | loss 4.64 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 81.23 | loss 3.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 83.31s | valid loss 7.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 79.73 | loss 4.77 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 78.53 | loss 3.88 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 80.33 | loss 4.23 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 76.98 | loss 3.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 79.57s | valid loss 7.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 76.60 | loss 3.06 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 77.83 | loss 2.95 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 122.65 | loss 3.19 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 158.94 | loss 3.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 104.74s | valid loss 7.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 82.94 | loss 2.01 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 81.67 | loss 3.64 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 80.59 | loss 2.15 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 82.18 | loss 3.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 82.68s | valid loss 6.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 82.90 | loss 2.74 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 82.07 | loss 2.73 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 82.47 | loss 2.66 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 81.67 | loss 2.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 83.31s | valid loss 7.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 82.23 | loss 2.07 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 82.75 | loss 2.40 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 81.87 | loss 2.53 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 81.66 | loss 2.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 83.09s | valid loss 8.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 81.69 | loss 2.35 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 81.36 | loss 1.81 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 82.07 | loss 2.68 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 81.36 | loss 2.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 82.82s | valid loss 8.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 82.31 | loss 2.22 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 80.97 | loss 2.01 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 80.41 | loss 2.07 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 81.83 | loss 2.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 82.35s | valid loss 8.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 82.27 | loss 2.31 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 81.17 | loss 1.63 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 80.92 | loss 1.82 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 79.69 | loss 1.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 82.22s | valid loss 10.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 81.34 | loss 1.88 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 81.55 | loss 2.02 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 81.65 | loss 1.39 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 81.75 | loss 1.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 82.70s | valid loss 9.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 81.50 | loss 1.84 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 81.74 | loss 1.61 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 80.67 | loss 1.81 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 80.69 | loss 1.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 82.28s | valid loss 8.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 80.24 | loss 1.49 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 81.80 | loss 1.21 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 80.66 | loss 1.09 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 80.27 | loss 1.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 81.54s | valid loss 8.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 82.26 | loss 1.27 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 80.54 | loss 1.04 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 80.59 | loss 1.33 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 81.67 | loss 1.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 82.27s | valid loss 7.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 80.78 | loss 0.95 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 81.88 | loss 1.29 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 80.82 | loss 1.35 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 81.44 | loss 0.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 82.22s | valid loss 9.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 81.98 | loss 0.71 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 80.74 | loss 0.68 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 81.48 | loss 1.29 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 80.14 | loss 0.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 82.26s | valid loss 8.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 81.07 | loss 0.77 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 81.17 | loss 0.69 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 81.23 | loss 0.98 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 81.02 | loss 0.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 82.26s | valid loss 10.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 81.66 | loss 0.53 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 80.90 | loss 0.64 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 81.64 | loss 1.06 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 80.10 | loss 0.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 82.36s | valid loss 9.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 81.29 | loss 0.66 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 80.53 | loss 0.54 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 80.75 | loss 1.27 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 81.64 | loss 0.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 81.92s | valid loss 10.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 80.75 | loss 0.72 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 79.32 | loss 0.75 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 80.78 | loss 0.61 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 80.69 | loss 0.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 81.84s | valid loss 9.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 81.16 | loss 0.64 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 80.71 | loss 0.78 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 80.34 | loss 0.65 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 81.48 | loss 1.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 81.79s | valid loss 8.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.90 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 84.14 | loss 59.34 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 79.91 | loss 25.75 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 77.15 | loss 18.63 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 112.21 | loss 15.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 102.51s | valid loss 68.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 81.73 | loss 12.68 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 80.75 | loss 11.92 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 80.53 | loss 9.82 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 80.26 | loss 10.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 81.78s | valid loss 13.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 80.46 | loss 8.82 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 81.10 | loss 8.78 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 80.21 | loss 7.38 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 81.18 | loss 8.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 81.53s | valid loss 10.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 81.33 | loss 7.93 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 81.01 | loss 7.10 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 80.84 | loss 6.95 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 80.50 | loss 6.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 81.88s | valid loss 11.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 80.89 | loss 7.03 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 80.50 | loss 6.55 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 80.66 | loss 6.20 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 80.56 | loss 6.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 81.54s | valid loss 9.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 81.91 | loss 6.91 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 80.36 | loss 6.34 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 81.18 | loss 5.76 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 80.32 | loss 6.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 81.88s | valid loss 8.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 80.70 | loss 6.27 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 80.43 | loss 6.15 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 80.14 | loss 5.57 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 80.68 | loss 5.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 81.31s | valid loss 11.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 80.48 | loss 4.76 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 80.20 | loss 4.92 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 80.35 | loss 4.78 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 79.65 | loss 5.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 81.11s | valid loss 10.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 80.10 | loss 5.24 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 80.45 | loss 4.60 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 79.76 | loss 4.56 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 80.42 | loss 4.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 81.14s | valid loss 14.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 80.80 | loss 4.34 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 79.34 | loss 3.25 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 80.63 | loss 5.41 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 79.89 | loss 5.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 81.03s | valid loss 8.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 79.98 | loss 3.78 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 79.86 | loss 3.77 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 79.50 | loss 4.22 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 79.65 | loss 4.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 80.63s | valid loss 7.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 79.92 | loss 3.07 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 79.68 | loss 3.82 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 79.98 | loss 3.51 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 79.85 | loss 3.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 81.00s | valid loss 8.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 79.84 | loss 2.43 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 80.29 | loss 3.70 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 79.00 | loss 3.27 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 80.20 | loss 4.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 80.78s | valid loss 7.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 80.00 | loss 2.91 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 79.27 | loss 2.66 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 79.21 | loss 3.53 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 79.39 | loss 2.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 80.60s | valid loss 10.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 79.27 | loss 2.48 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 79.10 | loss 1.98 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 78.84 | loss 3.34 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 79.56 | loss 2.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 80.14s | valid loss 8.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 79.93 | loss 2.20 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 78.84 | loss 2.65 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 79.29 | loss 2.50 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 78.96 | loss 1.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 80.24s | valid loss 6.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 79.53 | loss 1.91 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 79.79 | loss 2.90 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 78.84 | loss 1.87 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 79.60 | loss 2.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 80.45s | valid loss 9.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 79.35 | loss 1.59 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 79.27 | loss 2.11 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 78.99 | loss 1.70 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 80.43 | loss 2.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 80.50s | valid loss 10.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 78.65 | loss 1.63 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 79.16 | loss 1.50 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 78.69 | loss 2.15 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 78.80 | loss 1.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 79.82s | valid loss 8.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 79.27 | loss 0.88 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 78.59 | loss 1.67 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 78.77 | loss 1.60 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 78.33 | loss 1.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 79.72s | valid loss 10.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 79.15 | loss 1.29 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 78.76 | loss 0.97 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 78.48 | loss 2.32 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 78.87 | loss 1.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 80.01s | valid loss 8.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 79.77 | loss 0.70 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 78.66 | loss 1.58 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 78.60 | loss 1.16 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 78.48 | loss 1.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 80.00s | valid loss 10.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 78.40 | loss 0.49 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 78.98 | loss 1.68 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 79.09 | loss 1.90 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 78.76 | loss 1.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 79.74s | valid loss 8.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 79.21 | loss 1.09 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 78.25 | loss 0.83 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 78.32 | loss 0.51 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 78.19 | loss 1.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 79.49s | valid loss 9.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 78.69 | loss 0.91 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 79.09 | loss 0.54 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 78.06 | loss 1.20 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 78.78 | loss 1.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 79.72s | valid loss 8.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 80.01 | loss 1.05 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 78.54 | loss 1.07 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 78.74 | loss 1.28 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 78.31 | loss 0.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 80.13s | valid loss 10.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 78.35 | loss 0.68 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 78.72 | loss 0.43 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 78.26 | loss 0.98 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 78.75 | loss 0.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 79.45s | valid loss 10.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 79.09 | loss 0.24 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 78.67 | loss 0.55 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 78.38 | loss 0.34 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 78.08 | loss 1.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 79.77s | valid loss 10.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 78.73 | loss 0.57 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 78.62 | loss 0.17 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 78.17 | loss 0.80 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 78.91 | loss 1.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 79.56s | valid loss 13.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 78.90 | loss 0.67 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 78.73 | loss 1.08 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 78.47 | loss 0.53 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 78.65 | loss 0.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 79.86s | valid loss 10.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.63 |\n", + "=========================================================================================\n", + "Result over 5 runs: 6.4991774559021\n" + ] + } + ], + "source": [ + "num_runs = 5\n", + "best_model = None\n", + "best_result = 1000000\n", + "results = []\n", + "\n", + "for i in range(num_runs):\n", + " features = nn.Sequential(nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(64),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + "\n", + " nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(128),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + "\n", + " nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(256),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + "\n", + " nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=(0,1)),\n", + " nn.BatchNorm2d(512),\n", + " nn.ReLU()\n", + " )\n", + "\n", + " classifier = nn.Sequential(nn.Linear(512, 1028),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=1028, bias=True),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=10, bias=True))\n", + "\n", + " conv = nn.Sequential(nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.AvgPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.AvgPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.AvgPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.AvgPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.AvgPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.AvgPool1d(kernel_size=2, stride=2))\n", + "\n", + " model = ConvMNIST(features, conv, classifier)\n", + " model = model.to(device)\n", + "\n", + " optimizer = torch.optim.Adam(model.parameters())\n", + " scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "\n", + " epochs = 30\n", + " model, result = train_model(model, epochs, train_loader, test_loader, seq_len, digits_per_batch, optimizer, scheduler)\n", + " \n", + " if result < best_result:\n", + " best_result = result\n", + " best_model = model\n", + " results.append(result)\n", + " \n", + "model = best_model\n", + "print(\"Result over {} runs: {}\".format(num_runs, torch.tensor(results).mean().item()))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.064929, + "end_time": "2021-01-21T10:12:59.996862", + "exception": false, + "start_time": "2021-01-21T10:12:59.931933", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "### Max pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-21T10:13:00.121250Z", + "iopub.status.busy": "2021-01-21T10:13:00.113205Z", + "iopub.status.idle": "2021-01-21T13:19:14.014837Z", + "shell.execute_reply": "2021-01-21T13:19:14.015244Z" + }, + "papermill": { + "duration": 11173.967961, + "end_time": "2021-01-21T13:19:14.015390", + "exception": false, + "start_time": "2021-01-21T10:13:00.047429", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 84.14 | loss 79.52 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 81.70 | loss 31.08 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 80.45 | loss 21.66 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 81.11 | loss 17.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 82.71s | valid loss 25.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 81.98 | loss 13.01 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 82.79 | loss 11.19 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 82.73 | loss 10.64 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 82.76 | loss 10.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 83.58s | valid loss 11.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 83.19 | loss 8.93 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 82.38 | loss 9.08 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 82.99 | loss 8.67 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 82.17 | loss 8.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 83.18s | valid loss 10.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 82.49 | loss 7.16 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 82.97 | loss 7.50 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 81.94 | loss 6.61 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 81.69 | loss 7.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 83.37s | valid loss 14.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 81.67 | loss 6.77 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 81.73 | loss 7.04 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 82.25 | loss 7.55 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 81.35 | loss 8.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 82.70s | valid loss 9.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 82.80 | loss 5.75 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 81.22 | loss 6.57 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 80.17 | loss 6.21 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 81.92 | loss 5.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 82.52s | valid loss 16.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 81.69 | loss 6.31 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 82.77 | loss 6.45 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 82.10 | loss 5.18 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 81.87 | loss 6.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 82.53s | valid loss 8.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 81.52 | loss 5.69 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 82.05 | loss 4.96 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 80.67 | loss 5.00 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 81.66 | loss 5.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 82.27s | valid loss 9.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 81.56 | loss 5.29 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 81.69 | loss 6.30 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 82.44 | loss 4.79 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 79.75 | loss 5.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 82.24s | valid loss 8.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 81.90 | loss 4.75 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 82.00 | loss 5.40 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 81.35 | loss 5.33 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 81.57 | loss 4.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 82.58s | valid loss 10.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 81.92 | loss 4.27 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 81.48 | loss 3.50 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 80.82 | loss 4.54 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 82.16 | loss 4.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 82.58s | valid loss 8.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 80.92 | loss 3.74 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 81.24 | loss 4.01 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 80.81 | loss 3.81 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 80.91 | loss 3.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 82.29s | valid loss 8.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 82.54 | loss 3.43 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 81.51 | loss 3.97 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 81.41 | loss 3.76 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 79.65 | loss 4.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 82.17s | valid loss 7.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 81.98 | loss 3.22 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 80.64 | loss 2.90 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 80.10 | loss 3.95 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 80.45 | loss 2.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 81.75s | valid loss 8.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 79.25 | loss 3.62 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 80.50 | loss 3.22 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 80.17 | loss 2.99 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 79.62 | loss 2.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 80.99s | valid loss 11.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 80.45 | loss 2.35 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 80.53 | loss 2.91 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 78.49 | loss 2.66 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 79.21 | loss 2.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 80.57s | valid loss 11.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 80.01 | loss 2.89 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 80.02 | loss 2.73 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 80.75 | loss 2.96 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 80.37 | loss 2.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 80.87s | valid loss 9.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 80.34 | loss 1.54 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 79.17 | loss 1.49 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 78.06 | loss 2.64 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 80.04 | loss 2.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 80.23s | valid loss 11.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 80.86 | loss 1.47 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 80.66 | loss 1.82 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 80.54 | loss 2.41 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 78.82 | loss 1.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 81.34s | valid loss 8.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 80.65 | loss 1.87 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 81.23 | loss 1.35 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 80.99 | loss 2.74 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 80.90 | loss 2.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 81.71s | valid loss 9.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 80.06 | loss 1.67 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 80.11 | loss 1.12 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 79.37 | loss 2.12 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 79.38 | loss 1.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 80.71s | valid loss 8.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 80.37 | loss 1.78 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 79.05 | loss 1.27 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 80.49 | loss 2.25 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 79.77 | loss 1.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 81.15s | valid loss 12.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 80.73 | loss 0.79 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 79.16 | loss 1.82 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 79.93 | loss 1.53 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 80.69 | loss 1.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 80.77s | valid loss 14.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 81.52 | loss 1.46 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 80.15 | loss 1.51 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 80.26 | loss 0.91 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 80.07 | loss 1.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 81.32s | valid loss 12.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 80.55 | loss 1.14 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 80.18 | loss 0.95 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 80.44 | loss 1.26 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 79.20 | loss 1.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 80.89s | valid loss 11.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 80.17 | loss 0.82 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 80.22 | loss 0.51 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 80.27 | loss 1.02 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 81.43 | loss 1.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 81.46s | valid loss 13.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 81.45 | loss 0.64 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 79.89 | loss 0.72 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 80.38 | loss 0.67 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 79.02 | loss 0.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 80.90s | valid loss 12.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 80.66 | loss 1.27 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 77.87 | loss 1.00 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 78.91 | loss 0.38 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 80.17 | loss 0.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 80.60s | valid loss 12.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 81.33 | loss 0.82 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 80.05 | loss 0.83 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 80.95 | loss 1.28 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 79.44 | loss 0.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 80.79s | valid loss 13.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 80.25 | loss 0.56 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 80.42 | loss 0.62 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 78.91 | loss 0.32 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 74.77 | loss 1.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 78.76s | valid loss 11.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 7.94 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 79.36 | loss 79.66 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 78.79 | loss 30.92 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 78.07 | loss 20.34 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 79.43 | loss 16.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 79.63s | valid loss 33.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 78.26 | loss 13.30 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 79.09 | loss 11.66 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 78.72 | loss 11.80 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 79.39 | loss 10.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 79.44s | valid loss 18.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 77.17 | loss 8.70 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 77.88 | loss 8.63 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 76.22 | loss 7.92 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 76.02 | loss 8.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 77.40s | valid loss 8.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 75.12 | loss 7.39 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 73.66 | loss 6.69 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 73.73 | loss 7.14 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 76.99 | loss 7.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 75.58s | valid loss 9.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 77.25 | loss 7.11 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 76.81 | loss 6.57 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 76.07 | loss 6.67 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 76.47 | loss 6.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 77.40s | valid loss 10.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 72.96 | loss 6.51 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 76.81 | loss 5.76 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 76.42 | loss 7.62 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 76.24 | loss 5.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 76.55s | valid loss 13.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 75.03 | loss 6.08 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 75.90 | loss 6.65 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 75.86 | loss 5.87 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 75.95 | loss 6.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 76.60s | valid loss 8.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 77.02 | loss 5.19 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 76.65 | loss 5.89 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 75.99 | loss 4.82 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 73.90 | loss 5.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 97.48s | valid loss 7.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 88.55 | loss 5.36 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 80.86 | loss 5.37 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 81.40 | loss 5.38 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 81.77 | loss 4.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 83.95s | valid loss 10.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 80.75 | loss 3.34 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 80.45 | loss 4.88 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 80.75 | loss 4.24 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 80.13 | loss 4.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 81.74s | valid loss 7.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 82.09 | loss 4.31 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 80.59 | loss 4.21 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 80.87 | loss 4.76 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 80.31 | loss 4.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 81.87s | valid loss 9.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 80.64 | loss 4.19 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 79.72 | loss 3.83 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 80.69 | loss 4.09 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 80.81 | loss 4.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 81.33s | valid loss 9.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 81.16 | loss 3.09 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 80.10 | loss 3.86 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 80.65 | loss 4.02 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 80.74 | loss 3.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 81.73s | valid loss 10.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 78.67 | loss 3.50 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 80.45 | loss 3.54 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 80.34 | loss 3.38 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 80.33 | loss 2.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 81.02s | valid loss 9.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 80.63 | loss 2.86 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 79.40 | loss 2.20 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 80.74 | loss 2.92 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 80.79 | loss 2.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 81.42s | valid loss 8.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 81.21 | loss 3.02 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 78.30 | loss 2.86 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 80.94 | loss 2.62 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 80.66 | loss 2.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 81.34s | valid loss 6.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 80.98 | loss 2.00 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 80.59 | loss 2.69 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 79.55 | loss 1.90 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 79.39 | loss 2.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 81.09s | valid loss 10.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 79.13 | loss 2.46 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 79.85 | loss 1.71 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 79.65 | loss 2.78 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 79.15 | loss 2.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 80.60s | valid loss 8.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 79.75 | loss 1.49 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 79.90 | loss 2.20 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 79.81 | loss 1.80 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 79.40 | loss 2.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 80.99s | valid loss 8.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 79.99 | loss 2.05 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 80.84 | loss 1.41 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 80.49 | loss 1.61 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 80.60 | loss 2.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 81.29s | valid loss 8.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 80.48 | loss 1.39 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 79.84 | loss 1.56 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 79.33 | loss 2.42 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 79.76 | loss 1.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 81.29s | valid loss 7.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 79.37 | loss 1.16 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 80.15 | loss 1.27 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 78.96 | loss 1.60 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 80.03 | loss 1.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 80.88s | valid loss 10.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 81.38 | loss 0.97 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 79.21 | loss 1.34 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 78.59 | loss 2.08 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 78.47 | loss 0.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 80.62s | valid loss 11.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 80.45 | loss 0.93 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 79.96 | loss 1.12 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 79.21 | loss 1.18 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 80.43 | loss 1.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 80.87s | valid loss 9.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 80.75 | loss 0.86 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 76.96 | loss 0.84 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 75.61 | loss 1.50 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 75.25 | loss 1.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 77.75s | valid loss 10.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 75.12 | loss 0.57 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 74.62 | loss 0.73 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 73.90 | loss 0.73 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 74.32 | loss 1.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 98.24s | valid loss 12.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 79.02 | loss 0.48 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 79.44 | loss 1.12 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 79.89 | loss 0.81 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 79.88 | loss 1.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 80.26s | valid loss 11.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 79.19 | loss 0.97 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 79.60 | loss 0.75 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 79.08 | loss 0.62 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 79.58 | loss 0.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 80.43s | valid loss 10.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 79.58 | loss 0.59 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 80.01 | loss 0.63 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 80.16 | loss 0.36 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 79.22 | loss 1.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 80.63s | valid loss 11.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 79.53 | loss 0.48 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 79.67 | loss 0.92 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 80.16 | loss 0.34 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 79.23 | loss 0.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 80.53s | valid loss 11.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.53 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 83.61 | loss 78.80 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 81.78 | loss 28.18 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 82.89 | loss 18.70 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 82.72 | loss 15.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 83.46s | valid loss 27.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 82.80 | loss 12.43 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 80.42 | loss 11.47 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 82.02 | loss 10.58 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 83.52 | loss 9.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 82.98s | valid loss 14.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 82.40 | loss 9.28 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 81.70 | loss 8.82 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 82.41 | loss 9.18 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 80.58 | loss 8.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 82.86s | valid loss 14.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 81.32 | loss 7.83 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 81.88 | loss 7.15 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 82.43 | loss 8.00 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 82.24 | loss 7.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 82.85s | valid loss 10.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 82.20 | loss 8.26 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 80.94 | loss 7.30 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 81.63 | loss 6.33 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 82.05 | loss 7.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 82.71s | valid loss 19.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 83.17 | loss 6.42 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 81.90 | loss 6.36 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 81.65 | loss 6.73 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 81.79 | loss 6.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 82.75s | valid loss 8.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 82.75 | loss 5.29 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 80.19 | loss 7.22 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 80.63 | loss 5.58 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 81.98 | loss 6.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 82.42s | valid loss 8.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 81.58 | loss 6.06 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 81.82 | loss 4.31 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 79.78 | loss 6.51 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 81.55 | loss 5.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 82.31s | valid loss 10.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 81.46 | loss 5.16 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 80.60 | loss 5.14 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 81.59 | loss 5.08 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 82.22 | loss 5.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 82.54s | valid loss 10.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 79.75 | loss 4.54 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 82.21 | loss 4.52 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 81.42 | loss 4.81 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 81.36 | loss 4.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 81.90s | valid loss 7.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 82.63 | loss 3.81 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 81.62 | loss 4.76 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 80.72 | loss 4.78 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 79.99 | loss 4.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 81.64s | valid loss 9.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 78.70 | loss 3.64 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 77.99 | loss 3.77 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 75.30 | loss 3.38 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 76.18 | loss 5.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 77.37s | valid loss 9.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 75.93 | loss 3.79 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 69.06 | loss 2.85 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 67.93 | loss 3.25 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 66.91 | loss 3.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 69.99s | valid loss 7.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 67.22 | loss 2.92 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 67.07 | loss 2.80 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 66.77 | loss 3.51 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 66.84 | loss 3.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 67.60s | valid loss 11.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 67.04 | loss 2.15 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 66.72 | loss 2.96 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 66.93 | loss 4.33 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 66.89 | loss 3.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 67.53s | valid loss 8.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 67.12 | loss 2.50 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 66.79 | loss 3.33 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 66.70 | loss 2.11 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 66.77 | loss 3.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 67.46s | valid loss 12.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 66.95 | loss 2.29 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 66.68 | loss 2.66 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 66.61 | loss 2.25 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 66.73 | loss 2.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 67.37s | valid loss 8.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 66.85 | loss 1.42 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 66.70 | loss 3.10 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 66.50 | loss 2.40 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 66.46 | loss 2.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 67.26s | valid loss 10.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 66.79 | loss 1.65 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 66.34 | loss 1.70 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 66.40 | loss 1.73 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 66.38 | loss 2.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 67.11s | valid loss 9.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 66.79 | loss 1.72 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 66.42 | loss 2.53 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 66.22 | loss 1.50 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 66.41 | loss 1.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 67.10s | valid loss 8.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 66.69 | loss 1.23 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 66.60 | loss 2.50 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 66.44 | loss 1.98 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 66.40 | loss 2.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 67.16s | valid loss 10.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 66.74 | loss 1.52 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 66.45 | loss 1.57 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 66.18 | loss 1.46 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 66.48 | loss 2.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 67.10s | valid loss 13.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 66.66 | loss 1.12 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 66.39 | loss 1.45 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 66.36 | loss 1.58 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 66.35 | loss 1.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 67.09s | valid loss 11.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 66.52 | loss 1.17 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 66.12 | loss 0.78 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 66.28 | loss 1.64 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 66.28 | loss 1.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 66.95s | valid loss 11.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 66.46 | loss 1.19 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 66.04 | loss 0.87 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 66.17 | loss 1.54 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 66.54 | loss 1.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 66.96s | valid loss 11.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 66.53 | loss 0.80 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 66.10 | loss 0.80 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 66.24 | loss 1.02 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 66.21 | loss 0.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 66.95s | valid loss 11.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 66.45 | loss 0.61 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 66.04 | loss 0.53 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 66.27 | loss 1.05 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 66.22 | loss 0.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 66.89s | valid loss 14.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 66.43 | loss 0.52 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 66.11 | loss 0.50 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 66.07 | loss 0.57 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 66.23 | loss 1.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 66.87s | valid loss 11.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 66.45 | loss 0.70 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 66.16 | loss 0.88 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 66.08 | loss 0.65 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 66.08 | loss 1.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 66.86s | valid loss 11.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 66.26 | loss 0.67 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 65.96 | loss 0.62 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 66.09 | loss 1.12 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 66.04 | loss 0.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 66.75s | valid loss 13.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 7.53 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 68.06 | loss 79.84 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 67.75 | loss 28.99 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 67.75 | loss 21.31 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 67.72 | loss 16.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 68.38s | valid loss 36.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 68.20 | loss 13.14 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 67.83 | loss 11.34 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 67.83 | loss 10.78 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 67.81 | loss 10.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 68.46s | valid loss 11.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 68.09 | loss 8.70 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 67.76 | loss 9.18 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 67.77 | loss 8.15 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 67.63 | loss 8.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 68.36s | valid loss 14.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 67.96 | loss 7.55 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 67.61 | loss 7.77 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 67.66 | loss 7.38 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 67.71 | loss 7.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 68.30s | valid loss 11.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 68.06 | loss 6.46 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 67.65 | loss 6.79 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 67.62 | loss 7.36 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 67.75 | loss 6.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 68.34s | valid loss 7.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 68.01 | loss 6.71 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 67.68 | loss 6.68 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 67.65 | loss 5.81 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 67.64 | loss 7.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 68.31s | valid loss 9.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 67.99 | loss 5.55 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 67.61 | loss 6.68 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 67.61 | loss 5.91 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 67.54 | loss 5.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 68.25s | valid loss 11.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 67.79 | loss 4.32 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 67.52 | loss 6.58 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 67.51 | loss 5.36 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 67.50 | loss 6.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 68.15s | valid loss 8.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 67.78 | loss 4.88 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 67.40 | loss 4.84 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 67.51 | loss 4.97 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 67.28 | loss 5.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 68.06s | valid loss 10.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 67.65 | loss 4.47 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 67.16 | loss 4.87 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 67.24 | loss 4.52 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 67.33 | loss 4.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 67.91s | valid loss 10.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 67.65 | loss 4.99 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 67.12 | loss 3.86 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 67.11 | loss 5.40 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 67.30 | loss 5.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 67.88s | valid loss 9.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 67.36 | loss 3.66 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 66.95 | loss 3.59 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 66.99 | loss 3.40 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 66.95 | loss 3.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 67.69s | valid loss 7.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 67.33 | loss 3.93 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 67.00 | loss 3.44 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 66.97 | loss 3.94 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 66.95 | loss 3.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 67.64s | valid loss 8.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 67.28 | loss 3.33 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 66.81 | loss 3.42 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 66.93 | loss 2.69 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 66.94 | loss 3.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 67.60s | valid loss 7.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 67.15 | loss 2.60 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 66.77 | loss 2.79 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 66.74 | loss 2.90 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 66.81 | loss 3.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 67.46s | valid loss 7.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 66.86 | loss 2.51 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 66.70 | loss 3.21 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 66.68 | loss 2.97 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 66.62 | loss 2.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 67.36s | valid loss 9.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 66.98 | loss 1.70 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 66.61 | loss 2.13 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 66.72 | loss 2.46 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 66.64 | loss 2.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 67.34s | valid loss 11.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 66.73 | loss 2.20 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 66.55 | loss 2.59 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 66.28 | loss 1.78 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 66.36 | loss 2.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 67.14s | valid loss 8.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 66.85 | loss 2.19 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 66.54 | loss 2.10 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 66.44 | loss 2.31 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 66.44 | loss 1.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 67.20s | valid loss 9.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 66.66 | loss 1.98 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 66.34 | loss 1.89 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 66.31 | loss 1.68 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 66.46 | loss 2.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 67.08s | valid loss 9.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 66.72 | loss 1.81 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 66.13 | loss 1.38 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 66.36 | loss 1.87 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 66.41 | loss 1.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 67.09s | valid loss 8.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 66.73 | loss 1.41 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 66.28 | loss 0.59 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 66.18 | loss 1.81 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 66.39 | loss 2.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 67.03s | valid loss 9.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 66.49 | loss 1.44 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 66.07 | loss 0.91 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 66.26 | loss 1.64 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 66.76 | loss 1.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 67.04s | valid loss 12.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 66.63 | loss 1.40 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 66.15 | loss 0.96 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 66.27 | loss 0.66 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 66.21 | loss 1.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 67.06s | valid loss 13.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 66.58 | loss 1.16 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 66.27 | loss 0.77 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 66.64 | loss 1.06 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 66.23 | loss 1.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 67.08s | valid loss 11.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 67.09 | loss 0.92 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 66.30 | loss 1.63 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 66.11 | loss 0.47 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 66.22 | loss 1.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 67.04s | valid loss 10.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 66.52 | loss 1.61 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 66.71 | loss 1.25 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 66.27 | loss 0.97 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 66.22 | loss 0.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 67.16s | valid loss 10.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 66.53 | loss 0.33 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 66.26 | loss 0.49 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 66.75 | loss 1.42 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 66.31 | loss 1.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 67.08s | valid loss 9.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 66.87 | loss 0.21 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 66.22 | loss 1.21 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 66.12 | loss 0.53 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 66.66 | loss 0.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 67.07s | valid loss 9.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 66.43 | loss 0.85 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 66.04 | loss 0.31 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 66.16 | loss 0.81 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 66.04 | loss 0.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 66.82s | valid loss 11.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 7.41 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 68.15 | loss 82.72 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 67.78 | loss 28.69 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 67.77 | loss 19.62 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 67.77 | loss 16.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 68.43s | valid loss 26.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 68.20 | loss 13.39 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 67.87 | loss 11.83 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 67.89 | loss 10.71 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 67.89 | loss 10.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 68.51s | valid loss 13.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 68.25 | loss 8.53 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 67.89 | loss 8.79 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 67.90 | loss 8.13 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 67.93 | loss 8.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 68.53s | valid loss 14.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 68.24 | loss 8.14 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 67.92 | loss 7.83 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 67.89 | loss 6.99 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 67.90 | loss 6.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 68.51s | valid loss 13.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 68.15 | loss 7.31 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 67.85 | loss 6.78 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 67.87 | loss 7.31 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 67.86 | loss 6.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 68.50s | valid loss 12.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 68.22 | loss 6.93 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 67.85 | loss 6.00 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 67.78 | loss 5.92 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 67.74 | loss 7.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 68.47s | valid loss 7.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 68.00 | loss 5.80 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 67.65 | loss 5.23 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 67.64 | loss 6.49 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 67.69 | loss 6.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 68.35s | valid loss 9.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 67.81 | loss 5.46 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 67.65 | loss 5.98 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 67.97 | loss 5.07 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 67.56 | loss 5.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 68.28s | valid loss 8.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 67.75 | loss 4.69 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 67.49 | loss 5.21 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 67.38 | loss 4.74 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 67.43 | loss 4.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 68.10s | valid loss 8.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 67.74 | loss 4.88 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 67.35 | loss 4.86 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 67.47 | loss 4.20 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 67.34 | loss 4.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 68.07s | valid loss 9.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 67.65 | loss 3.10 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 67.24 | loss 4.33 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 67.17 | loss 4.54 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 67.29 | loss 4.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 67.92s | valid loss 9.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 67.52 | loss 4.11 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 67.13 | loss 3.85 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 67.06 | loss 3.71 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 67.31 | loss 4.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 67.85s | valid loss 6.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 67.43 | loss 3.16 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 66.95 | loss 3.51 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 67.21 | loss 3.74 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 67.10 | loss 3.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 67.76s | valid loss 9.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 67.27 | loss 3.05 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 67.01 | loss 3.03 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 66.99 | loss 3.80 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 67.13 | loss 3.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 67.70s | valid loss 6.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 67.15 | loss 2.27 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 66.82 | loss 2.54 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 66.75 | loss 2.48 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 66.94 | loss 3.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 67.55s | valid loss 7.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 67.16 | loss 2.64 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 66.71 | loss 3.10 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 66.79 | loss 2.84 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 66.68 | loss 2.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 67.43s | valid loss 9.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 67.03 | loss 2.23 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 66.69 | loss 2.94 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 66.71 | loss 2.12 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 66.88 | loss 2.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 67.46s | valid loss 9.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 67.23 | loss 2.67 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 66.74 | loss 2.18 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 66.79 | loss 2.89 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 66.62 | loss 1.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 67.46s | valid loss 7.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 66.96 | loss 1.57 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 66.61 | loss 2.08 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 66.58 | loss 1.80 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 66.67 | loss 2.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 67.31s | valid loss 9.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 66.95 | loss 1.80 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 66.45 | loss 1.38 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 66.54 | loss 2.03 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 66.71 | loss 2.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 67.30s | valid loss 13.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 66.80 | loss 1.38 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 66.59 | loss 2.11 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 66.58 | loss 1.60 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 66.72 | loss 2.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 67.29s | valid loss 8.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 66.80 | loss 1.42 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 66.39 | loss 1.31 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 66.44 | loss 2.07 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 66.60 | loss 1.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 67.18s | valid loss 8.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 66.78 | loss 1.16 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 66.45 | loss 1.25 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 66.42 | loss 1.39 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 66.45 | loss 1.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 67.15s | valid loss 9.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 66.64 | loss 0.63 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 66.36 | loss 1.38 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 66.36 | loss 1.36 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 66.45 | loss 2.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 67.09s | valid loss 8.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 66.77 | loss 1.28 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 66.29 | loss 0.92 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 66.27 | loss 0.74 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 66.28 | loss 1.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 67.05s | valid loss 8.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 66.56 | loss 0.67 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 66.26 | loss 0.57 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 66.30 | loss 1.21 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 66.20 | loss 0.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 66.98s | valid loss 10.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 66.59 | loss 0.74 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 66.27 | loss 1.36 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 66.25 | loss 1.09 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 66.24 | loss 0.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 66.96s | valid loss 9.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 66.60 | loss 0.97 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 66.26 | loss 1.25 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 66.15 | loss 0.77 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 66.17 | loss 0.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 66.96s | valid loss 12.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 66.54 | loss 0.45 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 66.17 | loss 0.87 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 66.12 | loss 0.52 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 66.30 | loss 1.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 66.91s | valid loss 8.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 66.64 | loss 0.74 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 66.14 | loss 0.41 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 66.20 | loss 0.70 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 66.18 | loss 0.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 66.93s | valid loss 10.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.22 |\n", + "=========================================================================================\n", + "Result over 5 runs: 6.899555206298828\n" + ] + } + ], + "source": [ + "num_runs = 5\n", + "best_model = None\n", + "best_result = 1000000\n", + "results = []\n", + "\n", + "for i in range(num_runs):\n", + " features = nn.Sequential(nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(64),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(128),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(256),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=(0,1)),\n", + " nn.BatchNorm2d(512),\n", + " nn.ReLU()\n", + " )\n", + "\n", + " classifier = nn.Sequential(nn.Linear(512, 1028),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=1028, bias=True),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=10, bias=True))\n", + "\n", + " conv = nn.Sequential(nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.MaxPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.MaxPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.MaxPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.MaxPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.MaxPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.MaxPool1d(kernel_size=2, stride=2))\n", + "\n", + " model = ConvMNIST(features, conv, classifier)\n", + " model = model.to(device)\n", + "\n", + " optimizer = torch.optim.Adam(model.parameters())\n", + " scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "\n", + " epochs = 30\n", + " model, result = train_model(model, epochs, train_loader, test_loader, seq_len, digits_per_batch, optimizer, scheduler)\n", + " \n", + " if result < best_result:\n", + " best_result = result\n", + " best_model = model\n", + " results.append(result)\n", + " \n", + "model = best_model\n", + "print(\"Result over {} runs: {}\".format(num_runs, torch.tensor(results).mean().item()))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.227794, + "end_time": "2021-01-21T13:19:14.482416", + "exception": false, + "start_time": "2021-01-21T13:19:14.254622", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "### Smart pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-21T13:19:14.956156Z", + "iopub.status.busy": "2021-01-21T13:19:14.954978Z", + "iopub.status.idle": "2021-01-21T19:37:14.643507Z", + "shell.execute_reply": "2021-01-21T19:37:14.644169Z" + }, + "papermill": { + "duration": 22679.934735, + "end_time": "2021-01-21T19:37:14.644374", + "exception": false, + "start_time": "2021-01-21T13:19:14.709639", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 149.99 | loss 44.63 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 149.22 | loss 11.12 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 149.19 | loss 9.61 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 149.19 | loss 7.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 150.06s | valid loss 9.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 149.88 | loss 6.61 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 149.15 | loss 7.10 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 149.22 | loss 6.19 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 149.15 | loss 5.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 150.07s | valid loss 7.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 149.83 | loss 5.02 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 149.07 | loss 5.61 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 149.06 | loss 5.73 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 149.08 | loss 4.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 149.98s | valid loss 13.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 149.81 | loss 4.61 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 149.01 | loss 4.64 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 149.01 | loss 3.91 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 149.03 | loss 4.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 149.92s | valid loss 9.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 149.72 | loss 3.82 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 149.06 | loss 4.22 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 149.05 | loss 4.34 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 149.19 | loss 4.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 149.96s | valid loss 5.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 149.65 | loss 3.82 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 148.87 | loss 3.49 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 148.91 | loss 3.48 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 148.95 | loss 3.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 149.82s | valid loss 6.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 149.46 | loss 3.14 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 148.72 | loss 2.52 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 148.85 | loss 3.98 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 148.77 | loss 4.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 149.68s | valid loss 7.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 149.38 | loss 2.70 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 148.67 | loss 3.26 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 148.64 | loss 2.91 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 148.65 | loss 2.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 149.56s | valid loss 7.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 153.65 | loss 1.97 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 148.66 | loss 3.20 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 148.59 | loss 2.45 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 148.52 | loss 2.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 150.37s | valid loss 5.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 149.14 | loss 1.75 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 148.49 | loss 2.88 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 148.42 | loss 2.27 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 148.41 | loss 1.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 149.35s | valid loss 6.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 149.14 | loss 2.09 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 148.27 | loss 1.30 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 148.36 | loss 2.45 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 148.48 | loss 2.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 149.32s | valid loss 10.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 149.02 | loss 1.96 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 148.28 | loss 1.64 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 148.41 | loss 2.17 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 148.33 | loss 1.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 149.24s | valid loss 6.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 148.80 | loss 1.39 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 148.20 | loss 1.11 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 158.74 | loss 2.04 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 150.41 | loss 1.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 152.09s | valid loss 8.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 150.93 | loss 0.69 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 150.04 | loss 0.98 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 150.24 | loss 5.29 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 150.23 | loss 1.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 151.16s | valid loss 8.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 150.94 | loss 1.21 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 150.18 | loss 1.23 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 150.13 | loss 1.08 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 150.18 | loss 1.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 151.15s | valid loss 7.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 151.02 | loss 1.09 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 150.15 | loss 0.93 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 150.09 | loss 1.16 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 150.16 | loss 0.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 151.15s | valid loss 5.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 150.91 | loss 1.12 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 150.08 | loss 0.50 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 150.09 | loss 0.94 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 150.05 | loss 0.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 152.66s | valid loss 8.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 164.41 | loss 0.66 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 163.32 | loss 0.86 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 150.19 | loss 0.93 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 150.21 | loss 1.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 156.52s | valid loss 9.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 151.07 | loss 0.80 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 150.14 | loss 0.48 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 150.14 | loss 0.40 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 150.20 | loss 0.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 151.32s | valid loss 8.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 150.87 | loss 0.33 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 150.15 | loss 0.89 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 150.13 | loss 1.11 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 150.02 | loss 0.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 151.10s | valid loss 7.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 150.84 | loss 0.34 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 150.02 | loss 0.18 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 150.04 | loss 0.54 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 149.97 | loss 0.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 151.04s | valid loss 8.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 150.90 | loss 0.29 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 150.04 | loss 0.56 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 150.08 | loss 0.53 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 150.10 | loss 0.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 151.07s | valid loss 7.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 150.89 | loss 0.54 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 150.04 | loss 0.52 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 150.00 | loss 0.25 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 150.13 | loss 0.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 151.61s | valid loss 8.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 162.27 | loss 0.09 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 152.77 | loss 0.38 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 149.91 | loss 0.35 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 150.03 | loss 0.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 153.81s | valid loss 11.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 150.64 | loss 0.35 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 149.90 | loss 0.13 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 149.91 | loss 0.22 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 149.93 | loss 0.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 150.89s | valid loss 10.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 150.69 | loss 0.36 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 149.87 | loss 0.18 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 149.90 | loss 0.25 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 149.87 | loss 0.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 150.89s | valid loss 9.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 150.75 | loss 0.59 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 149.89 | loss 0.26 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 149.90 | loss 0.24 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 149.87 | loss 0.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 150.90s | valid loss 9.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 150.61 | loss 0.21 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 149.90 | loss 0.36 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 149.85 | loss 0.14 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 149.82 | loss 0.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 150.84s | valid loss 10.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 150.70 | loss 0.27 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 149.81 | loss 0.11 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 150.28 | loss 0.20 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 149.89 | loss 0.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 150.95s | valid loss 8.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 150.62 | loss 0.08 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 149.87 | loss 0.09 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 149.86 | loss 0.10 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 149.86 | loss 0.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 150.86s | valid loss 11.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.67 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 151.59 | loss 44.72 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 150.84 | loss 13.42 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 150.83 | loss 9.18 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 150.84 | loss 7.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 151.77s | valid loss 13.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 151.75 | loss 6.44 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 150.99 | loss 5.59 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 151.00 | loss 5.91 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 151.08 | loss 5.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 151.94s | valid loss 8.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 151.78 | loss 5.11 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 151.02 | loss 4.97 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 151.03 | loss 5.19 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 150.96 | loss 4.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 151.93s | valid loss 12.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 151.83 | loss 4.28 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 150.95 | loss 4.55 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 150.97 | loss 4.46 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 151.04 | loss 4.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 151.92s | valid loss 6.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 151.56 | loss 3.76 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 150.77 | loss 3.88 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 150.82 | loss 3.99 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 150.82 | loss 4.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 151.74s | valid loss 6.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 151.42 | loss 3.28 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 150.67 | loss 3.30 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 150.75 | loss 3.83 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 150.78 | loss 3.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 151.68s | valid loss 7.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 151.47 | loss 2.75 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 150.63 | loss 3.62 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 150.65 | loss 2.66 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 150.56 | loss 3.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 151.59s | valid loss 11.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 151.26 | loss 3.34 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 150.45 | loss 2.70 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 150.48 | loss 2.21 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 150.64 | loss 3.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 151.49s | valid loss 5.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 151.26 | loss 2.45 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 150.38 | loss 2.19 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 150.42 | loss 2.82 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 150.45 | loss 2.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 151.41s | valid loss 6.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 151.14 | loss 1.29 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 150.48 | loss 2.05 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 150.43 | loss 1.76 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 150.40 | loss 2.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 151.39s | valid loss 6.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 151.22 | loss 2.76 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 150.37 | loss 1.40 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 150.45 | loss 2.53 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 150.23 | loss 1.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 151.35s | valid loss 7.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 151.18 | loss 1.85 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 159.01 | loss 1.87 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 150.90 | loss 1.47 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 148.44 | loss 1.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 152.38s | valid loss 6.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 148.94 | loss 1.36 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 148.25 | loss 1.69 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 148.18 | loss 1.20 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 148.26 | loss 1.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 149.19s | valid loss 5.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 148.93 | loss 1.01 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 148.16 | loss 1.25 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 148.25 | loss 1.27 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 148.16 | loss 0.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 149.17s | valid loss 8.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 149.00 | loss 0.82 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 148.21 | loss 1.16 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 148.21 | loss 1.67 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 148.03 | loss 0.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 149.13s | valid loss 9.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 148.72 | loss 0.82 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 148.03 | loss 0.74 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 148.07 | loss 1.12 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 147.99 | loss 0.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 149.01s | valid loss 8.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 148.76 | loss 0.70 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 148.08 | loss 0.91 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 148.07 | loss 1.05 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 148.11 | loss 0.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 149.03s | valid loss 7.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 148.84 | loss 0.93 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 148.04 | loss 0.64 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 148.11 | loss 0.92 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 148.00 | loss 0.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 149.05s | valid loss 9.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 148.78 | loss 0.94 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 148.08 | loss 0.55 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 147.97 | loss 0.39 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 148.06 | loss 0.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 149.01s | valid loss 8.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 148.73 | loss 0.57 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 148.02 | loss 0.46 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 147.97 | loss 1.09 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 148.11 | loss 0.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 148.99s | valid loss 8.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 148.64 | loss 0.34 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 147.99 | loss 0.44 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 148.00 | loss 0.58 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 148.05 | loss 0.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 148.96s | valid loss 9.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 148.67 | loss 0.40 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 147.97 | loss 0.59 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 147.99 | loss 0.75 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 147.89 | loss 0.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 148.92s | valid loss 9.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 148.63 | loss 0.20 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 147.92 | loss 0.55 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 147.96 | loss 0.52 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 147.86 | loss 0.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 148.88s | valid loss 9.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 148.64 | loss 0.32 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 147.95 | loss 0.56 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 148.04 | loss 0.35 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 147.97 | loss 0.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 148.95s | valid loss 9.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 148.64 | loss 0.24 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 147.85 | loss 0.12 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 147.91 | loss 0.16 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 147.97 | loss 0.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 148.87s | valid loss 12.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 148.64 | loss 0.86 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 147.88 | loss 0.23 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 147.89 | loss 0.20 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 147.91 | loss 0.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 148.87s | valid loss 10.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 148.64 | loss 0.29 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 148.20 | loss 0.19 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 147.89 | loss 0.54 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 147.85 | loss 0.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 148.91s | valid loss 9.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 148.60 | loss 0.27 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 147.82 | loss 0.13 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 147.83 | loss 0.17 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 147.88 | loss 0.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 148.83s | valid loss 10.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 148.67 | loss 0.07 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 147.86 | loss 0.23 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 147.83 | loss 0.28 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 147.79 | loss 0.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 148.84s | valid loss 8.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 148.56 | loss 0.10 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 147.84 | loss 0.21 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 147.84 | loss 0.38 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 147.82 | loss 0.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 148.82s | valid loss 10.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.19 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 149.61 | loss 40.65 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 148.86 | loss 11.71 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 148.83 | loss 17.70 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 148.84 | loss 7.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 149.77s | valid loss 13.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 149.68 | loss 7.52 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 148.93 | loss 14.16 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 148.94 | loss 6.42 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 148.94 | loss 6.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 149.85s | valid loss 43.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 149.73 | loss 8.64 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 150.64 | loss 14.03 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 148.85 | loss 4.69 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 148.85 | loss 10.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 150.15s | valid loss 6.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 149.65 | loss 5.24 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 148.88 | loss 4.07 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 148.88 | loss 5.23 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 152.72 | loss 5.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 152.69s | valid loss 5.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 157.66 | loss 4.17 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 148.74 | loss 4.00 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 148.75 | loss 4.92 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 148.79 | loss 4.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 151.29s | valid loss 5.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 149.44 | loss 4.41 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 148.62 | loss 3.54 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 148.62 | loss 3.78 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 148.52 | loss 4.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 149.55s | valid loss 5.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 149.21 | loss 3.63 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 148.50 | loss 3.32 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 148.46 | loss 2.98 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 148.56 | loss 3.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 149.44s | valid loss 5.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 149.86 | loss 2.47 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 154.54 | loss 4.26 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 163.93 | loss 3.02 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 159.68 | loss 3.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 156.57s | valid loss 60.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 151.43 | loss 3.06 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 150.71 | loss 2.72 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 150.62 | loss 2.53 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 150.69 | loss 2.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 151.67s | valid loss 8.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 151.37 | loss 2.45 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 150.61 | loss 2.82 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 150.57 | loss 2.17 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 150.60 | loss 2.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 151.59s | valid loss 7.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 151.30 | loss 2.56 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 150.58 | loss 3.06 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 150.46 | loss 1.99 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 150.61 | loss 2.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 151.52s | valid loss 5.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 151.27 | loss 1.98 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 150.44 | loss 1.45 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 150.54 | loss 3.23 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 155.81 | loss 1.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 152.04s | valid loss 7.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 149.38 | loss 27.05 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 148.66 | loss 20.93 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 148.65 | loss 19.45 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 148.67 | loss 17.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 150.13s | valid loss 4.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 159.57 | loss 1.88 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 150.34 | loss 1.27 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 150.52 | loss 3.14 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 150.37 | loss 1.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 153.07s | valid loss 5.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 151.08 | loss 0.91 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 150.30 | loss 1.58 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 150.30 | loss 1.51 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 150.25 | loss 1.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 151.28s | valid loss 6.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 151.10 | loss 1.06 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 150.21 | loss 1.62 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 150.33 | loss 1.65 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 149.36 | loss 1.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 150.58s | valid loss 6.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 148.48 | loss 1.34 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 147.73 | loss 1.39 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 147.74 | loss 1.08 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 147.74 | loss 0.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 148.70s | valid loss 6.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 148.50 | loss 1.17 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 147.76 | loss 1.07 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 147.70 | loss 0.66 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 147.72 | loss 1.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 148.73s | valid loss 6.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 148.38 | loss 0.90 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 147.73 | loss 1.00 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 147.72 | loss 1.37 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 147.71 | loss 0.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 148.65s | valid loss 6.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 148.37 | loss 0.67 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 147.65 | loss 1.05 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 147.59 | loss 0.73 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 147.63 | loss 0.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 148.60s | valid loss 9.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 148.37 | loss 0.69 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 147.60 | loss 0.78 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 147.61 | loss 0.91 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 147.55 | loss 0.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 148.57s | valid loss 77.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 148.33 | loss 0.58 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 147.60 | loss 0.78 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 147.64 | loss 0.75 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 147.56 | loss 0.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 148.58s | valid loss 7.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 148.28 | loss 0.70 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 147.53 | loss 0.40 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 147.54 | loss 0.61 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 147.55 | loss 0.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 148.51s | valid loss 8.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 148.26 | loss 0.16 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 147.52 | loss 0.55 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 147.59 | loss 0.71 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 147.54 | loss 0.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 148.52s | valid loss 8.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 148.22 | loss 0.30 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 147.53 | loss 0.25 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 147.53 | loss 0.86 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 147.60 | loss 0.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 148.51s | valid loss 8.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 149.03 | loss 0.39 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 147.46 | loss 0.35 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 147.56 | loss 0.26 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 147.55 | loss 0.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 148.68s | valid loss 8.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 148.29 | loss 0.50 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 147.49 | loss 0.24 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 147.52 | loss 0.27 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 147.58 | loss 0.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 148.51s | valid loss 8.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 148.30 | loss 0.45 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 147.53 | loss 0.32 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 147.54 | loss 0.19 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 147.48 | loss 0.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 148.51s | valid loss 8.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 148.24 | loss 0.13 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 147.54 | loss 0.32 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 147.48 | loss 0.17 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 147.49 | loss 0.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 148.48s | valid loss 10.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 148.19 | loss 0.13 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 147.45 | loss 0.27 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 147.54 | loss 0.42 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 147.50 | loss 0.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 148.49s | valid loss 7.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 4.71 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 149.31 | loss 34.05 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 148.55 | loss 11.32 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 148.56 | loss 8.93 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 148.55 | loss 7.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 149.45s | valid loss 11.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 149.40 | loss 6.15 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 148.65 | loss 6.55 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 148.61 | loss 5.68 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 148.64 | loss 7.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 149.57s | valid loss 11.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 149.39 | loss 5.78 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 148.78 | loss 5.99 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 148.63 | loss 5.87 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 148.60 | loss 5.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 149.59s | valid loss 9.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 156.69 | loss 4.22 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 164.08 | loss 6.10 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 155.76 | loss 5.18 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 151.14 | loss 4.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 156.53s | valid loss 5.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 151.69 | loss 3.72 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 150.85 | loss 4.75 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 151.04 | loss 4.43 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 150.92 | loss 4.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 151.90s | valid loss 6.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 151.68 | loss 3.68 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 150.87 | loss 3.59 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 150.97 | loss 4.26 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 150.92 | loss 4.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 151.87s | valid loss 8.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 151.64 | loss 4.18 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 150.74 | loss 3.09 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 150.75 | loss 3.20 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 150.71 | loss 3.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 151.74s | valid loss 6.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 151.54 | loss 4.02 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 150.66 | loss 3.16 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 150.79 | loss 3.71 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 150.72 | loss 3.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 151.71s | valid loss 7.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 151.41 | loss 2.83 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 150.56 | loss 2.35 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 150.79 | loss 3.72 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 150.67 | loss 3.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 151.64s | valid loss 6.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 151.34 | loss 2.36 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 150.52 | loss 2.39 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 150.83 | loss 23.65 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 151.14 | loss 25.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 151.79s | valid loss 35.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 151.92 | loss 21.28 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 151.16 | loss 19.26 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 151.14 | loss 19.55 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 151.16 | loss 17.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 152.10s | valid loss 30.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 151.93 | loss 18.31 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 151.16 | loss 17.31 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 151.16 | loss 16.51 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 151.15 | loss 16.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 152.11s | valid loss 26.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 151.91 | loss 15.98 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 151.15 | loss 15.93 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 151.17 | loss 15.56 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 151.16 | loss 15.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 152.11s | valid loss 25.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 151.97 | loss 14.79 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 151.19 | loss 14.75 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 151.22 | loss 15.10 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 151.18 | loss 15.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 152.15s | valid loss 24.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 151.93 | loss 13.65 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 151.15 | loss 15.27 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 151.19 | loss 13.81 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 151.16 | loss 15.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 152.11s | valid loss 23.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 151.94 | loss 14.36 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 151.18 | loss 13.31 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 151.19 | loss 13.81 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 151.18 | loss 13.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 152.13s | valid loss 23.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 151.96 | loss 13.46 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 151.20 | loss 12.94 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 151.22 | loss 13.31 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 151.19 | loss 13.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 152.15s | valid loss 22.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 151.96 | loss 13.29 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 151.21 | loss 12.89 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 151.20 | loss 12.71 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 151.24 | loss 12.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 152.16s | valid loss 22.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 151.94 | loss 12.15 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 151.15 | loss 12.19 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 151.20 | loss 12.00 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 151.15 | loss 13.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 152.13s | valid loss 22.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 151.95 | loss 12.18 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 151.19 | loss 12.07 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 151.21 | loss 12.52 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 151.16 | loss 12.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 152.14s | valid loss 22.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 151.96 | loss 11.78 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 151.19 | loss 12.06 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 151.22 | loss 11.88 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 151.20 | loss 11.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 152.16s | valid loss 21.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 151.94 | loss 11.69 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 151.17 | loss 11.76 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 151.21 | loss 11.59 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 151.20 | loss 11.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 152.13s | valid loss 20.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 151.96 | loss 11.35 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 151.20 | loss 11.53 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 151.29 | loss 11.42 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 151.18 | loss 11.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 152.16s | valid loss 20.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 151.99 | loss 12.14 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 151.21 | loss 11.24 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 151.24 | loss 10.56 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 151.20 | loss 11.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 152.17s | valid loss 20.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 152.01 | loss 10.98 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 151.25 | loss 11.25 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 151.25 | loss 11.21 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 151.20 | loss 11.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 152.18s | valid loss 20.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 151.95 | loss 11.38 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 151.15 | loss 11.11 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 151.20 | loss 10.29 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 151.17 | loss 10.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 152.13s | valid loss 20.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 151.94 | loss 10.94 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 160.21 | loss 10.11 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 151.22 | loss 10.92 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 151.21 | loss 11.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 153.96s | valid loss 19.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 151.96 | loss 10.95 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 151.17 | loss 10.55 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 151.32 | loss 10.70 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 151.18 | loss 10.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 152.15s | valid loss 20.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 151.95 | loss 10.30 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 151.21 | loss 10.50 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 151.20 | loss 10.32 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 151.15 | loss 10.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 152.13s | valid loss 19.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 151.97 | loss 10.38 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 151.18 | loss 9.84 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 151.21 | loss 10.29 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 151.17 | loss 10.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 152.14s | valid loss 20.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.52 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 160.14 | loss 56.05 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 164.54 | loss 14.05 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 154.36 | loss 10.30 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 151.07 | loss 8.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 157.03s | valid loss 12.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 151.95 | loss 6.87 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 151.18 | loss 6.41 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 151.19 | loss 6.99 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 151.19 | loss 6.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 152.15s | valid loss 8.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 151.90 | loss 6.28 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 151.12 | loss 8.11 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 164.67 | loss 5.38 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 152.18 | loss 4.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 155.00s | valid loss 6.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 151.88 | loss 5.26 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 151.16 | loss 5.08 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 151.18 | loss 16.91 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 151.14 | loss 12.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 152.10s | valid loss 5.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 151.70 | loss 5.27 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 150.92 | loss 5.17 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 150.88 | loss 4.62 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 151.02 | loss 20.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 151.92s | valid loss 5.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 151.61 | loss 4.59 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 150.94 | loss 6.24 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 150.89 | loss 4.55 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 151.05 | loss 8.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 151.92s | valid loss 37.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 151.72 | loss 4.33 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 150.97 | loss 9.37 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 150.95 | loss 8.14 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 150.88 | loss 4.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 151.92s | valid loss 8.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 152.56 | loss 3.04 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 150.77 | loss 4.74 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 151.06 | loss 14.78 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 158.43 | loss 3.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 155.46s | valid loss 6.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 155.07 | loss 3.00 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 148.21 | loss 2.78 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 148.23 | loss 2.90 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 148.26 | loss 3.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 150.40s | valid loss 6.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 148.87 | loss 2.45 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 148.07 | loss 3.45 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 148.13 | loss 2.87 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 147.98 | loss 2.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 149.04s | valid loss 7.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 148.71 | loss 2.96 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 148.13 | loss 3.08 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 148.09 | loss 2.66 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 148.26 | loss 3.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 149.07s | valid loss 6.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 148.82 | loss 3.06 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 148.05 | loss 2.18 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 148.11 | loss 2.81 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 148.12 | loss 2.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 149.02s | valid loss 8.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 148.74 | loss 2.28 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 147.93 | loss 2.82 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 147.89 | loss 1.80 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 148.04 | loss 1.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 148.95s | valid loss 10.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 148.75 | loss 2.31 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 147.87 | loss 1.64 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 147.93 | loss 2.10 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 148.02 | loss 3.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 148.94s | valid loss 9.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 148.67 | loss 1.82 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 151.65 | loss 1.75 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 148.16 | loss 2.34 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 148.19 | loss 2.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 149.80s | valid loss 6.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 148.83 | loss 1.25 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 148.17 | loss 1.67 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 148.17 | loss 1.36 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 148.24 | loss 1.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 149.14s | valid loss 6.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 148.75 | loss 0.91 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 147.97 | loss 1.41 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 148.08 | loss 1.86 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 148.70 | loss 1.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 150.56s | valid loss 7.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 150.96 | loss 1.27 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 150.23 | loss 1.34 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 150.19 | loss 1.30 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 150.20 | loss 1.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 151.22s | valid loss 6.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 150.91 | loss 0.78 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 150.14 | loss 1.30 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 150.21 | loss 1.31 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 150.27 | loss 1.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 151.23s | valid loss 8.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 150.93 | loss 1.14 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 150.25 | loss 1.26 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 150.23 | loss 0.99 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 150.18 | loss 1.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 151.22s | valid loss 7.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 154.53 | loss 0.56 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 149.96 | loss 0.58 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 150.04 | loss 0.99 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 150.07 | loss 1.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 151.79s | valid loss 8.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 150.77 | loss 0.76 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 149.91 | loss 0.76 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 149.97 | loss 0.72 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 149.97 | loss 0.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 150.99s | valid loss 9.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 150.74 | loss 0.84 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 150.04 | loss 1.16 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 149.96 | loss 0.80 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 149.91 | loss 0.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 151.01s | valid loss 9.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 150.79 | loss 1.25 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 149.97 | loss 0.54 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 149.98 | loss 1.06 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 149.93 | loss 0.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 151.00s | valid loss 8.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 150.72 | loss 0.64 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 149.98 | loss 1.00 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 149.98 | loss 1.02 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 149.94 | loss 0.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 151.00s | valid loss 9.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 151.82 | loss 0.58 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 149.98 | loss 0.95 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 149.96 | loss 0.50 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 149.92 | loss 0.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 151.20s | valid loss 8.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 150.66 | loss 0.35 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 149.94 | loss 0.74 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 150.01 | loss 0.56 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 149.99 | loss 0.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 150.98s | valid loss 8.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 150.67 | loss 0.40 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 149.90 | loss 0.51 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 149.91 | loss 0.29 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 149.88 | loss 0.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 150.91s | valid loss 9.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 150.64 | loss 0.23 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 149.85 | loss 0.20 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 149.91 | loss 0.30 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 149.93 | loss 0.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 150.91s | valid loss 9.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 150.60 | loss 0.45 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 149.92 | loss 0.63 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 149.85 | loss 0.30 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 149.83 | loss 0.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 150.88s | valid loss 9.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.04 |\n", + "=========================================================================================\n", + "Result over 5 runs: 5.1899094581604\n" + ] + } + ], + "source": [ + "num_runs = 5\n", + "best_model = None\n", + "best_result = 1000000\n", + "results = []\n", + "\n", + "for i in range(num_runs):\n", + " features = nn.Sequential(nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(64),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(128),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(256),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=(0,1)),\n", + " nn.BatchNorm2d(512),\n", + " nn.ReLU()\n", + " )\n", + "\n", + " classifier = nn.Sequential(nn.Linear(512, 1028),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=1028, bias=True),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=10, bias=True))\n", + "\n", + " model = DoXTimes(Smartpool(divider, 0.3), classifier, features=features)\n", + " model = model.to(device)\n", + "\n", + " optimizer = torch.optim.Adam(model.parameters())\n", + " scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "\n", + " epochs = 30\n", + " model, result = train_model(model, epochs, train_loader, test_loader, seq_len, digits_per_batch, optimizer, scheduler)\n", + " \n", + " if result < best_result:\n", + " best_result = result\n", + " best_model = model\n", + " results.append(result)\n", + " \n", + "model = best_model\n", + "print(\"Result over {} runs: {}\".format(num_runs, torch.tensor(results).mean().item()))\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-21T19:37:15.554953Z", + "iopub.status.busy": "2021-01-21T19:37:15.553760Z", + "iopub.status.idle": "2021-01-21T19:37:15.861327Z", + "shell.execute_reply": "2021-01-21T19:37:15.861742Z" + }, + "papermill": { + "duration": 0.755122, + "end_time": "2021-01-21T19:37:15.861895", + "exception": false, + "start_time": "2021-01-21T19:37:15.106773", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQcAAAMwCAYAAAAtS6WbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABGvUlEQVR4nO3dd3xV9f3H8dcng4S9ZG/Z4ABZIlZx1FVFbKvVOtBi0Z+7dVTU1lWtdrhHRUWtW0HcdaE4KoKgDJkiIFOWLBmBJN/fH9+bkMA3g+TO5P18PHjk3nPPvfeT5PLO55zzPd9jzjlERHaXlugCRCQ5KRxEJEjhICJBCgcRCVI4iEiQwkFEgpIiHMzsODObZ2YLzOzaOL5vGzP7yMzmmNksM7s8sryRmb1vZt9GvjaMY03pZva1mb2ZyFrMrIGZjTGzuZGfz8AE1vKHyO/nGzN73syy41WLmY02s9Vm9k2RZSW+t5mNjHyO55nZsXGo5R+R39EMMxtnZg2iVUvCw8HM0oEHgeOBHsAZZtYjTm+fC1zpnOsOHAxcHHnva4HxzrnOwPjI/Xi5HJhT5H6iarkXeMc51w04MFJT3Gsxs1bAZUBf59x+QDpwehxreRI4brdlwfeOfHZOB3pGnvNQ5PMdy1reB/Zzzh0AzAdGRq0W51xC/wEDgXeL3B8JjExQLa8BPwfmAS0iy1oA8+L0/q3xH7YjgTcjy+JeC1APWATYbssTUUsrYCnQCMgA3gSOiWctQHvgm7J+Drt/doF3gYGxrGW3x04Bno1WLQnvHNj1yy+wLLIsrsysPdAbmAQ0c86tBIh8bRqnMu4BrgHyiyxLRC37AmuAJyKbOI+ZWe1E1OKcWw78E1gCrAQ2OufeS0QtRZT03on+LP8O+G+0akmGcLDAsriO6TazOsBY4Arn3KZ4vneRGk4EVjvnpibi/XeTARwEPOyc6w1sIb6bVoUi2/MnAx2AlkBtMzsrEbWUQ8I+y2Z2PX4z+dlo1ZIM4bAMaFPkfmtgRbze3Mwy8cHwrHPulcjiVWbWIvJ4C2B1HEoZBAwxs8XAC8CRZvZMgmpZBixzzk2K3B+DD4tE1HI0sMg5t8Y5txN4BTgkQbUUKOm9E/JZNrNhwInAmS6yDRGNWpIhHL4EOptZBzOrgd+J8no83tjMDHgcmOOcu6vIQ68DwyK3h+H3RcSUc26kc661c649/mfwoXPurATV8gOw1My6RhYdBcxORC34zYmDzaxW5Pd1FH7naCJqKVDSe78OnG5mWWbWAegMTI5lIWZ2HPAnYIhzbutuNVaulljvUCrnTpYT8HtavwOuj+P7HopvtWYA0yL/TgAa43cMfhv52ijOP4/B7NohmZBagF7AlMjP5lWgYQJruRmYC3wDPA1kxasW4Hn8vo6d+L/Gw0t7b+D6yOd4HnB8HGpZgN+3UPD5/Xe0arHIi4iIFJMMmxUikoQUDiISpHAQkSCFg4gEKRxEJChm4bC3Z1qa2YhY1bK3VEuYaglLllqiXUdMwqGCZ1omxQ84QrWEqZawZKkl+cMB6A8scM4tdM7twA8HPjlG7yUiMZARo9cNnRE2oKSVa1iWy6YW9axRUozIUi1hqiUsWWqpaB2bWb/WOddk9+WxCocyzwiLbB+NAP9NHWonxKgUESnNB27M96HlsdqsKPOMMOfcKOdcX+dc30yyYlSGiFRUrMIhYWdaikh0xGSzwjmXa2aX4KemSgdGO+dmxeK9JHks/utA8rL91mOTnmsAmHjg2MLHO354HgB1J9cEoNl9n8e5QtkbsdrngHPubeDtWL2+iMRWzMJBqo/1b3UG4JteD+zx2M4iu6HnHvEYAM/2bQHAS+8fDkDenG9jXKFUhIZPi0iQOgepsIKO4X+9XtjjsX9v2BeAuyb+HID27dbwXg8/ReeZdVcCcNu5+wCw75/UOSQjhYPstdyj+gDw4YEPRpZkAnDP+i589Ju+ftEKP+dql/VTAEjLzub2SfsDcN0+M/3rNMyNU8VSEdqsEJEgdQ57Yd3vBwLQ9uwFzF3dDIAdOf6vZqvn/dday34if9rsxBQYJz+1qgFAWuRvyz3ruwAwYcj+5C2cF3zOgpt781yjf0Xu+UFvrd/R36Zkpt+OiASpc9gL11z9HAC/qr0eOu724GD/ZXHuVu5dc0SFXn/y6nYA1P5XfQAyxifDxa/21OA/EwH49RR/4Slb7y8SlrtycYnPOf+ED6iTpmHyqUSdg4gEqXPYC/dddzoAfzkgjYZz/Oie9d39Cag1DtgAwN/3e4W7W/iryL21tQ4Av6j10x6vtc3tAGBSTm0ABmfvhMjzOv3mAgC6jI/FdxE9ebPnl7nO4tv8fprhDf4JZANw5cqDAaj7wRz/OrEpTypJ4bAXao+ZFPm6a1m93da5v/lg/jqovX/s4wUA/H1wpz1eK2Obv5B27Rn+mH/jT8ayf43ITs3FmVGsOjE2nO1D4X/n/BOA+mnZTMxJB2DaX3sDUHNTTK8UJ5WkzQoRCVLnEGW5P6yi9thVwK52ufaYdSWuv+p8/xe2Z40M/vmjv25t+ycW+teKXZkxt/Ygv9lVPy27cNmwCecD0OVVdQypQJ2DiASpc0iQjHZ+oqwHrvNnMmZaOi/fezQAjVdOTFhdlbXjfX84dmK3ggFPvnM4cOIwul/5HaAdkKlCnYOIBKlzSJC5f2gFQL8sfyh01o5tNJq9NZElVVrGvu25tdPLADSM7GuYmuMfa3drHnnr1yeqNKkAhUOc5fyiHwBf/fruyBI/avD/Lr+cmp+n9o66ji8tp3eN4s3oGeMvBKDL9C8TUZJUgjYrRCRInUOcLTne53Ed8x3DGYv8ZCi13plOwq+KUkHrh/nDsTc3+xcFndCwxX7navdr/EAw7YRMPeocRCRInUMcpdWty9k/+wyATfnbAVh9u59OLSsn9bbJM1q1BOBnl/lh5UXPupw42w8Z77I+9b4v8dQ5iEiQOoc4+vamnry5z0MAnPztrwDIejt1/7LOuc4P5Hq1+RuFy46YeSqgfQ1VgcIhDjae5U9RnvGb+/gudycAP93ZGoAsViasrsqaOqT44ViA+hf5s01zNaYh5WmzQkSC1DnEUMEOuyv+/CIAWZbB6dPPBqDJf1N3c6I0O5v5Ke4yd7QqcZ28NWsBcDl++KRlZZHeZJ/i6zRpAMC3V9bY4/kuz48q7XbpAvI2bap0zRKmzkFEgtQ5xIBl+B/rgW8uA+DUOn4+h2c3N6XZn30e5yemtJh7a8zoMtc55OszAFi7ys+j1bDJZib1eW6v36vHDZew7zWpewZrslPnICJB6hxi4UA/o9OtTZ8utvjB20+lwfSq85fu5NlnAjB+vzFlrFnc572fL/GxrZGJd3e6Xb3VCTPOBWDjtOL7JVp9lspzZSU/hUOUpffowogXXiu2rMfoiwFo//QXiSgpZmoeuwiAnrdfAoALfJrqdvsRILjZ0PPT8/zzltQuXLbvmMhM3ZNnFi5ryLfFvkp8aLNCRILUOUTZ3IsaclKt4ofXWk/wrTIuVc+7LF2H68reVDqRPns+jxmxKEeiRJ2DiASpc4iS7Sf1B2D8Sf8CaiW2GJEoUOcgIkHqHKJkxSB/qbe2Gbu6hmc3NwUgc5Pf51A19zhIVaVwiIG/resBwMRj2wPgVs4sZW2R5KTNChEJMpcEh9fqWSM3wI5KdBki1dIHbsxU51zf3ZercxCRoKTY5+Dq1yLnZ/2Cj9X8fnNca9nWrm5c36+iYv1zSZWfQ2ni/dlJZqX+Pt8MnxujzkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJGgjMo82cwWA5uBPCDXOdfXzBoBLwLtgcXAac659ZUrU0TiLRqdwxHOuV7Oub6R+9cC451znYHxkfsikmJisVlxMvBU5PZTwNAYvIeIxFhlw8EB75nZVDMbEVnWzDm3EiDytWkl30NEEqBS+xyAQc65FWbWFHjfzOaW94mRMBkBkFWzQSXLEJFoq1Tn4JxbEfm6GhgH9AdWmVkLgMjX1SU8d5Rzrq9zrm9mjdqVKUNEYqDC4WBmtc2sbsFt4BjgG+B1YFhktWHAa5UtUkTirzKbFc2AcWZW8DrPOefeMbMvgZfMbDiwBDi18mWKSLxVOByccwuBAwPL1wFHVaYoEUk8jZAUkSCFg4gEKRxEJEjhICJBlR0EFRW2cStZb30ZfCw/zrVkfRPnN6ygWP9cUuXnUJp4f3aSWUV+n+ocRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISlBSHMqXqsT49AXjr9acB2P/fl9Dm1s8TWVJCpTeoD8C8B/YFYO4RjwFww+o+zDyzCwB5s+cnprgSKBwkJlb3qwdALnkA1FrhEllOwuV3aA3AzMGPALAz8uP4a9OpHHjKIQC0SbJw0GaFiASpc5CYWH+A7xiW5eYA0PjxiYksJ6Ey2rSmw6gFiS5jr6lzEJEgdQ4SVW5QLwA+PfEuAA7/5FIAOvF1okpKmCV/8fsS+hw3m7+3+LTE9eocsgaApX/26+8zIxeAmq9NjnGFpVPnICJB6hwkqn7sUROAFum1AGg1JjOR5STUjAvuB2Cnyyt1vQkHPutvRCZdHLelBQCjNw8l48OpMauvLAoHiaqjLvI7Hl/d0gCAOhPmAVD6f4+qJXOC/8+daellrvv1jnwW72wCwCm1fwTgtDr+ag6nPT2KE1v1iVGVZdNmhYgEqXOQqEnv2ZXbmz4PwOOb/KCfvA0bE1lSXG0b2h+A81q8DOzanAhtVuw3/kIAmozPImujf3zkYP+3euap9xWut2yk30nZ+m/xH12qzkFEgtQ5SNQs/3njwttTN7eL3NqWmGLiLL1nV/561ygA+tbYUbC08PGCnYw3fPQrALpf4y8rm7dpU+E6Xb/151hMHpINQP+s7fz3//4OwDHZ1wDQ/na/g9Ll5MTi2yhGnYOIBKlzkKjZ1GNn4e1pD/QCoAHVY9h0fo2MIh1Dcb/7/jg2/8Yf4u2yzA9sCh29KTgr86In/f6IKRfcQ4t0/7yvht8DwK9e8ZehddPnRKv0EikcpNJyju8HwGvH3M8ta/2ht0ZjZwDVewbo61b1BWDT+Y3JW/ZtuZ/XfuxaAP489GDuaB6elT0etFkhIkHqHKTSlh3pP0YH1Mhm2OL9AWi6ZW4iS0qI3Qc9zTioYA6L8ncNAPgr15ORlr/Ha6642X9tPrQCBe4ldQ4iEqTOQSqtyX5+uG+eyyfjtYYJriYx5v1frTLPoSivxb/0h4THNJnMTuc7h4LXbnmjXyce+3LUOYhIkDoHqbCMDn6g0z+7+uHCj25sQ6PR1ePQ5e5u+NkbFX5uRhs/1Hxzn5YA/Pu8h/ZYZ3KOHxhlO3Ir/D57XVfc3kmqnG8v8B/mg7P8/d9/dQRtqAJX4I2z2Tc3B2DWMQ/s8djYn/YB4OGrTgUge078JoDRZoWIBKlzkArLb7O92P1tG7ITVEnqypzQgr+1GFvi408u92dlZr8R/ynj1DmISJA6B6mwhwY8U+x+q/+WPfNRVZVuew5Y2vTbgwtv33zL4wAcUbN4t5Vp6UUOge7583NHLo9uoXtBnYOIBKlzkL22/SQ/49Gh2QXbwfoY3fHirzktcuZkgU/+8SBQfCaonYGrApY0eGq/8RfSma+iVuPe0m9V9tqSIf4TnmX+43PLWn8+RZ3XplJdr4i574trmXzWrkla9kbBGIZRPxwOwPqL/KHNbosWJHRiXm1WiEiQOgfZK+n16vGnQW8XW/bcfw8DYN/c6jk6EvxELX/54/kALD3Jn/kw//hHyvXci0b7yV3a3FYwiez6qNdXEeocRCRInYPslfycHGZv9cOmj17uZzrqfPssoHpduCak4NqWXV7z9w8742IAMs9dxTs9XwTgmG9OByD/yaYAOIP20/y1MpPt56fOQUSCzLnE71+uZ43cADsq0WWIVEsfuDFTnXN9d1+uzkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkFJMQgqv2Ftthw1IPhY3Xkb41yNkJHG5o514/qWqfB73ty1fqJLKJfQz7LU2seMCS5W5yAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCSozHAws9FmttrMvimyrJGZvW9m30a+Nizy2EgzW2Bm88zs2FgVLiKxVZ7O4UnguN2WXQuMd851BsZH7mNmPYDTgZ6R5zxkZulRq1ZE4qbMcHDOfQL8uNvik4GnIrefAoYWWf6Ccy7HObcIWAD0j06pIhJPFd3n0Mw5txIg8rVpZHkrYGmR9ZZFlolIion2DkkLLHPBFc1GmNkUM5uyM+enKJchIpVV0XBYZWYtACJfV0eWLwPaFFmvNbAi9ALOuVHOub7Oub6ZWXUqWIaIxEpFw+F1YFjk9jDgtSLLTzezLDPrAHQGJleuRBFJhIyyVjCz54HBwD5mtgy4EbgDeMnMhgNLgFMBnHOzzOwlYDaQC1zsnMuLUe0iEkNlhoNz7owSHjqqhPVvA26rTFEikngaISkiQQoHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRoDJP2Y6HtPVbqD12UvCx/DjXIl7tafF9v1T4Pdf+pux1kkHoZ1mR2tU5iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJqpbhkHfEQeQdcRD/9+2CCj1/828OJr1rJ9K7dopyZZIoG84ZyIZzBvL28q9Yev0hLL3+ECwjA8tIinGCCVEtw0FEylYtY/H7Y7MAaJResat7//CLHew82+dqoxOjVpYkQEarlgDc/JfRhctmXvQAACfedzgAbvPm+BeWBNQ5iEhQteocLLMGAEceOa1Sr1P362xOG/4xAB81aA1A3oaNlXpNSYzVx7YD4Oc1txUu6zvltwA0/WleQmpKFtUqHDafchAA97W6H4Dur15CZ8Jng5Ymp6HjsoZzAZhQt7tfqHBIKWm1agHw80v/t8dj2S828Deci2NFyUebFSISVC06BzeoFwAP3nkvAM9s8q1ktxvmk1eB1xt4TIqc2C8lyjnEd3y3Nn202PJtbgf1nvsiESUlHXUOIhJULTqH9SO3AtA6IxeAP176CwAy10/dq9fJaNEcgCfavsNOp1xNZYtPCX/0fz3/V8Dy+BaTpPQJF5GgKt85rPv9QF7e/x8A/GfjAQBkfrB3HUOB2be0AWCny2PY4qMByFu9JgpVSrwd1396sfub8rcDkHdzU9LUOQDVIBzShq6lZYYfEfn4c8cB0JrP9+o10nt2BeCZox4BIMftZMldXQConbP3h0IlsXJO6Mf9LR8ptmxpnm+i0z7+OhElJSVtVohIUJXtHNKbNAHghi5vFS5rffvedQwF5l7UAIC+Wf7A54Pre5Q4lb4kv1X9MvdYNvSNywEqNCiuqlLnICJBVbZzsFrZABxbayP9vzwHgObMqdBr7dP+x2L3n13Ul32YX7kCJWEye68vvD13Zw4A3e5fC1ChQXFVlToHEQmqsp1D/o8bALh1zUH8tuMUAD5p0RGA3JU/lOs1Mtr5Q5f/6/VCZInP0m1f7APqHFLO9hP7A/BlvweBdADm7mgGQN787xJVVtKquuEQmaDjveXd+LTXcwCsfLM+AJ8+MrDE523o4c/Eq9N+Iwe3XOxfa7erD1r1PlkvZW3bxwdCRiQYAEZOPQWADkwPPqc602aFiARV2c6hQMObszn8pjMAGLffkwDceePEEtefkuP/quSRRt8aOyJLrdg6be+fmRJXhZbitg/dUHi7YEdk68er/H+BClPnICJBVT82J8+k/gn+5tmDLwNgQ+esEldv/OiurmL5Kz0BmDrgyWLr5FfTCUdTVXoXvyP6y37/KVjC2z/tB1T8PJvqQJ2DiARV/c6hiPQJXwHQeEL51t+2uK6/MaD4cjeoF/a/aVGqSmJt1RFNgeJHKR7+0J9Vq+HSJatW4bDXIvsh03ZrsBQMqWV7o+I7lKfuyKP735cBkJuIglKENitEJEidQ2kig512HwQlqaXJESuK3X9940HkrVmboGpShzoHEQlS51CK/OziHcOavJwEVSIVYVn+kPWQljOKLf9xZ21czvZElJRS1DmISJA6h1I8c9y/AZizw3cQZzx5DQBt93IOSkmQPD87w6NzBgFw+SELAPh4SUdaMythZaUKhUMpblk0BIAtD7UCoO1YhUIqcbn+QGWHkT8B0PP2YQCkTa+bsJpSiTYrRCRInUNpjvIDZWqzLMGFSGXkLVgEQLvTElxIilHnICJBCgcRCVI4iEiQwkFEgpJih2R+w9psOWpA8LG68zaW+LzNXevHqqSoKu17kOiI92ehzkJ/eNR2psaVLkr9+YwZE1yszkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQkqMxzMbLSZrTazb4osu8nMlpvZtMi/E4o8NtLMFpjZPDM7NlaFi0hsladzeBI4LrD8budcr8i/twHMrAdwOtAz8pyHzCw9WsWKSPyUGQ7OuU+AH8v5eicDLzjncpxzi4AFQP9K1CciCVKZfQ6XmNmMyGZHw8iyVsDSIussiywTkRRT0XB4GOgI9AJWAv+KLLfAui70AmY2wsymmNmUnTk/VbAMEYmVCoWDc26Vcy7POZcPPMquTYdlQJsiq7YGVpTwGqOcc32dc30zs+pUpAwRiaEKhYOZtShy9xSg4EjG68DpZpZlZh2AzsDkypUoIolQ5tT0ZvY8MBjYx8yWATcCg82sF36TYTFwAYBzbpaZvQTMBnKBi51zqTF3t4gUU2Y4OOfOCCx+vJT1bwNuq0xRIpJ4GiEpIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkqMyL2sRD2vot1B47KfhYfinPq/1NKQ8mkdK+B4mOeH8W3G5fk11Ffj7qHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCXFIKjKymjXhqYvbgDg46k9AOj2kL+fN2tehV4zvUkT1h3fCYCGL34FgMvJqVyhIilEnYOIBKV055DRvBkAt0wYS9dMP0j5yHXNAcib9W2FXjO9SRMAzvzsKw7OHgfAxTMv8A9+Pasy5UoVlL5PYwDm3d2WwZ39Z2754TuB1O80UzIcMlq3AqD+i1sBOKBGOl0/uBCAzsO+qtRrz/lrewBOq/MOB91zDQAtv/68Uq8pVc/qSw4B4MbL/wPAL2q9V/jY0H1OAiB3+Yr4FxZF2qwQkaCU7BzWD2oDwKvtHyxc1v2G1QDkVvA13cADAVhw4iMAHD7zVNqMngtAXgVfU6qe9C4dAXjsynsA6FXD/xcqeubtyofrAtDiAr+Jm7vyh7jVF03qHEQkKKU6h4x2vmNYc/L2Ysv7/vNSmi+t2H6Bgo7hhmefKrb8p7eaU3vdwgq9plRdc65tCPj9XCWZ1Oc5AOZP3AHAL5/+I/ve9jUA+du3l/i8ZKPOQUSCUqpzWHpvHQC+7f8kADes7gVAqydmVXi/wPLBtQEYlOW3Gvf7fBgAbe/XEQopLr1HFz446p7IvZoA3LmuOwBTNrTlxY7vFFu/S2YNAB4982HuHH0yAPmLvo9LrdGQUuHgnAGw0/komLSuPQDp21bv1euk1fU7jObd1oNXh9wFQD6ZALQ9dWY0SpUqaG3/xrTPqAXAiKWHAbDs4J8ASKu9lT4XXgrAVb9/CYAz6/rP5WHZ8MbYJQDM/kXq7KTUZoWIBKVU57C7t7u9CsDwCUewZHMLAHY83rzE9X/4mZ8O9IQB0wB4veVDEOkYBk07HYCGVGxkpVR9eVmQH5lSdsYj+wPQiIkA5G/ZQot/+U3Rl07qB8AZdd/0T3T5rMrx3arbnjqjJtU5iEhQSnUOTe/3O4E+GpUNwBE1/WGhx9t+RBp+f0T+XSVPFl64TpEJxZ/f7M/PaHzdnoNZRIqq+6uVhbc3HrsFgEZP7LneX9q9Hrm162/vp193A6DL+skxqy/a1DmISFBKdQ4ZH04F4N5DjwTg1kPaA7DsGMeCk/4NwOQc3x2c9d6Fezy/83/89t5bL48uXPb32ccC0Gq6zriU0m0e2wJ6+tvn9vAXYfqkX38A1vSugzvxRwD2y/TdwZyd/uzMnpk1GHf8/QD86eDf+xf4Yka8yq4wcy7x1+ypZ43cADsq5u+TdoBv7d78rx/B9te1+zH5xH0ByF26LObvL6kto0Vz/j7Rn8ZfMIYhtKn6m++OA2DbZf70/1Oen8B59ZYC0P3j4QB0/O20uNRcHh+4MVOdc313X67NChEJSqnNispacqMfD1+Q8u/ddhh1ln6RyJIkheSu/IERV18BwBP/9IPnumT6Eba4fDq95zcZul3iz+bN3zIbgDs+PInhQx8G4M6+rwDw2IG/8OtMnxOX2itCnYOIBFWLzmHtiIEAzDjYz/+wOHcbADXX7EhYTZKa6rzsd0Sexx8B+PE0PxvZ9o1ZdL/6OwDytmwp9pyu187mqM6/BOD9nmMBuPFG/3e51S9jX3NFqXMQkaBq0Tls/flPxe7/etr5ADT9qHLzTUr1VdBB1Hl517KSzgzO37yZTeP283cih0LvPMB3EA+1GJy0J2FVi3B4pM/TAKzM8y1g43tqJbIcqYaaPOLHPgw4/rfArglhLr+qPR2vTM5w0GaFiARV+c5h2chDGJTlNx++yPEdQ7o2JyTe8v1GR+N/+c/g2qf9TvE5pz/ISc+dA4CbmlyjdNU5iEhQle8czjxjfOGgp+FTzgWgHX62p/TGjaCpv2JR3hzN4yCxl/axn2h28FNXAzD7dw+y+TbfRdQ71c/5kL95c2KK2406BxEJqvKdQ1H5eT4LCy5l9ovzP+XVhX4GqWQejCJVT6dR/kSsp09tzif7jwHguAN/B0DaZ9MSVVYx1Soc5hzmZ+bIP8xvZvT85Hd0usmPZtNVrSSeCs4CfumUwzn7gxcBWHu1n7yo6WcJK6sYbVaISFCV7xzevf5wZo/0mw4TJ/n5HLrd669+3PGHeeSl0BWIpOrJm/Mtv1l4DABv9H4MgOEHX+QfTPCEMOocRCSoyncO2W9MZs0b/nYn/NwNFb0St0gsbD3F7wOb9HlLANZ39XNENEzwVCPqHEQkqMp3DiLJLm/tOgBGdfHzmTaMXCgn0dQ5iEiQwkFEgpJisyK/QS22De6f6DIqpfZ3mxJdgkTBlo71El1CqSr6OSv1+3p1THCxOgcRCVI4iEiQwkFEghQOIhJUZjiYWRsz+8jM5pjZLDO7PLK8kZm9b2bfRr42LPKckWa2wMzmmdmxsfwGRCQ2ytM55AJXOue6AwcDF5tZD+BaYLxzrjMwPnKfyGOn4yfhPg54yMzSY1G8iMROmeHgnFvpnPsqcnszMAdoBZwMPBVZ7SlgaOT2ycALzrkc59wiYAGQ2scpRaqhvdrnYGbtgd7AJKCZc24l+AABmkZWawUsLfK0ZZFlIpJCyh0OZlYHGAtc4ZwrbSSGBZa5wOuNMLMpZjYlN2dL4CkikkjlCgczy8QHw7POuVcii1eZWYvI4y2A1ZHly4A2RZ7eGlix+2s650Y55/o65/pmZNWuaP0iEiPlOVphwOPAHOfcXUUeeh0YFrk9DHityPLTzSzLzDoAnYHJ0StZROKhPOdWDALOBmaa2bTIsuuAO4CXzGw4sAQ4FcA5N8vMXgJm4490XOyc0/ytIimmzHBwzn1GeD8CwFElPOc24LZK1CUiCaYRkiISpHAQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiARlJLoAgLQNW6n52uREl1Ep+YkuQKKi5jeJrqB0Ff2cVeT7UucgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJGgpDiUKbGR0bwZOzq3DD6WOX8580buC0CD2QZAoznbAUj79Ov4FChJTZ2DiASpc6hCNp51MADrTvAdwLW93+Gcem8H1318Y1t+WXccAA1PzS722Imt+sSwSkkVCocUlXZgdwDmXlqbT4+5B4Am6V/6x8rREA6vvwTILnM9qb60WSEiQeocUtSWDnUBmH/8w0DNcj/v3xv8Tshnv+9X4jr1WVCp2lJZWq8ebG9eG4DFQ/2O2l/39x3ZTpfOR0/3B6DFxxsBcF/PSkCV8aHOQUSC1DkksYzWrQCY86fWNPvc/xWr9/wXAKTlOADm79zB0twGALTJ2ADAud8MA2D9nMY0+9Kv1+DzpQC4n34CoP6G6tsdFOUG9QJg4cX+/nMDH6VPjfSSn3C1P3t421U7ABi1oQcAD00/nM7D5wCQv317bIqNM3UOIhJkzrlE10A9a+QG2FGJLiNppDeoD8CAj9cAcN0+Mxl03SUANHxqYvF1e3Ylb9Y8f7t7ZwDy5i30D+bnxaPclJN/aC8WX+RvvzXoQQA6Zuzab/P+Nn/7utlDAdiwpAEA3wy9nz+v8oeL/958CgATtmcCsCU/i5GjzwWg9d8+j2X5UfeBGzPVOdd39+UKhySSlu0PLe54sykA73T34xC6vnIR3a73LWvepk2JKa4KWPhcLwCeDWw6nLHo5wB8ObcD3S6PbB5s2VJsnWYT67H6snYAdHl4LgA3NJsAwKfbWjCk9noAhh58MgC5S5dF/5uIgZLCQZsVIhKkHZJJIr1hQ+be2gWAed0fAmBqjn+s2y0L1TFUQFptf0jy21v2B2DO4X4TIo10vozs0D3zNb8nsuvNvlvosmFKiVOx7V93Oe9ndABgyj/8KNLGd00CYGjtDYBF+1tIKHUOIhKkziFJrDirO/NOuR+A17c0BODxE/12cN6a7xJWVyrbMMR3DB+e+k8A0qgFwPhtWdxxkT/c2+k9f2g4tOvWMvx/j7SuHQF47NVG/OM/TwGwf43VkbX8a6ZbGvtP+i0ArVZXjd+XOgcRCVLnkCQ2D9hWePveRf7ITc35VeMvUKK4yAGJ7a74voDN+TX5YUANALb90g+H7tR5ZeHjG7f7o0antvsKgIsbPA3AlB01GJRVsEeiVrHX/N/2fFr91b+Py8mJ3jeRQDqUmSRuXfQlvWv4Rm59vh9hN/DlKwHo8PoO0id8lajSUlZaXX/+ybaxjQF4ptszADRLr0mm+eTIc8V3P+a4XLKs7L+ZuZENkcEzTgeg0cV55C5cHJW6402HMkVkr6hzSBLvrpjGThce0bjT5bHf+AsBqP+lb3l/au1/b/UWwj4zig/WWXuAP4TXbMJq8rRpUii9mR9cNuem9hzTZyYA8zf6Zd8v38evUyOPIV1nALtGQYZ0+Wg4AF2vXA5A3qrVJa6b7NQ5iMheUeeQJOY/0o/5J/47qq85Oce4YnZkm/jE+VF97apsxTh/puW0/s8UW744dytD778GgFb3+LMzXW5ufIuLAXUOIrJX1DkkCcvIYMfgAwE454E3AKiV5g+JnVhrTeHe9b2VHxkM3PO5ywDoePXE0lav9hbdPpCvzrkbgJpWo9hjfW+/hKYPptYZl+VRUuegcQ5JwuXmkvnBVACe71b8WhP3/fp08jL9MfRDrvLt7B3NvyzX6xZMNtv6wJVlrFm9rbj6EADePfPv1LTiYxjuXd8JgOZPTCvxvIuqSJsVIhKkziEF1B4zqfD2GwcOBOCOs33nsNXtoM8n/wdAu8f8psfay7YCMKVf8R1qsqedx/hu+tVL/g5A24xdXcOSXP9zfP1PfpM3a2v5urWqQp2DiASpc0gxbd+NjNs/23+pZTWYc/jjflE7fxbn2+3fjay9K/uX/NAIgM4sjkeZKWPxib7bal+kY1iZ5zuGc67ww9drvTVpzydWA+ocRCRInUOKyZzyLQAHf3UGAF8c9HzhY0+3fz9yy2d+jtvJiZFBUN0u88OoNeWsl97Yd1Jf//KeyJKswscGf+Yn8+04rnp2DAUUDikmf/NmAJpf6ieEOWn0EK5r/xYAA7P8f/2xP/nzBK5/+zd0+kPJk5lUV+kNG3LFpE8BqGNZxR67c113Ov/eB3B1OmwZos0KEQlS55Cichcv8TeOhMsu8xdh2NzPTxjT7Ya1AHT6/ouE1Jbs1g7pxjG1PgIgb7cBwm/fPJjaW6r35kQBdQ4iEqTOoQpodp8f798scj/1zxOMrV9d9cEeM0B1esPPl9FlrLqGAmV2DmbWxsw+MrM5ZjbLzC6PLL/JzJab2bTIvxOKPGekmS0ws3lmdmwsvwERiY3ydA65wJXOua/MrC4w1cwKjpnd7Zz7Z9GVzawHcDrQE2gJfGBmXZwrYZojkTg7sOYS0s3/Xfxiu/9Y9vi7n8lJXdcuZYaDc24lsDJye7OZzQFalfKUk4EXnHM5wCIzWwD0B3SusCSFK54dztzf+6uK/W70pQC0WVj1TsWurL3aIWlm7YHeQMGG2SVmNsPMRptZw8iyVsDSIk9bRulhIiJJqNw7JM2sDjAWuMI5t8nMHgZuBVzk67+A3xG+YOAeM8qY2QhgBED2btcAEImldjd+zrE39gKgDeoYSlKuzsHMMvHB8Kxz7hUA59wq51yecy4feBS/6QC+U2hT5OmtgRW7v6ZzbpRzrq9zrm8mWbs/LCIJVmbnYGYGPA7Mcc7dVWR5i8j+CIBTgG8it18HnjOzu/A7JDsDk0t7D1e/FjmH9gs+VnPJ5rJKTBnb2tZNdAmVlmy/j1T9mcb751jqz+mtMcHF5dmsGIQ/QXimmU2LLLsOOMPMeuE3GRYDFwA452aZ2UvAbPzO34t1pEIk9ZTnaMVnhPcjvF3Kc24DbqtEXSKSYBo+LSJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkaCMRBcAYBu3kvX2l8HH8uNcSyxlfZPoCiov2X4fqfozjffPsSI/J3UOIhKkcBCRIIWDiAQpHEQkKCl2SEpqS6tVC4A+n2/mxibTADhm9i8BqPHz7xNVllSSOgcRCapWncP2k/oDUPO/XwHg+vZg0ZDaAPzsyJkAfPrh/oXrt5iYB0D2G5PjWWbKKOgY5o/qCsCrTUYVHqJbOr0FAB1R55CqzDmX6BqoZ43cADsqJq+dvk9j8l6sCcCTnV8AYFVeJgD103bSNqNWic9dnbcVgBV5NQC44PbLAWj86MSY1JpqFt82EIAZ594HwJkLj2fdbR0AqPFOeNyKJJ8P3Jipzrm+uy/XZoWIBFX5zYr597ZlXrfHI/d8l9A03d97aEMXvtrcFoBlWxoUe1665fNW1zeKrf/iDf8A4MI5l5D22bRYlp0SdjTNLXZ/xqed6fCOuqqqQp2DiARV2c7BDTwQgBcPeYSCb/Odbb5zuOPqYQDUnbUW1vwIQNr6pcWfn5ZOl39dBMDs0+4HoGNmHQC23bCJ+uc2AyD3h1Ux/C6SW2adHQBszvdf276fk8hyJMrUOYhIUJXtHHbW90cYetXIIB9/RObqJ34HQJtxnwOQV9oL5OfR6Q9fANC9xiUAzDj5XgA+3n8Mg472XUX9Z6pf55DeyR+RmHXYaAAuX+GPNKV/9FXCapLoq7LhkJdthbcP+PxcANre9nmFXqvzxZMAePNof+z+1Drr2DBkCwD1n6lEkSlq3k0NovI6Ocf3A2Bzm10fwyZTNwHgps6KyntIxWmzQkSCqmzn0HXkrr886VPrRuU1r/9yKACnHvE4F/f8BIA3aRiV104ldw94sdj9/z13EADNKb0z++7Z3gDcO+B5APav8RkAzdKzCtdZsNMfHj15zB8A6HjVF1GoWCpCnYOIBFW5ziHtgG4ADG7wPgDzd25nnxk7o/LaDT/O9jeOiMrLpaT0evWoneYPWb63zZ+X0vzuPTsGy/Q7hHcccQAA1z/8BIdlTwUg0/yossk5vmM4Z+6p/LHDewAMqe2HrD801A9cu2f0KeTNnh+T70VKp85BRIKqXOfw7bAGAJxeZw0Ah844m3olTF4re2/RFftxaPZ4AHp8dA4Anfi68PGCw5zzLvaDxAoGkAGM3+YHkV307rkAdLt3LQBZ87/jQboAcP/4NgC82e0VAP7Wtj41ZsfkW5EyVLlw+MPxbwF+cwKgxoONge8SWFHVYgdsKryd+V3NPR4vOMw594gHgV2zLJ+58Hg2XdMKgM4T/aHh0DiTBQub+xvdolKuVII2K0QkqMp1DgUeWXcYANlvaqKWaOrWtOQRodanJ+MOfThyz8+Z0XPCCAA6D5+DbZ9e7vf5y2o/QCp7wsyku1ZGdaHOQUSCqkznkN6gPgB105YluJKqrXWtDaQV/E2x4rOIzb8si+6ZvmPo8+VZAHQ80++sLO9f/4IzPbfk+sOc+du3V7JiqSh1DiISVGU6h2XDewJwZt2PAPhqS/uov0fOCRsLb2/NrxH1108F+S6N/II+wFmxx1o021D4WI8mft/E+nK+7u5neh424zQA6ulIU8JUmXCIpdwj+wDwQu8HIkuyGHenP025Phr7X6DB8O1M+tRvVjzQ1k+xN/DOqwDoct/35C5fUeJzu7/oZ6lelbcNgOx7G0UeUTgkijYrRCRInUMpCjqGHy/3czd0y/Q7yS5aPogGL0aufZGY0uKuoO0/rP6HJa6Tu3wFdx49FIADxy4E4Juz/LT1Fx1+BCt/4buBvHV+ar4NZ/up7Q+9YhJ/afY/APq84DuNju+oI0u0MjsHM8s2s8lmNt3MZpnZzZHljczsfTP7NvK1YZHnjDSzBWY2z8yOjeU3ICKxUZ7OIQc40jn3k5llAp+Z2X+BXwLjnXN3mNm1wLXAn8ysB3A60BNoCXxgZl2cc6XOylZZ9Rb7l1+cuzUqr2cZGWz4w2YAphzkL4bz/jY/XHj+n3tSY+eUqLxPqshbsAiAF37ozykd3wGg3aFLAH+mJkDepk3kLlwMwNTe/u/OYWdfBkCjGRuwffzZsYse8OdPzDrM78NZlbdtV8eg+RuSRpmdg/N+itzNjPxzwMnAU5HlTwFDI7dPBl5wzuU45xYBC4D+0SxaRGKvXPsczCwdmAp0Ah50zk0ys2bOuZUAzrmVZtY0snorKLYLf1lkWUzVHutP5nnn1u4AdMz2Z2V+23o/cpctL/P5+Yf2AmCRnzeWX3Wfxu1NXyi2zu1X+Snta75bfYdkbz+/HneN9WdFvdntNQAuHz8IgMn/HkidFcUvdLOmnz+02e+yhfyrpZ/5qWAQ1aiN7QF48p8n0nG0LoaTbMoVDpFNgl5m1gAYZ2b7lbK6BZbtsd/OzEYAIwCyKfl6lRV1UQPfBq96sx5Tfmxb5vp3dBgF+NmqC0zd4TdVzp48HICOH84Fypi1uorLm/8dn5zsx5Q0fMvvqL275af+wVs+LVyvIADyA2Mj9/vsPAA6/dGfst1ouYIhGe3VoUzn3AZgAnAcsMrMWgBEvq6OrLYMaFPkaa2BPQ5wO+dGOef6Ouf6ZpK1+8MikmBlXmXbzJoAO51zG8ysJvAecCdwOLCuyA7JRs65a8ysJ/Acfj9DS2A80Lm0HZLRvMr2j7/zh8dOuNxPAPuXfWbu1fNzI33B9B1w1ot+Z1qHa/WXLSS9md+SXHJeJwC2dNjJu8fdA8Cx717hVyry8er6mD9Pwn25d78Tia2SrrJdns2KFsBTkf0OacBLzrk3zWwi8JKZDQeWAKcCOOdmmdlLwGwgF7g41kcqRCT6yuwc4iGanUOBgkE7R7w6gz82/LbM9bt97K+GVWOm3//R+m8VuwCOSKopqXPQ8GkRCaqyw6cLBu18sF9dPuCgMtffl2kxrkgktahzEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIUFIcynT1a7H9sPBZ3bUW77r82tb29eJVUlQUrT0VpNrPF1LvZ5wopf5u3xgTXKzOQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJykh0AQC2cSvZb04OPpZf5Hb2N/GpJ1ryy14lqaTazxdS72ecKBX53apzEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISlBSDoKR6y2jejI2D2gOw/OcOgEVDRgGw0+UxaNrpAKxZ2hCAHnf8AEDu4iVxrrR6UecgIkHqHCTuLCsLgIU3HwTAA79+jMNrbi22zk7n/27lk8+nvZ7zC3tFvjT+HQBtT419rdWZOgcRCVLnkCTyBh9Exl9WAfBG19cByLR0oPh2d+PrMwGwxcsBWHdSDxq96s+qyd+8Oa41V9SSq/sAMPPse0tc57zvjwLg8Xbv7/HYtENGAzCEfjGoTgooHBKkoLXePKQXADf+bXRha11wpuFOv2+uWGt90J/PBeDA5r7pe639A/RrcCkAze7/PPaFV4IbeCAAo393f4nrHPDEZQB0uPUrALrdfTFzT34w9sXJHrRZISJB6hwSJGfw/gB8eM8Dhcs+2lYHgL/81e9wy9zqCh/b1M7neI3IfrtrrvKt9cb8XOqszIt5vZXlBh6I++uPAPTxTVNhhzTup6aMPncIAO0n+Xk9XL7/nrr+YTrHv/p/ANz6b394s2+Wf+zobzbzwX5141F+taTOQUSC1DnEWcF2998efqTY8jO+O4FNN7YBoOFHE/d4Xv1OHQDo9fJ3AHSv4XO922t/oMuYSTGrN1pW96vNl918t1Owo3Vj/g4AbnzpdNpP3PN7BnA5OWS+NwWAs969EIBZJ/lu6+pG3/Ho88MA6HDG9NgVX02pcxCRIHUOcbb++m3Aru3uE+b+EoD0q+qR/vVXJT5vQ59mANzY9KViy9u8F4MiYyDt6HXkR/YyFByFOW9hZD/Dn8Ndw+66/J/fH3H/oT0B+GOjuZzZ40sAPqdGNMsVFA5xteiFA5jV+wkAluX6kEi73p8v4L6eUeLzLCuLTlfM9utHmr2CcQA1Xw1PzJssMlq1BODKrh/s8djClzsD0Iw1e/Wao187GoA/nje3ktVJacrcrDCzbDObbGbTzWyWmd0cWX6TmS03s2mRfycUec5IM1tgZvPM7NhYfgMiEhvl6RxygCOdcz+ZWSbwmZn9N/LY3c65fxZd2cx6AKcDPYGWwAdm1sU5l/zH22LsnB6TC1vr73Pr+YVflN4xAMy750Bea+sHAhUc/vv+H10BqEVy74xcf2hbAH5d57XCZSOWDgagVWTnam4lXn+/mssAmLzvkf61Fi6uxKtJUWV2Ds77KXI3M/LPlfKUk4EXnHM5zrlFwAKgf6UrFZG4Ktc+BzNLB6YCnYAHnXOTzOx44BIzOweYAlzpnFsPtAK+KPL0ZZFlUk7pPX1XMOfS+gDMPWnX8OGCgVJ1P18EQLK3Y2sOsj2WfXdHdwBq/lD5/SUn1l4HwF19mwNQR51D1JTrUKZzLs851wtoDfQ3s/2Ah4GO+BNpVwL/iqy+56ch0GmY2Qgzm2JmU3aSU4HSRSSW9upohXNug5lNAI4ruq/BzB4F3ozcXQa0KfK01sCKwGuNAkYB1LNGpW2mVBljF/Xi6sYzAeidtQWAn83Yvsd6/Wu9AsARNf1jRS/5duX0XwPQetWsGFYaPXm1fPVpRf4OResIS6alFx4WlegrMxzMrAmwMxIMNYGjgTvNrIVzbmVktVOAgqvxvQ48Z2Z34XdIdgaS+3hbnDQ/azlDXj0FgDe7+R10BWER8rM/+bMt889YV3hWZtNHa8W4yug64IDFAIU7YqNpp8uLyeuKV57OoQXwVGS/QxrwknPuTTN72sx64TcZFgMXADjnZpnZS8Bs/I7oi3WkQiT1lBkOzrkZQO/A8rNLec5twG2VK63qyd+8GY7yE7IcecpFAKzus6vdbjjH98j1n/X7c9c87ffFzO31Ao9vbA9ArVm+WavM4b+q5Ptcf35GzTU7ElxJ1aNzK0QkSMOnE6TWOD94qf24kteZe+RjgN9ef3De4QC0XDo75rUlu/OH7jqh5OQnrgag7UfJPQtWKlLnICJB6hySUMEgKD/uzG9XN7svO3EFVcKWv/gTr6Y8kV44g9OSl/0sWG1PLflITWn61fQDwCbnGO3/4edx0DGL6FM4JKGFNxY//fjUr8+n+Ucln86dzNI+/hqAi++5hC//5CeWfX/AwwCce4SfTDa9nN/bohcOAGBQtg/NQ74+g0Zb5ke1XtlFmxUiEqTOIYkUTCH3+oCHIkv8poSNb5igiqKnxYQf6XvkWQBM6fcMAMsG+++v3UelP3fLrwYA8NKA+wCYmOPPVm3019Tc1EoV6hxEJEidQxJZ3a82AB0y/F/EgqHBGdtT/wSC/BlzaXW939E6blwjAF4/9x8AHLfPH+l8cfF5KayPnwpu1cD6PHKlvzJW4aS6b4wAoMsXGpUfS+ocRCRInUMS2b6P7xAKOoZ7fuwBQONHyzcBa7LLmzUPgKeOOwKAR0b57/OdE+/ipZ/562e+8Jyf0emxEf7IRu+sXQcpj5vtz0jt9rAfgq7Dl7GlcEgiZw0tvmeuYCLV9lSNcChQMJVb1hlNALiw9+Vk/ukHAKZe6jchur1xceH6HV7xMZD1kZ9SL3+nzqOIB21WiEiQOockMnZRL6D0OR6qkrw1fkr6zPfWQOR0iSH0A6BLYAqQ1N8tm1rUOYhIkDqHJOLG+0N817X2g36aTdEcOZI46hxEJEidQxJpdp+fk+AbP0qYmpp6UxIoKcIhv0Etth0Rvu5N7QWb4lxN5WzpVC/RJQTF8ueYrN9zgVT7DMVCqb+jcWOCi7VZISJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJCgj0QUApG3YSs1XJwcfy49zLZVV85tEVxAWy59jsn7PBVLtMxQLFfkdqXMQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEhQUhzKlPJbcPfBAHz3m38DcM73h7Fq4KZEliQVkHtkHwAWnZLBlUe9DcCI+osBSMMAyMdx4+reALyxeD8AWv4t3b/A5Jkxr1Gdg4gEqXNIMYMOnl3s/n/afcLPTrkAgFrjJiWiJCnB8j8dAsCWzjs4o0/xQX43Nx0FQD75pEX+RudHhmt1nzACgKavZ1H3xS8AaEnx33s8qHMQkSB1DinmP+0+2WPZisP8NmqncfGuRkoz/bIHAL/vYFXeNgAeWue7iS7/9d1e7W9rkL3WAdD48YkAdOTreJcaVO5wMLN0YAqw3Dl3opk1Al4E2gOLgdOcc+sj644EhgN5wGXOuXejXLcU0ekPXyS6BAk4bOavAfhw/xcLQ2Fqb9+sd2FKwuoqr73ZrLgcmFPk/rXAeOdcZ2B85D5m1gM4HegJHAc8FAkWEUkh5QoHM2sN/AJ4rMjik4GnIrefAoYWWf6Ccy7HObcIWAD0j0q1Iimkwe930OD3O3hzS2OGNpjK0AZTSe/emfTunRNdWrmUt3O4B7iG4me/NnPOrQSIfG0aWd4KWFpkvWWRZcWY2Qgzm2JmU3aSs7d1i0iMlRkOZnYisNo5N7Wcr2mBZW6PBc6Ncs71dc71zSSrnC8tHV+8kI4vXlhs2YK7Dy4cHCXJI3fpMnKXLuPacWfSu0YavWuksaN5XXY0r5vo0sqlPDskBwFDzOwEIBuoZ2bPAKvMrIVzbqWZtQBWR9ZfBrQp8vzWwIpoFi0isVdmODjnRgIjAcxsMHCVc+4sM/sHMAy4I/L1tchTXgeeM7O7gJZAZyA8zZPstYJh05JCbNeQ6HU9swFoZH32WC1ryrcA5G1KjuHwlRnncAfwkpkNB5YApwI452aZ2UvAbCAXuNg5l1fpSkVSTEab1gDcMfRZ8iNb1l+MvBeg2KjIgtuDZ54KQM7LPYFd4x4SZa/CwTk3AZgQub0OOKqE9W4DbqtkbSKSQBohKRJlBR3DCe9OB2BI7fV7nF3pvmhQuP6Q0z8D4I/7fgDA0Fs2AJB/i+O4s/15FonY5NC5FSISpM5BJMp+6tUSgBH1/T76w2acRr3jvwPCZ1dOvdP/jZ7e+mcA3HB+OwAOPm4m7zztz958cENHAP57nl9H8zmISMKYc3uMT4q7etbIDbDgvk3Zzbsrpu2x7NiWveJeh8RHwZwQBfslhjbwYxFHDr+QjA/LOy6xdB+4MVOdc313X67NCpEk1urOzwGY/qwfV9ji3Y0A3PLYo1x+28VA7A55arNCRILUOYikgNxlywF4+bpjAVh50xc8dMN9AAxrczkAbW/6PKrvqc5BRILUOYikkJqv+dOUpk9tU7j/Ydrv/ZDsITf1i+p7qXMQkSB1DinmnO8PA4pPNFswl4Pmkqw+cpct577pRwBw4eELY/IeCgeRVNR/f54++HFg1+jJaNNmhYgEqXNIEVtPGQDAf9o9kuBKpLy+v9mPbsxeC83uj85hxvQeXQDYdMsWWmf4a2G8c27kfAuie76FOgcRCVLnkCI6XDOn7JUkKawbPhCAmeffD0D3CefT7P7yP79gPojvf9u2cNm+J/idjte1eR6AL7Z15JSbrgag0ZcaPi0icaTOIQVsPWVAifsafnbxBXQap0OYySgzcqG3OYMf4+tF/pIvv534e2DX9RsO23cB8zb4S758tP/LAKTxFeCvsVkwMe1DGzoAcMaH/hqbPW5aSaNlsZ1jUuGQYgquWVEwpqEWkxJZjgQUnCV5yBb/u1p90q6LNj010B9+7J/lp0p4cENH8iMB0H3C+QDkr6sBwL7jdhY+r8bUBQB02eSvsZkbs+p30WaFiARpsheRaq6kyV7UOYhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJCgpBkHlN6zNlqMHBB+rO3djnKtJjM3d6sf09avLzzFeYv37iquXxwQXq3MQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISZM65RNeAma0BtgBrE11LxD6olhDVEpYstVS0jnbOuSa7L0yKcAAwsynOub6JrgNUS0lUS1iy1BLtOrRZISJBCgcRCUqmcBiV6AKKUC1hqiUsWWqJah1Js89BRJJLMnUOIpJEFA4iEqRwEJEghYOIBCkcRCTo/wFj7mcc63NAeAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "model.eval()\n", + "with torch.no_grad():\n", + " for data in test_loader:\n", + " data, targets = get_batch(data, seq_len, digits_per_batch)\n", + " data = data.to(device)\n", + " targets = targets.to(device)\n", + " output = model.visualize(data)\n", + " \n", + " fig=plt.figure(figsize=(12,8), dpi= 100, facecolor='w', edgecolor='k')\n", + " matrix = torch.empty((2*data.shape[0], data.shape[1], data.shape[2]), device=data.device)\n", + " matrix[0::2] = data\n", + " matrix[1::2] = output.view(output.shape[0], 1, output.shape[1]).repeat_interleave(data.shape[1], dim=1) * 10\n", + " \n", + " plt.matshow((matrix[:matrix.shape[0]//8,:,:].view(-1, data.shape[-1]) * mnist_std + mnist_mean).cpu().numpy())\n", + " plt.show()\n", + " break" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.455162, + "end_time": "2021-01-21T19:37:16.773157", + "exception": false, + "start_time": "2021-01-21T19:37:16.317995", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "### Average pooling - one layer" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-21T19:37:17.648069Z", + "iopub.status.busy": "2021-01-21T19:37:17.647547Z", + "iopub.status.idle": "2021-01-22T00:39:10.871379Z", + "shell.execute_reply": "2021-01-22T00:39:10.871845Z" + }, + "papermill": { + "duration": 18113.668852, + "end_time": "2021-01-22T00:39:10.872000", + "exception": false, + "start_time": "2021-01-21T19:37:17.203148", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 122.11 | loss 52.53 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 121.81 | loss 20.15 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 121.81 | loss 16.17 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 121.85 | loss 13.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 118.91s | valid loss 16.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 122.54 | loss 9.93 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 122.83 | loss 9.60 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 122.95 | loss 9.75 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 123.07 | loss 8.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 119.77s | valid loss 19.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 123.66 | loss 7.76 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 123.19 | loss 6.50 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 123.82 | loss 7.33 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 123.12 | loss 6.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 120.30s | valid loss 8.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 122.82 | loss 5.82 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 123.63 | loss 6.44 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 123.03 | loss 6.45 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 123.43 | loss 6.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 120.28s | valid loss 6.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 124.10 | loss 5.21 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 123.36 | loss 5.61 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 123.44 | loss 5.51 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 123.90 | loss 5.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 120.59s | valid loss 6.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 123.68 | loss 4.94 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 124.06 | loss 5.52 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 123.42 | loss 4.79 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 123.79 | loss 4.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 120.65s | valid loss 7.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 124.05 | loss 3.44 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 123.58 | loss 4.87 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 123.78 | loss 4.75 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 123.48 | loss 4.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 120.67s | valid loss 10.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 124.12 | loss 4.14 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 123.55 | loss 4.22 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 123.49 | loss 4.34 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 123.46 | loss 4.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 120.63s | valid loss 5.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 124.10 | loss 3.55 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 123.42 | loss 3.47 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 124.00 | loss 2.53 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 123.84 | loss 4.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 120.72s | valid loss 8.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 123.66 | loss 3.04 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 124.10 | loss 3.32 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 123.88 | loss 3.29 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 124.26 | loss 3.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 120.82s | valid loss 8.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 124.61 | loss 3.02 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 123.66 | loss 2.70 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 124.48 | loss 3.32 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 123.49 | loss 2.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 120.99s | valid loss 6.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 124.00 | loss 2.09 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 123.68 | loss 2.64 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 123.53 | loss 2.57 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 123.84 | loss 2.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 120.68s | valid loss 7.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 123.80 | loss 2.28 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 123.30 | loss 2.25 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 123.55 | loss 2.59 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 123.42 | loss 2.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 120.50s | valid loss 8.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 123.50 | loss 2.51 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 123.81 | loss 1.83 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 123.27 | loss 2.12 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 123.57 | loss 2.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 120.48s | valid loss 9.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 124.10 | loss 1.72 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 123.51 | loss 1.69 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 123.44 | loss 2.64 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 123.74 | loss 2.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 120.59s | valid loss 8.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 124.03 | loss 1.45 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 124.12 | loss 2.20 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 123.57 | loss 1.33 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 123.44 | loss 1.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 120.72s | valid loss 8.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 124.22 | loss 1.24 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 123.30 | loss 1.36 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 123.58 | loss 1.55 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 123.64 | loss 1.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 120.63s | valid loss 8.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 123.81 | loss 1.64 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 123.76 | loss 1.63 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 123.21 | loss 1.12 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 124.02 | loss 1.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 120.67s | valid loss 9.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 123.34 | loss 1.39 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 123.49 | loss 1.15 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 123.97 | loss 1.26 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 123.49 | loss 1.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 120.45s | valid loss 11.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 124.14 | loss 1.41 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 123.70 | loss 1.27 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 123.40 | loss 1.27 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 123.62 | loss 1.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 120.58s | valid loss 10.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 123.81 | loss 1.01 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 123.37 | loss 1.58 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 123.85 | loss 1.04 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 123.65 | loss 1.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 120.72s | valid loss 9.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 124.29 | loss 0.51 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 123.62 | loss 1.08 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 123.93 | loss 1.18 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 123.92 | loss 0.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 120.77s | valid loss 13.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 123.87 | loss 0.85 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 123.13 | loss 0.75 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 123.47 | loss 1.12 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 123.54 | loss 0.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 120.43s | valid loss 11.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 123.70 | loss 0.58 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 124.25 | loss 0.93 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 123.28 | loss 0.62 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 123.51 | loss 0.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 120.60s | valid loss 8.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 123.90 | loss 0.45 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 123.61 | loss 0.67 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 123.35 | loss 0.54 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 123.52 | loss 0.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 120.59s | valid loss 9.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 123.32 | loss 0.75 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 123.98 | loss 0.76 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 123.51 | loss 0.46 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 122.86 | loss 0.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 120.44s | valid loss 7.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 123.81 | loss 0.66 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 123.42 | loss 0.38 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 123.02 | loss 1.30 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 123.32 | loss 0.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 120.35s | valid loss 9.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 123.70 | loss 0.31 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 123.08 | loss 0.57 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 123.48 | loss 0.61 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 123.89 | loss 0.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 120.48s | valid loss 14.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 123.10 | loss 0.81 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 123.28 | loss 0.27 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 123.49 | loss 0.38 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 123.40 | loss 0.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 120.27s | valid loss 12.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 123.52 | loss 0.36 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 123.17 | loss 0.37 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 123.82 | loss 0.42 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 122.92 | loss 0.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 120.25s | valid loss 13.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.83 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 124.42 | loss 55.54 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 124.22 | loss 20.31 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 124.05 | loss 15.68 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 124.20 | loss 13.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 121.23s | valid loss 16.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 124.67 | loss 10.57 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 124.44 | loss 8.62 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 124.58 | loss 9.83 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 124.63 | loss 8.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 121.41s | valid loss 11.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 124.69 | loss 7.36 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 124.03 | loss 6.90 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 124.93 | loss 6.83 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 124.32 | loss 6.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 121.35s | valid loss 14.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 124.55 | loss 6.28 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 124.49 | loss 5.49 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 124.03 | loss 6.05 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 124.36 | loss 6.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 121.19s | valid loss 9.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 124.76 | loss 4.89 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 123.76 | loss 5.43 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 124.75 | loss 5.77 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 123.91 | loss 5.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 121.21s | valid loss 9.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 124.74 | loss 4.40 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 123.86 | loss 4.89 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 124.21 | loss 4.96 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 123.77 | loss 5.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 121.03s | valid loss 6.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 124.66 | loss 4.12 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 123.77 | loss 5.04 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 124.26 | loss 4.29 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 123.95 | loss 4.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 121.11s | valid loss 13.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 124.71 | loss 3.46 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 123.68 | loss 3.46 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 124.01 | loss 4.32 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 123.76 | loss 3.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 121.05s | valid loss 8.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 124.11 | loss 3.13 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 123.70 | loss 3.61 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 124.52 | loss 3.82 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 124.12 | loss 3.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 120.98s | valid loss 6.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 124.21 | loss 2.37 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 123.98 | loss 4.08 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 123.84 | loss 3.44 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 123.92 | loss 2.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 120.85s | valid loss 10.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 123.90 | loss 2.03 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 123.72 | loss 3.51 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 123.63 | loss 3.18 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 124.20 | loss 3.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 120.67s | valid loss 8.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 124.42 | loss 3.09 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 123.43 | loss 2.54 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 123.59 | loss 2.60 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 124.28 | loss 2.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 120.79s | valid loss 8.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 123.95 | loss 2.17 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 123.47 | loss 1.70 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 123.08 | loss 2.81 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 123.99 | loss 3.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 120.53s | valid loss 10.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 124.09 | loss 1.85 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 123.51 | loss 1.77 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 123.41 | loss 2.09 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 123.91 | loss 2.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 120.66s | valid loss 7.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 123.27 | loss 2.19 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 124.20 | loss 1.86 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 123.53 | loss 1.83 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 123.51 | loss 2.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 120.49s | valid loss 9.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 123.85 | loss 1.40 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 123.82 | loss 1.70 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 123.56 | loss 2.90 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 123.80 | loss 2.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 120.60s | valid loss 10.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 123.54 | loss 1.51 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 123.85 | loss 1.56 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 123.55 | loss 1.51 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 123.74 | loss 1.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 120.55s | valid loss 7.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 124.15 | loss 1.05 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 123.59 | loss 1.24 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 123.14 | loss 1.68 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 123.32 | loss 1.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 120.48s | valid loss 10.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 123.86 | loss 1.84 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 123.84 | loss 1.64 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 123.47 | loss 1.55 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 123.42 | loss 1.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 120.58s | valid loss 5.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 123.95 | loss 0.54 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 123.10 | loss 0.74 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 123.33 | loss 1.45 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 123.45 | loss 1.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 120.40s | valid loss 8.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 123.75 | loss 0.88 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 123.62 | loss 0.83 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 122.96 | loss 0.89 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 123.56 | loss 0.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 120.44s | valid loss 9.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 123.99 | loss 0.75 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 123.10 | loss 0.61 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 123.14 | loss 1.10 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 123.16 | loss 1.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 120.38s | valid loss 8.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 123.94 | loss 0.58 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 123.45 | loss 1.23 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 123.58 | loss 1.23 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 123.59 | loss 0.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 120.51s | valid loss 6.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 123.11 | loss 0.55 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 123.77 | loss 0.56 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 122.97 | loss 1.00 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 123.09 | loss 1.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 120.25s | valid loss 8.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 123.41 | loss 0.70 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 123.10 | loss 0.58 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 123.31 | loss 1.42 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 123.26 | loss 1.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 120.21s | valid loss 7.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 123.09 | loss 0.81 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 123.31 | loss 0.43 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 123.16 | loss 0.49 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 123.12 | loss 0.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 120.09s | valid loss 9.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 124.12 | loss 0.19 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 123.29 | loss 0.37 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 123.20 | loss 0.78 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 123.67 | loss 0.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 120.38s | valid loss 9.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 123.76 | loss 0.27 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 123.78 | loss 0.72 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 123.07 | loss 0.44 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 123.83 | loss 0.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 120.39s | valid loss 9.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 123.85 | loss 0.51 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 123.06 | loss 0.55 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 123.37 | loss 0.17 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 123.10 | loss 0.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 120.29s | valid loss 9.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 123.79 | loss 0.54 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 122.92 | loss 0.35 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 123.48 | loss 0.35 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 123.27 | loss 0.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 120.38s | valid loss 10.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.97 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 124.63 | loss 49.17 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 124.04 | loss 19.22 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 124.50 | loss 15.00 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 124.71 | loss 12.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 121.26s | valid loss 33.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 124.72 | loss 10.16 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 125.48 | loss 9.13 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 124.33 | loss 8.64 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 124.75 | loss 8.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 121.61s | valid loss 19.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 125.21 | loss 7.50 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 124.64 | loss 6.99 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 124.38 | loss 6.52 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 124.20 | loss 6.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 121.52s | valid loss 7.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 124.70 | loss 5.96 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 123.92 | loss 5.56 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 124.49 | loss 6.29 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 124.40 | loss 5.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 121.23s | valid loss 12.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 125.06 | loss 5.09 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 124.00 | loss 5.80 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 124.96 | loss 5.57 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 124.29 | loss 5.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 121.48s | valid loss 6.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 124.86 | loss 4.18 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 124.35 | loss 5.24 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 123.90 | loss 4.59 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 124.40 | loss 5.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 121.33s | valid loss 5.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 124.88 | loss 4.05 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 123.78 | loss 4.10 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 124.55 | loss 4.91 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 123.73 | loss 3.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 121.20s | valid loss 8.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 123.69 | loss 3.53 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 124.10 | loss 4.79 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 123.65 | loss 4.45 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 124.19 | loss 3.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 120.80s | valid loss 9.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 123.85 | loss 3.60 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 124.23 | loss 3.39 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 123.48 | loss 3.74 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 123.76 | loss 4.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 120.71s | valid loss 9.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 124.31 | loss 2.72 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 123.72 | loss 3.63 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 123.72 | loss 3.52 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 123.81 | loss 3.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 120.78s | valid loss 5.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 123.90 | loss 2.75 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 123.87 | loss 3.46 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 123.74 | loss 3.35 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 124.11 | loss 4.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 120.80s | valid loss 9.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 123.82 | loss 2.53 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 123.44 | loss 2.67 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 123.98 | loss 2.67 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 123.16 | loss 3.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 120.59s | valid loss 6.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 124.51 | loss 2.14 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 123.73 | loss 2.35 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 123.50 | loss 2.42 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 123.61 | loss 3.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 120.84s | valid loss 5.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 123.64 | loss 2.28 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 123.74 | loss 2.57 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 123.61 | loss 2.82 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 123.91 | loss 1.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 120.58s | valid loss 6.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 123.79 | loss 1.81 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 123.17 | loss 2.58 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 123.49 | loss 1.61 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 123.73 | loss 2.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 120.57s | valid loss 8.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 123.47 | loss 2.36 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 123.04 | loss 2.22 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 123.25 | loss 1.80 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 123.43 | loss 2.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 120.19s | valid loss 8.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 124.05 | loss 1.44 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 123.26 | loss 1.73 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 123.58 | loss 1.68 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 123.61 | loss 1.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 120.49s | valid loss 8.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 124.05 | loss 1.34 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 123.78 | loss 1.82 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 123.50 | loss 1.09 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 123.52 | loss 1.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 120.55s | valid loss 7.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 123.49 | loss 1.75 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 123.65 | loss 0.67 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 122.87 | loss 2.06 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 123.78 | loss 1.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 120.34s | valid loss 8.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 123.75 | loss 1.73 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 124.20 | loss 1.64 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 123.74 | loss 1.10 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 123.40 | loss 1.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 120.63s | valid loss 9.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 124.49 | loss 0.92 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 122.77 | loss 1.53 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 123.56 | loss 1.33 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 123.25 | loss 0.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 120.55s | valid loss 9.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 123.74 | loss 1.11 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 123.23 | loss 1.04 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 123.53 | loss 1.11 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 123.31 | loss 1.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 120.36s | valid loss 8.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 123.64 | loss 1.01 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 123.63 | loss 1.05 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 123.30 | loss 0.67 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 123.28 | loss 0.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 120.39s | valid loss 11.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 124.05 | loss 0.81 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 123.28 | loss 0.84 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 123.39 | loss 0.85 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 123.42 | loss 0.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 120.41s | valid loss 9.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 123.99 | loss 0.46 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 123.15 | loss 0.67 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 122.98 | loss 0.51 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 123.43 | loss 0.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 120.40s | valid loss 10.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 123.41 | loss 0.47 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 123.36 | loss 0.34 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 123.35 | loss 0.65 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 123.97 | loss 0.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 120.46s | valid loss 9.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 123.88 | loss 0.55 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 123.52 | loss 0.35 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 124.16 | loss 1.32 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 123.15 | loss 0.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 120.62s | valid loss 7.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 124.03 | loss 0.48 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 123.35 | loss 0.19 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 123.91 | loss 0.37 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 122.94 | loss 0.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 120.46s | valid loss 7.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 123.37 | loss 0.73 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 123.52 | loss 0.43 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 123.31 | loss 0.52 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 123.08 | loss 0.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 120.25s | valid loss 8.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 123.71 | loss 0.40 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 123.33 | loss 0.26 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 123.28 | loss 0.42 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 123.24 | loss 0.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 120.31s | valid loss 9.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.75 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 124.35 | loss 55.93 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 124.46 | loss 20.38 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 124.23 | loss 16.43 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 124.48 | loss 11.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 121.33s | valid loss 21.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 124.46 | loss 10.21 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 124.13 | loss 9.43 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 124.01 | loss 9.17 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 124.50 | loss 8.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 121.14s | valid loss 11.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 124.88 | loss 7.54 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 124.10 | loss 7.42 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 124.08 | loss 7.42 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 124.28 | loss 6.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 121.27s | valid loss 12.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 124.56 | loss 7.33 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 124.22 | loss 5.58 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 124.12 | loss 5.96 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 124.16 | loss 5.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 121.09s | valid loss 10.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 125.03 | loss 5.16 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 123.92 | loss 5.86 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 124.16 | loss 5.09 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 124.30 | loss 4.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 121.17s | valid loss 7.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 124.40 | loss 4.05 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 124.42 | loss 5.40 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 124.20 | loss 4.54 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 124.04 | loss 5.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 121.16s | valid loss 7.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 124.14 | loss 4.00 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 124.30 | loss 5.29 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 123.99 | loss 4.74 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 123.98 | loss 4.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 120.98s | valid loss 6.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 124.89 | loss 4.34 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 124.11 | loss 3.76 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 124.57 | loss 4.59 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 123.85 | loss 4.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 121.15s | valid loss 5.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 124.16 | loss 3.84 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 123.95 | loss 3.72 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 124.11 | loss 4.39 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 124.16 | loss 3.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 121.04s | valid loss 9.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 124.29 | loss 3.34 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 124.04 | loss 3.54 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 123.57 | loss 3.27 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 123.91 | loss 3.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 120.81s | valid loss 8.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 123.87 | loss 3.93 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 123.87 | loss 2.81 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 124.00 | loss 3.35 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 124.11 | loss 2.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 120.87s | valid loss 6.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 123.65 | loss 2.51 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 123.97 | loss 2.86 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 123.55 | loss 2.49 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 123.53 | loss 3.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 120.62s | valid loss 6.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 124.22 | loss 2.26 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 123.82 | loss 1.82 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 123.37 | loss 3.37 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 123.88 | loss 2.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 120.68s | valid loss 6.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 123.75 | loss 2.12 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 123.65 | loss 1.93 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 123.65 | loss 1.61 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 123.81 | loss 2.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 120.53s | valid loss 6.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 123.78 | loss 1.50 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 123.64 | loss 2.18 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 123.40 | loss 2.14 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 123.66 | loss 2.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 120.46s | valid loss 9.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 124.10 | loss 1.79 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 123.31 | loss 2.51 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 123.86 | loss 1.80 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 123.68 | loss 2.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 120.55s | valid loss 6.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 124.81 | loss 1.66 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 123.82 | loss 1.80 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 123.75 | loss 1.53 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 123.86 | loss 1.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 120.94s | valid loss 7.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 123.72 | loss 1.36 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 123.88 | loss 1.69 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 123.44 | loss 1.43 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 123.54 | loss 1.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 120.54s | valid loss 8.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 123.60 | loss 1.00 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 123.35 | loss 1.41 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 123.50 | loss 1.43 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 123.24 | loss 1.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 120.38s | valid loss 10.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 123.44 | loss 1.21 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 122.84 | loss 1.28 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 123.26 | loss 1.55 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 123.29 | loss 1.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 120.23s | valid loss 7.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 123.85 | loss 1.24 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 122.96 | loss 0.79 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 124.60 | loss 1.40 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 122.92 | loss 1.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 120.56s | valid loss 8.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 123.90 | loss 0.60 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 123.46 | loss 0.57 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 122.87 | loss 1.18 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 122.78 | loss 1.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 120.10s | valid loss 9.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 123.56 | loss 0.73 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 123.13 | loss 0.71 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 122.45 | loss 1.11 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 122.73 | loss 1.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 119.82s | valid loss 8.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 123.70 | loss 1.09 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 123.34 | loss 0.82 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 122.76 | loss 0.74 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 122.43 | loss 0.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 120.05s | valid loss 13.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 123.57 | loss 0.82 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 122.50 | loss 0.90 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 122.96 | loss 0.45 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 123.13 | loss 1.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 120.00s | valid loss 9.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 123.65 | loss 0.39 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 123.18 | loss 0.56 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 123.49 | loss 0.77 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 122.85 | loss 0.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 120.22s | valid loss 9.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 123.52 | loss 0.83 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 123.31 | loss 0.35 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 123.18 | loss 1.08 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 122.79 | loss 0.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 120.20s | valid loss 10.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 123.51 | loss 0.21 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 123.18 | loss 0.60 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 123.22 | loss 0.57 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 123.13 | loss 0.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 120.18s | valid loss 12.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 123.70 | loss 0.49 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 123.94 | loss 0.42 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 123.19 | loss 0.62 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 122.95 | loss 0.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 120.29s | valid loss 10.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 123.95 | loss 0.53 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 123.17 | loss 0.50 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 122.72 | loss 0.55 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 122.96 | loss 0.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 120.10s | valid loss 8.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.59 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 124.17 | loss 50.42 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 123.82 | loss 20.06 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 123.77 | loss 15.18 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 123.88 | loss 12.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 120.77s | valid loss 15.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 124.52 | loss 10.32 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 124.09 | loss 10.22 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 123.77 | loss 8.48 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 124.12 | loss 8.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 120.98s | valid loss 9.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 124.23 | loss 7.59 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 123.89 | loss 6.70 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 123.77 | loss 7.29 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 124.25 | loss 7.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 121.06s | valid loss 8.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 123.67 | loss 6.49 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 124.62 | loss 6.41 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 124.10 | loss 6.00 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 123.87 | loss 5.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 121.02s | valid loss 12.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 124.36 | loss 5.73 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 123.96 | loss 5.64 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 123.99 | loss 4.54 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 123.65 | loss 5.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 120.87s | valid loss 7.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 123.95 | loss 4.25 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 124.45 | loss 4.77 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 123.67 | loss 4.54 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 123.55 | loss 4.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 120.89s | valid loss 6.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 123.98 | loss 4.54 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 123.90 | loss 4.00 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 123.69 | loss 4.29 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 123.68 | loss 4.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 120.74s | valid loss 5.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 123.88 | loss 4.25 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 123.21 | loss 3.90 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 124.22 | loss 3.81 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 124.09 | loss 4.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 120.78s | valid loss 7.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 124.60 | loss 3.86 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 123.37 | loss 3.12 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 124.36 | loss 3.96 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 123.16 | loss 2.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 120.85s | valid loss 8.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 124.58 | loss 2.90 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 124.33 | loss 3.15 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 123.60 | loss 3.21 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 124.47 | loss 3.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 121.04s | valid loss 6.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 123.85 | loss 2.99 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 123.06 | loss 2.97 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 123.61 | loss 3.26 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 123.44 | loss 3.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 120.32s | valid loss 5.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 123.87 | loss 2.33 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 122.91 | loss 3.13 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 123.55 | loss 3.07 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 123.37 | loss 2.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 120.30s | valid loss 4.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 123.33 | loss 2.58 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 123.34 | loss 2.42 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 123.52 | loss 2.66 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 123.01 | loss 1.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 120.23s | valid loss 6.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 123.88 | loss 2.04 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 123.39 | loss 2.76 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 123.43 | loss 1.78 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 123.04 | loss 1.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 120.31s | valid loss 6.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 123.97 | loss 2.41 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 123.68 | loss 1.75 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 123.39 | loss 1.72 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 123.02 | loss 2.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 120.35s | valid loss 7.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 123.97 | loss 1.83 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 123.07 | loss 2.40 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 123.28 | loss 1.45 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 122.96 | loss 1.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 120.26s | valid loss 8.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 123.74 | loss 1.27 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 123.19 | loss 1.45 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 123.15 | loss 1.13 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 123.57 | loss 1.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 120.28s | valid loss 4.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 123.77 | loss 1.26 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 123.15 | loss 2.29 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 123.16 | loss 1.27 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 122.87 | loss 0.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 120.20s | valid loss 6.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 123.58 | loss 0.64 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 123.53 | loss 1.47 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 122.76 | loss 1.74 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 123.31 | loss 0.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 120.25s | valid loss 9.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 123.44 | loss 1.23 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 122.87 | loss 1.05 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 123.07 | loss 1.07 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 123.16 | loss 1.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 120.14s | valid loss 8.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 123.18 | loss 1.23 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 123.10 | loss 0.85 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 123.45 | loss 1.35 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 123.22 | loss 1.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 120.14s | valid loss 8.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 123.66 | loss 1.02 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 122.76 | loss 0.44 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 123.38 | loss 0.60 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 122.93 | loss 1.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 120.15s | valid loss 9.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 123.41 | loss 0.65 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 123.26 | loss 1.06 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 122.92 | loss 0.73 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 123.46 | loss 1.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 120.15s | valid loss 8.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 123.76 | loss 0.67 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 122.71 | loss 0.84 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 123.21 | loss 0.72 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 123.73 | loss 0.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 120.20s | valid loss 10.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 123.29 | loss 0.35 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 122.84 | loss 0.64 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 123.45 | loss 0.65 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 123.11 | loss 0.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 120.07s | valid loss 8.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 123.36 | loss 0.69 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 123.27 | loss 0.61 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 122.93 | loss 0.73 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 123.41 | loss 0.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 120.16s | valid loss 10.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 123.25 | loss 0.73 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 122.79 | loss 0.45 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 123.10 | loss 0.53 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 122.78 | loss 0.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 120.02s | valid loss 9.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 123.44 | loss 1.09 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 123.15 | loss 0.42 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 123.05 | loss 0.93 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 123.75 | loss 0.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 120.17s | valid loss 9.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 123.62 | loss 0.24 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 124.15 | loss 0.64 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 123.08 | loss 0.54 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 123.37 | loss 0.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 120.31s | valid loss 9.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 123.34 | loss 0.25 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 123.56 | loss 0.24 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 123.39 | loss 0.38 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 122.76 | loss 0.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 120.26s | valid loss 8.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 4.38 |\n", + "=========================================================================================\n", + "Result over 5 runs: 5.386036396026611\n" + ] + } + ], + "source": [ + "num_runs = 5\n", + "best_model = None\n", + "best_result = 1000000\n", + "results = []\n", + "\n", + "for i in range(num_runs):\n", + " features = nn.Sequential(nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(64),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(128),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(256),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=(0,1)),\n", + " nn.BatchNorm2d(512),\n", + " nn.ReLU()\n", + " )\n", + "\n", + " classifier = nn.Sequential(nn.Linear(512, 1028),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=1028, bias=True),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=10, bias=True))\n", + "\n", + " conv = nn.Sequential(nn.Conv1d(512, 512, kernel_size=64, stride=1, padding=32),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.AvgPool1d(kernel_size=64, stride=64))\n", + "\n", + " model = ConvMNIST(features, conv, classifier)\n", + " model = model.to(device)\n", + "\n", + " optimizer = torch.optim.Adam(model.parameters())\n", + " scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "\n", + " epochs = 30\n", + " model, result = train_model(model, epochs, train_loader, test_loader, seq_len, digits_per_batch, optimizer, scheduler)\n", + " \n", + " if result < best_result:\n", + " best_result = result\n", + " best_model = model\n", + " results.append(result)\n", + " \n", + "model = best_model\n", + "print(\"Result over {} runs: {}\".format(num_runs, torch.tensor(results).mean().item()))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.638745, + "end_time": "2021-01-22T00:39:12.167902", + "exception": false, + "start_time": "2021-01-22T00:39:11.529157", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "### Max pooling - one layer" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-22T00:39:13.463733Z", + "iopub.status.busy": "2021-01-22T00:39:13.463213Z", + "iopub.status.idle": "2021-01-22T05:47:47.940706Z", + "shell.execute_reply": "2021-01-22T05:47:47.941441Z" + }, + "papermill": { + "duration": 18515.138307, + "end_time": "2021-01-22T05:47:47.941700", + "exception": false, + "start_time": "2021-01-22T00:39:12.803393", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 123.82 | loss 55.16 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 124.42 | loss 18.01 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 125.06 | loss 14.23 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 124.58 | loss 12.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 121.43s | valid loss 9.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 124.76 | loss 9.70 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 124.47 | loss 8.95 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 124.20 | loss 8.79 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 125.35 | loss 8.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 121.50s | valid loss 11.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 124.79 | loss 7.21 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 124.81 | loss 7.04 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 124.84 | loss 7.67 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 124.71 | loss 7.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 121.60s | valid loss 9.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 124.47 | loss 6.07 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 124.25 | loss 5.89 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 124.78 | loss 6.67 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 124.34 | loss 6.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 121.34s | valid loss 7.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 124.74 | loss 5.30 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 124.58 | loss 4.79 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 124.50 | loss 6.08 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 124.23 | loss 6.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 121.41s | valid loss 6.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 124.26 | loss 5.59 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 124.70 | loss 5.53 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 123.94 | loss 5.08 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 124.91 | loss 5.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 121.34s | valid loss 8.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 124.40 | loss 4.38 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 124.11 | loss 5.09 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 123.94 | loss 4.96 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 124.63 | loss 4.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 121.20s | valid loss 8.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 124.90 | loss 4.91 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 124.65 | loss 4.02 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 124.04 | loss 5.25 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 124.28 | loss 4.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 121.41s | valid loss 10.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 124.15 | loss 4.26 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 124.21 | loss 4.54 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 124.40 | loss 3.97 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 124.42 | loss 3.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 121.12s | valid loss 8.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 124.58 | loss 3.59 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 123.91 | loss 3.51 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 124.45 | loss 4.39 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 124.06 | loss 4.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 121.25s | valid loss 5.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 123.90 | loss 3.11 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 124.17 | loss 3.25 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 123.92 | loss 3.32 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 123.64 | loss 4.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 120.89s | valid loss 6.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 124.71 | loss 3.50 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 123.78 | loss 3.39 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 124.07 | loss 3.06 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 123.59 | loss 2.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 121.01s | valid loss 6.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 124.15 | loss 2.28 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 123.52 | loss 2.48 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 123.90 | loss 3.12 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 123.67 | loss 2.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 120.70s | valid loss 6.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 124.36 | loss 1.78 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 123.63 | loss 2.68 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 123.84 | loss 3.45 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 123.67 | loss 2.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 120.77s | valid loss 10.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 123.76 | loss 3.42 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 124.31 | loss 1.92 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 123.73 | loss 2.49 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 123.85 | loss 2.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 120.82s | valid loss 7.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 123.86 | loss 2.24 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 123.45 | loss 3.16 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 123.27 | loss 1.67 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 123.99 | loss 1.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 120.62s | valid loss 10.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 123.84 | loss 1.71 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 123.61 | loss 2.30 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 124.00 | loss 2.43 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 123.55 | loss 1.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 120.63s | valid loss 7.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 123.91 | loss 1.94 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 123.27 | loss 1.59 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 123.68 | loss 1.67 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 123.70 | loss 2.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 120.57s | valid loss 9.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 124.25 | loss 1.55 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 123.78 | loss 0.93 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 123.10 | loss 1.87 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 123.86 | loss 2.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 120.62s | valid loss 7.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 123.96 | loss 1.34 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 123.62 | loss 1.34 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 123.63 | loss 1.12 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 123.75 | loss 1.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 120.64s | valid loss 7.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 124.38 | loss 1.05 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 123.32 | loss 0.72 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 123.27 | loss 1.39 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 123.68 | loss 1.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 120.69s | valid loss 9.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 123.94 | loss 1.61 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 123.02 | loss 0.65 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 123.66 | loss 1.29 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 123.56 | loss 1.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 120.51s | valid loss 12.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 123.63 | loss 0.83 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 123.87 | loss 1.03 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 122.76 | loss 1.54 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 123.15 | loss 0.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 120.31s | valid loss 11.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 124.05 | loss 0.83 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 122.92 | loss 1.05 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 123.53 | loss 1.14 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 123.25 | loss 0.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 120.44s | valid loss 10.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 123.61 | loss 0.85 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 123.11 | loss 0.73 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 123.83 | loss 1.18 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 123.17 | loss 0.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 120.37s | valid loss 11.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 124.31 | loss 0.84 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 123.46 | loss 0.38 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 123.37 | loss 0.56 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 123.59 | loss 0.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 120.57s | valid loss 9.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 124.68 | loss 0.88 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 123.38 | loss 0.42 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 123.26 | loss 0.66 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 123.39 | loss 1.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 120.54s | valid loss 13.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 123.75 | loss 0.55 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 123.40 | loss 0.22 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 123.78 | loss 0.63 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 124.18 | loss 0.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 120.60s | valid loss 10.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 124.21 | loss 0.35 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 123.73 | loss 0.66 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 124.90 | loss 0.44 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 123.53 | loss 0.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 120.80s | valid loss 11.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 123.69 | loss 0.69 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 123.70 | loss 0.41 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 123.01 | loss 0.34 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 123.46 | loss 0.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 120.24s | valid loss 12.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.94 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 123.95 | loss 56.54 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 123.80 | loss 17.64 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 124.06 | loss 13.55 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 123.77 | loss 11.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 120.84s | valid loss 12.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 124.10 | loss 9.33 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 123.92 | loss 8.59 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 124.10 | loss 9.39 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 124.49 | loss 8.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 120.92s | valid loss 9.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 124.98 | loss 7.25 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 123.56 | loss 7.81 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 124.12 | loss 7.41 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 124.09 | loss 6.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 121.13s | valid loss 7.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 124.07 | loss 6.67 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 124.90 | loss 6.57 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 123.87 | loss 6.47 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 123.70 | loss 6.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 121.00s | valid loss 8.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 124.95 | loss 5.92 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 123.32 | loss 5.37 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 124.20 | loss 7.39 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 123.76 | loss 6.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 120.94s | valid loss 6.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 124.90 | loss 5.41 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 124.54 | loss 4.07 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 123.71 | loss 5.86 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 123.88 | loss 5.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 121.01s | valid loss 8.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 124.32 | loss 5.27 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 123.74 | loss 4.37 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 123.77 | loss 5.59 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 124.16 | loss 4.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 120.95s | valid loss 5.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 124.61 | loss 4.09 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 123.93 | loss 4.33 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 124.28 | loss 4.06 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 124.39 | loss 5.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 121.08s | valid loss 8.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 124.53 | loss 3.13 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 123.49 | loss 4.66 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 124.64 | loss 4.48 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 123.89 | loss 4.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 121.05s | valid loss 6.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 124.14 | loss 4.18 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 124.25 | loss 3.99 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 123.87 | loss 3.92 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 124.07 | loss 3.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 121.05s | valid loss 9.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 124.02 | loss 4.00 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 123.79 | loss 3.22 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 124.21 | loss 4.27 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 124.11 | loss 3.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 121.13s | valid loss 8.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 124.05 | loss 2.55 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 124.64 | loss 3.08 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 123.75 | loss 2.74 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 124.47 | loss 3.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 121.09s | valid loss 8.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 124.68 | loss 2.42 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 124.50 | loss 3.27 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 124.42 | loss 3.03 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 123.80 | loss 3.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 121.23s | valid loss 8.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 123.59 | loss 2.21 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 123.55 | loss 3.35 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 124.23 | loss 2.50 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 123.42 | loss 2.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 120.54s | valid loss 9.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 124.14 | loss 2.95 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 123.26 | loss 1.96 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 124.89 | loss 1.71 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 123.84 | loss 3.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 120.93s | valid loss 8.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 124.91 | loss 1.33 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 123.74 | loss 1.70 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 124.05 | loss 2.96 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 123.92 | loss 2.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 121.01s | valid loss 9.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 124.40 | loss 1.84 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 124.17 | loss 1.63 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 123.87 | loss 2.71 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 124.13 | loss 1.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 120.99s | valid loss 11.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 124.20 | loss 1.77 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 123.33 | loss 1.57 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 124.04 | loss 1.61 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 123.85 | loss 2.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 120.72s | valid loss 11.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 124.01 | loss 1.28 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 123.75 | loss 1.92 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 123.86 | loss 2.15 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 123.68 | loss 0.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 120.72s | valid loss 7.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 123.91 | loss 1.18 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 123.94 | loss 1.25 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 123.80 | loss 2.06 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 123.39 | loss 1.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 120.77s | valid loss 10.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 124.38 | loss 0.70 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 123.51 | loss 1.32 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 123.44 | loss 1.33 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 123.97 | loss 1.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 120.88s | valid loss 10.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 124.17 | loss 1.12 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 124.28 | loss 1.06 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 123.82 | loss 1.57 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 124.53 | loss 0.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 121.00s | valid loss 10.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 124.61 | loss 0.79 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 123.61 | loss 1.14 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 123.73 | loss 0.81 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 123.90 | loss 1.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 121.01s | valid loss 12.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 123.84 | loss 0.73 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 123.66 | loss 0.82 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 124.00 | loss 1.75 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 123.61 | loss 0.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 120.69s | valid loss 10.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 124.58 | loss 1.39 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 123.20 | loss 0.56 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 123.62 | loss 1.35 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 123.63 | loss 0.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 120.68s | valid loss 9.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 123.82 | loss 0.84 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 123.41 | loss 0.22 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 123.80 | loss 0.98 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 123.61 | loss 0.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 120.58s | valid loss 9.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 124.28 | loss 0.62 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 123.39 | loss 0.66 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 123.52 | loss 0.64 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 123.75 | loss 0.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 120.57s | valid loss 8.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 124.15 | loss 0.42 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 123.45 | loss 0.40 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 124.30 | loss 1.14 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 123.15 | loss 0.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 120.68s | valid loss 11.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 124.40 | loss 0.58 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 123.48 | loss 0.75 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 123.59 | loss 0.41 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 123.58 | loss 0.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 120.73s | valid loss 8.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 124.17 | loss 0.64 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 123.96 | loss 0.31 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 123.72 | loss 0.35 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 123.55 | loss 0.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 120.74s | valid loss 10.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.80 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 125.06 | loss 58.58 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 124.74 | loss 19.52 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 124.48 | loss 14.45 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 124.81 | loss 12.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 121.65s | valid loss 13.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 125.46 | loss 9.94 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 124.66 | loss 9.81 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 125.65 | loss 8.89 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 124.42 | loss 8.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 121.93s | valid loss 8.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 125.13 | loss 7.61 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 124.66 | loss 7.28 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 124.56 | loss 7.60 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 124.74 | loss 7.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 121.63s | valid loss 6.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 125.15 | loss 5.98 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 124.46 | loss 6.35 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 124.88 | loss 6.31 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 124.68 | loss 6.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 121.62s | valid loss 9.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 124.59 | loss 6.74 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 124.80 | loss 5.04 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 124.40 | loss 6.97 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 124.24 | loss 4.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 121.45s | valid loss 7.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 124.46 | loss 5.07 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 124.17 | loss 6.45 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 124.61 | loss 4.57 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 124.02 | loss 5.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 121.25s | valid loss 6.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 124.46 | loss 5.27 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 125.02 | loss 4.53 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 124.31 | loss 4.72 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 124.35 | loss 5.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 121.38s | valid loss 6.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 124.44 | loss 4.34 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 124.33 | loss 3.97 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 124.29 | loss 5.48 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 123.94 | loss 4.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 121.10s | valid loss 7.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 124.37 | loss 3.79 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 124.54 | loss 3.83 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 124.31 | loss 4.63 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 124.47 | loss 4.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 121.31s | valid loss 6.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 123.98 | loss 4.41 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 124.11 | loss 4.43 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 123.99 | loss 3.95 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 125.33 | loss 3.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 121.17s | valid loss 9.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 124.02 | loss 3.95 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 123.90 | loss 2.68 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 124.15 | loss 3.55 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 124.13 | loss 4.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 121.05s | valid loss 8.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 123.80 | loss 3.15 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 124.13 | loss 2.59 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 123.68 | loss 3.48 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 124.31 | loss 3.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 120.86s | valid loss 6.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 124.38 | loss 2.79 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 123.97 | loss 2.57 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 124.63 | loss 3.23 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 123.63 | loss 2.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 121.03s | valid loss 6.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 124.37 | loss 2.17 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 124.01 | loss 2.78 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 124.05 | loss 3.09 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 123.79 | loss 2.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 120.94s | valid loss 8.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 124.56 | loss 2.58 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 124.06 | loss 2.54 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 124.26 | loss 2.19 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 124.73 | loss 2.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 121.19s | valid loss 9.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 124.26 | loss 2.05 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 124.10 | loss 1.69 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 123.54 | loss 2.15 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 124.03 | loss 2.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 120.91s | valid loss 8.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 124.64 | loss 1.97 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 123.53 | loss 2.26 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 123.92 | loss 2.25 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 123.84 | loss 2.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 120.97s | valid loss 10.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 123.68 | loss 1.57 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 124.06 | loss 2.26 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 124.31 | loss 1.78 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 123.38 | loss 1.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 120.81s | valid loss 8.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 124.23 | loss 1.62 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 123.62 | loss 1.60 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 123.71 | loss 1.94 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 123.80 | loss 1.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 120.83s | valid loss 7.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 123.94 | loss 1.34 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 123.92 | loss 1.23 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 124.16 | loss 1.68 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 123.63 | loss 1.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 120.87s | valid loss 7.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 123.71 | loss 0.78 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 123.25 | loss 1.28 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 123.97 | loss 1.46 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 123.69 | loss 1.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 120.63s | valid loss 10.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 124.02 | loss 0.69 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 123.38 | loss 1.29 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 124.01 | loss 1.19 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 123.65 | loss 1.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 120.69s | valid loss 9.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 124.71 | loss 0.44 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 123.42 | loss 1.49 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 124.30 | loss 1.23 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 123.97 | loss 1.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 121.04s | valid loss 7.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 123.85 | loss 1.05 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 124.52 | loss 0.90 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 124.09 | loss 0.94 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 123.50 | loss 0.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 120.88s | valid loss 9.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 123.80 | loss 0.74 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 124.10 | loss 1.07 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 123.70 | loss 0.45 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 123.75 | loss 0.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 120.85s | valid loss 10.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 124.43 | loss 1.07 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 124.60 | loss 1.00 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 123.62 | loss 0.40 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 124.66 | loss 0.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 121.13s | valid loss 11.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 124.64 | loss 0.37 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 123.73 | loss 0.13 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 124.68 | loss 1.11 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 123.47 | loss 0.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 121.20s | valid loss 9.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 124.02 | loss 0.87 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 124.53 | loss 0.36 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 123.65 | loss 1.22 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 124.44 | loss 0.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 121.11s | valid loss 12.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 123.92 | loss 0.78 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 124.13 | loss 0.49 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 124.05 | loss 0.56 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 124.40 | loss 0.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 121.00s | valid loss 11.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 123.94 | loss 0.45 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 123.55 | loss 0.56 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 124.08 | loss 0.67 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 123.37 | loss 1.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 120.59s | valid loss 10.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.10 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 124.37 | loss 59.08 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 124.44 | loss 17.79 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 124.56 | loss 15.09 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 124.63 | loss 13.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 121.45s | valid loss 16.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 124.97 | loss 9.57 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 124.41 | loss 9.58 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 126.11 | loss 9.25 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 124.62 | loss 8.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 121.91s | valid loss 9.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 124.69 | loss 7.86 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 126.98 | loss 6.93 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 132.50 | loss 7.38 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 131.17 | loss 7.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 126.66s | valid loss 7.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 131.87 | loss 6.64 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 131.06 | loss 5.81 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 127.39 | loss 6.99 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 131.42 | loss 6.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 127.77s | valid loss 5.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 132.30 | loss 4.94 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 131.08 | loss 6.79 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 127.60 | loss 6.43 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 126.43 | loss 6.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 126.97s | valid loss 8.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 130.91 | loss 5.89 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 130.49 | loss 4.52 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 129.78 | loss 5.81 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 125.86 | loss 6.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 125.36s | valid loss 7.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 125.47 | loss 4.91 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 130.73 | loss 3.99 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 133.63 | loss 5.12 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 134.23 | loss 5.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 128.54s | valid loss 6.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 133.79 | loss 4.09 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 129.47 | loss 4.53 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 131.05 | loss 5.04 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 130.57 | loss 3.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 129.38s | valid loss 7.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 125.49 | loss 4.11 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 135.18 | loss 3.85 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 131.46 | loss 4.84 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 131.17 | loss 4.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 127.35s | valid loss 7.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 126.29 | loss 3.90 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 127.92 | loss 3.34 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 125.78 | loss 4.46 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 125.72 | loss 4.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 123.25s | valid loss 5.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 126.37 | loss 3.02 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 125.93 | loss 2.61 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 126.36 | loss 4.71 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 125.69 | loss 2.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 123.21s | valid loss 7.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 129.86 | loss 3.08 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 133.04 | loss 3.01 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 131.86 | loss 3.87 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 130.53 | loss 2.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 128.38s | valid loss 8.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 130.01 | loss 2.75 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 130.31 | loss 2.59 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 135.09 | loss 2.59 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 129.33 | loss 3.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 128.41s | valid loss 6.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 131.11 | loss 2.07 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 130.61 | loss 2.95 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 129.68 | loss 2.57 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 131.65 | loss 2.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 127.75s | valid loss 8.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 130.16 | loss 2.08 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 126.88 | loss 3.17 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 128.43 | loss 1.74 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 125.17 | loss 1.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 124.29s | valid loss 6.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 127.97 | loss 2.18 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 131.05 | loss 2.05 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 129.23 | loss 2.01 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 132.23 | loss 1.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 127.35s | valid loss 8.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 133.50 | loss 2.28 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 125.50 | loss 2.36 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 126.39 | loss 2.07 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 130.64 | loss 3.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 127.29s | valid loss 6.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 135.27 | loss 1.36 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 132.17 | loss 1.78 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 127.42 | loss 2.03 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 128.53 | loss 1.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 127.80s | valid loss 9.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 131.15 | loss 2.02 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 130.78 | loss 1.18 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 130.71 | loss 1.60 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 132.60 | loss 1.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 127.85s | valid loss 7.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 133.29 | loss 2.16 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 126.98 | loss 1.71 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 130.44 | loss 1.32 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 127.14 | loss 1.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 125.52s | valid loss 7.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 126.20 | loss 1.63 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 125.20 | loss 1.14 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 126.99 | loss 0.97 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 129.78 | loss 1.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 124.63s | valid loss 7.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 128.66 | loss 1.31 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 131.23 | loss 0.98 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 125.08 | loss 1.49 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 126.14 | loss 0.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 125.61s | valid loss 7.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 134.22 | loss 1.29 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 124.43 | loss 0.94 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 131.25 | loss 0.91 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 125.89 | loss 1.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 125.91s | valid loss 7.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 131.85 | loss 0.41 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 130.28 | loss 0.89 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 129.88 | loss 1.22 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 129.85 | loss 0.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 127.72s | valid loss 9.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 125.84 | loss 1.27 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 131.70 | loss 0.63 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 130.63 | loss 0.96 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 130.65 | loss 1.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 127.23s | valid loss 11.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 131.52 | loss 0.71 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 130.69 | loss 0.84 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 130.55 | loss 0.93 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 130.64 | loss 0.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 129.30s | valid loss 9.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 130.37 | loss 0.80 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 130.37 | loss 0.69 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 130.45 | loss 0.65 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 132.03 | loss 0.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 128.38s | valid loss 9.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 130.35 | loss 0.65 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 130.55 | loss 0.71 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 130.86 | loss 1.11 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 130.14 | loss 0.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 127.70s | valid loss 7.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 133.82 | loss 0.63 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 130.22 | loss 0.38 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 124.62 | loss 0.51 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 130.59 | loss 0.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 127.03s | valid loss 8.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 127.28 | loss 0.59 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 130.38 | loss 0.63 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 130.49 | loss 0.24 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 130.56 | loss 0.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 127.06s | valid loss 9.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.89 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 129.39 | loss 55.36 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 125.84 | loss 18.03 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 128.60 | loss 14.05 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 128.61 | loss 11.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 125.66s | valid loss 19.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 132.23 | loss 9.93 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 127.61 | loss 9.54 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 133.35 | loss 8.80 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 132.77 | loss 8.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 128.77s | valid loss 7.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 131.88 | loss 7.27 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 126.18 | loss 6.90 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 130.16 | loss 7.49 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 126.10 | loss 6.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 125.07s | valid loss 7.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 128.99 | loss 5.52 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 125.66 | loss 6.23 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 129.46 | loss 6.87 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 131.68 | loss 6.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 126.83s | valid loss 9.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 132.20 | loss 4.68 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 131.87 | loss 6.31 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 132.28 | loss 5.96 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 132.31 | loss 5.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 129.28s | valid loss 6.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 132.43 | loss 5.51 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 130.43 | loss 5.26 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 134.07 | loss 6.35 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 126.66 | loss 5.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 126.75s | valid loss 5.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 131.78 | loss 4.73 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 133.51 | loss 4.66 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 126.57 | loss 5.73 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 125.98 | loss 4.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 127.30s | valid loss 8.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 127.34 | loss 4.60 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 125.57 | loss 4.80 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 130.00 | loss 4.67 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 131.60 | loss 4.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 126.59s | valid loss 9.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 133.26 | loss 4.27 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 134.27 | loss 4.41 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 127.38 | loss 4.06 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 128.03 | loss 5.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 128.17s | valid loss 5.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 131.52 | loss 2.42 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 131.63 | loss 4.07 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 131.54 | loss 3.11 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 131.66 | loss 4.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 128.50s | valid loss 5.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 129.03 | loss 3.82 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 130.91 | loss 3.35 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 134.31 | loss 3.53 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 130.84 | loss 2.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 128.51s | valid loss 6.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 128.05 | loss 2.65 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 128.25 | loss 3.41 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 135.52 | loss 3.30 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 125.53 | loss 3.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 127.76s | valid loss 7.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 131.90 | loss 2.25 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 125.73 | loss 2.67 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 125.46 | loss 2.62 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 125.42 | loss 2.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 123.99s | valid loss 8.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 127.75 | loss 3.32 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 129.94 | loss 2.83 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 127.17 | loss 3.05 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 126.08 | loss 1.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 124.28s | valid loss 10.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 127.38 | loss 2.40 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 125.75 | loss 2.10 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 125.40 | loss 1.89 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 126.04 | loss 2.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 125.93s | valid loss 6.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 124.75 | loss 2.28 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 129.98 | loss 1.53 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 130.71 | loss 1.64 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 130.80 | loss 1.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 126.18s | valid loss 10.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 129.29 | loss 1.20 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 130.34 | loss 2.39 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 130.82 | loss 1.63 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 132.70 | loss 1.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 126.60s | valid loss 8.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 125.94 | loss 2.19 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 129.64 | loss 2.17 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 130.52 | loss 1.64 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 130.43 | loss 1.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 126.25s | valid loss 9.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 126.99 | loss 1.83 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 130.39 | loss 1.67 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 132.15 | loss 1.58 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 130.80 | loss 1.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 127.53s | valid loss 7.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 131.36 | loss 1.26 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 137.10 | loss 2.16 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 130.35 | loss 1.45 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 130.77 | loss 0.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 129.91s | valid loss 7.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 128.65 | loss 1.01 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 131.78 | loss 0.95 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 129.26 | loss 1.09 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 129.87 | loss 1.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 128.46s | valid loss 7.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 130.81 | loss 0.74 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 131.80 | loss 1.10 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 131.93 | loss 1.54 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 128.97 | loss 2.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 129.52s | valid loss 8.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 127.07 | loss 1.28 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 125.53 | loss 0.61 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 130.10 | loss 0.79 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 130.19 | loss 1.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 126.38s | valid loss 7.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 130.38 | loss 0.94 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 130.32 | loss 0.70 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 130.51 | loss 0.84 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 130.31 | loss 0.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 126.37s | valid loss 7.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 138.57 | loss 0.62 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 130.70 | loss 0.57 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 130.72 | loss 0.54 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 126.94 | loss 1.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 129.03s | valid loss 11.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 135.87 | loss 0.82 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 130.62 | loss 0.78 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 130.65 | loss 0.67 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 130.63 | loss 0.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 130.24s | valid loss 12.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 131.82 | loss 0.81 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 125.89 | loss 0.70 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 124.77 | loss 1.10 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 125.23 | loss 1.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 123.85s | valid loss 11.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 126.77 | loss 0.35 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 129.15 | loss 0.67 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 127.82 | loss 0.65 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 126.79 | loss 0.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 124.04s | valid loss 10.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 126.06 | loss 0.57 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 124.61 | loss 0.53 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 125.52 | loss 0.57 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 125.21 | loss 0.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 123.31s | valid loss 9.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 125.76 | loss 0.50 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 124.56 | loss 0.35 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 124.84 | loss 0.69 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 130.18 | loss 0.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 124.57s | valid loss 9.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.26 |\n", + "=========================================================================================\n", + "Result over 5 runs: 5.667505264282227\n" + ] + } + ], + "source": [ + "num_runs = 5\n", + "best_model = None\n", + "best_result = 1000000\n", + "results = []\n", + "\n", + "for i in range(num_runs):\n", + " features = nn.Sequential(nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(64),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(128),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(256),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=(0,1)),\n", + " nn.BatchNorm2d(512),\n", + " nn.ReLU()\n", + " )\n", + "\n", + " classifier = nn.Sequential(nn.Linear(512, 1028),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=1028, bias=True),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=10, bias=True))\n", + "\n", + " conv = nn.Sequential(nn.Conv1d(512, 512, kernel_size=64, stride=1, padding=32),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.MaxPool1d(kernel_size=64, stride=64))\n", + "\n", + " model = ConvMNIST(features, conv, classifier)\n", + " model = model.to(device)\n", + "\n", + " optimizer = torch.optim.Adam(model.parameters())\n", + " scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "\n", + " epochs = 30\n", + " model, result = train_model(model, epochs, train_loader, test_loader, seq_len, digits_per_batch, optimizer, scheduler)\n", + " \n", + " if result < best_result:\n", + " best_result = result\n", + " best_model = model\n", + " results.append(result)\n", + " \n", + "model = best_model\n", + "print(\"Result over {} runs: {}\".format(num_runs, torch.tensor(results).mean().item()))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.871959, + "end_time": "2021-01-22T05:47:49.806647", + "exception": false, + "start_time": "2021-01-22T05:47:48.934688", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## With gaussian noise" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-22T05:47:51.560950Z", + "iopub.status.busy": "2021-01-22T05:47:51.560267Z", + "iopub.status.idle": "2021-01-22T05:47:51.562736Z", + "shell.execute_reply": "2021-01-22T05:47:51.562316Z" + }, + "papermill": { + "duration": 0.882894, + "end_time": "2021-01-22T05:47:51.562850", + "exception": false, + "start_time": "2021-01-22T05:47:50.679956", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def get_batch(batch, seq_len, digits_per_batch):\n", + " batch_size = batch[0].shape[0] // digits_per_batch\n", + " width = batch[0].shape[-1]\n", + " data = (torch.zeros(batch_size, batch[0].shape[2], seq_len) - mnist_mean) / mnist_std\n", + " choices = torch.multinomial(torch.ones(batch_size, seq_len - (width - 1) * digits_per_batch), digits_per_batch)\n", + " choices = choices.sort()[0] + torch.arange(digits_per_batch) * (width - 1)\n", + "\n", + " a = batch[0][torch.arange(batch[0].shape[0]),:,:].view(-1)\n", + " b = torch.arange(batch_size).repeat_interleave(digits_per_batch * width * width)\n", + " c = torch.arange(width).repeat_interleave(width).repeat(digits_per_batch * batch_size)\n", + " d = (torch.arange(width).repeat(digits_per_batch * batch_size * width).view(digits_per_batch * batch_size, width, width) + choices.view(digits_per_batch * batch_size, 1, 1)).view(-1)\n", + " data[b,c,d] = a\n", + " batch[0] = data\n", + " data += torch.randn_like(data)\n", + " \n", + " return data, batch[1]" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-22T05:47:53.399258Z", + "iopub.status.busy": "2021-01-22T05:47:53.398506Z", + "iopub.status.idle": "2021-01-22T05:47:53.694083Z", + "shell.execute_reply": "2021-01-22T05:47:53.694486Z" + }, + "papermill": { + "duration": 1.176984, + "end_time": "2021-01-22T05:47:53.694635", + "exception": false, + "start_time": "2021-01-22T05:47:52.517651", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAKoAAAOfCAYAAABYKVgBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAEAAElEQVR4nOz9ebBl2XXeif32PsOd5/vmKeexqjJrnlBAFQACIAlwaklNiVRLbinksKND7ZbbbjH0j+1wWyHb4bCj/7CtkFohcxBFUZxAEjMKRAGFmiszKyvnfC+HN9337rvzcObtP9bNBzYNClkgEcgK14nIyMyX+c4795x19l7rW9/3LWWM4aPjo+NBP/RP+gI+Oj467uf4KFA/Oj4Ux0eB+tHxoTg+CtSPjg/F8VGgfnR8KI6PAvWj40Nx/MQDVSn1OaXUVaXUDaXUP/0Rvn9JKfWyUuqyUup9pdR/Pfl6VSn1NaXU9cnvlQ94Xksp9a5S6o//qudTSpWVUr+rlLoyuc5n/4rn+28mn/WiUurfKaXSH+R8Sqn/USm1o5S6+Oe+9pd+v1Lq1ybP56pS6rP3eb7/y+TzXlBK/b5Sqny/5/uBhzHmJ/YLsICbwCHABc4Dpz7gOeaAxyZ/LgDXgFPA/xn4p5Ov/1PgX3zA8/4T4LeAP578/Uc+H/BvgX84+bMLlH/U8wELwBqQmfz9d4C//0HOB3wceAy4+Oe+9gO/f3IvzwMp4ODkeVn3cb7PAPbkz//ig5zvB17zTzhQnwW+8uf+/mvAr/0Vz/mHwE8BV4G5PxfMVz/AORaBbwCf/HOB+iOdDyhOAkv9ha//qOdbAO4CVcAG/ngSFB/ofMCBvxBYP/D7/+IzAb4CPPvDzvcX/u0Xgd/8IOf7i79+0lv/vZt+71iffO1HOpRSB4BHgdeBGWPMFsDk9+kPcKr/O/C/BZI/97Uf9XyHgF3g30xSiX+llMr9qOczxmwA/1fgDrAFdI0xX/0rXN+94y/7/r+OZ/RfAl/6q5zvJx2o6gd87Ufq6Sql8sB/BP5Xxpjej3xBSn0e2DHGvP2jnuMvHDayLf4/jTGPAkNka/1Rr68C/Dyybc4DOaXUr/51XOhf9iN/wNfu+xkppf4ZEAG/+Vc53086UNeBpT/390Vg84OeRCnlIEH6m8aY35t8uaGUmpv8+xywc5+nex74OaXULeC3gU8qpX7jr3C+dWDdGPP65O+/iwTuj3q+TwNrxphdY0wI/B7w3F/hfPeOv+z7f+RnpJT6e8DngV8xk33+Rz3fTzpQ3wSOKqUOKqVc4JeBP/ogJ1BKKeBfA5eNMf+3P/dPfwT8vcmf/x6Su/7Qwxjza8aYRWPMgcn1fNMY86t/hfNtA3eVUscnX/oUcOlHPR+y5T+jlMpOPvungMt/hfPdO/6y7/8j4JeVUiml1EHgKPDGDzuZUupzwH8H/JwxZvQXfs4HPt9PtJiavGQ/g1TqN4F/9iN8/8eQreMCcG7y62eAGlIQXZ/8Xv0Rzv0i3y+mfuTzAWeBtybX+AdA5a94vv89cAW4CPw6UkHf9/mAf4fktyGywv2D/9T3A/9s8nyuAj99n+e7geSi957J/+t+z/eDfqnvr8gfHR8dD+7xk976Pzo+Ou7r+ChQPzo+FMdHgfrR8aE4PgrUj44PxfFRoH50fCiOH1ugflBWlFLqH/01//yPzvcAnOuv7Xw/Jmz0A7OigLf+mq/ho/M9AOf66zrfj2tFfQq4YYxZNcYESCvy539MP+uj4/8PDvvHdN4fxJB5+s//h8l28I8AlOs+7hQqZKeXjI4gscFoUAkoA7EDOoLEBXsMiQY0GCVfj1OTf3fAWAbtK1LpMpnZJeMMExJbEacVmMl5jZwbA6ocEQ9tOa8jLa7EBSuQP1sBoMAuy/XJxUNigYohSYH25Xt1NPk+DbErP0OHYO7dZSPnViGkMmWyM3K+xALLhygnvyfW97/PKNCx3BPM5N9dSFz5nEnGoAKFVSmTOrhotKdIbPnZRk+u/96P12CPDEYrVGIIcwodyf+NMvK5tA9OsUJ6ccnoyc8CUJFch0rkfjuDhCirMXz/+owF9gjiNFieISwodCDny04vmXv33vINQQWskVyrv7neNMZM/acC6scVqD+UIWOM+ZfAvwTIzC2ZR1/4r9k7bZHbNPQPgA4U3lxE5ZyFsRVGy82cfd2jeyhF9wg4fbnRw6UYFSlm3oDeiibbMHg1RfVSCAq8isXeIwqnr5h5K+TWL8LSnyp2HrMJDnhMf82ldVox92qMV7aIU3K142lFumWI0go06EAetBUYOsc0s6+HNJ5w5EHnDdZYkduUj5nbitn6mIUOwPIUiStBUL8YsXfKJiwaylehv6Jwu9A9EVE9Z5HfihnVLUZz8mK5XRi/2OfAP09Y+xtFtK/kBY6hfCPGaEWcUqTbMd0DNkFJAlKH4AzACgz5zZjBnEVQknPGKXkGmV2DMzJEKUXnBBjboBLIbGsKdxMGC5psIyHMKdJtQ+ukJrNj8MsKewTT58Z0D6Txq4q5V3psvVAEoHA3RsXQOmFRvhEzmrHwy5DbNIxmFZYPxdsxfkEznlFc/uf/5PYPC6gf19b/gRgyKob+okViG3oHZUVxBpC9bZNtJmQbMTow6BC2nk0znlLk78qKE6fA6WmsQDGc1XjTCcM5xeBQzGDepn3UoXtYU1yFOG3YPeuQW3MYzFlktwyVV1NkmjFRPqF91MYKDSgYzSlGSxHZ3QRnZAjz4PYMrUcMOga3B40nHcJigjeVoEJI70KcUozrit6yjYpB+4owbyisSQBHGY3bMyQWdI5BlDF4dUP2rk3iKPoL1uSlNFg+BCUofTFP8/Eiy1/2yN81OH0IS4atFyDVjRnNKIYzFokj99P2wJtKGC5IYIRZje3JSjdcSvBmY3QkwaxiuY/ppmLmdZh+A/yKYfs5ud7+kmY0p2gf1fjVRF48H8o3I/qLKforCssz7D1SoH8gQUWw+YWQ1gkLgMG8hTMwJK5hPK2Icganb+gcthgsKabfCX5gTPzF48cVqB+IFWUsGM0bjCWrpNNXhAW5Ic1HNINFCx1DbjuRm2sgcRT1CxFBOQEDlUswXEzI3dXEGUNm3cIdGPJbCdUrCamOobAGhdsJxbUEYysSV6FiGNcsZr+rWPhGm85RTZhXxGlDZssmzChajyRktw3t0zDzOmR2QtyuYep8RKahKV3V2CNF/b0x3if61C9GjOYMbldW0vp5Q2LLNtxf0gQlWRXzdyG3rrCHCrcngRQWFK1HDLkzLby6wfagv6ywAmidTBNlFYkDxetgDzT9JZswD+lOQrplyG0YMg1Deldj+bKxbX06RkeG7KZh+k2ovaupXo2wfUPjpwP8iuwcQU6BgtymonZOM5421C5H2AO55wf/MGTvyZjyzYjGkzajGc3Bf7eN2zdEWYU9Vtgjw/RXXXQoz9abMpRueiQ2DFci3I7c91THoGKI3fsLwR9LoBpjIuC/QmQGl4HfMca8/5f+fwtK1yDVVmSahlQX7AGEBbCHit6JEB1A74DGHkke1D0V0Vu2KV9VKAN+SW7yaNagA3mw4ylN84zkQZ0jGmUgzCkaz0KYl208vxnRPayJXcXN/7xMUE5ItQy59clqV9VkNi38smLxmyFRWnHnMy5BSTGctrCH4E3D0tf79A6mKf5Rnr3TklHl1xPcHgRFyZFLazG5zYTslsHtSf7mTSEBOTYMlhMWv9bB8hTPzd0iKiREGSjeThgsKqKconQrZPa1EX5NUT7TpHtUUpLRlCbKgFeT4NQhpNoQZwzZVQe/qNh7LqR9QtE9Bt0VmyitqHw3RbppGE0r4ozCqyhy2zFeTZFqyWrsVw2ZXcPO4ynsrsWobmFsQ6ptWPvbszTPgjU2lK9A8W5I4xMxxdsxQVnSod7BtCxCbYvEhtpFD5XIQtRbub/s88eGoxpj/tQYc8wYc9gY89//p/6viqB7RAqlcV2R3UlQk/zM6UPxisNoVlYhd2Dw6lB72yLOQOzKilS5HkKiKF+FVEvOOVwwZLckvy3cMRglQbPypxEqguKtgMGCjTOUXE7FkyIlNuS3Y9JNWRFrlyPiFIzrNp0TULgNhTsxQVlWNxXC7Z8uMFhUpPoxbscQLISEWSV54MAQZWA4owmzisJ6QPlGjI6huGqovg+DJUX1PUX7/xQQz/k0gxy5OxbT74TokP3cd++0w/qnsrKz/HadhW9FePMSWP0V2fa7RyYFZgBOVxEW5PMU33Mp3biXjkCUUXSPSkpVvxgRZZDdZLLTKANeVVO/YPCqivSewR4rco1YKg4F9YsxzkAznlY0H0vYPeMy/YpN65TF/Csx9fcjdp9KSO8qVr7kkWobvLpDlFGk2ga/en/x9EB0puTtUgQFyDYMsSNfy28m6Mgw/c6Y8o2Y/N2E0bTC6cnKmN02DJcMxVsJKkqonzcMlhReHcIiZBqKMAdGKforCmdsyO4kbHzckXz3YymitGL21SHjqiazo7CHmp1nDFvPWqT3DH7V0HzYpno5ZjSlWfqa5FS9Axbah/GcIU4b0k0Jps3nLcazCjWwKK0FxK6s9uXVmPL1gKCkGMy7BDmNSgz5zUDyyLyh88kxje0y2bzPO3cXGRwN0WFClJEiqHAnJtUy6EAKy85x2HnMYfbbCqcvW7ZfBh0pgkoi96FkZMcpSq44WJD7FhYN3WMJbkcznlJ0D9mMFiS12n4pRiVQvxASpaF9QpPblBc9s20YzFvMfyfCGcmfnZ7sisUbliAbNkydizC2rNwqUER5WP9UBiuA3rLk07X3h2Qa90czfSACFcCfigiLksv5ZcmPEgu8umLrmQz2OMGryvad3pP/F+QV+dtgewnjaYfhrKZwRx6MimDqvEe6aegdVEy/K9seBiqXDYkL/lSMV4fBSgZnaOiejrA8Re6WhdtVZJsR2U2FDmHj0wa3bwgLFt2jhuCJAdPvjCleh8WXA4IS7J0xZBoKawz5Wxbrn3QZTylyjYT+gsXe6RT94yHOOGE4r+gvafpLrhRdEWSzPl945DzjkYt1KU/+ukNYsCTYy4qt5xV7z4RYPlge5DbAGULnqMYKZNVe+LMxfi0mf0dTuxQz992YQ7/TQYeG2FV4xzzCgkKtDHni8euM5yOcgYEEjv7bPoOVBEKFMzS0jjsEJYO3ELJ31uAODGFRERQVncMO47om24hldc4ZRrOG7I4UcK0TNttPWxgb6ucU3kyENx2R6iXkthLskeHup/IMln54bMADEqgqhpnvalQCia2wAoM9UPQOamrvRyx+vUv3oIM3ZXC7hnQ3Zvb1MWERxjOK3TM2fkkRlA3DOQUJTL8bsnsmTf8QZLcNt3/JUL8gq1NmLybdNLh7mvI1gWHCnEKPNX49xhka6hcihtM2lmeIU7D4NQnY1nGbymVF+tU83cNpUn1D54hLqi3XrCMpGqIMkAgE1HhaU73sY3sGp22z87hmfNyneiWW1fTJPtaRAVP5Ia9sHCZzMYMOZdXcPSvogV8Ft6OpveZQvRLi9gSnrFwTCK5/QD7n1nMZqhc0QQH2Tll0D9jsPVrG9g3eNFS/k2I0m5DLBJQcj6mVNsf+7lXGc4ar//Mcpx+7xcOn79D6mM/gyTHm6BCra+G2Ne0TmqBo0BE4Q0PvcIJf0hjbULgFU+8mNM8a7JG8ePk7Uge4/YS5P9Nk79o0ntCMZjSFdclhneEPQjL/f48fF476gY57AZrdVOjY4PYkua9cjQkKmp2fL5LZhvxtaJ1JUOdkixkeC8hdc6W4OmqYfgtyWz7DOZfOIQe/ashuKKpXx3j1DE4/wCtp/LJFupOgbmr8ssYaC7CdvyPvbeIIXNZ5KMEeaOwB9FYEbrHH0DlhKE0wUFDUL8RsP6Olgh8Z7J5Fdtvg1RX9A4YoY9j4hItKFDNvxKz/dELlNZfOf9Fl1EvzsaXbNL08l68scvrkXRqNCjqWe+PVpPhJtaWJMJ5S6NAmyir8isEvWQRFKc7SnYjBsg0KMk1DYkshOVhSDLAE1sorStfArFb5xrESqhrQ6WWZPtugnB5zILfHn7zyONPHm7R6WU7NNTjfWSEKpGi1B4reQwGFyy65dUVQBGsMue2I7gGb+rsGZRLyGwGDeRe/omk+YmGP5d75VQni1kmb/F3QwYdo6zcWNJ9MiFOQWIreAVldt55XdI5qVCx5p46gcNNiOKe4+1konnelcxMY5l41+CXFYMEVAD4HxZty7o0XMpSvx4xmHHpHIMwqNl6CdCdmuCAJfaptsEeGmdeGRFlAwdTrmtnvxdI1SktlW7kakN7RhAVFbt2QasvPqFySAM82Y5y+onscMjuGKGcoXdVULxuiXEKYUeSvOySOIvXHJYxn8X5zlraXYeFgk41uCWNBmJV7E6cEsivfiMltGvxawu4nwgkwr/BLisKaInYlr47TUhQKdin3wRlAbl1yzN7RiOm3+oQ5WDy+Q6k4IvN6jo2tCrfbFb747lmSTEIQWSzVOxwvNjh2dJN0U5HZVhgbaq87xC77nTa3B42nLOyxQIzpVsztz6UIc4rKFZ/CLfnZIHl0fiMmsaUp4lfub0V9IAIVoHDdYuGbHTpnQxJHAOukGKEMzLwVUrmaMJ5RBGUorSZk1228aSMw1Iwi1Y5QCXQPafpLmuJtgXZ0aIiyhnFdU1wdk2kogoJi9ruK0bRFcU0q/TCv8CsKa+jj1RPCvCK/GWB5CUHRkGlKfrb3kItflRw5vxXh1WA4a0nOeTRiVLeoXo3JNBSpXkL+tsavwWBeUz8Hew8rgpIh1THYniFV8XCshIGXIuOEBK9VyW3H2B7YnmG0EBOcGjH9j1fp/9SQwtEOs3NtBs+NmPv524xeGtA9GRPMhdQvGHSosMeG0YxGRZJW9Y5H+9itNdYMl3OkW4bul+YA8GqG3JUU1stlnKZNesumvVvgSHGX//DeY6zt1KQtnIL573iAQEsqlq/ZY0P9XMJwQVFa81j/pMXs6wmWD3sPpegdktV4uGAIs7D3kIWxwKvo77eXf8jxYATqpPe++VKZzG1pr6SbGiJNnDaMazaJA2FO/m+YVxRuJ5Suy02uXk7oHHGJMlLR6giaZxTVKz6jecFijaVoncpQWJeuTOeYPMigqHB6iuCTXawxNB+vYLTke6t/R7H+kkP1snTMrLFUtIU1CMqG1gkHow2DZXlhyu/ZjOYUreMW5ZsRzUcs0i1D5UqMVzeMpzSF23Dw9/vSDw8hDGz2OnmWyh0slTBaibj7OYX+W7u0/uaQmSNN0umQlpcj3MkQG4UXOByfb3Bze4ootEnPDcmuuux9YYzTk5cBJT313JZh5hXN4EBC4kDpmmL9swl+STGaNRijCGox5RsxpdWI+nmDsQz2rsMwSrE40yaONOPZBJXAYMHF7RmcoSG3nZDegyir6B62mDofE+ZtCquaIK+JshLMQTXG6RmY9zD2JI1JGeKU2m8M/LDjgQjUxIEoK0GnEgGpw6Kh9qZFak/hjBIGi5rqpQTbk9Zj4giGWbgFW58wJJYiyoBfUcQuZDcUUVYq5uqVGBUZ0h2DM0hIt6SbpWPZtjO7BvVmSd76ecXcq4bhgqJ4waV+3lC4NZbt/GyAMxAkIrWncLuGzI6SFEMLJJbblIbF+qcUmR3oHIXBokVmRxAHewwbnyzi1YQYsjDVQVsJXT/NtfUZsAwqUjQ2KsyWe+y+P8XobgHHirH7mvj1Cr1ehs1/f4B6pY/WCZX8iPF8jH0ph18z0i5OQVCW3aN3SJPaFcSke8xQfs+Z5IyKILIo3LTpHraIU4rchs/s6xGWrziQ3WNzr0SpNELFEvxRStFf1iSWoBaJLS+vNZbe/vqLwjfQkaAMOjJgICgprLU0KpJnPvtaTO94TP38/UXqAxGoykBQMkydSyjckTf/3odLbGg8pXE7ht4BzWg+YeHPYsKcYjincAeG6jkNWhoEbtcwPBLiV6B7yKF8FXYe1xitaJ3SjOsWwzmNNyOgdfRknzCn8KsGyzeU1hK6B4WYYnuG/opm48UciQOZWy4qMdQvRljBhM3lSU8+dtmH05yBBJsVCEA+mhHoKN2SXC2/nhCUDXunLLzIJp/1GHgpTixtkyp5HHp4A923aHQL5NY1Jh2zdm6B4qoA+qaVQsXQbBdINjJsrlc5dmqdKCNb8HhKkW4arDEMlg2FWwZnJNfotjQ6FCgJYLbUx69KXhvkNZsvZBjMCpvse82DxD2Xbi9L6miP6qUQZyxdKr8Ks6+N6D0UkN+IqVwLGa2ElK9MmGBpIbJYvtQVzlDgtPSeoXg7onXC5tB/DDHWhyhH1ZF0e9pHLVBSBBVvQn4jwViQ3ZIPHeYNpSuKzlF7n/LnlxRhThGnhFwxXITSBQdvJiYogF9VVC8J0WT6rYgoo7A8uXmdY7DyLwx+TbYiJp2YxIby9YTRrCJOw3g2xlsIMRpSXUP3gI09FhZRUFB0D8sDSHVkq2ufALetSSwo3ElY+HaEcQRu2jtr8Mqa6nsGtw9Hyk1amyWWyh16fpp6cciN63MY2zDup4Qx9i2LuBIxXFAYBYsvJ3h1hdlKc/DRDZxcyHSmzyMvXGewaJHqmP0OXnZL4dUV42lDUDJMvxuSbhsKt6VFfHunSpwyZHYMw3lhNnVOSAu2+cVFjh3dpFgYk7xbIspJS7q/ovDrCZ2jGTK3XLyqpnPUoXDVwa9KjjxYUnSOS2NmcFB2sDgtbeDOYZtswxCUbPzST7DX/0EPHSQklsIZSku0cFeSf4Da+zF+eRKQ1QRnCJWr4T6byaspgpIEyHjGYPmSuB//l10yu4byZ7boHhJAvPmwTX4zIs6AXxFSSPdoHsuH+juKoKToH0yoX4wmaAIklqFyUTPzbYs4Y0h1YlBynbktQ2E9Ib8ulECQSn/+OzHGNsQZRfeIJspqooyhdxjq7yo6Z0PCvATdm6+cIF31KDoejhWzWOig0jGVg23OHFqn/IltzN9tsvK7ivRjLcwLHbZ/2cOvJXzi+YsMApdyccT1zhS11JBU29A5lRC7ipUvDVj42h5uV+5LYU2x9bxN57Am3U4oX4XcG1mcnqZ/QCp6FUN2WzGeMfSOR9w4v8hgmGZ8MGBc1Th9RXoPpt6SNCm9Jy3Y0azwFUbzCX5ZAr50TeCo7LrGHhucnsEvGyxPXvL+gk1QvL8V9YFwSsnMLpkD/+CfEOVla0x1FLEj21y6afAriuLtmHFVE6cUyhi6xw3Fa5ruqZjyRc1wAbINRWFdyCqWZ0h1DVZo2PzFgNo30pSvjrj7UzmivKGwKqnFvW5YnE2ovaPpHRGycv4u+7S0xW9GhHlN66SF5YNfNpMHJrlsUIk59usjNl4s4FeE0nbwiz4bL2T22V5OX1qf/YOSMkR5AUqtoYWOgINDnlm5xSsXj4NlyBQ9xv0UTjpiutKnPcxgWwm9rQLZ6SG1/Igw0Txa3+BLFx7C2XGovm8I8op0x9D4vI/Shqkvpmk8A+kdTZw1ZDcV6VaCV9G4fcPOczHFazb2UO6XVxWyTJST9nNQMVgHBsSRhfueYGaltYTcls+dzwjZREUCobldQ34rZjgrmHOYU+S2EtonNIlr9nHXzI7scP2Dcq9v/tr/+m1jzBP/qRh5IAB/M1nXU3uy1eoAwrxhVDVEWU2mYeitCPPGq0uQqCkP62Ka0mWLbDPG2BZRFtpHbfyqdImEua6xb6fRkeHGr6YpXzQkbQXK4FeM8AMWNYljYbSh+r6hdVKR2IpUSwq90bR0vuyhdFpym4Z0O6JzyMEZQpTVNJ4uMDgcMfttjVfVbLyQwfLkelMdOV+6laBDTXYbsjua0bSGBEZzhuNzO7zy7glULkI7CXGs+ZVH32B1WKcXpmmcm6F4Zgc936OzWSSbCtm7XeGVl2dI5SC3YfDLmjAnL8bcH7gYregta5a/HHLnZwz5WxajWekGBSUwlqL+psV4CoaLhlRTKJKWJytiumUYzycU0gF+aFO+IUE4nNFsP+dSvgxu35DfCNg9k5ZdcFuQlMLdhHFdCN060Mx9L2IwJ5xbv6Jw+wa3I8SU+zkejK0/gvyGFByF2+Kdm92C+jsCE2VaAo0kDsy8KZV/+eU0YUHYTZ3D0qkqrsUkDuQ25EZnd2OmLowpXxWmUO1tTZxWdE9FIsEwMJzXlG/GQlQOoXlW7bf1gqLkzsXVMVFWSDLd48L498qWUAUHQk/0qrDwdcXOk1A/N8IZSIE4/XaCPRL4rX1SYw8hKIgSoXckYbQg2/LaHx8iMz3CvZ3CcWKCborffPcpLnzxJO/fWMA6NKBxs473ThUMdN6vkVsTVCO7JYhGUITKdQHRw4zk2joUgnd23WI8I2yyvTOGKD8hck8YXuUrQo10+kKQAWGyWSNN506Z8XqBxBEcW0fCafXLinFdc/dTKXQIYdEwqlvk1xM6RzXefMzWczYL3x6z85hDfjMk25DGTulmwOCU/4O1ID8oRv7ao+5HOIwF+Y0AHUkVndjQfiogKCq8mqJ10iIoSjCP65rxjORD2pcqX03ajTqG8o2E7omYKAeNJy3WX8wwWFL0Dwh05XaFTZ9fj1h8WTo8eyctsttSGNlDxXg2QYdSHIV5RfNsFmcAO0/IFprfjASMXxKSxb1VYfeMxliGzrGMSEWuQeeIhVc31N73yN8xGBvcvgRQuqEJSwnl63KO3J8UyN+B6GYeu2Mz/bKLPYTsTZf5f5Uiu27hrQRU3rNI7wlcVH8vwh2Y/aCMHcXi13v0DikSS17kbMMwPBZQf9egI4M9UlQvSuWe2xL0w/KkINSB5Kj20OAMJGc9dmqdxRMNuoc0xoLCRiQqh77shqmWwq9AUItxRgavoknvGpyOJr2r2Hgxg9uD0bRD86yQxO98zmHhT+z93fSHHQ9GoCrY+IRLlIXmIxblmzHF8ykwMHUuZOrdiMq1RChqOwnFG1LMjD4xoHMC0KIQsDxZjSsXtDDddxRuR7bFVEu2GWOBPYTuQYfBvIPbFUgqccGrg7ccoEJFWFRkmonQBLV0X9y2xu3DYMGmfdxCxYooI9u6SsDtKixP0zsoLczhgtrfSnfPpHGGQkmMsvLv1asxy3+a0DsgK32UEWYSCkrXv58SOQPYfN5l6t2A4gWX0ZzsNG7X0DliE6UUsSOwmBUYmo8WmHkjJLMXEeah9XjEkf9PzPbHpXskojpF6UbCxqeF3NJfEfSieDcmzAl6Yo9FpXDj3SX+j0d/n6CSkN00jKYkDes+6ckL15JWcv6mTeeYZrgoXbywlDCeMZRuJoxmDJVLPZa/GjBcMOhIOK/t08lfGhd//nggAtUeC/tm/rsBlifAMQrSbUP7mIPbD+ke0sKVfFozWJHVMf29PDqUlUT7Cq9qMZqWf+8cn6zUWwntU2APpSjzqoooC1EeUt2EvTOG0dSEVuga5r5mU7kEbsew/bMBxpaOWO+wtG5jF0Yz0gE7+usjYdiHwhhCQXZTOj3jGcGAo4wUD27f0DmiJ0C4tF9bJyw2XpywoyqSJ3rThoN/NGY0I7uJKGJlpdt8wSHdmuTosaF3SLBRtBBPuscNO09oOscNcUYzmLPJNCB7y6G3nKb+hkX5WkLtUky6I9vQ3Lc0hbuxVPwG1j9tSLckf3SGBqPh137mD/hvL/9NjIZ0O6FzXHaQzJU0tUs+YVbh1SYv3/sx5WswPO2R2bDIrUuXMNVR3P5CiaBkM/2OoXhd9FRTb91fCD4QxVScFqwvTjnU34sYTU0EbmnZqnceywjBVsF41rD05YDRrDvpfAigXr4GzshgezFh3qKyntA9pBjMS/GS6gm0Zfty88d1zWDeYvnLIe1jmkxD8L/mGUmaqhcNB35Dk9ghncMO2QZ0j2ghd2iFMzLsPiYvSrpjaM/ISjjzegjKIbclBJTSqiEoKKKswpuJ8Rqa8XzEbuwI231atuLZ18fsPJ6R9u8jGZy+5OSltZiNzyakNxzcjsIKErLbE+mykXRo72F52WdeAytI2DtlMZgVYaCOYemrPbY+ViTKirxHJYr8HQmsrZ+KcJoOuXUpguyBJYvAnqF7RILo/3H1JarZMaVHNrjpzFN9V54XQPuYS+dJn+I5IaF4NQu3B7VvpRhPw2AFMjsCxSW2Ib0XcOfTafLrUgMM73NsxQOxouoASORDhTlNlFXkNyPRmceG4WLCeFpWGLetWf0lh84RzXD+nswECus+reMWO09odCAFUJwxxGl5YIN5Le3EAxqjBINNtxPaJxxSHenUFNYjStcF/xsuaNpHXeKUrILZ3ZjqpZjC3UCEcV6CX1FULhsG8xp7DOmmMO7DLHgVwXPHdU3iyDZXf1sTZSQYyjdijFLU35YXZO+hNOmWoXA3IrEFx9QRBAVN/oZDehcK6wmjKc1gUdF4GuEoFKT4S+8qUp2YxlOaytWYsDBpHcfQeLpI70REdlvaq4VbQqvUEdhNh6m3E0o3Q9yepBz5dVGblm5O+vvdDGGiub1TZfFrosLNbSTEGWl5177jir5tzARjluZLYks65FcMxdsxOlJsP5khtwH5jZj0niF/5/5i5IFYUe+tHHtZi/6S3MDBvA0aircD2sdThEVD6boUN1ZgMZ4WXbxfkbyueyRFdhNyW8L47x3KkNpT2GPRXUlLFvyqAaRrlN0WfDPTiukkFnunHLINQ6ob44w1jadhPKNJHAhKNtYYRk9pijflOqbfCRgsOPSPRVTfscjtxPQPatyOxihFlP5+q7J4UwBytwduRzGctRjNG4ZA+Sr7DYDBgjyS7LYis5swmhW+7OCAIdcwhAWoXIvZmlFEeUOSUpSvCi7aPWAz+1pMkJNr7hy2JqRqw9KXYDA3KSj7TDi/GmcAzjCBiWmGV1OUrxlGCwodKlJ7MJ632W0XiHbT9BeF+dQ9GaMyMSpJ4bYlqL2qpnI1ESy8G6MiGyswtE8pOkcsKpcTtj4b4TQcvJq0aWuXPkS9fmMxKU7k7+MZ4VImFozrDnOvRuhAVh5pCBhSbckBg5KhtJoIJtcxtE+JWjVKC2klysn5+oeFdla5BKP5hNqliMGSonfknvZcUbgjEpHRlMX2c4bKRUX9QoI1lnM5Q5FcRzlF9WrInc/a9FcUtbcsMnuJeAVsilK2eDfC7Qv0lGrJ92d2DIkjgH/vRExuQ6Th2UZElBNSS7olBV9uO6G/rMluJ0Q5SO8qdh7XZLdkFUuyMdZYoX3ZsjuPRHhTinHFwhkbxvPRxDlG5NrdAzZ+VSQ52UZMnILiHaFG3v5FaB9zcAaG+nsx5fc7gGj3kxTU3rKwruRY+qogBX5FCqfM1RSl6wk6hnQ7Jr8V0z2oGc5pOkccBiuKnWcMlUuyCuvIkFl1scaK8g1pjyfOh6nXH0KqJQ9gcCgi1Vb7DiCdoxZ3fkaDgqCgGCwLd3Q8G+MODPYE86xeidl9OqZ4QwI83UmkAwJMvx1Tviw9fpXA0lfEOSS/bqheFJVoqiUWN149YTSjMLYhtxPTOWxRvi5BHqeFBJNqGXpLNtX3pCkA0Dpp0TtsUNFkZSpbggg40mKMU4rBsiAJ3myEcQRRiFPQOumQ2Zb2YmZPHmDrhAT8cF5TuxiR2LIt9w5JQZldc5h+OyQsSTcpf8OmciXG9g2dQxbV8xbzr4yYfstnOKuZfmc86apZbH7SEDtiWuEMoHjRgYl8uXPI4tYvVaU9XbHoH4yJcnLvuofsibBQAq92OWL3SVCxYTBn01ux9llwUQYKtwzFaxZWKDCiV9Fkdg3GNgRFRWZHqH73FSM/lsj7gIexZMuZedOn8p5Fqi3gde19Hx3BgT+MyG4ZRnOKpa+H5Ndj8relwo+ykiNuP6PJr9oS9B0hM7tDw+BIiF+y8GqKqfMjhvOK9nFH+KmPCD1wNKMp3o2IXbDGSrbDroU9irE88QfIbhv8suCKYV6R6oqWajwrq6Q3G1O8qchtS+7m1RWDpUnRMjLU3hem+2BR47Qtipcc8usJmQb7/XmRdiuql2JpGJSlXbv9rLXPKivcEtOKoGzYes5m+i0J7DgFRiuy26Fc15SifyDN7tkUS3+4ze6j0inrL2qK12wGKyJVCUqyU0RZqQfSewZ7KC9l+yQc+KMItzspQKcM2W1DblMRpUXZuvS1mLCgcAfy4i19fUxuA4bLMd2jYkCx84QQh1QsDQZ/PqR3CKzQ4Bc/RKQUjJCIx9MOw0VZETM7htGMQ+FOQu+AKy3NNnQOu3QPSZclTsHc9yKiPBRXJQ0Yz4hmPSgI27/+PZs4JatF44ms5KxDeRFmv2vYO2vIbyZsP20znhW3j2Qi177zWRdnJBY0/RXhk7ZOKdyeYe+MItMyOD3BJCsXNKW1CL8sbWCjJyx4A4MluPWr8pAsD3J3pZ/eX9HYnsH2DdXzsmt0Dmuaj2ipziNI74nOarCoxfrokLzYbkdgsPZRC3sE2R3DcE4zWHSJcobxivgKeFOG3iNTjObMPlBfuBsTFWMGB0R92jqbMP/qmNgVdpM9Fjiv9p5hsOgSp+VlSrcUzccTuiclZYjSsHfKESeVBPJ3Da1TaUprIfk1C3ugyDQUM28k++y28ZQhteFSvCGuMfdsiH7Y8UAEqj021C6GOKNEbs6K4Ix7jwhJN04r+ocSsjsJSUpuWvmKbPc6MuTuCgQU1GJyG4Y4I3msDiDdTXAGhtJaxHBJOkCpToLbh83PxVQuKroHpYNieVC9LG3YymVDnE1on5LV5MBvbzP3PZ+Zt2LG04rshpCCC3cTjBbEwqtatE8ZokLM4IBonBJb4JniOynSrYTZ1wa4fUP/UILblc/ul4S5NftqHx1B+TpULyfkNwTT7C/YZHYkNZp+SypyY4kJh9FSGEVphVc37HxMJN+pLZvyjYDy6T12/uaYVEvRPSCF2u5ZjVUKiMoRQSXBpGOc/12D1kPCybU8YaUltqJ7BDDQPW5INw1OX+N0LRJXctzBkZBMK8HtxzRfCtAh7DzmYo8N9lhe2NYJuc6gIAYh5esJgxXuuysFD0jVH2UUvRUbY0krbvb1iL1TDukmjOYSZl6XG+VVtJgoLMH8twO2n0nh9iwGywkzbxjSLbkh6aZs0ffMyEYzGh1pVr7kM5px2HlcU7limPuqTWILSUMl0gVTUUKqo2k8l1C8JrzX0Rys/cosUdYQpwy5uzD/pS3u/uKcGClcitChzXBOFJ7jaS0pQx3QAtNUrsUUX75O63PH6C9LEaUSaQmjhNzReKYgxBdPmgL9FcXMGwGjGZfElZU+KAuvwRnCaF7hHxuTbqUprUVYvsVA2RgLps4lrH/SpfCHNTL5iTp2DIMFRVhJ0ECmOub5M2u8trmCH9u4HU2Uhv5BcTbJrwe0TrtYvih0U70Ip28Tu5BpGPpLNm4ThjOKvdMOzl1Jc3KbomOrXQwZzNkoI6kCCnqHFM5AdpT0nqH1WHxfMfJArKhGi0RiNCcGX4klHlRhwZDd0oynNLV3NLlt6UvXLhrCgg3m3uoqsoigqIR2FoumxwomRczjY+yhoXVC/BaNYxhPqX3is1+R7+kesNl7OIXlweHfiSjfiMQ/qguVywn2SFG6pimsJ9z8+7NSYKVEe48WF7w4DeGhMfYIyqsx9gjcjpDCW587RlCUije9C6NZxWjeoAOh2FWvBIzmDF5Z0T2s8WsJQUn0Yk5fOA3DBWlLzn63T27dUPxeBq+uuPs5wZRJICgntI5bFG7BcE7RPxwTpxSj6QlbyYDZSVPMevzZ6hHq+SHL+TbJ6QHd0xGVy4rZ132aZ1LE5YjRrKL2foBfsoiygkqEBTlfeldhewa3r0jvyQ7oVyceCCccnLGhv2KI8oKXi9+WELXHU4qjv/6TdfP7QIeOoXhdKG/jGUXrlMXeGTGMUImAw9lmTPMhW/ifOUXjKU12x9A9Cu2HDF5NKvzhnKJwN54YpinaxxX5tzK4gwS/PHFXuSX5nj2W7cseCeSUOKIUyLRj9k6l6S8JrzK7k9Bf0RRuGeyfbpJYYq7g1Q2Vq2K1mNlNyG2I7Y7puUI8XrZE9jEvARynpIXrlxWl2yGWNyGntA2NJxU7j4n2P8wrKlcTCqsiG7mn7xKYSpPdUtz5XIHBssKvCmfX6WqRHmsorGmp4I9Dcc2w/KdSiA2ORPQPwMrxbaj5ZJwQfT3L7SuzvLGxTCk/Zma5xXBBsfNYSoSTb0mLd+dRl3H93kIguDfICxS7ikxDKn0Vg1cVgD/TNAxnNaXr4Lah96Q3MV42zPzZLrktw8bHs/cXIz+OwPugh1GCIRbX5O9OX6Qc46pmuJQwnNVsfNyicEesHptPx9QuGAp3QnLrUhShpXEwPhSwd9rGCoR8UntfmFZhVpg/1UtjvLpgmbuPG3Yec/bFac5AOKqNJyyGS8INcIYJKjFkt8TfM/h6ncanIn7xv3oZfyZi8Dd7VD+2Tfh3W7QfD9l6AUqLXVaeXmfw+JiVX7lBWEnwpyPan/KI8gKDbb5go2PZohvPSsAmtlD2vClJCXrHYtrHLMKiIcxKkGQbQkaJM4bZ1wLijGE4D4vfDCjcTYiy8sJmG6LSjdPCyVUJZO7aZHYVo1+fx72eoft781ieYvY7iuBGkeZalc8vXkTFsi1XLwn05fYk1x4uJWQaimxD9F7jFZG2W76heMdnPJvgdgxxVvgTiQXzf9alfwj6R2LSV9PEKZh9FbY/OUXz+ZCpD5O4L7GFZla70EdFgrlFWcNgeeJ+vB4z86bcrHHV4sDvC2u8fdz9vgVQX6pyNbLIbRl0aPBe6LN7Viwfx1OK8vWEKGeTv2MYzceUr0jel22IxmjvrPAJjIU8cAP9BRu/KGa2p3/+CtnPNsDT/I/nnhPW1iDFOLTpDtIcOrBDfrnH2HfY6edJPJt3b6zw0EO3UdmIJNSE1YjRQoLlCX7YW7ZQoWLqnPzsKKeICqIxyt2xKK1JcMQZ6ZU3n0zwqwq3o+gedLBGIiHZfipFYisWv5EQFCe27JEQvaPMxInPg6lz/sQI2ZDdTcivC7zmdhSVlTa/ceVJFl68S5yGvUekS4gSpW7tvNwvo6UTVbzkEJQVU+/02fh4mvSOVPGJPZEJ1RWth4tkNxWZLYs4Kyyr/ooWN8BAY48+RDmqjsUqu3W6QKptGC4q8ncnNt6TLbO/JO4klm9oPOmSXzd4Neg8HBFlNOmWPOjlPxXN+s4Tmtp/yJLqKHqP+ZRuxeTvemx+3Jl41Wtq743FojKB9kkoXdPMfi8ht6GoXlSUbyb4VYgzggG+cf0g3VdnePShNeg4/OzZCyRDh0IqwLqSZ/X6LIulLpaV0O/IllauDbh4a55HDmxgQs3KwV0ee/o6yy/c2Wd+Wb7AUoktq3p626J8MyTMi7SkdCsiykp3aua7k8IEJm7ckjNWr8b0l8VUY/b1QCzhq8I3aJ8QoV1+I2HzhZSsemlpsUZp2HkSRisRrUaRYOSy9dUlpt/s47bFsjJ2J3qyFSk6/bJiOKfFJaVraDxdZOaNkOIt6VKVrmq8mmbmTY/RjGC1QcmQvy1wXXFNvBVmvqMZT7v3FSMPhGYqX1kys//df4PTn2CYBmoXY/qL4mlvFGS3A259IU3tvCHVTdh5zKZ+UXDEVMcwWBQChEoEr8tMXOVGJ3xUx6F4Q2MPhegR5gTTNDaMpxOcgcbpiSN04kielWmK4DCxJdfN7Iog7V5bMrtt6JwwuF0R7qkY3L54CySOEKQxgplGecEcs082MUahlKHdzpO+npbcLiuUOhVJ56lyVQyEe0diitctSmsRu2dtau/HGC2mvTqUgiZOTbxQQxgtRdTftISSeAPy6wFbz6eYfS0gzFv0Fy38msjSMWKdGU7si7y5mCfO3GCtU6NzqUZuXZHeS9h9EuZeMRSudrn7eTEztYdQu+Sz+iuKyhsyw6B62aO/nCLKisykfVzuiz1SpFvQOyK5fG5D1sbc5r3ZAAmv/t7/5odqph6IFVUgJcXU+ZCwYHB64JU1qa6Z+IjC5gsZqhehv6zZeVxQtc0XJJBKayH2SDE6Oya/mVBcE91/+XrM7JcdVDzxOaqqfezOrwoElN7VLH1lKFaOY9kGc5OXw6srxjMK24O9J2LiNMy95pHeM+w9G1K8qURmPWFwib2NFFqHf6uNSoRWl9ozOENo367QH2QYjlOcPXCX6uWY8s2AdEt8rwRYV0Rp8TnNboiyoXPYxvJg96wlwjk1mdziyXkzu+I1MP8tESeGxYQwq2g8lSIoJ2w/49JbsRguGspXxT/fq2iCothFnvrsNcpLHXpBmr1WHssTVWnsKvJrGq+i2fx0lSjD/lyBrWdS2LviM1u8HdI9LMu8V1XsPAEL3/ZY+maIjqB7VArD8mVF72i0vyP4ZfXh6kzp0DCeMahYyMvVKwH9g0I6Hs6LLsftSps10xTLmcQxlC+LZHn9RYewaJj/jy69Fc3Ok7KN9hcttl5MWPmTkM4Ri2xDSCdxWpFbZ58L0HgqR+WqT1gQAV/rtIVf1vQPJAyXIyrXQjLrNsVbCVvPpMk1EqZecbB8WcWXv+LvD8iY+86Q3E5M47kKqZYUHmFB4dUM//ilr/DxQzc4NrPLu9dWCDOa7adSZHcijFZk9oSM0zsC3myEXxOzB2UmRrmr5n8yFqd3RFqf4ynRyu88Lvjs4jcM41kpAPN3NPk7kxehGjKenvTbLfAWQ3Kn25Qcj8VSl2uXFklfT1O+ZogWfdqnDbmGGFr4FQHq003BY1NtSZn2nohpPuLSfNQwmgz7qL6nuPUzKVrHXSrXYvK3xbY9LCqWvjoZYWTk13jmQ9TrR0sLtHnGJdXWNJ50RYdTltE29giqV3zGU0bsx9sKlSiGi8JsT7VE65TqhMRpWPpqQOJIfmv3LLaeSxFnxMXP7cpKpEORY6tEQOqdx1My9iYy+NWY8bQMlph5VSh7tUsxvYOa0lpC40lNf0VWcxRsPp9muHDPfCLL7hmbVNcw++pQnK6HhnAq4n945yWWMy38yEb3bby6onYpovmQQ+Vij53HxLGleB10oLGHis5JI74FBnLbofTVO5KPZ3b0RK1gyDQTlr4W4HZFYJfZVoQFxdx3+nSOQ/vRiMpbDpZnGNcVs79yi08+fBmAP/vuQ1z/xiFUJK6Ag0VN7mKayiXFuCaGczqC3ScQBGQ7obARkW4qileFrjf9pqgi5r5jcEaCOAAM5iz8qiG/kZDZSdh8QfjCYVYxnDfkNj9EUpTEFt/40mpM4siqek+WHBZlMMTmx1IUbknQkghKEBalQWBsmH3NZ+3nXBkTM+WgYzEyS+8oUh2Y/7ZPdlf0SsMlQEPvgKJ6JSIoSFWc204ICgrjitV5YktB13zYpr9o4XbFgExNKuHEEe174sjfjZ7ktzsCQa39fBZ7COPP9Xj+9HWqlSGbfolra7OUr4gJ72jawhkaNj5dElKKJbDYwssJmcZEHeoJI8keRuKtH8ssg8SC3F0j3bC6Zvdsikwzwa/JUIrYhav/KE1mV5FbdfDqCv/TPdIvNtnsFdkYiUPL1Fsw91pA7ZyiuAbl6zHFtRivrsjsiVJhPB+R2dLsnVH0VzQ6kLFDmd2E8bRh93EmbooW45q0vVMdQ/FOxMybMYMFIa4vfj3ed2VRCYxrH6atP5Bgax8TS/I4JRqlwaJwRI2lWPzGiPaZBKcv2qLElZlI7sBMJMEpCqsat61onpHOSKahiHIwWDI0HxGpRJQTonJQlKFevWWbwUpC75CQJOIMrHzRULkW4FUUVigt0ntCPxKoXDHk1mV+QKotW/Lys+vEKWFWjWekR37gSx5+zRDHmtfWDvL83Cpffedh0ndcwpxi6zmbMC+NhtxmQq6RkGpqBocj/JKYlsWu6LPcjqJ7OEOqK/cqt+5NZgxM4LfZe711RXEtZlwXMs7Ud2xUBN4jYw588haWlbC3l6fTyrP6+jKNf32Q1ilF4wmX1sOG0cTC06tqYkcCae/xmPmXtShl1xWl1YTdRx3yt6HxQiIzv7SkZrWLAVFO0X0opHMM2sdsOkfsCWQoxPDsnvAgrIMDmaB4PzHyY43A+zyirLDbUy0x9oozCeUrTIR14s5341ddtCfbqNuTOVA6Eo9Ne2QIS0I8qV8MyW1Ib1tMFqQi14HMBtAh7HwipH9aWncqMSx9LSF/B3JbBq+e0F1x2DstxIqtFwxRWjOuaYKKQcdiYakSmfV0D6Xo/OYiM2/5jGcM6V1Y+bLPjf/cIZwO8fspcjmPO8MKeizXMJ6eWKkH0uEaT4vkRcdCVB7X1f74TGecMP2ODKpILAmK3UezjGfkpQ4Kgi6EefEO6C9Z9A4papdjeocUw6dHlIpD2l6GI9Umxrewt4TB5JfUviJ07rtCQje2KGLnXvPpPBRRft+me1DTOySMrnFNC5VwRjH9qoU9MpSuyedqPCUzCRa/rHGG0qQwWsaEprvJZPidTfsUWO8WiDP3FyMPBCklSQkwnZp0QJy+FrZNUeANlcDh344ISrZMxIsgnA3YfcwluwnRZztY5yqoxLDzqENYEAUARnLf4Zy0Gr2jHrn309ReczAKco0Ir2zhlzTuQDDL4k1NkpqA/hlF/W3N9scS0DFTr2taD5mJUG2S3/bl79kdcbROtRSpbkL3vx2gmgXK1QGuHbNzvc5Fz8HtaBkX5CvRQNU1bkd09P1FS9rI04r6xYjOISGYjGua7kGLxS+1uPU3qnjTEQvfUPgD6avHEwRAxUBPrNtr58VVmySBwGKh2GMqNeCb750ke8uZzEk1+BUB741WtI+KfXrzrMPcd+V+O20hArl9kfm0TxiKN78PicUpqRPczmQ2rCMqgN2KRfWydBVTbUP7JPRjjT1QBBWxVLJ8wczv53ggVlTLEx+o0uU+rYdEtNY+LRqp3FYiA3KjhNG0AOFuHyqvuzg9xXDR4P5Jmfxtw+bHbHHVa8lqVL2SkN2N9t2RT/z33YlJl2z93QPCHShf6eP2E3I7CVPnx1Suhjh9UYiOZhT5NYuFb4hzSHpHYYXSNUv1DN6U3Oj+ksW4ruXn/oM7NHcL/PKZN/ECh521GtZYMfvbaeKMzChAyZC23jGRmky908cZCq3vwG/cQUUisXa78rPCPLTPlrGHUH9LICozeVmmLoSMp+Uzj6fUpMpW5G9ZFFa6HF1qsLpXY7Vfw23YsnN5EqhhQdhlrVP3RlAKDzUoaoKcZvaNmN3nIlqnRW6S2VEMl4SAMv+dkDCnwIinQJSVdnjlsqFwW6iHg2XBS1Nt4aZ6MwnGMrQficXM4taHyNKHRFpujedKlK7DaDnmyG8PGU+piSbc0DqVIXYUrZMOXlXyocLthNy6YjSrGKyoyZuuKN2SCXkqMfhFi9JajBXAxk/PYHkKZ5wQlAT/zK8b/KksgzmLzV8KaJ7OcPsLitE8+wa9RoNf1Pse9dntEHuoaJ2UKSFhXogwbk9mpq5++wB2w+XfX3ycF5Zvkr0jTKat5yzq78oKPPWuT27TcOh3PYGKpjMCcQ0Nmz+/zGDeJrGh/t6Yg38wEHb+srQem8+HtI9aYum4lDCckcq7/1ND7AnRxfLkWoLQ5rMzlxi2Mwx/Yx4VKzoPCQNr+2OGsJjQPyGTVfzDHtFcQOM5Q+eopntEXsLcmiMIxCGhPbodQMHuo660ZkcyOza7bahdFBOMdEeaCnOvCilHFLYx1fMKp6vJ3bGpXPXoHr2/EHkwAlUJ5zMoCrk2vWWx+3ie2uWYqBQTZUU/pSMZw1i4bfDmYjrHZUWNXWkSDBcnU6hTiu2PG8ZVTaoX41U0w0WRbFg+dP7OgPJ16D0UkFjCUi/dinBvZJh+Z0DlgkX1fSFMl9YSxrMJe49IsZJuGXaeSJHdNiS2ofaeiN3WPyVpwfBwyNxzG1iHBiSezTe/dZb5V4boWHydtj4hhdudz7hgJNcEgcVAQHAAvybw0s7jWRpPFyZzYqUVWXzPpXBHmGXVi4rcdsRgJSH77bw4DQ4Fa3UGEN7K8y8vPY/u2Yx/ocuxF1cx2ZjDj93FZGUqd32uiz4wJJUJoesw923Zyg/9zh4AyRM9orzk/DpA+vRGWFteVVG/EJJuQmEjFMv1KU2YkZZr67hFeleKy71TFlZgqF+U0T23/5cJ+dv3FyIPRKAmNvvFTFAS4NwZCBO9eFlMc9OthKCsKKyJrp18RGZbqntjSaep/l4srcW8YulLhsRVtI47DBcUqaYmv5Hg9gy1f5OjdQryV13cQYI3begcFqJ252iO4p2I7eeN2FJmFdktTeWyYjQjL4zlSSFhjxT9ZSkYklxM5nAP5WuOlXZIuRHYCW5bsfFiToap5Qzli7bgj2uyTQ8XDUYbtp51ybRkdeo8FGEP2Z+C53YFsvNr0hq1xwavJiPKQRSkc9+VVKF4U8go4r4i6VO9OCR3sEslO2Z1r8bSwh7JxJ1s6ewmg3GKoO9S+OM89XfVvoL3zudr+EWL4HZexvLEcj2jecNwKaawHpHdSdh5wiFxxSZpPCN2SF5VcvGF74zJbwmumm3IHIO90xbLX/Gp/Gn2w2WShob2KZFkpPekIxXmxNFPx9Jrb50Wl7n8hnBNVcvBr4mjsj2UGVVeSTM4IGyhMKfpnA0m2iqDNy0Oc4mtaB23KV+DyrWI0qUOtQuSq9kjybNuf15RumZJG7UmNuOJxaRdGDH1zojcppGhtL5MVcFJGDTy6KrPxdYc/oUyuutQuCtbYFAQZnvvaELvuDhfl1dj3M5EANgHjKgz3aaFV2PfWzUoyme0B3I9g2XZ2kfTUn1joHtAJrO4Q5GTDBcM6b2JCcbvzaC+VaHvpQgvFwkTTcqKsNIxg9+aJ/1ygdIFV8SOthSfKOEzhFlpM3ePaMo3Eio3ItJ7isJNi8G8TZiTJsp4xjD1Vo/UnrxYYUHSuRt/22FcVfvmd34FolNDGk+k2X1Sdpf7DJGf/KF9yN/VBCUhgxRvJWSbCZ0ThjAH9QsR9li6UTtPyJtauaSYejciKEDxVkJQEkKH2xZJcn9Zs/RFLeSMu5Bb18y+EUxGdk9MfLOa7ReqeGUtHZ0ZgckK122ssbDS001ZWUW+Dd0DNhsvZcUwtynOf0E15syhdXBkhdu8XSPTkGkrUVpRuT6ZF5A15NYlP8vsJrjdiOl3xMUwysL28+KLNfVuQroJcUZ67okrO4aO5HfLU8KNLd8bkCZSltymYVzT9A5q8utiddQ59v0x83ylytQ7CdvrVbZ/8wDFV9KUbvn0DxrG04bqhcnkwcMhmR2xN7d8aQAYDc5IXA7jlBRxUVasfVJtWQyaZ4uMZ4TvkNmVKYqVcxaJoxhNa8azYsDhvJejci1i+nVF9f0PUTGVuJD5qR38wx6jKU3jsyE6Mph0TFg03P05kYO4HRHKqUgcP1onJLkX3qUQr3UohY9KYOMTmnFdWPWDYyFrv6RpPiF+pUaLR+loHnKNGL8slLYoK8ZkvcNQvSTM+OGCwDj5DUP3VERQkhfDHoo5G5ZhGLmcPrwBGxmsfETxdoQ9VLRPGbafFhhJxWJ+kW4aOsc0m8+n2H3UFfcXTyQyOoDNlxBDi2OBCPreGVO6Ljm80VBcTfDmRFIdZ8U/NruT4FVkUqDblYbF3Pdi0nsSyOkdodt1D1vkrzvktkSv71ccZt4Uo4zRnOSVlXcEDZl+OyTKKfrLonS483nD3kmH7LbMyYqyQrAezQrSEKcVhVtGBnK808NbCRguQOW62HuSwNx3ByS2jD/3S3rf3+qHHQ9EoFoeWP+mxvK/t0i3DembKdrHbIpXHLKbioU/FRcVowW3zDUSGUrmsM9oj7Lylvs1gzclSf/0mxK4RkNq06H2tkXlPdkuY1cGQRz8jx3ax6z/iQ9SasJt9UtiF5S/o6hcE8Xr0leYDB0Tn9DMrsJt2PiRzaX3l6mc3KP4SprWKZvSWkJ2UzP1jni5WoGC5J5Ln+SjsQvd4xOo5rpFqmtI7Yp5RWZNXEVax9MU1iMSV64x2wiZ/ba8sNWLUjT2VmRic3ZLkr7WCZfRlEX9fMhoRhol+Q35mYMTAeO6WEe2jlv4BZGii81nRPuJkKBouP0LoiVLHCjdSNBji6Akswis0JDbEPsey2ef6Z9px7RPKG7+rSKFSy4zb8XsnnUIC0w8ZvPEafFICIsfMnjKWLIqjms2/cl2c69Tld+K2T2rmXm9R/WqwCijaWHKZxsiTW6d0kxdCNGhkHMXvzESyXRXiqtUF6bOx+hAfFWn3wkJixKMrUfKxBkxSCuvhuS2EtyB4cAXR6hIiCthXlai9gmFX5Qqtn80ZvesjfYh3VLcXZ0ie8fC/s0q7cciyUvzUmj1DsiEOhVB/u7EXHdLkd9IJBfcmMwGqIrkJL3HhCADaGk8DOZFzGhsJerbQcxoXoqToCSBcm+nCIqyegtWK3xRezTh4Wag9qrDYFFRWhOic/c4+CWh/Lm9iOL7LuVrsPBVTeeEmE4kjmLlj0OMDYVV2P6YYTgvcwDcniHKCWLROmET5cXBxh4ZxlWLsCCFZP+UdAPdnjjIpHcNrYfuL0YeiEDFQOeotT8G0a8YxrOJEIMdReWq4cYv52mdcPDLMkuzeFnmcc6/LNM/2kccdp6C0TxsvJSViSau+AF4VeERBCUJnM5RsXD0S5rdZxKKq7JytI+6DOeEPLH+Uo7cTkS6JbBWfj0Bo7DHQmwpXRZpdliA4WLC4X8fUVqNSXVjyudFRZDuJEy/G1C+KQXVaDkS+CwrbCeZQCjmvnEaDnxxIGz9y/6+5U1uwzA4KKOEahcF/ai9H3P7C5ID31MJjOaSfVJ3fj0h1UlEWz8yuF0JUIz4HQwXZIhG7AKJvDQ6FgXB+kspxtOiD9v4tGHmdQH6u4ch1RwLqhBCasdCB9A+oSdoiKH+XoDlw8z3xB+hfSZmuCDu1m5XMftNm8QSslCqLVacR367d18h8kAEqppgclFWtPWFNZh5TYYw2F5CdieifFWS9mxDCL0qFmJz56jF6i/kSFyoXlBMnUuwxhPAO6sprGnq78VUrsqgWJnUlzCaT7B8Q+WC2Ff2Fx38MhPrGslXR1PSYQ7K4ikQp4zMRVKTcZdqQrfb0uw8nsYvazY/bu/PDfWLmsGcQ+ewJr+ZMP09C68m2/e4phkuKJqf9El1YoYHIrpHcnQfCWmdSOH2pGgZzSqK18XWMtMMxZKyrpl7WVO4k9B5SMYRZRoix9GRzMpKd2J2noTSLVnF/IqQV/yKvADWWAo9HUr6kW4aeidiUntCmyzcTph6zaK3otk9a2OPFdf+fpHS2kTerOWXDqRwmnonoXPUJSjJDuBVNPPf1Cy+PKJzTDp22y/Fk0FqYocZlOHG3yncV4w8EIFqLGnn5dcT9k479A/IrNLshmLjRU3jcXeyEgmu2Tsgg3DHKyHl6zFuV+2z7ttHLbI7IiyLUorCuojHGk+LhNiryUif2VcFJUh1DE4fZv5klczuhBTckV68X1L0VkSebHuG7LYi1RPHkNr7YuYG4rOEkdGXM28klFdD3K7kzK2HDW5P/FnjFNgjNQm6BLcN2UtpwrzG7kpbdO4bkmuOpxXGVhOJjNDp7nzWpXNUCi2vquj+wlBWtknPfe+Mwe0YKtdDnG7I/LcNdj8QV5VLEf2DgqjcIyvrSPJKfyI1r76r9w3rRjPCda1cj3G7EBbkno1mBDNN7wIJoAXOsscC4WFEfxZlpNLf+EQWtzdZ6a85lK8lNE/bZHekqMxs3V8IPhCkFBCc0aspoqwh3RSYZOrdIV49R6otfeP+sZip18QY1+lDas9hXBc5hldVuFsOmR2DPU7I35VVZfeMjV+PsUbClKpfEKlvdltG9XSOaGqXYrZ/7hBhQU3kGQm1CwJVhXmZlTSaFi8AlcjWaSyb/HpM76CFDqXXrWPYelaT2bEYHohJ7VrUzxlG09A/ElG+ZFO4I9CZSgSf1aFi7yGLdBN6K7La+RUoX5PUQAdixe4MxA7dWDIcI7NryH4tD0ggA3SOSeXeOqMxGYf0ukXvQEFy/esd0g/XSbV8ijddvLq89FEWKu+LulUFYiix8K0R1/9eivS2TZSVLb6wBr1lTe1ySOukzcqXhhhHs3cyje2JKV1uO8FYmoVv+8SuqITLNxJRWyxJw2U4p4kK0qJd+saIWz97f/SpB2JF1YHY9NTej2XIgi3+oI2nc6Q6Mg8pcaH+hqZ3UO0P29WhwViK0qrMJo0KhlQvoXXKYjSr2HrOwgqgdE2miNhjKXDskWyFo1lN4Y5o6EdzMsY8LCakWnpC2FD4tUQobocT8a8/In372J1wQpsynwmkyKhelG10+nuKwpoYl828OSa9bZNpJnSPypbtDKF7VHLcTEPGPzqDiRGGK+a37WeF21m+IXnx0tcC7JGa4MaKbFNGGY2nxfFl6pwE8vTrisO/FZNbF9okwNrfqpPZMWw9l5po1AzZphRTXk1RPaeZeluKrvVP5bB7Fukd9vVNykj72C9aGAfWfi5LUHQwtqL5RMJwUTFYlG5Z+1iKxBGIsLciYykTV3bJwUpM7bwhKCc0nspS+DC1UI0l7cjOYUuIxBZsflIq4qAE3VMC38RpGRkTFMTu0Qpg+q0BUVaT3UlwO5rNnw+pvxcTFsRUTAeSrxZvxWR3I8ZTYhnp1wRvTBzoL0OcNri7FoVVTXpPKH+Z3YTaedE9Tb/JRLJt6B4B9fE24f+hQ+8wlG8IG340r+gfFGlx84ysbqMzY/orModp+wVDcdVQWPeJ0tL798+McIbSQu4flgKodkECzrnroiIYzupJ7iw6rfYpxewbY0ZTmvx6gPal4+OXxLJy5zMBrRMpusckDxwuSgoRp2SWVGLDcEERZmQiooqhewwq73WIJqrUqXckRfLrCamO9PSdkXzN6Soyu8I+s4eG4lWhB45mZbByum3Yes4i24gnLW5DcTUhKCbkbwn0VliTVKhz4kMET8VpyZfKq7GY6nYM+VWb8o0IbyYmvSk4auWqT5SZaKEGUjVufjzPYEGTbgkk5NxOkd7xJ21JkTmPpySH2nnMYep8KOe6JCMOxaNKWrBhKWE8a8htx4R56evbvgTm3iNiJ+n0FFE1YnizxJ3tKlHG0D5ukb9rSLUEb8xvJjgD2doXflfgIb9qqL0tt3v7adnu/FpCvJfCq0mq43Y0o2lF43MBGPFCHS4Y6hd8wqJMuh4uxeTWYefRDIml9guYdGuyS8xYFM6lGSyJ8LB8XeQ9bk9mTKkI2qcFTw3ziubDFtmdhNxdRfdkWfLnHXE6MUo6gMqIa+G4phnOWPsmE8U7EXuPJTKXtiuVvHGg+bB07UbTFn4FFr8ptpnlq4psI5GRS0r4CMf/Vfu+YuSBCFR7JPBJdn2EOxmzYw9Eq1S4Ltu3sWA472IscevLbhvymzGDQxGWZ2g87sqso3XY/HgWa8LyybQiMruGvYds4pRh75QjfqyBwW0L+aJ6KSbTTJj5nrT0RlOSl42nFM4gYekbAbOvJeQ2DMOjAcrXTJ3a5eTSNkk6oXwjEewyEofo/qKmcFuMh7eetRjNaFItRfOpmNxmAIlUypXLiunXZAfxy4rcurRBMYqFb4UUbwekm4q9h1K4bUX9fMDsq/dGZ8qkF3sMwbEx5RvS9uwfSPCqMqFPJnCLjMWvSvNi6Ut7LH0tpnsqYub1nqh7K5rSLRnuoRIJ4DgN5ZsBw3kpXsc1TeeMDEsurhlGSzF+ySK7YWF7MFiWVrbbFbjLrwjBKHENGy/apNpm3wZdDIsFhtx7vHpfMfJABOo9S5/m2bx4LGnpc9sjyeHu/R97wmJPtxNGc4reikXpkhAjnKFs8blGTH7d0HxUpMZ3/7Zs99ktQ5w2E9MJcZ0bLsd0D2uCgvjO+2VF4xkhmqAFIWg8aRNlLFrHLayfb+LkQnK3LYZfn8HVEfm5Aa1TMufU7RtSe5NOWQbGB6SPH2ZhdDSg9pbF7mMpOb+C/oqkPdIrl7bt3hMJqVspOkddOkdd0e/7hsVv9OkvyksGgsX2lzTZRoh7LcNg3qZ/QJFb12R2JdAKtwyDBYvhkngd2CO4+9M1WqccKucs7ny2SJSB3E5M4gjslW6JXMTyYefR1MREQrgCtTeFI5tpRVTPaQYLQiU0CjgxwBkmFG/HMuvgljy/TEPhthX5zRgViQiwft6nfsGX/PQ+/U8eiKrfWNLOTGwhQRstJl9+SZNqy6rQOwjDaY1XTyjekml8pZtjGk9lyW2JALC4Ombr+SyltYTaOXFC0ZsTY4S6IndXMZ6TrkmcNsz/GWQ3h4zn0vQPWEQ5RfG6hRUm9A+InMMvQ/u4zWg5Qvku8VYGK4DK1YD+z6QJ3i9RuCOK1MHCJD3oi8t0/rK4NRdvJ7g9F2eYkOomxKmJBfr7huLaiOFiBr9iUb4KcUpjtKF32FC8qah8fJtyesz15QNYx/uYVIAximCzRORpbv2iRqV9vBWF8izCnEbHkL8N3cPiWF07L+4qVmCIcop0Q5QJfi3B2Ibpd2LaxySFcHvSbvWqIgc3tkB2uXXFaFqGwvWWbYaLAAbbUwwfHaPu5gizMkt2+SsB659ycXpMCDeGOC0uhctfCdn4RAqUjBIafJikKG43ZucpIaekm2aSV2q8GnSPJUx/a4vCbVl9Zl8Te3IrNKz9Ypr8hmy7UQZapzKM5hMZoe7LEIl0UyYd5zdkQG7+9mSSXkfM1Vb/RobMlkduXfxYh4uG4YzGbWvJhceG/pGIqeU2o16a7JYmzMNgwWH9lSVyj7Rov+gJvrkr5m62NwmQS6H0t5+UfKx7WNM+Jka8w3mh0q39fI7BnMVwOaZ/QF5UZ4j4ws4oHCvmyuYM9ccbxLEm40T0+lncikeSTqjMd5mZ6WBlIs4+vErhWJvamR16L4zFd+tYxHBO0T0p+X9mR+zjowzoSEb0DGed/ZwxsSHVSbACwZNn3hDZeJiH8UIkdu4jyf9X/mTE6MkRU19Okd3Q9JeFrdY6maJ4Q1hU2UZCZlfh9GOxXlpw8Kci/Gnx+P9QTZcOiha184rWWSFHD5bENC1OG1ItTfP5OYKS2tfk6FDhFxXWWPKuxBGG1HBRHnD7hMLtSwBjpGruL4uUundIJgKmWtK5WvxGzO6jOTJ7ImtJN0VSbRzBZpf/ixuk62P6ozTKThidGWMUNJ8LSWzDpxavMfPFFJkdqfKn3xYuqLFgNGXj9gymENE9KcC5sSRg03symUTGRUJmWxhK9hhap+W+JLZh/G/n0LcybF+ahus59l6bJe47WO/lOfh7Cd6bNRrX66QuZrnw5mGc36/I94Yar2ZwuhajFdmq/ZKMNbJHkL9jyG4K66lzXH5eqi1UR78oyt7hkqF1QoYkT7/tU7wm1kLKQOVqTPt4loP/g0hxEleaJ61HBK3xqyLPUYl85r3TLl5NTEVyt21qb1vMvRrdd4w8EIFqBZKfpbcs3J5h/rsRreMWUU7ett0nk/3VYLAgl+zVFJav6B5lIouG+gXRQs29FtE+KrlraVWc4yxP0ILstrDqu0elT731vE3xTkT3kMX02yO8mpms7FLJXtyYJ0kU/maO6T9NMfXlFJldQ2bNJbcO77SW2HpRXP+CckL+rkflMoR5KQqNhoO/ZSheF8O30k2R0pSv+0IkriY4QxkJ5M1G5DYSZl9LxGE6gXRLBHVHf7MvatWB9NnTTcP6J8XbtXhDir/KRcVwTrG9XuXE8japttyjynmL9kNC8BnNiUo0moD9zuD7fqpxSrHyB7uM5hSd44bKJdGUzbwZs/Wx1D4n1vKNzKsF7vy0SGlS7UkR3Je5slLwSXfw3rBkZaC8GqGM7FS3vyCGavdzPBCBGhYEO8xuC8N977RD4kD5spAddCh8TWdsGM2LVqi0KuPD51+JSbVk8MTOEwKpNJ50Jn73ht1HNeUbMsWjuGqYe6VPkk2Is3KeoBLTOSJmDY0nsmR2FZltgzU2PPr336NUGBPspSld0zSeldZi75AoD4xW7A5yZO/YpFpiAdR4Kkt+IyCzLdW5DmXqX/FOxObHofGJmNx2yO6jacKCItXSjGaFMWYNNZ1jms5hi6CsJjNbbaIMNJ4pkdtOqF0OqV8QgnZ2U1hIxpqkTd2E0aGQqfkOay8fEDlLwj5fNMqIBMbyIL8pQ9FS3YT0nhL1gwNbL02R2RXPKnssoz23n7FQoXSnEgu2Pz7hFEwpyleFQxtlZZdyBhM+7/GI5iOyywQFEf6pGIyS+bZhVnHkN0PaJz9EOaqKFc2zMosos2P2GUYoqFyRAOgfULROaXLrGm8mxgoM2UbA3imb8s2Q1klZVVJthTcdoUMpKOrnE7aet6hcC+ichPF8hvyqRXpXQOrZV6SDYnnQPxxPxkkqgi90eOXlh2k2iuRXbbwaVM9pYheyDbU/Y/VItYl3eixzCA5q0k3Drc87jGeEI1u+McZYCq8sUE5q22brORenJ82GyjVhz98jt+Q2DIPjIemmoXJZ+ueZXcFAw6witeczmtIEZWGGoaG0JgPTRlMa5SQsF9vkNmU6jNuX6ju3JQ5//RVpP/dWLGoXY/Yeshg9PiK3If5Q3RMxUUYxXEoIc5ra+xG1C4KYdI/DaMEw+4q8WKm9ycqZE65F4goKkFtX1N60qJ+XmVXJZCK32zP0DtzjJiiG8ymS+7NHfTAC9R5VLddI6B2WrWLp64GQJyYCNx1AmDP0jkUc/g8BrZMW3QMpRosxnSMOhVuG9K7cqJlXtUxe3oroHLUEzD7ooA8N8EqihcrsyIvQeFZG6RTWEw7/jo9fNQyPBfS3C5QfaaJ79v7KeE8KkjhQWBOo69yFQ5hI5kJ5dZGtaF8RFhOB3B7JCNCeCHG4uAr1CzJ1EGDzRXB78nL4NaHXWd17xBRN62FpWHhVxdSbPW79bI7irYipdwPq7yjCYiwv6/WE/kEoVYa8ffUA+a2I8bShdyrEGcq2azRwsi+jHpsJg0ULbz6k+O0M3pRiOK+xR8Jvzd2VmQA7j9vsnRFdfmFNOLWjaS3stOJkmsw7Q/rLitxWTPch8bwazwipJyxAdlvSm9bDCf2DMfZYitbt56Dy/n3GyI8t+j7IkYiuafesonhTgPLhvEt2S1p2lm+oXo1JMgnZuzbtI2minGGwrLCGYjFZvOUxnhXm+GBRU7wtAVxaTQgLMPVGD3M1j1+V6XpBUU0EaRrtS3G2+2iWKD+x6jZg/UaNQ3/gS1sR0QwV1mQbc0YJfjVh5fg2Bxd3Ccryb4MlGQdUf2fCUApgXBVJjEoM6XbM5ouw9ZIEstHy+XJbyf64zNnXpKjpH4opXVOERcF2b/1CCbenGCzYbD3vEBQVtXeEdK0SyJzo0L9RJnvTpfGEQ3rv+y/6YF7LILhGlsKdCL8sc1Zrb9gUNuL9idvF65Kf6wjy6zEzr4e4HXF1SSxFblNRvB3hjAz59Zgwq9h+NkdmVzDb8nnpxGW3hKQz+5pP43kZ0ZlqWaR3LHoHNYVbosAICx+mrX/SDcltSE6T3ZK2pQ7FPa+/bOGVNIXrFrXLEb0jAo+kWlC4Lf+vv5IitTfBR789ZP1nYma/2xVRXc6w8akSIGz9MCsdqeINIV1ExZjBktyw4lWbTz8kdozDOU3zoQxT52V4bZTRoGVW03DOwlRCSq6H9/+eI66F2GPJ0TDQenhCql4SPPKei8nmxywKNyzcPYvq1QC7N5kAvaj3ffe7B2Ua39wriuKdiNSeYMJuTyajxK5wRt2uET+oeGIf382gQklL0ntQvxCS2r43v1S885e+Yth53CHVEedC2zO0Ttj7o45sz9A9GUMCWy/AYNEWPm753hxaMRburQiLLcoJp1VFkoJkd2VAXPsUdF706B5yyWyIGZw1htymkG9Gs4LM3CO9/LDjwQhUA5mdhOm3hySWtOOMEoaS2xHqXVAWILq/aJNqy6pmTxyXu8cSstuhVJsGgrJL+V2XtV8qEmUUlnevsJGXINNKmH3Nxx2I/qf6roXRBq8OvRMR7SCz3zGJU9BfsGSqc2wICiJP6Z4NsNyY9+7OYyxFek1mWGU3JaVI7woRBAO5HSF2KAO1C1C/6DP1TsJoymbhzyJaD0/MJK5IjqwDKNxNZCjaoi0eqJNf9hB6R6UDZAUGp6/oH4nZeS5G24bZN6TQym3H7DzqiHtMGuGUFg2jaYvFl0dEk7kEQVEkMToUjuxwTrP4NcNowVC8IWZt2Q1RRuQ21f507/LNhMG8xhrLqPfxtEz6a53S+BVh9OfezOyv9uXrIV5dXGJyWzFL3xzTeMoi27i/qv+B6EwltvTIdz5rkbox6RWXpWXaO5pQuqYYzgsGV1wVpahUxGKnnt5xCAu2jKtJqf0hFYktMEhYEI+A8fSEEleyKMUQO4qZNxO6h4RPYI0hc7aHrRPK74tBmZn4oGZ3EorX+sSpoky1Hjks/J11rnzvILtnAQzt0xKkM68lbD8nzPzSTRHA7T4uYzG9qmI0kyIoGzI7UhHP/1nC7qOGVsoif8fQP6Dwq5oon4i/a1vUDk5fUVqLmX9lzN1P5yncmTQXtizsJ9vEb1RQSUSqJRCS0bJT5TfFvytqasbTABnhJSxIdyq9K8OPw7x0/pyBhTUWIk+UVfSOxZQuW9hDSUGColy325eXr7gKrTMx46bG7cjKbwUJXkXWQfVkl2ZSIreJGH5UNc0zaVJ7isK6d18x8mAEqiMQVen1NJ2HIoprFtZYmONxJkYHFlPnY/ZOi8lCnBFrx+KthHHNxhlJgj9YhsQydB6FwhW5gV5VoQPFeBoWXglxWwFbH8vReEocnYOKItOQjphfVvRWy0wv3OR8DcITI4qvZIjTiuajiuaZIvZI0Twc4GRC3ru2hK1lPLnbFwA/TkHzEcX0GwbU993sKpdk8JkOE4YzWoYD22IXrhqQ25DtebAk2vfMTkLzMVlZh7MayxNEZDhr4RdzFNcSmo9KCrT88Tvc/eYyuS1D55CN2zd0jtiii8pAUNR4MwnFGzBYNhMDYiGOp1oaHRu6NTMZ0iEdOTXhqaoY7KHGr4qKVofAvYEezYSgIPBa4aZF97BBGTFgi7ITry4NyTslotIkLVLiUxXMBaT3XKzhh2jOlEpEH+9XZBDtcF5yMmcA89+SQWZRSjoq6ZaRN/FuQFBQtE99fwRO7YIMCsvccbDGsoWmWxOz3dsJdz5jcfvzWXKbCam2GNJmNzTDlZjC3ViM04AvvvEoPNyn/qfpic7IkN2QHHp82IdQk8QW1bds3LaivyziwOxuRKaVsPLlMYNFzWBRRtx4dUX/oMBHgwWL3I4I8cZzhjBvaDyl6ZxOyDSlqaFi2PlMQPGGpCypjpjgDpcUfkmYVo0XRGxYe3ab61cWcPuTqdN9s7+dz7wl0wR1JPREyzeUL0NxLSHdkjHtOhINWv1dCCsRmYaZwHxCug6K8pIFZWlC9E6EMkUwK63V/rLsYF7dsPBKRHEVsXMvynXeQ0ymzhnydw35ux7VSwkz37SJ0rD2i/n7ipEHIlDtkSHKyxsdZSUgU11D6VbE3mkxNuhNMMowr+idjGg8mZoYh4k9+WBBxGqZbS1D1QYiGfFL8oCGc5riTS3DxVLig99f1pTWYpQvxVHhNuQPdvnCU+/iN7I0XohZ+YVVKk/u0D0TwGdaWKmY8kyf1MWMkGgOBwS1WLazhxy2ntUMF9KkOsI60gFUL0eSQrjscwniFIQlqbYLt6B8UUjQcUrywemvuYzmJD3x6oqFb4/2q3G3b1CxwloesrFeJXfLYjwlENhgScZc7jym2HlcM1yE0YKoXMOsonsM3H6CPYDxXEznlHhWtR5SWH1rvwq3fEk38uuCgS5+I2Y0q5j+rjQgKldihnOaVIfJxG/FuGrTOwi182p/Tm3vkNr3VhjOK6KMxe6j0voezyZktj9EVX+UVax82cNts+8UPZjXNB+yMZaRiXyJfNC5V/pk79hkmrLFYMThrrSa7K+uM28H9A5KVytOK/JbMU5fpMZGwWBZZCl+1eCVJYBbzwcYpeivF/niuTMY2zC/ssfNZo3eKM3S0h5e4JDL+pivV6lcj/Fq4Ow6lC/ZRGnhCNTeE26BX1FMvTsiysDeaRt7KMMV8rckmFQMTkckMomlKN8IaDyZIrsjxVDhtk+mIVslBnYez+KXRaTnl8XxMGinyd4U68dsQ+EODNlNmepSuQwk0k2qXhATjd6pkChr2DtlM55BVKS+TDVMNRWZhibbiEFPyOyhrIpGw2jaJr9u6B5WFG/FbP5UQqYpRaJfi/GmJrNQE+FSLHxrKAtCV4I0Sgs+vns2Rf2cYfrNPoU1/SED/AO4+8kM41lD8U64v/1ldsVxuvOwrEhuH1b/Rl4sZbriw+kMZVvpHhG5xtQ5n1tfEMZ9cVV62M3TNkHpnmJUJjuPp4WA4oykaq192yXbTMhsWGRuu1iFkM27NT5/6H1SXyvSfGUOc6mA9ZUyxhYIaep8hOUJ1OWMDH5VhGxGQfVSRPPhDHrC7Tz4H3bJbotkOX9HHmj9vKFyLZQXc9EV3ueM4MK3vpAmLIgsOz3huBZuG8Y1C/V8G3OmT+W8jKAEmHm9z+aLCe0zMU5fAPd0U9hYnWPCW8jedrAnIylzm4blP04oXRP3mcHJgMSG7WeFKXUPLYlyQvNrPSxdsul3hBfhNG2ZzF2S/n7+Nvvmwt4U3PhfWKQ7CdlGwnhOhIq2B7VLITtPKLafKzKeluC+rxj5cQTeBz2sUPr8qT3F2n+maD0Z7UuNZ1/3mfqexdT5kO7xmNy6WIGPpjXuIGGwktA+qglKCbX3hGJWWLXwK2IkZoUyRDYoGXafgEO/5zFYMRNKHngVRf4uZPaklWl7YhUe9x3mv6r56r9+TlSvQ8g0Jiv260PCgqG3YjN1PsKry+rudmXbdoaCTfoVxWheDH67D9cIC1IwWcEEO85Ix8zYYvJQuhWR3hMZsdGi6TJK0XpYJpDYXkL4i20KaR/31QKWL23bxAFvKi3ynYv2fhv6Xn7o9hQ7nwgJSka22gkiFBYsWVUjOPSbhuDhEdlNeVEyTdFLycpqmHtVrOt3HrP3nV5KqyHOwFC5IoGdbiUkjqF2MabwTpogr9h5StwAE0uElY0nHMGKT0XiH3DnQ0RKSWwpNnJbCbW3bJymTaojxl2tEylUIsVKas/CHgtml+okQjKe9nFGUL4ixGW3N1l97ko3xC9qCquw8K2Axa8nXP+HFm5bcqTFl8cy0qaq2Hncor+kCQrSZFj4upCbvRoyDzUlUum9J2IaT+fIbQh5ejgthOfeQUVQlY7XvTwvcdmfSapDg1+Ra2o+JYSQ1hmzPwC38aQlKYIn7K3Cmqb9REjvqAz4zW7C1sfBthJGfzSL0zd0j0keeM9tUNS1gj+Xr8eiDgikXVy45JLdlB0kccW+s7eiJwYUiu2nUxReyQiJpiyfo3JpYgXvi1dtei8ityEDOXRouPsZm72zhnRLrEC7hy0yDY1flGAfTymym5rhitD5grJcS3FVGGD5TVET3M/xQASq0XDgTzz6y5rYVUQ52ULr56SvPlhStI+7GCUfvnJNhiMkFiRDh/G0+Nzf65mXViMwZn+cpFcXv6bmwzaZGykyTdFbJZYmTstox8SR/6sSyGwrhrPiyT/3vYDimsGrGYxWpDdtGUhmoHXKZTTBd90e2D0Z26gDGB6MmToXYRzBGv2SwGHZRkL5PZvcpmHuFWkyZHZCKpdlG7w3cjHdSlj6omb2e5JnfuwfvoUzPcaPLFJdkUtPv5UwXBL92MaLej8V8iuK1imL/tFIirZJ98jy702IUVhejD2E8tURiStsNB0I7S9/R5PqyDPI7kj3K7sJzUdcBssy3NgKJgPjXoe7PzXBSyOYfttnsKKYuhDgDCQwnbZFdjfBGciLM1wQC9Htl+4zSnlAAjVKw8YLGea/MyZOiWmYX1EMZ2U0oVGi/bc8gayy2yI6s0LD1PcsonmfmTdjdCirafuYzWDRovZ+QOu0yJnzdyVgxgdF4dk5bBEWLGnFigM4lesRB35/D2/aTMwnxBUvKCgql6XAyK+LnCO/GVO5FpLeFQ+Cwt2Y3IaoSfsHE0qXLdrHbdF0FaBzAuZfiegd1DIivKboHrII6jFJSjNc0KRak1GStyP2HlEM5ixUbBjNK671prEu5gnfrUjhlhXz3toF0VvpQE0muYA3LX6li19R+ywqeyzMfq+uKN1MaB9P4QwN28/myG8mdI4IHupMsM/eQU39/Ii9RwyFNXFnWXi5h9uRojGxYecx0Z5V35OOVVAx3PmMS+l6QuNxl+m3R/gVRXZb4ZVllW2fFvhr7js+K38g9/h+jgciUBWyKrSPpfHqhlTbMDgYU16NKNxJmP+uR/e4SHbdrmHzhTSFOwmbH0fGe59PM5qyRLoxIeLmJ1Y+B/9wzN7Dlsxr6sHUK0LmMBrWPyk6/TAvfkijukVUzlBYEybXvTHfbk/Gz0yd8wnzUmx0jtrsPuJg+VJkbL4ow4MtH/J3NKNZsf7pHU6I04YoYwiKFtXLMd0jssLZI8ivWsQpTWZHprDsPROye9Yhd1eGQrSPiRHE3q8vy8RAIzJvEhidHdN4RrpP6aZ4sY5WQvHX31X0li1KN0ZStc8Ksyx2JxS7JSE6q1jkzTqSAmv3MRgcSEjvGnaeyGEP9WT0uWa0kJXRnQ7kt+J9cZ5flULpnsQ7TskO03gqi5nY/ER56B+LyN/SjI4EtE+k6Bx2ZBLjfRwPRKDqEPoHwArFqNcew+HfDeiuCOyz83iao785oPWQ2p8QPa5rpt8QI4Ti7RjLl6ETiQtLf7RDmNX0lx32HspMICxFWIT+QXkomV3D8pdjUUuOEYl0BOsvitR65/F7/kyyJadaiuYjKQZLsuIEBSFPhzmpfI1l8MsW9kBM2OyhDFioXlAUr8PS10QXv/uoOGZ3T8UMlgT73DttkW1ETL2bMP8VSxSkGxHFK5bYFdWFpGx7ULxlCKox5ZsJzs0MqaYmzMmqnmpr0lsOUTGW2a9FuPO5LF49IdW551Ytq672xZlbRzIXwK+JpXy6qXEXhjhDSYeqlxPhp84rGk+INPreDKzuUdmloqy8AMVVQThS3YT8ZkyUFUZW4ojD4MoXZRE5+c9bMsnbFp7xfcXIjyf0PtiRWGJKNpyXLW24oPBqrnRURvIwu8fyWJ4A9ZYH6b2E9glFlJYkPigo3J6cY/1np3HG0qHRkchChosyL764KiYMUVrhdiQNuCdqG08LbpndiYhKMam2MI5EFSv53cwb0lWqXE32PUjtMaz8ibD/h4sCnocFQ+WyTF82Nmx8wiLKS4MidmDpS4bcpiK7LdcV5jVGi+DQ7cDeKeHB5jdi0i3J9fLrMV5NkV+zcbsxmYYEhg7FwS9/W7bV7F1xFMxuirgxt66FKvm+EV9YI9/XPCtbepSRFThxpC1d+JM8vYOKwaGYvYekYZLeg8yu3O9xzcIZGYo3NWFeTIrvzYEVBatm+1nN3Pd8dCCoQm9Fs/mrMkx5+1Mz2L40eCz//mLkgQhUlbDvkSQPJMEexrh9sXsM85MW6qRQqF2KZCW4LgVU9bKYJ9xzXAmLwm0NStIeLN6UlU0l0DkqngHGBr+Wontc8qR7nMxsw7D1nE1mQ7yijJoY6k6uM3EUpRsJbj/GGktlbzRsfMKWYcK+UA2z24r+AYXdtRgsib491RbSSmktoXXSxq+KH5MzgOZZjV9SdA/akwl4kynYsRE17hEICpooDd6UoXVSXJzvDQzW4WSKS0FodPZYGGd+VQK5dxjaJ4QA7QxFWeC2Ne1jGmOLqwsTlUJuW+6nXRPlwr3pgL1jscB3az5eVV7qzLZs8/PfGRBl5PMU7oSkdxSjaVG3dh8Kya8bzJ2spGAj4Qynm+bDV/VHuUm1vCBGCM0zLu1TMFiRFXCwJNQ+yxdv/uJNuTHGEie5MK/IbpqJS188GTAhQZnqSevVHot9j//UAMszbD9tEZUinIFQ+Zy+dL/uMdnHVU2SMnRe8ggK4oE/rmmciT2QNyUw0GhWUX9XWPPpXVh4WVZdtyNBXj+fMFyO6R0SU4v2CUVQFLtLtwd+TTizsSM/w6/KKKP2acPuI6LHKq7KvXF7ULom98sZCnVv6lxI8Lkulm+YfiuRSj8rY8lzGzJ3Sgx4Ff0DCYNFcRBc/OZQ5m6djugcl8bC3Gse/UVbXLhfzlFclQUgTkFm08IaG/ZOpck2DN1jkmJkdxM6R3PkGmIy3F+UaTCDBc1wTpG/7kzMMUSyEhTkpU1sgSTv53ggAtUey5vldmVbEpdkQ+X9iXlZH4KKQDKWJ/DJxktFOqcTSqsR1UtjqUQdoaC1j4mf0nha4ZcNzUeknZk4wljKfDcvIxljuYleXd5yty9DzvyKEue6mhQHqYsZCusyvtsKDM2HLeoXI9wuDOZFA9R4BrpHZXxQ4sqDCPMw+3pMmNWkd8UFsHxNVldjw3hKxuOUr4rBW5wR76rENbTOJJSvCFNMR8LsKt0UVz/bk5c3sYGDQ8b/uM1gK89wHlonLQanfOKMKCDskbykTMD/0nXN6JhPegfCokN+I6F6TrBggM7hFKmewe4rKld96dMrCWK3K/eotBoynlKUL8mc1N4B2fWaD1sM5qWo7R7SjOYTxish2e3vt8FRUojZYxisSGp3P8cDEaiJI0ym4aIizgp041fUxIpHM54S+KW0GmKs73vR233N1nMW65/MUnsvZjQ/IeneFC6ksSHVUdTPG278z2SwWliUnEzFonua/84AYxuaZ6WRkNsUp8A4JQrP7hE533BGjNrGdVnV2sds8pvJfv468zqEBWlJrv+UbMNuF3YfsYWlNHngfllR2IionzcsfzUgqCS0TikyDcPMW2Mh3+wocreF+Z9pTKa6HIjZ/kRMqi1WPn5deu3xdpZ6dghugkqkilcDm4VvBURZgzuQ8e0StDBYNMx9xUHHhihr0Tkm7oZBWZhofkXRfER2r9X/zGH3+Wi/e9U7mpBpRvRWbGrvh+S3YvFTWIkxWp5h/6Cs6Ok9Q2ZHM/2KvQ9BJe7E4LeuGU8JF2P7mfsLwQciUI2+58hs0L5Ut9YY8psJ4xmpTJ2+on3cwe0LFS7KylS6TENRuSpGu/k7MHh8jFeRKXvW+PskicL7Lv0VcczLbhtKawI/bX4sj9tRzLwugZTfjInyMb3jMjrI7YlJmO0Z/KoEqVczzL4+pndAhHDtk9IOrZ2TYq7+hrVv/e32xazBHkn/HQUbL2oGC5q90ykyO5r8OgwXYP3FjJBlaqLzD/OyA6T3DCoSyx4dCmKROFKlWzNjrjemqL7pULmSUL8YUVi1uPtpF2egiDKa/LowuaKsoCq9FZlTsPeQiAi3PpFQuBPLJMRAdpGgpJh5VVG87JDYAs9NvyGO0/c6heOaTJOZ/p6YDMeuonhD4U9FjOYV+bsJu08YZl7tkN0S9MTpi++sisWppbB2fzHyYASqJblW/o5s9b3jETqWfDB/W+P0RT7dOy3+8XFGMf2OmPYGJQmwoDQh+a6lye4mFG8n+6MWw7xiNDepgDcmSs9AVs2pc77Y7RzSBGVD96BwybN3LbKbmvL1mPSuIA8guVrtoiHK2IR5Q+lmQu09Q/thMzFym3jY18SDqvtwSPu4mDUUbouJWn5Nk1+XProUYGIBFOVkSw8L4g2lA6E7+lVF7bxi4c8MzceEEVY9rwX9yASU/zjHaFbk5nd+VvLr2nuSn46rWpy2L4aUVhMWvtmRYmskvyqXYOHriuGMwHOJK4TpTNNM/BMijC0DjFsnFe2Tck13f8qWVTEt2qfsnys8s+syLcYKYeptxc4zZXQsNUh+O2LmjTHF22Lo6wzuL0YeiEC1R0LoGCxDuqnIbIhOKLsr23TpVkTlCsx/xSLMymx7lYglT5wx7D6uGM1Jfzq7rYhdRevzI9FdpUR0ZgWCb4a5yar2KRhPwcaLLsXb4gCY3RI9+9TrstLULoZ0D1oU7wS0T4oKwBlKTtY+6hBlzb4LYGZbmPGWJ+54mQaMpw3lc2KGMZ6R3DcsioTYqyjmXxmw8HKX9ikYPjmidl4KsvxtzfBYQO19n8xesr+67p20yN3VkIgyYO9zHsO1EntnDZUrCf0DULhmk26LBms8pcjsJfgVTeuEQ5hVtB4pMZ4WbDZ5vovRsPfLI5IUFNYjnIG8cINJURu7isSW9nHlivAvjCW+qX7NULwTMa5ZhBlN7whMvdklu2UYHQnINnwGC9K1aj1kyO4YGk867D6awStrRrOanZc+RAz/MCeQS2ZXcE9rDLmNBBULxcwrWwwWJhPklFTmgzmL5mc8pt41pFqCFU6/Mya/Ja1U50Ke4SOeeKOeFlyx/0t9jv/NqwwOROhyQOIagpmIzZeg9l/epntMCpndZ2UqnlcVX/qtZ1JMv2329fn3hjtkdjTVyyGDJZh9zcOvSLGXaQQEJZGAhwXx48/fkUDXE5KIV1esv5Snd7RA/o6CrTSNFxI6x6TFOvs1m+5h9/9L3X/G6pal953Yb+385nTek8PN+VbdWzl2dWCTbJJNiqRIiZI8tERBNmDMYMAxjBnPB/uDBhhgYAgGxhpYA1nRYBAlskk2Q+dQuermnE+Ob847Ln943nNKHgjqO4Zo3NpAo26dPnXPe/Zee63n+T//gJ8ziDJicuY2ZcczQ/FBLeT7mDMDKldlnHz4Gz0Ky5KmbQ9ktr53URqq/eZr78XxvD+E+HqBvVdjjMs5SGBYFigtvfsZT6Jz2KTwQF6u3rwh2QIpUQc4TRkCGCGUr7co3YF7fz8nMegrDptvpgiKmsxuQvGe9BxWnzG9TzF5ecjMt55ODfVMLFQjFiOCIM84YEzSk3dflkiaMCMitNSuSDISV1jpk3/u0p8STVLxYczGF1Os/0zCYEp8q3L5IXtvh8z/zAre6Ra+b/PJzaOcOLmJYcZUL+6gAoPCQpus7ePO9ikdauJVhvBOkyil0EoxOjEisz4kvRPSfmdI92wgrtJdTfuoTeJoWkddyU0tKnZfTDFYiMRJehy8q2JwOzFBfqyvcqVO7ixJ55zeNFj8Uyl9tJJSJLFkUTptTWbFpPg4xKtrGm/5fOnVm7TaGbyPs1i+prNgsvVWjtZRU4w8Fi2BjrYV6V0xfai/FEuaYcRY/q0xRgZhXvgAw0lF4aFkCxiRTOWCvCa9J6yv4sNYwujGGVj78hdtwpNfLWKEmolPhbeAgtGkDEUGE8LS2qcXZtZl3Fw7myLIfI4mU1qJERrIQxxVZBew24rsRoARSdit0prqjzZlp8pIuIE1HDOhbEmfnv2uyagMwWREr+thpSIefbKI86dFJoo9nNKIjW8tkn4/y1Kuibtr0qpluVubpJrv0dgq4DkhfmBRfyOge0iRuuvROZJiVLGZ/GOP0ic2YUrhNTXVS30SV6Ys2pBRYudMKI51tyMyWwnZVdltN75okFtLqJ9XZDYk0W9UTRhOSPJK66iFWhwwmFWMpgXCCXJjdlZbs/xrAvMUigM+2lwidSNF7MowJLGhenUkhmQGpHcTeifHZOhXTZrPJ6RXLLoLBvnlhMTUFO7D3I8SnLYMR9LbGqebkHvMAafV7ol9enYjZjghu33sSJ1q9wSfDdPirJjY4paYmOLinX9gEC+MGFUUrZNQWI4JilLG1J6zxVwkeLo1orT+j+NYSqn/F/ALwK7W+tz4a2Xg94BDwDLw61rr5vj/+2+A3wJi4L/QWv/lT/oQmcqCPvKf/bbspGUF43GcORLXlMTSuC3F7PfadE7mBCWYN8hsJuy9JMK13BPZBRpnofBo7AvgqYPIne5hKN+QnzeYlgdv+oJJ9mfFgW723YhR0aRxVmEe72FZMW/MPeFbl85T/dCkuzQ2qVjWkrL8d+pU/lubu/8gK2NLk4OmS6AYmVCpaMxn1bKQ9+Mg3bFZWGLLZ89sCx9A5N2yUNonoHxL6mIjErGdd6ZFbz1P9WNhPCVjcwq/IHGaVl9T+0JA5T2H2BVfhPS2xulLXNFwOia7bIpf6YLCq4knql8Y0/USoQvu/3nvJSjfkBANEmFnaUN2UhJp3OyBNG7ZrYjmSXEZzK3K72P6mtxGRG/WQisJaevOOzi9BDPQvPvH/6dLWuuX/mNr5Gl21H8O/Oz/4mv/NfBdrfVx4Lvjf0cpdQb4m8DZ8X/zj5VS5k/8CYqxF5IaG4eJV3xiiYfTvinD+s8U2HtB0TircOuywCYuixy6Pys3svBAsM72yc86da+pyawLNts8Iw87yI+5BZ4w3p22YusNi96CBKAZl3NorbjXmsKpy+Rr7kcjzEBq6sZZRfJHE9QvFMisWuTWxF1wOJXglzRhTlzyhpMim4k9ebBxSjNxQxSvKob+wthSvanZ+pIQkPNj6Kw/r5h5Lx6fHAKLqaN9uvUMh/9dROew1HyZTU3rXERQFkLzcFJeKFHQMvbYVzROjRMAt2SRDqaUJGXnobcgx3WcElx7ODn2nJ0xqFyVeztx3Se/GjP9QYzVU8x/NySzIaEfnSUTpy+j3dLdELurya4NMcefZ+OL1jisGLZec+kcFdgwSv0nwlG11j8CGv+LL/8S8C/Gf/4XwF/7977+u1prX2v9BHgIvPI0H0RmxAlBUbP9imB1gxlN7pFB6Y7gm0v/eoX0hiIsJnSOCwzTOaKY+iTAq2syW+KFmt6WUoFxKMP21wJxW8kLqeTwN7qk9qTGap0Shk9Q0NgdcVIJilInn5ncZvX+FC9+6S5OV7P3vEeUFl7n5CXJG9UGzP6oT2zLEV24Z2ANFCSy081/P6Q3L2VJblWO2e1XDRa/JUdi+aai8VogUJWlGUxJ7tVwQhEUEza/YJLa0dhdzdEX1gh6DnY6ZDhpi4frPjvpoYVbM2idlJfC2XBkrt6TJiqzLrLxMCvYbn9OFk3pthqPeUNS9QSnJRM1t6kpPo6wRgL9JRa0DznUz5is/rxAaFtvOHiNBL8oDtZBRhCZ3ZdshlOKxuk0vQWB31I7Yh3avCg8DbunGJUNNr/0VztCndJabwGM/zk5/vocsPbvfd/6+Gv/0Wvf6S63MiK7BpVbiWRxXhWDs+xGwKhksPnXlkhsyDwRwrA2xOJm76KD29bkb9RkR1oJyGzKrpRbS7C2XOw+uE3YfdEgKIvhUZSC2NVjAoho4VONGNOH3jxcXllk8kidD24fE6pfWxhI/kRMf9qgdcykewjaR9NiVZlXtF4IcBsQ5yMxD3aEYaQSsIaS0+o2FY0znsSW70QoQ0ak2bsOdl8wXmson8lpikAvyihqgzTK1Bz5v0m9WL0qxORkjEOnt8Xkwe4hOQWuLOTYg+Jjn8SB7Jqm/cqIyi3RiDXPaXrHQurn7PEoVH3mEn3GYlQWfdNgLmE0oajcibG6op6d/jikMx6VGrE0xbGrmLgRsfCtPrGnmHk/OsCz7Z4m+1DSFcOsaP/3qX8/6fpP3Uz9h1q4/+AnUUr9A6XUp0qpT/24j9vUrH4tRW9emgNtitKzciviya+a+OWxPv+cWEPuvgS5dQH101tCLau/NolfEJ3VcELoeY0zBioUHkFuI2LyUsLWG5KtOpxKSLxEjv6OcFyHZRO/rJm4ofnVM1fwQ4vUsi2Kg1mZYR/7HeGmRVnR/0j2537epyMz/h+KTdDeBYvFb/lYfcVgwqS/IDlaiSVapM6SRflHLn5F01+UEWmYVhSeRGTWpalJHOjPJQxGLplbLis/l2cwpXn8yx5BQZhSdkeNo40U3ZMhcUqLGsFTzH9vQHfeZea9HkFOMf1nDu1DJvnHmiQfUX3fErpgIE/QbWr6s7LgK7djijct8g8M+icDIlcx+Uki4sZFgf3K1xXDqqZ4X1jQQcZg74UMdk+z9lVTTsZ1Oe2CnLh9Jzbk1jTdhb9aeGpHKTUzXnAzwO746+vAwr/3ffPA5n/oL9Ba/xOt9Uta65dsLyN0tX2GuAf5x0PMADa+ZFC+YjKakBHd5HdsMKS4331R+JFKw6gijKHhtOwg/bf6zH9zl9yKxohkl6idt2ieMFGR5EwlKS2qzbqYhEUpSemwupK8997OEZTSVK9FGAEUHyb4lYTmyRSjitTPfkExeTkZLxYOqHyDKQMVyWLZu+CR3tX4FUX1srwYXlNm4rGnDkJ08/dN4cl2tQgXTSX8B08z/9w2o6aHOWKcyqKpXFc4LXnocUrw5dhFYtgRS6TCSsj2q2laJ6BxOkOQl4lf8XFE+zhMfVcs5KO0dOVeXfRn1gBa5yNaR2X40T4TM/0dC7+o2PyqWNBHKXWgHnDaike/lhLv0znx/R9OKpyWjLbrZ00GU4r0lqJ0F4p3GTeBf7U76h8Dvzn+828C3/j3vv43lVKuUuowcBz4+Cf9ZVZfGgojlMmNNdTUz6cJcpDeMIjHbzFA7aJY90QZReLKAhlMKbQlsFb1sjRf5p0Me29UhSUVgt0XgnLpfozdR1xS7pioRNM+auI1ZKSpLRjNxATlGA2Escne89LRy1RMRHSJDd62yI1HBQlnGE3KZEqPtf1eS2QqTlsAd6sH7SPSW9r9RGb2zngy15TFEmZl4dWek4mOtweF83VW702RWrXJbsmR3TpuMCorgiIUHgeU7srvte/Jrw1Z0M0TNoO5hMLDcYnVF/+p3rRJ7onAX5kdScfuLJki6NtW9Bdjqh+aqARyazHz35ZTwBqOU6LvSFzl9Ieh5Entady6Qeeo/GynLchKelNybu2eIB/VK30xJp5QNM5JGuHTXD/xu5RSvwN8AJxUSq0rpX4L+O+BryqlHgBfHf87WutbwO8Dt4G/AP4PWuufSI2N0mP1afozOthgRjFxSzTzheWI3ZeN8dTHoDdr4Jdg4rJxYGDg7clNjF2FXxbnlNGEQENeTbz8Sw9ihhWD6tXRAbQSFISS1zpukNqRYxQNx05ssblZRn9UBGTxtI8J52BYNchsirlw4UlEUBCboMQSCxunJQ/FiDSt0/ICqYSD5JDSfZ/Ik8W97zyY2hO4LPZkUQdzgZwUk1BbL5LeMFn8ZpsgK+B58X4i7iQZjV+yCHIGVl9Tuh9g+lKS2D2ZhOUeGWS3Qvpz0vTM/TDCGkLpoWjRNr4qi9saaAbTglRMv6eoPy/kmPo5k9YRi968oAEqEXeb7IaYJe/HsgelZIyuSFZC83yC15IGUuwmNTuvZYgdqVnT24Yoev9TLFSt9W9orWe01rbWel5r/U+11nWt9Ve01sfH/2z8e9//32mtj2qtT2qt//xpPsS+XY+3p+jPJ+y9AFMfhdRPW5JC5yiclqL+VoBb11RuhahI8um1Ne5qdxL8iqZ4p0t6S9E4o8TacTwtctqKjZ+SKdeobEs37MiR2TksPFCUsHvMgcHjrQl+66V3JSDBFoKJNsV4ITGlsdp73iBMCwie3Yyx+gp/MqZ8JyBxoX7OpHhH2GBOT3gLgxlF67jL3kVF66RQ78yhPPigMHY2zCoKV1z8AoxmQ+ziCKetufdfePTmBBve+5qPORJD3603BdazRvDkVyxaJ4Wu15sXeKp1LqJ23iGzoRmc8Nl6w8IvK3ZeTKEigecEo1U4Xeno0zsBpq+ovxLh7cnnSm8LjOZ0tfB9Jwz6C1oUqzci7K5waku3ISi55B6Z9KdMhueHY3MPLZZNRU12U8qHwcznaTJlcKDKLN1WZFcNdl8aC+211KBOC5x1hyij8Esmo2pC/pHB1McB1kg66iilWfl6ge7RmPwTmP9egIql9lQazJ4QfLffUORXY/yyqAHS27Ijek2RPWdXFcVCn3966U0Gzw3xKzFuWzP7owhroKleDxhWDGY+kI42txbSnTcpPYhxaiZRxsSfiAlzCXFKlLHakIedWZOjMM4m5J5A+1RM7+XhwZGdfywuf4NpcSYsTnWxbmWJXVm8AOW7IWp7PLLNaUq3xljmYcX8tzQzHySyK34UktnQzH1XYCO/rFj8t6ZMovLj8aetqX5kHujGJq4NyT80WP45F3OkOPL7CZntmNGE1JxhWrxRcys+/dkxa78m3qpOU1z9+rNK/FErmuEUlH7gSfBFR+zVjQDcRogRCG3yaa5nYqGiZJFWbo/oHoLioxC3IW5znVMRG1+W2bTXkB1iOGEcSJKjjMgjzGGMNVQ4TSjcE4BexZ9NtqyBuI+kapribUV3zsQIFJtf0XSOxYITHjFonLKwe5pWO8P5IxtMV8VevXZBs/OKzXBa0Zux6S9IRxu5iq03bcxA010wcRuwe9HAbhrYbXFeQcHeCwaj+ZDekkhCZn8gL49bN5n6E5e533zM5KWI9heHmENF/jEElZjWdg6/nOA1BOJK1TSNUzZGODYsTgvdMcwK9LT5tsHmLwdMfZwQpcWsrbMk8ZlTH/XpzZlYfUjtCEtq4pIsvCgtcUf1cx79OTEdtgbQOuYQ5AzmfyCSHXHu06x/JUX1akT7TET7pMbPK4xYTENiD7Ze88bqhLHmLBGy+X5cUH/WkRyt2ueJ4W9KPVc/52EEitp5m9xGhBEq3F2T8jXxLoo8mcJ0D0njU3gArSMmufWYxmlX4hIXxeFj+r0u2695DJ8f4nQU7ePyMPtzQr7oHJNGrHTVpHDfRBsCBe3T44x1j1trM/T+YpryDQOvZhAUE0bTEXuvxXi7iukPhMBReCjcA2sgJYDdk1IjLCTM/WhAf0a63dxdm5n3QoK8pj9pUr7ZpXQ3YVRSPPiLo+xdtNCxuGp7rYTZ7ymK12y8XeMAVE9MhdICQ4mri8HOa6IQMAOoXFdM/7EjkpRZMeEI87Ig6ufTJLY0YNpQlO/GhBnoHknw6pIqo2Ipb1AildnfaYOceCOYQ5E4O01oH7Jwd020LfRHa6CFZNOV55reEk+w/qyi8ERiK5un5T61TgoG/rQmaT9x1v//jys1taCP/8ZvU3gc0TomO5o1EvijfRSs0biOCqQgN0LNYNqgP5cwcVXROAdxKiH3yGTiuk9nyZHQhGOyuKY+1PgFg/pLMZlli9gREnOUgvxahFv3GVXFxKL2akRq1cbpiPTXDDStE0K60JaI8KY/FB1U+4hB6V5M55B5YEqWjGFBfyL5rP4LAQXl25rtd2K8TeGyFu4LCbx822f3BRe3Jez3yPsMOTBCROO04hLkE6yBIrci8NbeBYPiPSE69+c0uSfCk0gscTEclYQj6zaloTNiGExpJq59xh0YTok1euFJQuuo2EAWHo3drDuGePDHIrdJLNHst46b9A9FpNYtCo8S+rNjb4ENwaoHVYvGeclf8OrCw3WbspgjV9E5Bpk1oSZ2jsCj//q/+k8y6/8rvxJX3t7hhMlgVlN/ReJlopSiekUTZjS5ZS2g/KTM+gezCbknBrtvR1g9hQoVnXMBuy+69OfkodgdSDyJlWlcSMg9kNz4KCsv56iqiB2D+3/foT9p0lsU6cXSnzZRsZhiRCnJjTJi0fKnthSDqilyjoLGzxsHKs35v2xgDYWMsY8JZzYVxQfy0MKUIndPJMQTVzWtU0JgXv+yQ/Wqj9sRe57iwxi7I/ZBThcO/Z7BaHrcrHgav6AIsgbmUGEPRBNVvC81pzkOK26dNPArAlGNKoLNhhk5hlsnZGHmVhOmPxBXw9p50UllNjS7r2gK9xSVm/HYQkm0Y4Mpxe7LIu8uXTUJs5rBpKAQUVrEhMMJeWm9mjSKjdcCnJYem/gqIZXvKkr3fWJHybj5Ka5nYqGaI7D7ivpzAtfYdeuAnW/E0k2JMYXc7OmPYkq3BEaxGhZTn4TkHxsc/n3R+NtdicNJbChfEQjL2zaJ0rLQZ94X8odXE1vymW9blO8M8MviuLL9VgmVSLJHbMtMfPZHPWrPy44VZuX4Te3ITU5vy/hy/WfL9OclWcRpGvTe6Y93oZgwJ4tEUIgEI5LfK7MhTVP9rEvnkMHw3JD8/Q6jSaHDxS5svmXhlEbklmV4kaoJ68nui1Q8cfWBQ7cRijV5blm4pFZfi4tKW+O2RLpdupfQXTQYVg3aR02ctox1i49HxI4iu2yK0HFtQHonJrfuHwxi3JqA/P150bF5DeEGaFtMKuy+HOujisZraNL3XYwI5n7oS07ra6bUqyfdMXb+dGvkmVioxph9Pvuuxm0qolxClIHhdMLWW4rJTwVucVtinVM7L0yczKaUA61jNqW7PutftNl6SwD1UVVyU01fomuy67Iopj4O2H7FFKFZTxbbYNJg+etp8g9Mmi+Ld5M2xN06zElmU+dImuwqkIjYbuJGSG5Nhgv9GbEa8mqamfdkR1v6RoOw5dE7GrN30cKIZIJjDRPcmiwQty74cemWMOjdhiZ1I8XjXytSvimE6aCgmfo0xns/S+2tUFAMbzw2rsgULHalTAgzBs1T4isQ5BTF+9KwBTlhQXUPJUSegduKcdqayV9eJcgJfpvZStj4Qgq/BOU7IeYI7v+9lORpGcI3mPpE8Nfig1C8GHKavVcTwrzG25OltPEliHJCmM6t+Tgdeb47r7iYvjgTOh0pqUp3egcmxT9xjfxVLb7/NVdiSyMT2yK+06l47Hkv7KmdVwQIr9wMKd+Wo9vujokca3J87l10yT8SRpUaDw2Kd4XgMZyCzHaENmDtqxbpbUXzfExQUKT2EjonY6Y/ivGaCdPfsaSWSksQgjUUDmn9nFDdBjOa3pxi50U5wqfeb1J4lDCYkUW79YZQ6e7/vSLF6xbZxxLJwzjsrLso0x67Kxbudl9qt9SepvjIJ72jKd4Tn4AoJQ+2cdLCCCF/fexxOnaMCaYiwiwHCSvDCTHM8IvG2GpoDCnlxDoos2HQ/Pk+/Wk5sfr/eA5O9NGmlAOxK7jz2ldNpi6NsFsmdlez+6JLYkN33iTMCj/B7iqiGZ/qR+I/G7vQPZRQumlg9QymPkmon/HoHE2oPy8vw/THI7xWIsLFEO79fY/03tNZpTwTCzW2RUrcPmrQXVSUPrWxhlC5Ygj1rG1gd4Qv2jijmHk3xmsJuWNUUeSWNX5Jatu5H4n2f+Jqgj3Q7LwO/lREmDVondagFYNpTWrLkiCvCYOp9xS1sxb9aYNR0aB8U47e1K4sgMJyTFSQEWz5llAAE1dTu6DwJzP0FgzKN8WL36uLnghDGg/TZ+xmJ4sxzMBwVnRdiSkGv7EjiYXN4y5BXoB6MaQQG0gVi0eUMbb66R4SLDJ/26ZyKyIxxX3FLwgHon5O4jN7cwb9s5Lj1J+RUkgvZ4g8EQc2Tpu4H2Xl5U4gmA3H9aVBf8YhsTSpZoxflEQTM5BhgF+Smr38rkvzzNgjYUWz8F1ZdKltxbAiXrHTH2isvpI4onMe278cMJwUemTxmk137nOkmTJ9yD9OcJuCuyW2OjDd6i4ITS7MQfVqQv4xtI9YeI0It62ZuBkRpcVRpPq+RfuwqD4HVRM/L8Tq4g3ZQea+n5BbUVRuaNyGSJQHs9IBz/9gQJwSmMyfSGEEAl7HrjiAlK4ZVK/59OZFLWt3xY25ecJlMBtTvxhTWA4ZVUQPlVkzqJ9X48Q9hVuXqJzMlsbdNQnyitgz8JoxvXlF5OkDj6f8SoI/HdI5GeG2YvwJzfabmmjctFl9mcRpE3YvWngNhREJecUvSYPi1mVaRddGxeKRkFtNOPyNgQxIBhxkJRiRZuqTkLlvmhQeh4Kz1iKW/jxkVDCZGHuw9mcEcivfFeK3X1JMfZyIZuuI1NLDqqKwEjGcVMy+OyRKiYHxqGxIHf0jD21Ig5euJaT3PkeWPiqBxvhoVRpmvtfA9D+ziFz4biQufLGMWq2BpjdrE7nS/XrNhKmPhATdWxDyhzXStE/FtI7vx3CbrH/FILWX0F0yKD4KqdwKDjrWnZfSuE1hM9Wec8iuCnUuccQqfDipaB9yxMx3RerbUVVLUNjAIL1msfe8zcRVCcjong7Qi0OCgsBp/UPxAfXQaYvspHXclAjKLZFqZzY02oTgNxrkp3pkJvv8xj/6pmiXugbZjeTgBd766YgwA7lVib0Mx5TDwbQ0a1K7arwtk+6xiOzqkEHVwJ9w8UvSpA2nBA0YzCgap2wGVYPBpE2Yh+GExfYrLlFa8hK6SyJzmfpADJaNULDb1PaI3a8K8bt4V55l66hF5VZE7fmUMLoudvFLYhEfpSR9xRpA4WaD1rHPkVOK0pBdlWmJOYTtd8qCpQ5gMJPgrXcZVhXdeTEyG01Ice92ZFeJPMWgKkqAxBKIpjevmPpQcfiPOnLUZSC9abD9hbEr9XGbzTcd8o/Egc/paoaTmvaZiOHFgRBatNS83p7AQoMZCQq2RqIWyK7C3A8GRPmY1J4msylNQvlezPw3TdwbaZy2vGzT78oYczCtyG3EDKYMUjuaziFLhhkZGP1qi+Bij2qmT7/vMeh4/MuV1w4cZMK0cEQxIHvXwW2KkfG+gYXbSUjtillFYkmXbvqw8JfQPpamPy/JfoYvuqrsmsBGmQ3pvoeTCnuYEKX3M7hkxwszkF2TMIm9F2Q3NAJJ2374N1PM/6FYZPplIQbll2NiRyyY/LLGupQDQ0gyqb3kINe19kqFqU+fTt33TCzU/YTh8q0eZqDHqXDgl2SOvf61MrELnePxARowWIjpLphoJbRAFOy+GZFdVXSPyC41Khqs/XQBv6wZLMb81v/2z0itmwzmY8IsYygH/HJC7fWIX/r6B5w+uc7SZIPhXEx6SyAWiV5U5J8khHmZpKW3ZYffeCeN0zRpH5OdW8aoIZ0lk9SuxupB6wSkd4KDJkIrwTwHM4I1Ysi4GCDeTLPWLJLEihOL2+y2shiBYrQQ4rUSio9CRmVhRWU3Y2bfFZ9+t6HZu2CQ2ZJddzAf47RkwjSomrK4ItGPdY8mTF7q0Z9REmhmy4uS2tNsvSG6s8ympnCnTeuoOQ4U9ukuGExcFU3X5KVAgjyeGHQWLLLLsviH05rGGZPckz71M7bk2pY1mTU9ZsgpqldjSg8igpyiO2c/1Rp5JhbqfnLI6s/lCLPCY1QJlG8K73Tq0xG5VU16wyS1p5n7fkL+rljQ7L4RE2YUE9dHeFu24JOJJBgP5sS4Ns7GeFN9fnflJXJv75Kb7zA6M+SLv3SZ9IkWb75+m0xlwLfXTvKLU9c4X9pEmzK5CjNjs7TnAurPKapXIulcy4rKjQFuC6qXEwqPJF0kvZvQOO1KRlRWEWUhuy6NklcTq8nWcRMjkHrTDDTdQ5CZ79LreZy8uMovHLmFHpncW55BPcqgp0d4xRG7Lxu0jtl4NcFvRyWD9pJN9VpE81yC25KG1GslWH3hNZi+wFMT13wqNzTDqsLbM+gvpFn4ThcjEIsgt65pvhCR3jRw2wn9GcX6T5coPoxJ72i6Cw5GKJxcv2gwKlsERaH6RRlIHEV/XlCKKK1Z/0peaIahxMx3D0P3sKb+coTTjgiyBtXrw4P4+Z90PRsLFVj60yalO8K0DwqK3pJ0lrm1iNWfdsmt+2TGEuK9i9ZYMq2ZfN8kdhTrX3Yl4WQsY9l6UxKSTV9x8ewTTlT3OFHapTPwcO2IxDf58yvnCSOTx+0JkkSRdkJu9uf5xvXnmVpsEOTF6c7bU9g1W1yaz1sMJg3mftDh0a+nsHuarbeEfRVmxfk5sRXDeckMDXJSo+7/XqYvIr/OyYjCQ9n9Ln7xHqX0EB0ZGErzb959lfwdm9InNsFERDY3opgdEE6EhJkxE2wkDYw2YfcFi8mPxCHabULzhEnplsTJhxlhp3WWHJx2TPeYaLmGZYP6ubH9pi+1ubtloS1h44dZAfJrz4lMvHNYBix+We7vzs9IXkGQlbjM4oOQyUsCK9odgdB6S0ACE5+Y5B8JmXr6Byabb7s0zil2L6Rwm0+3Pp6JhaoS2H2tSLCfz2QJoN2bNRiVTMyR7Ejto+I7n9jCBXBbiv6cHG/Fe1oamZHGHCmZgU/HHP/SY9JWSDvwqI2yjHoujbsVjK7AInOlNrVOhsC32b08xfv/9AWW/sBg8J1J/MoYionE8750V5PdEGv2+nM55n6YsPdGhDkUyx2/MhYcDjXlSxbdwwmL3xqRmFJTm77ATH7RIH/fYuenQvZegvVukbW1CspK2GgXSG+Y9OcScusxr597SNoN2F6poCyB5PazXt2mIAlhQaCrhT/X9Gc1pfsxQU6R2hY0JMqMyehTFpVLJkEBukuCxaZ2pSG0+yIMDPKa0WGf9I6UKW5TyhxtahQCT03cGHHoX0N/VkquMCvO2NuvSZaqM/YWULFwFmJHGrf8k4T2USGqRNmEzunoqaUoTwdi/VVfiRyxzth9I/aklrGGmu6i5NUn1hjozhtMXE3YfckAJfY3pXsJjVOm+DKNPe0b5zSpTZO7E5O4bkQcG5hmguHEMJ2QyYxIPizxaDSLWzOZvh3TPqJwegntQzZOR+bsfoExn1TROi4OesMZMTKzBwbOniI6NKL6l56oNG8H+EWLyJWg252XU6R3EoxYXjIzEJjJiMHac/B2FY3aNObZAaXvpBhMpUg8cNoGexcMVq8dB6A036a1VmQ4JXVtlDUOdGTVTzTWMKE3K1CSNgEDgoIwu3pzMhVDyQZgDaB3NMYILUZVGatGrqJ32mf6WzbmZYvITdBK7IbcToLTUdSfU0xcTVj+BQenaRCnEvEDMOSeoYTkklk3aBwTQzUVQ+XWiO6iy7AqTen89xK68yZuR+O2oqdaIs/EjqptGeHtjyPzT2LMkTCJrKE44aXqQsSNUorOYZPiXWHgFB6IDql0PxYb8B4MDoeYQ8XP/uqHhF0X37dIEkUUmbyytMLh6RpRZGK/3iC1ZRJ7muZxk+Gk2HlbQ43XFHc8f0IS8vqzmsJDyfUsX1OYQ7H4VrGi/D2P3oIiTMPOiw6dQwbdJUXrqI1XF+mK29Z0Dpk0T8iUad8GM3bh0B/skv0gfZDX6u0JtGR3QWUizGJAs5ZDpyPCXHJASXSaQjxWiYxPGxdienOGeEblhPFUuyiM+sympnw3IszJzl65ZIpDSkOR3glonYbCZVdMITyF206E7NzXbP5iSG9BRHrbbygK9xSL32xTviEGGNVLApnlH8k00fQ12RWD9Kb4XjVPeqT3xCmweiWhdt6iPy/8352XP0fNlAoZkzTG4rOxvXZiSkeqDcnRPPavmgRFQQNGY0tFayTY4c7LBvVXIsyRxtuw+W9+7d/yx995lYXFGtOlLrm0jz+wOZPbouAMSRLFVK7L8KRP7MmRmn+omHlvIH5OnlD7Jj9JRI81EK+r3GPjIFdJG5Bd1dQvClsqvyz4pdPW5JcT7J5MzlACmXk1zagqQb12V3REAA9/s0rlto/VF2hLG2JyO3ytxwtHVlFrKZShcTYdKtckGWZUkQXnl+W0MUJN5ZKJ3dM0j4kYMbsRkVkzyD8WSKl9yMIIpfa0exIugYbdi57sfLdF+9+bMwgzBqX7Ec3jBoVPPNJbgjQc/f0hnSPQOZEjsccyomlJ8ktMyD0xGZUV1WsjKrd8RmWDwZRi74IjI92KLLlgImY0FZPZ/BzN+vcZ8NnNmMlLQ4bTWuwJ52OitCa7Ihjf6tfLuA1xmvZLwm0M0xIrM/fjCGfPovG8WEf+wz/5VfJnJJJvbbOMH5koU/PPrr3Odj/PS/NrzKXbvHRsGaclTPjWGz7rX0kznFT0Z4Wv6TVC+vMJC9/uM5xKxoFi8gK1TkoNm9oySe9oMtshdk+0RztfjOgcEZqdVxsD+XnRhdXP2sz9SIYB1lBCdXtzDo0LMbsvGuL8VwedGFy6dYS/9bM/InvdxeoJ99Yajgkstc/sIfszooh1ugKpWQNY/drYnPi4ZjA7pvmFMqzoz4pboNOVIDYjUNTPuOO6VHgUzRMWqT2NX4DGeUExmqfT2H0xDfaaElnfPR6RakRoQxrK4ZEAZ69P65iDOUJ+17GNksRYgrNnYvYNeZGf4nomFqoZCF9zVDRpnPGwetJhTr4vu5o8zHH47Lp0ltmVcbDExYjynYjdCzaxJ7aHUVrj1RWNvTxpOyCV81kstjgyUyMJTTY2yrx37yjfv3QWgOLrO+RWEtzHHlFGjsn+onAxl3/BAa3YeyFD4b4aM9nlZ2tTDMMyGyIajF2DzsmY6hWfqe9bGKEa2wxpWkdNrJEkEg6nNFuvp/DOtOhfHOIXNV/97XdRiRL5ybmI7tGEiVKX/+z19/iXH7+B6ct9IhE7ns4RwZ+DJf/ANCLMKEYlQ0J8Tcg9NEErpj+U5O4wr0nvxmQ3x/P7zYTWKY3VE+VslBmn0kzuE3ASUnVh5qc3FKPJ/aBg+b7cgy5xSuNtWXTnLKIMoGH6OxY7b5bpLUJ2K2TzrRSJA+3jn6ldjVDKtqdN7nsmmql9h7vhlKSYpLdBJYrmKUhvS3eaONA5pgnzFtn1mNoFg4mrMdbIYvei5Cuhwe0muDUTb09jBA5Pthaxeoq75RxOS1Fsy888/7dv8v7joxhKs7Nb4MhmwKjsEWaFLFK6KSSO6fc1zZMGw0lQsQDkme0Epx3jNizSezGbXzA59CdDWsc80qtiL7nwHR8jdugcEkphblVTPw8Thxt0Bx6vf+ExV/fmOHdkmQ87J/jztdMcPb1Jygp5sbhKwRrw//w3P8eHb/u4Wxado5IGOP/9kN0XHLy67F50LJnDz8puBeItJXlOmsRW5B52aZwuUrgvWVWWryk8hNZRkYgbgUyxhpMJ2TWD/qwgKHZXsf2akqlhLQEMGcaY4JcSnvxKnuJdqaeVhuClHub1LL15g8qtkDBnY/piNZneEq/Z3gKU7yakr8WEmaffJ5+JHZWx5EIrIXuoWN40cySjTWsg06rCfUWQk+bJ21V0F0xx8SsnuA1pbGrnzHHdKu5zmTWJVsw/FmOJ/rx03R/+8CwvLK1yeXWBF46ssvWaR/FhyOSn0D3CmLUlpgtuQ6AWrz528jttYMRiHtE5ZJFYmt0XU2hTplMqgVHFpnXMIL8cYwbjWMf5AYv5Jocn6txrTdK+XuGTH58iM9ulsVlgZa/E3fcO86dr5/jH19/hjZ+9zoOb86hIkdo1cNqK1jEHEDgpvxKTXTEpPvCxhgIdOR1JNdw/avMrMf2lLNoUXLM/a9CbNccaL8b5CDCqJqS3DcyhSFr8wtj5b1cxmBNJdJSRBg1DQjusoTqI3xlOKCp/mCYsyL/vvGzLhGzKFRhtnOA3cTNm52VF5BnUz6mnXoHPxkLV45s1meBPJPglgVaEdCtv68S1mOGUYuqTECOS2sruS/Z8/oG4Jjstce8wQsHvOksOo4q4qFRujtB2QnYFMKB4vsaV1QXCtkvO9tGWELD7MwZuQ/DJ9nFxfE7vJeRWNYUn4TjzKmZUsohS6oDwHb7VIbsRETta9FWmGGoYkSa1oxl+tUvYcvHMiKzts75VJrMuA4k35p6gUjHF3JDc+Trtnodlx7xaeIw1NWB02Md4rUnvaETm69u4TcFyR0UDt6ExQqEgpnYUrXMRiS2RRVpJY9o6ZlG5GVO9KsLH6qU+pfs+w2kZRnh1TfGOHPVGCIM5TWZXJn5+ScoGcySll+SiCrpi+pL053Q0k1dCgpxi8lNpupyOPNPWUUOyaPNycnbnTGY+EAfA9JY6wM5/0vVMLFThQEo8TOWqonQ/IbcigWRuKybMSmR4blmCce2epj8jdLv8I5FZT1yTcWF/MSL1tR1GMxGtrwwlbWXFYO2rHmZXOlK7q4lig//ywncpz7X4dEvssoxI01uQhinIGEx9IuSK2nMmg2nFzss2fkF4smYoO2p6J6E/ZeCvZmmesgV8D2HrC8IRGEyYYmGzmSE92efW3jSXPziBuevQetlHnenyrcvnee3oEwreiKlsj7DpET7OMW21ODWzC0OTry7cw8yHbDfy6K81x5MiQRR2XkofMOVT6xaDaUWQld00VRcL91HJwAg01Wshm1/IsPWmy9QnCaOK1KPpWkx21aB7RFxn2kdMRlXJIdjHXQdTEv+e2hWxnl+SGKXevEF33hqH0Rl0TkeoSNxc3JamejVgVBWExgw0uy8Kz7j4OBSvrqe4nokadZ+RjxKNVOuogdJSB3WWLJzWZ3xIZwzrpGqa4oOAjXdcVCyCPvflBofSQzZqRbASDk/VGZZtNnITTHxq4pfEvOuv/fX3WB2W2fBLeHaE/mYF05ZZffWypnFGdvH2IRkp2r2xcG4Eg7kYu28ymhAN1s7rmtxjg+yasOg7z0VYI5PMikluLZFwsS147cX7GEpzrzFJ7kQT24qp7eUp5/qcWFjmB9dPoZwEc9eh/BA6x+Af/sPfxP6NHVRo8Ee3n8e0Zdet369AOSG7bIgSwpasU7srkTloqevBoHMcJq5oWicM6i9Aat0eIxHg1QMSO4XdE46v3dPjkA0hh6tIunt/QuiCYU5TP2sSpTUqkRyvyu2YnZcNUrvS3O1L0E1fjJbNEQyrFuktg/xyxNYbJlE+GQdeaFKNzxHD3xzJ7Lp+Vm6MGYgS022O9ezjQAJtiMAvty6da/uIQ/HBOK9JQX/gstXM8/bhR5hNmyfbE0ymu6h0jD0QjZFa7POwX+W9e0f5ncuv0Oqn6H5hSPuMBNV2Fw0SR4+xTE3xYXSQTO01NE7TxBpqqtcCtILKVRkQjMqys5Q/dAgz8jvtXRAys7bgyrdPc79ZZanQoL1WoH15Ah0YeFbEe8uHKV22KH4kpgxBUYKGh1WF/wdTzP5Iowxh57evTJCkEgr3FL2lhP6cIrsp/k75lYT8k5jBXAKGxAjlH0haYOwKzpqqiYyne0iz+WYKI2IcXxRjDTTDCTnug7ywnlI1TXpTRrFxShay21IUHsfYA9h8exyZ5I0tkhzGSgAZwaZqInZ0W5rWcYvFb/ukNkzaR0z6UzaDiZ9sSA7PyEL9jMQr8E+Ygdz9NlOfSE3YOiMTldFMzNwPQlSiaZyyaJ8Avyja8+KDBNuJyKR8jqb3MBf7vHP0AVv9PHpksndRES+OBPy3fBgJadn4oEDlmx7unuwCUx/7pLZFqh0UJCbIbYxlHAomrsdEaUV33mbyckRvUVF8EFC8LzPxxFKSNhhqKjek7DACOT77I4dRbFNcbHH2nYdYLYutH8yT+iiLNiSsIbUlD7j6aYegIMD+9uuKuGvjtKVhXPpjjVaKmXc1XgMaZ6RWbpw2iFIG5kAcrpvHLUFMLIGCBtOK7pLkd538H9cPAiWaJ0x2XxSdlUBdsluW7gq/djCjOfLfXSO9JV1/+U5E/ZxYgaZ2xCZJK0Vqh3Gqn/QYnRMxreNSR4/K8mLtPedRuR0JtFeRUfnTXM+GAcX0gj7+N36b3lKC6Qscsj95Sez9SHApzlUkOJ5KRLhWfCDxkqX7IcMJi903YlLVARkvoF7LcXxhh8Vsk5uNaRqXJglLCVZlSNh3sPdEpuE2hNTcW5Ka1xpJl2+GwvG0+oJBHv29NnsvFai/GuFuicBt+uMhuxdS5NdiwpQMChJbcM3EVnh1kSZrU47Fys2A3ReEMpdfTYg8Res4QoZeFb7ozsuKyo2xZ1YivzNKyDr2QKQzTkccTuy2waE/7bL7UlYmVSVF8WHE3vMWuRV9QIbZlyVHGfCL0oRWL8luq02p2/fDMGSzEB2X6Y9fslkhAZFIo5reEutNt6PZ/KkYZ28s9/lhSH/aJnEExtpPAewclsFM6X5EZ9FiMCvNb2Yz4ZN//X/8fBhQmGNuJlqJM/O0ANZayYNK7WgKTyTd2W1D+3QsN2BP3FA6JyPWf8qkeQa8HYthM0Wrk0YPTRazTYr2gFOlXYyTPdBQ+eM0uVsOaJj6JMYcimOJtyf25u0jBjtvaupnTMq3NIPDIYUH8PjXCqBg8RsyiFAxbHxBkqd3XjKovSDwTZTStE9KR906KQ55+0bFuy86kv5SlQVnDYVHmt6Snbk3JyfEqKLozY9zYhcl29VrJmhDcgZMX5NeNwhKMf2FNKOyonFeFBJbr5tMfRpKSYTopSq3R+ItEEBmXZFbFqm2kFyEV2H3E5rnNKl6RJgVMlB2PcEvQOkOuHVNUJKuPkqLQUfrqMHiHyuyK7ITb75t0z4hA5HWmYTS/Zjmmz6ZTXHC7s6ZqFgwWq+mMZ6uRH02FqpKZOyX2pOaLrMhN9waSYJf+d6I9iETbX7m3YkhwLtWULxpkVkTpao5hOJVG+9KGqducq81SSfyeNSeQF3PUb5msPO6pnrNZ/JTCQ7TJlRu+KR3kgPpdvaxJNINpg0W/mycdL2p6ByFtZ+W+s1raMq3xd7H7imKdxSjiki4xfdUkd6UJsNpy6mQ2pMYyZkPYtLb4uKSXU/IrUdsv+ZiBBCUY8yhpjie3Hh1xqRlqdkn391jVBbx3sQlg90XRSadXRkfzbdh4x2L5rkEpaVWDvIWYUZsecKcxGuGGSVfs6VHaJ4wqX4qngbVayPMkT5Img4K8vPcuqJ3OB6LAGVDidKiBesdD/HqityySIiMQIkK4IpH8UFI/pYQUPJrwokNiormyadbgs9E1x/bEq2jldyI7FbIsGoTWqCVQeR5TF7xqZ1zceuK4YRol9pHTJwuciw6UHgiVuVRWlg91lCx9/4M26lpjAAmr0fEjsIcGMSuQe15Q2J2dg1iz2QwLf6e7Sl5490WdA/HqFiI2tZAklOqlxV+To7mUdmkP5+QW5Zaq3wnpDdrkdoxBNts6LF/gKY/p1CJorcwxn0bsms1zirSm5LUPCorso9l2pQ4HPBbk5yic0QxeSVi962qSG0qivReRJQWD/7uktTJo7LUi35J6kOrp9h4R8J7Uw8FIciuienx9IcJm1+PKHzi4jY19XOKeCam9J5LUFBMfxTTmzMZTItLy/SHIRM3NatftVGxJrcCuy8Ibutt2pJE4ymKd8Y5VEogvL0LNqk94Qtvv2qRWR87WD/lCPWZ2FHFB17YReV7IVtvWIzODlGRvH1+WbH2VQd7IB105Am0gR5nLyX6IP6wc1gK+dxmRPslH6svc+rEkeCH4YTB9EcJdjdCK1j8y5iwoEntDCXC0ofSLYlWVxE4TWm69rPmtQG9GYP+nOye+dWY6iUZQQZ5RW/WIsipcUIddJbEm6lzVGCuzKYmvQXla1JfawuCgia/EkkayfiJGJHscoktNer0xz4z7we0jlo0z4ndT1DQbL5t0Z+F5lmJ5+nPS9YoiCWluFhz4Pxn+Rp/IRjXj2JakbnpEhREwzV5KWHqLx1MX/4epWUqaA4VlRsxuy/Y9KdskrT0E4XHIXZPMZgyMCSDQ/6uWbE/TyxoHZMXczCt6C5IDR+74g6zX578pOuZWKhKi8qxvyDRg4kFs//OIcxBZ3FMdhjbQUZpgYncjrB+wkKCVorWMZPYkxQ5pwXDkknmpuwSpq+ZeTem8DCRG5RWrH85RfVqwvKvwuJf+Nz/zTTNMzBxU6zOt78iLiRhXuCvqU9DtAVuQ45OrwaFxwndBZOdtxKKD2KG0xIy1jkdUbgngr3iI/Gdmrwcix/+qo/pi6hwVJbb7zYNtl+38Md81OyGoAbtk0J9bJ7VGEHCziuSuOK0Zdqj4n0jXUVmdSzFbokJhjYgsyYdeXZDnAWFytjj8P977LRSSmQX3tUHETvWUGzi90169y5YmCN5cayRmCX3FhSZZRPTh603bXIr+gAHF6nNZ/Hm2pQXsnJdjz2o+lQvJ0x90iO7kVB49DnS9Sc54W5mV7RAJqamftokqMSoWNymo2zCxPUIcyi11fbbmsTWzP5Q7M3NIQxmE6yR2HZra78cCEjvJexdsOguim9+blncQ4YVA2/NZjjp4G2ZZFfkxSjdj8jelabHHCpKDyI23rZE6psZN38peXjZ9Zj0mkVvziSzLjKM7COL0t0B+YcmnUUTpwPtwxZRCtqHXRrnRICoDbHBKTxIZJe9F+F0YFQx6C2I9/6+tWaYs3BaUj8H+QS3LQ84dqWTz2wLF7ZyM6TwOMbp7meZMiaDK8Ks4tGv56ifc9EWZFcMIW+PUZWlP2mz97xFlNEkrnT35dsJ7Ys+qV3Nzos22TXGBhKa/BN5AYKcIr8SCcPLl+lTlBL7SmugqT8nVvXpLU13KcWgavLklzL05gwG1c+Rrl9HUlf2Z+VmJp4mt6aZ/7Ymtx7Rn9NUrhg0Tlv0F0Xq7G2ZzP0goX3ExG1IyIJbN+gcMnG6iXTJNU3nkMPGFxXlOzG51QS3o9l+LU35dkznmCy25gkDv5JgD+TBj8rjCMYZhdNRdBYtsqvi9pEep+jZPZmgbb8pDUWYFcKK6Uuo7crPpzEC4SN0D8dktmS0OpyStD6vLuz+yjWBtAbTijAtx6JXT5j6NKZzLCG7arD3oibImfQOyUTH7hpoQzGaEam005ZsWKcladdGKERwcyTae+CAFLLPlMo/Tugei+kvJoR52clXfrEwjlqXiEy7q9m7oHA2HFKNGKWl7jRCmP4ooXFWXkx7oNl8S3yp2j89wM8rCo8Tdl8WD4bcE8mndbsJ229LUJ3dGWvRPk9dv9VTtC4GxCmpOwu3TYYTito54TnmltXBqC21LZOM/BPRCLlNTe3NkK03bapXI9LbItVwm1IqxK7kAgyqhtRjWz7ahOZJk8KDMd7paDIbBp0jYtY7rEptFXma9JZmWNW0TwuvtLuoyK0LBwAgsyquHyqSh9ifUwwPhTgtRewJ1JZdNhlMGvhlTeVmRPuEYKr9QzGRp+gdD0kcIXxkNxN2XoPunMXcD8XmaPbHmr2LitSWQZRR5J/IjjX1Y4Moq+kelkTAUVVwzu3XTXHvOyxBcbEri8scyZTJCMBrxZhdg6U/i+gtJIwqkFhSky79WYS3Z5CuJVhDUTe0jlrkljWpuj7wL0jtKronYrqLMk1EQfmbKRJL3LDNoXARRhUxJw6yBtnHFlFa7rtWiHX8U1zPxEKNsppDvy8Uv9j9zIx2NB0z9UmXMAt7z1sk1mfBt51DBt1DkN6LMZs2k5djzOG+bY5iVIHRcaG/GYE0CqNJTfOYJ+keLfnZbkuT3hTMdt8dWivoz0LxHrgdaRomLsmCjD3NsGyixnMSbQimaETgVxLxSn1sy9FrQnpTfpfEEu/+3Rcs0htic243Rd90/J8HmEOF14ppHzbIrshYtnncwu0kDMsG5VvjEmc01mqNhwBWX1G5Lv4B3p4oTjPr0DkdYoTy4gQF8dXfb0ZR0DpqM/1xwqhkkXti4LaE0qdNxc6LDnYXaufEf8D0xShOJaKqMEdyrzrHYqZ/JCyx1I6i8CBh9zVN62Wf+jkZrUYehAXpKWJ7PxdVZO3WSCJAn+Z6JhaqOVBsv+6IdeK0vLF2X1O8bTCqemLL7WiGcyJNKd7TDBeEobP3vMXkp5rejEnzhEtuXazK/UpC5pZLf0bROi1NEVrqy+6SYIK9BWgfNejPScqzX4156W9cp3fWFxVnUdGbM/EnEsKspC2DHGejCamB9yPNu0dico8NekvS7FlDKQcmrvdIHAHV05uibI0y4FdEpWn6sPmFDOW7Cd6uj9OW2ftgWiyFdl4xyK1HlO50CYoJ/Xnoz8sUqT9tEKWkFi3di7CHstMOq+Js7XQUo6lIgisKmkHVwGnL96hIFuJwwqB9MSC9k+A1E9I7kgU7nJLPmtnU9OfklAoz4jQYpQWqK98waJ6U+ncwLSQUb8fk2D8XfnD3sOC4pdti2+n0pH9weprusZjCk5BR6XNUo5qhOCRjQHZZMNXYERqfimU3mvokZvIDmYDsvqYpX5ajx2lD7YJiMCMW6zsvG+RWY+a/nxykiOQfwsx7Grchhb+YQTDmjYrQTGciMnNdfvj4GEtzdQxfUb4dUHgUUr42Nl8YL64wAwvf6pPe0oQ5TWYzoXTLIPlKE3Mgn18lgj7UnstSuuvTPSRHc3pbXhg9FjLaQ41f1Hj1kNGkK3KQaVMshXYSZt6PiVIG938zw+E/9sk/FJ+pzmGIsuK+PZhS1M9Y7LwpflLahN6RmDCjOfa7IYXlkPJNudetk1J2WCNN7InmKXfLYfvtRMjU4/Gx4UuJEDtSE2+/atB4QcR4C9/q43YSOkelPkYJLhvkpVRa+3KKIA+Tn44zCKYVydKIrS8l1C4ohlWD6XcVo5Ipos6nuJ4JwD9yxXA3ysgxGmUSgoJBNBHiFwVsbx+xSO8kZLdj/LJF94gmu6zxs5JAMnk5YFS2JAh23pRs0BMB3qqD04J20SSzKc2IXzLIrGua70j3b5ma8/MbRNpkpVlis5En93KN3XBCjkoN+cfCgTUDaZo6R1JEKRksgIDXg0cFbMG8BaR3wapp/LINi0OCRCJAJq6HDKsWiSkv4fF/WafxQpmdtxJIItyaycQnBvXnoHhXyDKq5LP6VQ+vJupcayjNSH/GJChJ3M7wZET7uEP+iaaZV1SvJtTOpvDLsjidtiZuf2bykVuRjWBYVVQ/NNl7LSaxLCYvD2kf9egcVpgjMVEDsIYmtYua5pk0bkORe6zpLWgymwJ3VW4llH7wBOsrhzFi6E+Jfi08MuJvnLnEH//eW+PaX5hpexdlN36a65nYUTEEKkrtKOZ+EHLq/1GneA+KVxx6cwbNN31Kd0N6cwZbr5kMDol9uT2A2R/LaKOzJL5T3QWT7GYsjKVdm8yaHud7atK7kczSAxh+rYPjRVTKPbRW3NubZL1dIPykRNjySNkR53/hLmFOjvH9TlpFUqd6DUlLVgnULirMkZA80puawrjRm35/QJBTbL2tMR+kSe0IcuCXTMkg3Y3Yfivh3j8o05s3xKH6iYVbV9QvCGmjt6DIrfuw50radE1gqMympnc8pHMYqpdl4uM9cImyCaW7I/KPxMAiKAizLMyKklfFUidqU3xoe4sCeflFRf6exXAS1r+cwmuKSVx6WyZqxUcBe68lTH7KWJkg5dnMhxGlX93grb92RU625xfozxlsvyEjZoCvnbzN73z6qjzqcUJMf8YEBf7R0dMukWfjCgpQuh+yd9HhyW9M4nQTVKxJ7yTYay7DqiVj0YFi/s8NoqwYlTVPpcityp/9nDGWQQhCYHcVxUc+/cWYxhlonHSw+5rBXEIlO8C2I3KuT+bjFM73C/RvlPFerYPStAYpPrp/mL/1Mz+ieyqkPyfqTrcl1jd+0aRzXLKqsquKwnKAEYp8OvLEY6q34NE5pindlOnUaFI6dK8e0X1jKGZuKxZuzSB2IMxLUzaYlbGn1YelP+vSPO6ReyLhxe2jMhaNHXB2LMq3Nc0TBnsXFKUHCaktE79i0z2ckN5RZDfEeXqf/F25E1J4EjKsKnrz8jK0zsgETJpVLWEUCxaphjhOFx5quvMOkx8oas+Ngzb2pLSa/28fMIosvv3hc4SFmM23bCY/9SneMdh7LcGfiPnzH18k/cTGL0t2a5Afu8UEUPjo6UZTz8RC1YawjgZVi8FCTGpbKHKxIzWls39cOWMD3aJBatNkVBVHk9iWWXrjvBYPzwuiJ88vJ/TmHSpXDZyOwi8LjU7bmkP5OqORTa2XYTQhRX75lqb9sMThf6vp19IQGDwZVFAjg97RkOrVkPZxScizhprZH4oTdutsxM5LLk43YTid0Dolmq7OYQO3bgjiMCMhavPfi9l+1cF87NE+YuO0NcUHCU5bGpwwLWZikr4Cw+kUrVPC0R3Mju0mt2NibxxBPiVsq8JDOWpHkwndWYtDfxJQuTnCiATLjR0o3Y2pn7YPpOXWQOA4bUgtn97RYz6p3MfYge7f7QhKcFpMQIr3wBwKSXzqf/+EUWyxtVsk98jEqZuEhYTdl1xiT3H45BbuxBBrKOWDtyuss/S2LHZrqOgvfM4MKAaTAvin1k36cxIDnl8VvVRiCRC/H8ldvyDueHZHsfW2LOLIEyfj/oyBV9ekawnt44aY/E7JVMTpSD2snYQbu7NUCn2GIxtroKidV3QXDYr3FKs/bWF2TEgUH60e4hffuITdtFj9OxGZDejNKxqnBBuNPY3hG+M8V/Fdmv1xLJjomia/klC5EZNeM0nVEnqzFnZfoKDCcshgRnZJbX7Waae3NKltwSI33zRx2gbVayG5ZRHNNU5aokm6B4XHUub4RYnfTNyE3pJm73mPrdc9WicMwqwwoFrH5LiVEac0oqV7ETPvSf2d2Qoo3tN0lkxaJwyCrCL8oCycVSWE66CoaLwdMPPTa1x7uMDdvSmsDZfO+QC7K2YcKoLorTaPVyb5GycvS+bWekB+NWHmgxFoMeXIbGiKd55uiTwbCxXxn0rtyVtdvZpIcskJk96xiNSupnKth3GQNypHqRFB9RNw+gmjSSg+ivEa0iQ1zkghv0/+HcyI1FobYGVCevdK7N6rMvlvUmQ2JViicksUnOUbisyGQfGmhXkjyzeuXaB4vkYhP5DIRReiCz0y2zGV65rFv4gJcxpzJCVH9soG6b2YzhFFZ0nGhBM3Qpx+QuvkZ/qwQVV6Wa1g9odttCU7mbYkmW/yckB6RxZJ45RNqiaGEJVbYizWn5Wc0zAnO2KY0+TvW+QfQ+9QQuFxwtQnIcOqpnlOuJ9uS9M5KrWq3deMSibdeZPKnYgwb9E5IksiuyKElux6gtPR5J7IrL/4MCJTGPLoziyGGzMa2Rz5t13MlkX1aoA/kTD/9WXSbojRsfhXP3wLp6Xoz9ioRLP+JY/2mVjGycfG5m1PcT0TC9UIIczI/DuxReJbeCKWhHPfEbVl61SWwYyYLlRuxuTXYnpLMUFeRo+pbXHcMwON2vYYTYn7dFCU6U5uNaFyO6b2RsTfOfsx5197SHrDOIiF7C9otl43GVWgeVYW9NRHHQmetUWI11wvMJhV5J5o8n+RYesNk8ZZxfqXLHLLkP/v15l5r8fuVxfpT5mMlgK8hiZKi469ecyieFeYYqlaMs7K0lgj2PhKgfnvST5UkJd0wvZhm+xGgt0XYsneCwZBTrH7osVgRk6I2Nuf/ws5JbspduaVq3JC1M7bRIWY+e/GuHVN/YWY6Q8lOcZryc/TJqz9rDS0safHi150UNpQNM5J7d0+bLP+ZYPnpzaxq0NMM8F1Q+r/1wC7azCcsChfVyRakfdG/Pzbl5j9kTzT+nOSt+BPRhz5gxBtKmY+jHDan6OjX6zCRVoynFRMftonShmk6mKq5TU0g0lFbkUejjZEHz71geiM+jOK7mEIU4rtt8QiMftEDMFASBJBTtE4aZKr9vhXt15hMdNkVNXUz8lxqJVklaJg6iOBsTrHsqS2Fcfndjl3eAOzEOLtyo13erLrWwNFlI9pn9Dc+5MTMqDYE6Oz/DWHyFNkNxJGRZlstY8jPqRZmU715sVHaljVrH9FmsHSvehAjhO5itK9CKXFzbl9VGrk4n0xSHNamsSR+ju9I3zS/pQxzjKQBJX0qgRhJI64+DVO2iLtcdVBep67axK7CrsrpZZWYgTcn5VAOiMWsnVqqcu9//k0Qd0jqnmEoUkUG/zcL37Izhdiwl9pcn99ikQrvvvvXpYc1W15Dn5B4dRNnL0+UZqxV9XnaEdNUloyObMauwOrX0tLuOuSyCW0gRBwd2NiF+rnDbyG7AbCWNfkluXBLnxLmECJBendBLtrkN4J6S3IKE8BppmwOSxQuA9o6M8rcsuS4Oe0oH3UFHb7hbHcpF3gxp1FXC/Ar8hkavPLms7JiKAg3kvFe3JEm37CcMLEa8kCMH1N87R07H4JZt+VmjKzE0ut1pAdbf77EVMfylCgcdqi8FiEjf54Ojaoimlb8Z5glrl1n9I9MfBtH0/EoGNSmhU09Bdk+tZ5Ucw1jEi4qoktWHXvaMxwUhQDo4qMXcVfQZPdkBdj5n2BqGJHMapoZj4Iia8WaJ8Ed8+i+rFBHJl0Oinudac4e3KdRCveOv6QlBUSX+iy8cshpbsDpj6SkImZ9yPu/92SoCAlk/zK50gurXxFd9Ek/1CoZSoR6YdXEz2/NZSAifWfMshsambej+gujuGipnhueo2x/1FWAhcyGxJ2NnEtprvoEGUT4nTCZK6Hbcdc+vg4/Xlh8U9cF+Oy4ZRm5r0u/aUYbYqGqrek6W1lMTIhs8UOUUpw2ekfGZQvm1h9Ad7Lt4cMZxLWfsoGDdYwQSXSRU99Iso6lQjdz6trerPC5wzyAhttvm3RPixOfn5Z058RRCAoigd/4vD/Nd3a+IKHn5OXo3RLYQYJk5/6BAWNPRA3QHMEc39sYQ3EYHfy04TBDAxPj7CbEi1vjjQo8Z/dr52bJ21UBM3jFt1DMq9PbyqW/7rgqqktRXZdY40SKt/xKP3A4+7GNP3QobOX5ea/OMvKtw5hXMtR/qHLk19Ko03ZcEYlk+yKQW49JrMd0pv7HMmljUistIO8pEaHuYSgqMdgtwQfpHYMSjcECN9+xcbuSd1Wezmms2TSPioUud2XZTzYPCM7z9abwjSf/EhhlAKBo0Y2p19YIXY/s73xpyMKDyHMOXibJtUrfdI7sri/9vJ1Ti9ss9vNkt5WTF4JSWzxG1AxzHwwonPYw/AVTkuN7SVHNJ+LaZ3WNI/ZDKc11evhAZtoWFXMvttj4obIr7MrjE035HToz8hOK3FFUr/nH4tU2S8ZTH0cEnuK9gmRujRP2Gy+7aJtgbLsvhaFgMk4NUWyBdy6InPTI70pMpCZv9zEHErjavryDIIiKC3YauVWPGbra6rv2vhlRee5gMG0onlK/pvGCwnpzIjVGzOULgmq4XQkMj6xYeaDmDCtmLgZsfOmkMK7CyY7LzpPbY3+TCxUbUDzQkz12ojqp1D9VI4iayQp0WFeiBSDaUXlqmL645DEhsZpxdx3JU4xs6kPJL/9SROvJgwj05cH0j5mkH/Xw/hmiS8dfQDAhbfvM5qA3HpEasUmdsWntHotIvYs2kcMvJk+f37lPIuZJsM7RfrzWnBIF3EkqSQkpoTiOm1xgHa6mvrZDPl7FsU7UoO5TcX2KyJui17sUrqf0DidoXlCdpT9kLL0tqJ9jIMMVpUgfvqx1I17z1uQQO05QQGMMXM/yMmY162Ju3VQEM5Bd8E8MNztzQrj363rA3Oz+/+7WaHcWQK7hVnGaIkscD9vUL4tdf9gWnKuZv/SIr0tfNfukoE2NFE0rvUNRfOsjEidtoR19GbFSqmzaGF1TLQlu3R+WXjDT3M9Gwt1fJRuvuWx/cWY9jFD6ilTLNOrV6SJyq9Ip9w4aY9Jt4r2YWHWqwSin2lhdyQ20i/J/+a/FzCaEGRhWJWR4cPOBJZKWG6XMX054iavhvglcDsxraMWK7+VEJ4aUMgMMTMRH24tEVYi8o/kBvslNe64DWrPu+L858jEZjih6ByXaduoosSMzIfUjoD4wW6aziGD9ol9ytuYXmjKy5ZM+WQ3ExJbvJ5yazG9eU1mNyJxxuZwtkTp2G0ROzptERsWniR0loQlZfdlspfekpPDaUsfoBJonBUKo1cT3mzsCDHIHIHTVFRuJIymI1onYe+iovgopnQvpnMkYfOLiRBNbBicGZHasBjupcmuGuIJUBSerTUQxpXXGEei7yVMfZoITGiJkiFxnm6NPBML1RpA85QcDbn7NqkdTfm2LM4gp+gsGnROyaw/So8bgpRgfaNJzeTlAd0FhfMnRay+xH/P/jjCiBR7z7uQyERr8S+6eDXFZqNA1vYxjYThCV+Y6YdsoowWycqRBLZcMmmfge+gNQSRxdIfQfn2iCglljZiL6nJbiQMZhTTHwRi3DAryS1eXbxKrYFiNAl+SXRJqU2TUUXSW+oXBT4bVTWVm5rGc5ryj1y23hT+qVbQPG6KK/WsRfVqgteKiTKCUMSelE4Yn7nrRWmpo6O0YucrEd1DMP2RLy/WjiAr1cvCkhpOjdUAvqJ5VjP98Ygwp4k8g5kfGSz9hU/pNnSWTNa/OrZTVzCqxhz/mUeQKNTFNpXLJmEa9r4QUrpqMpzR9BeEQ9A+YrD3mjRN7SMmmU2NX0oYVjnwdP1J1zOxUMP8vmmsSewy5n5KOrHpC+ySu28fTDPCvGY0oam/lGD40FvwSNUkA0lpaB9VhFmD3LLsPJktOeY2viy+80HX4f2HR5jLtrHciK9//QO6hzTVCzt89euf8HNvXIEZn59fukWnkaGQHxBfLbD9mkVQtKm/EdKflCNbG9A5ZGB3IfYMorRi4duCWya2pOvN/WBIdlXod8VHAl1NXJPdK7tsylQrgMYpg+n3JcW5+qkit5IwnDDIrSUMDoXUX4zx8waRZzD1cUL7GGQ2ZDwaZiG9GxFlNZkNDjxZ575pomLYecWleTEiseRkAvEsyKwr/IL4KSQpzfqXXDLrIs2un1PsPe/RPDt+IR9b2B2xFHJnBtzZnuLCkVX01YL4o96JKV5xiNKK8k3N3A+jsQBTU75i0vn1Ltm1hN6iIrsmRnifK7m04St6RyKqlxJyKwnd0+FBBGR/TtFdEifq+ushbjchta3IrilyD02MSIgSXiOhvxhDIgqAndcEo1MaOkckgifIa4ZTCaknDrkrHpaRYNkxf/DRyxjzAxTw7uYRLtXmsZ2IP7h3AdUzib9bYf77Q6KMDAUmfmwTpeXvzC0rZj4YEqWgO28RZqB+1qZ11Mbuy9iy9nyK3rzCq0mt6NU0/RnjQL7hlxTpHUXiaPy8QXdB8rNax0TF0J81OPRvNekVi/yyT5iR2tOrSUOX3ZCpVJSRJJLuEWHzx57YTeZWBa7L37ZF/uLKLue05eWxB5rSHU3uoUn+sRiCGAEsfnuE09ZUL8vkzvTFknLl1xPeWHhC0PR4/AfHqdyKqV4LaR8Wd+3sZoLXjGket0kcTe6xSMnVhwV2fjbA7owdWzZEiv1Ua+SvcP099WUEMPMDA7cZ0T5ukHloYw4ht55QujfGG2uK1IpD/YxJb0k62/R2gtWHhe8EDCsGmVUZuXrH22TWpf7JP4kp3oPamyHFexLc6zbB8OHxPznJaCeDShRRYJKyQ7RWdH40hf1BDvtaluyySXYrQVsSzWj6sjiqV320knianRdTpGoi90hsSZ1OHKi/GsrUa1YUCtoUQk2YkwZIK+Fmlu/GZDdiUjuKOCUWQP2lCDMQGbnT1my/5jCqJgQFC7edYA31gQ6qfKtHekvq9dxGhNNU7LysmP9+gNdKDpqg9E6Ct2XilxSH//WavDQNTXdJZM2xA91D0rhZA1j/Yori4xFaCQc3syPCvPNHNvj+vRMYWWlqvb2A+jlpMDNbCfXzimHFYvLSkPwDA6cnp4fhQ/a6R1CE3qIeB/t+jqQoiQXdBYPGaRdvT3Ydpyu7Tu15g/yKyKCLDxLyTxKyK2KxiBJ+QFCwiD0p7ue+o7C+XRSJSF/TOG3Sm1OkHzmYgab0IJKM1Jz8nOJNg0N/GFF6z2XjewtE75Zx2jIsGFUT/JJIOLZe95j+aCQJ0r6mc9iht6TZ/LIgDdKoSdp1akemUVM/sCTod1K8q1DSXKCARJqbqU8DWkdMejMmTldTvu0zmNGYfeOgTOgtCpPeGigZzZaMsbW5WHBufDFH96i80DsvmSz+WRO7rxhO2LQPmUQZQU3cdow1gurVEct/awEjlImdOVSkdyP6xwMiT+O2EkZV8dDaei1F/mFXoL8XRd1we32GV44tw56LX9Jsv5rCeKNJmBVPgPSmwG+151MUH4UMqgbVKwO6J2KcjhhlFO6LhWjwlJOpZ8LNz11c0If+3m9TeJwwrBpMXh7y6K87aEtTui6ZU1Mfhaz97Qj3XgprICB5eksIFqU7Cd0FseOZ+iQkzIzzSCsGKhZiSmoH3LYcSb05i+GE4JTDwwH5644MGmKp9YoPxINqVDIYzGiiQsLhfxfRPuTQX5BuNr2lcfpCkJaUD9l1+vMCRakEJj8Z0FvwGFUM0rsJm19KSK9ZUld68jJYw4RhxaRzRGLRe0vyEFONmPWvGLh7YgdpBuJdEOYEKegeARIRDCauhKP5E5rMukymjEg4BkGOgzGm20moXTDIP5RGa3/EHHsKpyPR8eZIxqVGILTKwZT8btl1ebFjW4YP5kgClcNxfOU+VXNUVtJTrIf0Zmz8ksKrCz5rBpruotAY45RkdPklxc1/9NufEze/Ecy877P3IrIrvJiifM2g+pFJYivsDmy9aZF/P8XUxwF+UZopSX+Wm2CNYOJGxOabFnsXDTqHROHZOj12jz4tuvTdF23yTwKsEQwXQ7w1h95SwmAmETipmuA1I+rn9+fgirnvQO2cS29RMf/9IeZQdF7rPyvHVv1CQqoeMfVhm+qVhCgl4rfVn00xmDZwWwnN44Z4sI4kvDfylPjrH7UxQ4jO9OktQOW6Htv/RMz9QLIDciuaMKtpn44wIllYE1cT8o/HzeaG7EqpbUl7bp2Pxgl8MPuuL8l+ecXWF2XCpC0hxkRpRXovIbsZ010aB+0eiegei+kcT/BqmvSOIqjG9Gelbk7VxZ9AaZi4EZBdF8jLGDtdqwQy2xGNkw4ogeiap0XQaISirUpsedm8Vsxg9nME+McO1M+7JG6CEUD/4hB7IHhfYkP3mCgp82sRjVMOiQPVSxqvpZm4pg98lDoLFl5d4TZltJqqJ1KTNiB/38RrxMSeprvoMHHdx90Wmp3pKzJrBtMfhZRuK/aed8aGDZrKrYT2YXN8pEP7kCcqzGFC9oF4+k+/L3GS/aUs9fOSbw9jQ9yRpn3UkFpMyxjR9PdLD02Ql7JHPUpTvqPZeRUmridi4fOyiV9U7L0ieGf+njXWS8nLl92KsPtyuuRXYqmNbcXUjw1Gk1IT18674idww+fk/9QmVRMMOLsutbQ1iEnG4830dkLlsknugcnkRzJgiTzwtixSewKVBXlFkBd8+MmviTI3P/ZHMCIIikKA90uC1c79aET+MRQeCTwVFORnNZ9L8PPm54uPag/kbbM6JqMJsB+nJE7ckuI8vWZRf8dn/UsGfnlsKVNStI7Kzjn5SYelb3aY+6NVOici7K6k1TktcTZWiUQqDiYtpj+M8ZoJexdc0Ir847Hld1+z+Xd9opQQMKKsmNm2jhr0F5MD47HegqLwJKF1VIwUnK5gl0FeItJT2wKf9RZk18rsJONSYgwB2UI0UbE0MNqSI10bMJg0yC0bDCYMVr6WIrUjiyL/wEQrWViZTWGZjQoGnUULp6NpXwjoz5jCTZgxGE4apDckSC5xYOG7EbXnXR7+nRLtIwZz3+swrCoK92H9y7awwdqQfzKQILU1SafunQ7IL0vwB4zdtlNizOs2NPlbkvUa5hWd43KMT30iZZPbkt2+u+ASpRR7F42DeMv0tgTXgYy7n+Z6JhZqmBl78y8zlihI4IHTFZmJ04XMLY/ZdzWpXdCWJiiOjcA2NXsv5ln/qTzLf2eR8lVTplaRYvNtF2sAflFqvv6sHLnbr+5bfEuMTOHe+Fh7N4tfkhrK7gg5u/gowa0ZlG/DcEpm2KPxuDOxNN1DkvyR3pExZHpPIr7T24rciqJ11GRUNije7REVhfCsDaheDyW0rAOt58MDL/0gB4mjyK2Mg9e6gsWGeXHDrr0gMJtfUjReDdn7coC77owdoYUc3T0kEUjpnYTcasLuRZvBbMLENU3pQczWm3nSOzJm9WrCGuvPJ/TnU1Qv98ZML0X+mkN/xmBUVqhIo02pf1sn5bRKxnm7UQqsnlAUt18VzkV+VbgTsS2bhNNS7L0oL4TXFnvQwZTYGT3N9UwsVCMUibTbEcjG6otbXn9GHPoy2zGJCY2TJhPXh0x9JEdJnNJ4LYlAdNrC7BnMyExahWKAm96RbNT2IUsM17oJxftivGb6UusaEeLX6cjb7rTlZekeAmsgTKwwLXHrwynJm5q8HJC4cOQb/jgXNCEoiOGC3ZS0EnM4nhClYPmXshRuSlT45OWArTcsedAKypcsrL5iWDYISgmDaYmC3HoTyncDmifEVDixIbtqMP1xDAZ4Kw7uE5fsmia7KnWqOd6h0tvyezVPGQwOh0x+LHIdP2fQn9c0zwqGGbtSOtg9RWwras9ncTrJwaJXMfTnE9K1hOYJA2skKTCNs/Js/LKMb4OJmM7JmNwyNC9EhCmDYVWw7PqLohIoPJAGFi0qifSO8HCfao38VSy8/1+u3GNh4jgdGE1IEZ7aS4QdlJXOd+oTHxUJD7U/bVK9mtA4KZbo2a14bAck48L0lnTeo5JBdk3I025TY3dDevOyIzZPmlgDxcSVLp3jAmdFaYURyxFbuiOO0P05cZoWm3axvEk/qDH5qZgtaEtGg/mHBt6uj9LgdOSItvvjz/SBmLtZfdh5ySG7KpwBbYqXaPGh8ElLt8WxunbewmkZdJZsBvORjFkr4lKy8Y4x7vhld/ZLiqAArVPy73ZHILDa84JQpFZttr6U0FtQ1F9KmLiqOfQnIdMfjhhNxVgDKN4Xmx3Th40vGXSXwC/ICZFbNuhPS3zPPscBDaOpmGFV47YT3D2T8lXRZ1U+tugcFv5slBZDO68lsvP0tgw1tBKGltN5OnjqmTCgkMgaSLaFTqbN8Y1zFE5D4RehdD9m6w2XxHGwe8LDbB82CQqazJoicoWfGmY1fsmifTbG6ggUU38lonjdpnExZjiZJk6LF33+kYHV1zz+lRyVa1rCJeYN+nMat27gNcSDqnQ3QZsi7QhymvwTWP4bs2NsUyZNXiuhecJk55U0hYea9jGo3IhFOz8vbs/FRwk7r0BqxyC9l2D3YoKcTZgROiKTQ8wghVcfO/H1hBWvEkViCjOrsBzSPGHTPRYz8wM5dayhKCDsthhoZddl97U76iAozQitsQZKjubHv2pS/cjE3RPiy761jhFpKtcUmc2A1a/ZtCxDZOqjMU9hTKKJPcWhP4roLDmEaXWQd+U2xnDgnixqa1x/d+cNUjVN65TYK7k1oWamdj5HTikgO41W8kvlHwoZJb8qdLveAhiB2KCbI3A6CemdkL0LLvknUs8GQ1Fa9k0lYrVLImnuHDJQqZjgywPcxOCNlx7zoF1lFFlMv9Dl5ieHKd2SOq34OCJOabJrYgs0KhvM/yCgP23Tmxc1QW71s841XUvofLWD+YcFas+Z+NWI0g2TwZQ0Tn7eEP5BDL1ZizAniS+i0xINkhHKzjj5KQwnUvRnNeU7EUHexhpo+rMKd9fEiMXpcPcFm9K9BK9mMCoLAtCbV+LzH2i6hyC3DObP1wluV9CWEiioJiSW+js+lR+6THxiklsLyK3D7osu/TmR3xSPNzAUbH06QeLEFO8Ltp3aSxh+oU+/m2M0mVC+AZ1DDn5ZHAQzm9LAFu+osaEv5FbEJsjuiy+VX1S4dSjflCCNzJakwjzN9Uwc/XFaHsJgyqB0F+E1WorugkWYVRiBmJJlthLCnMIvGey8LNnynSOiwFQJNM7L8ZLaS6hfTGj+cl+63t+3CHyp/H/8l8+xvlei894k168fYuJMjVFV/Kiaxy0mP5Gud18CUzvvSkib/1lghF+SEmVYUbjfLIiu6HJE8aZ10HA4LUXjomCRubWE3NrY3aUrnf+woqh+3KF5Vo8tiwyxYFTQm7PIr0SkGgmcF+5qYskM36+IMUdQkF3WL4hHlDbH+Op1uQf9DyfIrMvC2PeaapzXFD90iR2IMorevEPtOZfe8RBroDBnhkSxycC38S42oBCy86bU8VtfStD3svhlzcx7YnNefymmcjPE6YpYceLK2B07FFeW/oxB/UURYJojaZrzq5I9hZYp3bD6OVqoKhTbxMGspnNEhHqpmtiY55djig8SvDoEGcXEtYDBlB5LVsQprn1MGh8jVHSek/x6q2eQPM7iV2I6ixa2EzFqu2TXYOLPXNw2VC4bDL47iVeTqY1X1wQZkSAPpqSU2OeURmkR05mBsJMGMxq7D80zGrsr8eKphnTCdk9evIW/EHTCGia0jjkC1rvjBmUW2qdyoGG4EJFbERKJ21DUX45onrToLJqEqxmGFXlMlWuaxBOhoNtMCPNCADFDadyGUxprmNA5LEl9ILac2hSj3+JdeYm0pUjvJqT2IoZTmupsi2jWJ6p5WH9eJP2neXr3SphbLl995TpLb66BqbHOdFAJ7D0vuHD1Q5O958ViczBpYIbg1g2Wfncdw5dFqVIxgxkZ9yoNjVPSGGa2koNp1tNcz8RCRUP2pRqJJR2305JYl9GEpn3UZPdVabDaJ2DnFYd4bLUY5gR2yqxLPpU5Usz8pY3pS7edewL5hyZ2X5NN+UzNtmgfhVRdZs5BXtE9FlF7O2BYFUcTe/gZo8fqyy46nE4YHfUJigm158Uoo/AA7L5E1OwHl9WeFww2zEtt2DhtjdEKC2sgLPwwqwQ2Wob+jIHVN6h+YLL7hWjc9IFyYwk/62ucpjFuzIROmL9rERQVrZOK/mKE24yZ/4sGmR3hFDRPWAdcVauv2XlFJNbWUBq8zjGpMbvz8ujDqYB2L4XtRUy/N3anyQvGGk2EfP/xcQDMTIj5foHKDVHAdhfHwW8tzfZbQmgfVhS5NU3ztVm0NTb7vecKKpBX47A1qb33XhI8eTTxOdpRjQjaNyvkHxr4JVlk2VUx2B1NSGTOfsy324DcskHlZozdk2lU//UBpi96/86SIdlGT8QOaFQR84nuJ1UsI5FgrgWZ9QNMfmBS+tjBHGmctqY/bZBdk1LCbQnxRWkov+tQvCOpIdrS1C9o+lMmE9fHOvjU2NSh91kuk0rGZmSvt2mdguFsLAB/IB321MeDA5NhIyWpKHZH4933yD9JxFoyram9qEldaJDYUHwk37fw7RHetkXzhM3OGyWaJ0y8NQe/LPfLHEnyXnZVds/OsRh7qCnfUGN1LOy8aqNMTTCwCeoevTmD7mEZq1m/ssfZoxsksfBurYcpnJZIWMxQEJTBlCK3FpO/Z5HaE4KM3U+kL4ig+DAiKGq6F0aMJuUEGk7Ki3L4G4HoqlJPuUb+Slbe/9pLQXZFPJXyj4QU0Vvaz4uSdObsugDawyk5VocVg/aZSFjzyylQsP2GBM6OqjLxsQZCMLEGoM906Xx7msGsTI28hizG5umxu52h6M/Jn51ugjnSdJckkkaF8lmitKhlc8ugXXmgzZMG7WOMyRdCyQty0vVqQ4767DfylO6Atyt2N+0j0hw1T6YwQxkhZz9Nya40JTS71F6IVxPfgJkfa3L/LE/pfsLq12Vn7s+6HPpHN8CQ+6W0lBTlmxqvrhkeCVj4VlemeGXFxGVxbEksqF4Z0roQ4tZBhwa50gBv2yLIg9Mx6JySwcRfn77Eqbltar+/QGIJjLafprjvjdo+bFFYFoeZrb89onVcOKlBCVpHLfKPYOJ7LlMfC7urdFvuMYgCdz895Sddz0TXbw0EEB4VBHtDQ5QSt9vifSF8bLxt47YUh77Rpn0yR2wrpn9kkFhiJ7n+ZYvE05TvKAZTkNmKiTwDv2DQvuhDI0WwFGP4BvkxkThVSzADRW9RiC9hMWEYmwxmDAoPZDer3I7pHjYp3/YZTkp2ano3gMSl+GjE+pS40eXWErbf1NhtA20L/yCzKY1ilJKf5bRlFBpmEVqepcndtcmuC3XQLxqYgSa3JgvFaWuROlek/h1WDJb+MEabEfVzFu3//DypXcFr3aaQV/rTBqWHIbHnsPuKI3TFkkHsivSkcxzM0KPysfjnlyc79Icuqbpkq8aFCKNnUfSG/MNv/grpDYP+qQRzbHQWeYoopQgrEWrVZjQJTdcivavxb2XpL0bYHZPqZU3+XpvHv1YkccAaWJIRkMhYePs1j8KThFHpc0TzS00v6On/83+J0zTwJ2KKt2WjTyyhhnWOwrF/2eT+bxWpXBXAeTAtneRgQQzIqlcD/KJFdnXIys+nSW/JDfHLmngiIJ0f4doRWTfg7y/9mP/LB7/E1HdtGmcUbksd0PQyGyKiW/+yiVsT87L9oj9KKQYzmsyGOghxi4oxs99VjEqGhCzo8dAghMotn63XXYaLIeVPLbFlX1SkdjXFxyGdJZswrQhKkHuiGU1IHWfEjCM2pUQwArH9QYtS1AykYYvSYkleviUOfLuvawp3TII8DI4FGB1RwTbfCMhddekej3HqhqAoMzGpdZMwrzn1xhNub0xjmgmHq3Xqgwz+9ybIrSf0ZgwSV4J+o4wms25QuRnSXbDE8r2v6R4SQ7jOESHiRCkYTUcUb1lj45CE2q8OcD7Jym4/KXzd7EZAZ8nl0j/7r34ize+ZWKjpKUmX1oYcK4NpTemuEKf9iniFSk68YKN7FxWFR5Kkl9mKaR0TdpPTgd6CpnhPoA+l5SbFtqLy91Z4UquQ+2aWzmGFPx1itS0q1zV7L0Dhgagx7a4syP005zAPM+8HkGicdsDWGzmG05qpjxMap02ya4J1xi54DZnE+EVN4aG8TGFe8rDiTMLsd8Unq3lWOvhhVYwYgoxBd0lq2qlPxTOgdURILxM3I5onLOyepnkuoXxdZu/ahPKdmPoZk/Tu+DNrhAoYSBR6ald+H68tSEF3SXZ5vyjK0OGkYrgYQqzASSBSqNBA2wmly5b4ALQ03SXhmIZpxXBGnAa9hsbuJ+y8YlC9LDzUMCemIb0FNe4XhHeR3knGjoyK2ivxZ2qOIzZeU/PR73xO0qUBvIaEoiWmJNL5RQO7p1GhdIrFRxH9WYkydNpKXEMq0JszsXtidjCa0FRuSDR3lP4shrJ7SLH57w5hf5g7iFfH0kR5mcjMvC9H7L4JRpQWxeioKihE64jN9msuqz+bwxpqvD2FEWmsPtReHOuV0prKzRGDmYTZH0djKp2mcB+SiYD0ioVflNtdvSSp2JltedDdJclERcHuRZtR0SQoCgLQOiqcgFFFkVkRkrYZiHyls2Qe1HjZDVEDFG+JAK94HwpPIpE0Vw1ZyMY4JtOU+z2aijFSEa+ff4DhxJw8tsnxfzXk8O9L5pbXSOguKooPxSi5+CgiuwyleyH1nx2y/YYE9MaOcHcTU16CqY9DGqcl23Uwo2kfMdCm1PgqUOMYJJvYEW+tp7mejYWqhZPZn1VjObTG6WhapxMmryR0D8e0D1lk17UwoWqCN+aW5UZMfdJj4qqgBLULckJktsTdrz8jGqn+ggDR6Z2EwXEf1bMw8wHDSU12uU/sQuFxQOJAmJWgCKclcJPpi79VakfTfNOXbKtzFqUHIfkHBtlViGd81r/kUbivWPlFORK1gsGUInXXo3wvpnVizH7KKfyJmDClmLiix0wmGC4FBAVN4dGA1K6oS72GJruWHPACOocMwjQ0T0jGlpgQxzRPmnRPiJ3jvnu0Ee6nQksZNP+9gNJ9SQcMMwYqVGSyIz68fALbiXjy3iIPfz3NqGKz93JC64RBnNK0x3P77VfNMQxnk7qSxmkJ1ly7KCed29Q4Xc3GO2Jb5BdlV5/+yGdYFW5u7okpadx9aYxHlc/RQk0cMF9vkluVqY1Xk3Glt2sS2wpvR25Qb06YOoNJNc6CEjwzytj0Zw2sEaS3DPYuKpqnBZQ3R5pRRVKrh1XJScrddEUO0bMJl3y238iTqidsv+7i7Yk/a2LLIjFCRX4lILMhevj5PxQ/pqAk/Fa/LF74xQ9dElf4CZgi0wApR8KcZveiZBRk1sUwze7IizmckN3OiKD6Y1vYR6fSZLblM5i+Zvc1TePs2AbSlAzWoKBpnRTS9/arJpXbEYXbJqmawFP1l2L8gikxmSOZVm2+6RLkpO7uzyp0OaCa7WMODJQCt6Uo3ZFgXbcmI2EQs4gwI3lgw6qi8EheiNI9YZaV7oihcpwSZ0VtywsCQrEMChaZLRngBDkZj2tDwi6s4ecI8LcG4P5RkSAr+h9gnJ0U47YinC50jiXk1hI6F3xiT4/HdkJu3nzTFcOvQB84/7lNMTnzS2JXiUKyVZN9uhycPLaJ9k2GU5ogZzBxPSIoKjZ+JsEvS7DE1KchnUMOownpmrdeN1n4znBshJuQ2tEMpvcbIRklFq/aDGYSmdO3EjIbssiiNBIkdjRh+oMYty0SDRWLK0ziCI8gyCnWv6rJro8x5WWT8k0R24HIp522WIwbocT5tI5Ywt4vCBPLbpnEjqJ9VGrD2BMZ+XDCYDClKX5hmzNLWyRakT7ZIrmfJbFFK5XdFPwYT1SnsaepXouY+6FMwUYlg/nvDejNSr1cudHH2xGCT1ASVUWYEdK404GdXx/RmzfoHBdIL7cuL0BQULROf45UqLErD71zIqF0RxZh6d5ofNw4ZLYTrL7YUBauuCSu7GL9WRGaqVg65ciVnXjqI3nT/YqmcjtiOCnQSnrNon5BSB/dw3BveYZ8tYfVl0akcdrCaWmy923snmQu7bxk058T39B9Xfv2aykhiPyyT+wJydkcQfGuEjNhXxhIpQcxw6pB+Y6P2/zsJSzcl4wlry5y7zAHk1dCwrTsQkpLCEVmW/DJ7umAKKUo3RXVrTmmy+VXhf6nEqQenBSyd3/aJL0pZPD848/0SuZIPpteHNIepLh9a5H20CPr+QQTYqqrEkEbElNj1m3mvxdRuqPZu2jRnbWwO5reIjz5pRTZzWRs6e4Rp8SO3hwqig8jiUafNOieDCl8O01uJUHbAvr3ZqXJ8sua1NbnyM1PK9h90aB0S2ENpYM0BwGbvxKIkCzRpLfFnTnIyTEeO1B4JBE2Xl1SNmqvR8y+N6JxSlCAMKfZesMktStTlMkrIXZHsfOySWYdluZrlNJD0ttypJXux0QZdbB7xo6iejU6iPCxexq3LfWzV1dkPkzTOZoQO4rKrRC3rekcFq7CsCqcWW1C65hDfjnBaSoyWwmJI7X4zlv6gDq38Y5F4oon/z4fYOUXFaOKwtmyGVUU3UVFmBGB3aisCFNS61VuBfhFRedYQvt0TGYnwasLIcYaQXZNGtXRfEj7nRG5zIjBVpbcXAf7D8psbpVwdyzxUJ0VdWp6R5F/JKyvYdVg6pOQoABuW1CVzLoYZIR5GFYknG79p0XDtve8xcSNkNx6TGrVJr0Xi+FFTQYevQVp8LLL4pX1NNczsVCVFmqaX1LUz8rbVruQR217EsdtKHJrEUFeFqsZSEfcPAPZ+zbJOEDX2bUkZnItEWeORObV/RmF3dPsvmAz//2A6lVhYa2sT7D6YIrGeU3vWIQRyFFrBlKOhFlF/azFxBVNai/B7gsM1Hg5ZPr9vojs9gWICxat4wbluwlOSzBepy1GbUYou31QEKyz89JICMkPTEkQDMQ0N7uekNoLcVua/nMjypdNojQEcyFRWnicQUFgIpFQazpHDIKckM4nPwZvx8TPKRJL0XhBtE+DafFdLU11+IVTNwDQhia8VKJ1Aorlvvy942CP/qSJijVxSthNRgibb1nCp31TH0Rtlu7FuHVZ3H4RJt8XMk/hcUJ33qKzZJJbkYU798P+mHgj9ysoyui4feRzJEWxBpr+guyso0MBsSM5pOktRXZF010w2HzbILOT0Bxn3fcWtHTMu5rMdsJwJmbhOz5GCPXziu6RhDMvLXPq/95jsBSRqifELqx91WHzFyL0a22ypQE6HTH/vYTKJyb9aZPcWozdEd15NDYp8wuSXNKfNWifiihecdh6K0PkKabf1+TXYuy+1MHN46IAddsJli94aus09BYVTleR2jQp/9CldUzsII1QIB0JczNoHXVxuhpz08UvC4Uve9fBPzpCRTJSHVUUo6qmc0h8q9qHTey+IALpHWlIG+cgtSbiv/6c+Pp3HpT4xvXnMQxhPnk1TVhIaO1lMSL5XRNXH3hCuQ2p+TtHEtymIkwrijeFjFJ8HBGmDUxfuK77DVd/XmKL9rX+UUqasdbxNOWbPbLrCXZXMf1RQOVmfKC7+knXM7FQw4yYauXWEtIPHOweTH08xO5raq/GhDnIbBiMigqrZ4jb3nje3zou6k2rZ7D7gif45Zpi7vQOdz84zN3/PAdOwvbrivTFOs7JDpYXMui68H6RyrsOnUVLtPZpRW/exIiFN2r3tcziPdlV3Kam+rGJXxKjscxOQvuoybBikJiK8u2Y4TGf9Kai/ryiN2ew93pE4YFAbrmVBHMkLDC3MR6PziQ4XU3jlE3pgeRHeY2IyUsJTksWzdz3OrgPPVJ7Y8JLSTNxVWP1peFLbMFzy3djUvWEwpOQ1K4EpA2mFEf/zZAwrTB8hbvqcrxcY+8dccFGA7Ei8TTZTfEk8Ev7WLIMWrKrBqOyTKCaF+R5bHzBFDl1RgYlpTsJWkkDO/2BT+zIqaK0mHVkt0KWfylLd9GgdC+hfdimN2fidJ5ujTwTCxVDnOXClCKzpektJTROe4wqAhC7DWFReS1x86tekhvrV2Dihh5rpWQSlTji52T9D2VUBBMzbbycz8uv3qdZy1H5Zxmmfi/F0u+YFJ4Im6l1IaRyK6A/Kw/eqwu+NypLep5flrQ/MwDGNz7IKPy8YjAjFDenl+A1Qha+YdJblHrU7mhS6zZRSkLd/KJYUlYvS4lReBLitAz23oyo3AzoLJkifz5k010wKd/1sfpw/3+TxWsITS/MSX0+KgmigSFCufI1xfYbit60Sf2sQ+94iF9Q5FcTtt7K4HYSCdMAPl1e4mvnb9J6NaCw1MbKhRTujDVrLeGOdpfkZQ2zjNls8hycmsn0hyEzHyRUboVi6dmV0yDKyMu8/Zp74FlgBIyVv47U6JsSIBLsE76fDkZ9NhaqtqWO8yuK+ldGzH8/prc41vpXR6R3xMGvcUYaFSPUkm9fEzC6v5BQvj3OPx1Aele8oSaua1L/c5HK76V5/E9OMvdnJp1FCz9vsPI3E/YuSpBZ/qZNd94Wq8aMLAK3IfN9r5lQuamZfl8wze4hwVNrr8SEGUXlmtzpxFK0jrq0D1lUriuGc3KsRRk5Pv2yxu5pcssGO69B9wgEOZPSvYTZbxlsv+pgjFlF/TnBGR/9bZPEgeO/M8AcCqguxGcxxc1sSjOW2EK9y64YYtyhofKJRfnWgOYJg9SupnXMQivBa+0HKT7cWkK1bLq9FKylyK1HNE6baGvMBGsIN9fpQH8pJijIi1K5ofFLcooEOdFrDSeF+G73RCdVuhczqirCgkz6hjOa/JOE/qFYAH4FlVsh3TeHny9rdGOkSNUS4WLeTlE/bWN3FUFec+h/FFDcCMXcN/dEcMF9D/qpT30ya8JBTe9JSdA4J0By+4jBsGISpgy6hwQ6SixoPKdJ3XOZvJSMXxBNqpEI+/+aHImt5yVOsjdnsPlTMbsvKurPqQM/z9wDS5yfLenahxXjwADNLyiO/q5PlFHYHUX3eAyK8Ww/wghFiGfEms6iwaAqNaY10hiRRJnHHqihSDY238rSPcJY7q2pfgqN0yZuN6F6LUQbsPvKZ+PiyJNR6fqXM2Q2BeNFS804cT0it6yp/A9pEjch90GKqBCz/ssRdg8mbsQERdk4gnFzlb9vYrzSElPhkwa1XxjRm1d0DhtUbgwORsjdJYHNdl8yKN5PmPxEGlCrK1DV1PtqXFrAytcV2Y9Sny8pClrcRlAS7jWcTsRTaVux8YU0TlfMw3ZeEUFd65RQ0pxews7LrtR7vqazZFB8GOLWFX7BOOA9GrEE0e5dlCZp5l2BbfqTAu9oQ6Qko4qic8hiNKEpXbEoPBBSt7dhU3gIk5cS+vMyMdKWQFBBUZHek92zdyhh5t0u/cWE9S+nMYeCERPJQkkcaB4TD36vIQ90OCXJ0IUnYic+nJBdPrumKdyX2X5qT5PeEobWcEJKnOyqpjsnJ0SUSSjdMnC6Ym9uDUTW458cMpwUX/4wKy4viaVonoK1n0pRuWRKONuGRel9lzALe8+bzH83JLOVMJxU+BVNYTki93t5tCW8C7WSkkbIhNGkS5SRxtYaiJdt4mgGUwbdRZPIE/QldqXZm31vxGgypnDLEo3YU17PxEJN7H3/pIjGOz5uXZw7hlN6bEGZ8Mo7d3CbYnejgdS2QW9WTBCCokiSzRH0p60DV77cWkz9hYT0dkjrmEin5SVIiMbmCaOKkIkTc+xtWlYYkSJdkweVmJBbFpw1tT1i8Vsxpg/l2xGz7w+pXg3ozcqIN7NmsP1ajokrQgiW30uUmW5TzIpVApWbQt6euCoaKJVA7TmBqlQCjRfEtGw0IZ8xSsk0Z+a7uyJxOSP1qdsSMjlKRI5eM8YvChZt9RTZT1MHGbLpbbF73HvBILuqmPlAclhLd8dlV2kM/Y2gP23TnTekOdtRDCsm3XkZJgymNdkVaB0zidOawYT4Gey9JMOI4ZGApT8XQvt+ppW25H4YAQymHTJrQkWMsrITP831TCxUa/gZwO0su8QpTfmm6NljVwRhN3/nDNMf+IwqMjos34tkZKqkmC8+lDqqPyuYqVcTUsrc92D7VbH2yW5HTH4M1jA+GE8mpiQzJ44c4ftpH+0jpjjVZSSUYbAYsfNyhs6CxfTHI5onLLrzLnvPO7TOSNMRvNalPyc7pBnAcFo4nJ3j8vlwY8xABgnlW3ICeNsmqVpM4UEix30sNuLWCIJ8QvFhLKFnhy2aL04wmlA4HUXlpk/7mGL7nYTMqgDu26+KDXrsir+BmJbJ9CgoqAMKZH41wt0ZEJ/rUXteYXfFWmhUETpf4kDhibCztMkBR7Y/o5h9V15Ur66pXk5onZZod7trYPch/cBh73mHxNbMvNsfG/0KMXwwl9A6ajCYkfj4/CMRZD7N9UwsVJUI2ym1ZmGESuJjpkXq69VgcMLH7mvaRxxKDwSzHFRMMjsxuWWpwXZekqah9CDBL4+ja+YNWkckeMxtaXqzFv0Zg9gxhIDsyxEcZhXdJWid0jj/H+r+O0jSbE3vw37nM+l9ZZZv7+10j3fXe9x12F2CIASSoiBChk4kpQiC0j+MkBRSKIIUQxEMigyIBgQIgMAa7MW66+a68dMz09PeV3X5qvQ+P3P0x5NdFyFRu40IINSTERtzt2equirzfOe8532f5/f0NYXqHQ5pnIf5dyfUz7sc/J5aLel6zO5zKTksp73C4m1NtWb/dhpvJPJz6V5A7Gtni3IR9XMumbtJvIGldGe4r0tIdKBzyKO37JBblZjDHen75lYdhhVhK8cltfHyq7LWdA8mWH5rROG2R+9YiIkk5kk1YkoPAvy+dAaZDUe96NMBSz8JGcwaNt502X2pQPbHWdyRoBZ+Xzh07JTil5YDuHdIc3+dUDHNk55i3WcMW686VD+B8k05JEZTFmqQg/JNy9rXsoxqMUFB07SZTyV9zD9SWTYumX92LlRjzAFjzI+NMTeNMdeNMf/O9M8rxpjvG2PuTv9Z/ie+5m8YY+4ZY24bY7715/0dsS+XYmbLUngQk9nUj9U9NiVNf5Ck8GgioW/FYTBnCAqG+lmP9inlgRYe6AKxd0F+pZkbAe5I/pwnySHZrYj+gZjusk/+cTzdfbQjp3cMsx+oo+D1Lcldj8yGIBHpbYvfC5W7VFIN2Tnosv26xe/pNJjkHda/4lC4L5Ng66iPMzH4PUP2oU/s6eTwB5a9Sxm8kWrJJ4sAhJHMbsgynnsc0zsU0z2qi9OkIn98+6hILr1lQ3c5CRaKNzyGs9pBU42InctKeDGxjuP2KVj8vsvuJV8KLM9OnQvyea18d9rqMnqfhocnpHdDNr4kbbCZZlyld0O8vlwQ+ccxhfsQu5pedU9petY9ovraOoqeL902JOtPLCxQWBlLFTZ1LQTZf3Y7agj8+9baM8CrwL9hjDkL/AfAD621J4AfTv9/pv/uLwPngG8D/5kx5s9UHkQJtT4G84JqhRlB0JJ78hBNirDyXZ8gZ8ltCI5WuhsKifhQ9VWQN9SujnDHElW745hUc3rxMTrW6mc9Eg318Aazeqr9vnZRfUhWF4CMoXQ3JrceEeTNVI/q8fJvXSX+1QaVb2/gfbmO39GiTTVjuocMxVvCs0dJmQtLt1V7TwqWmWsRpbsh278yxh1aGqel4M9sx9LOugqpaJ6bimxCqF554mwFv6WLV3pHNWJ6R60gZ6K2WeE+DGuW1glfdX3L0Dns0D2oRn/9vKF0X+9H4b4oNImeatzCHU/uikh1bu5mgtaxBKkdl9yqAHBhztI6liDMgjf1d/kDKD4aUf0sJLMiBOWhPx5NbTqGKClIRucENKd1dX8xQeVWQOXGEHes9to/k4Vqrd201l6Z/u8ucBNYAn4d+G+m/9l/A/zG9H//OvB3rbVja+1D4B7w8p/1dzyRuZXvxnpydyzto674RSXpJ01oiDJCkRfvqpab5CWJS7b0AQ5mE8x9GFC4D1svJ/f9TgvviIuff6wodL+nMWPxYUjrlMPmN0PKtwPlsa6ISmcdQ3Z9xLhs2Xs55vFvh/zkwXFaD8us75XgDytMaiGFxyGFu13hLcuaYPWXtauOqrqgFO9J7dU86ZG+lqZ1Rjf5VGOaJ/rBHoMjAf0DEV5PO1uQMey8HhFmNCErPPgnMpnML+N5rKuTJMxodwyyU8duQ8qx7KbFTk11e88ZKjc0QQrTDqOyhhrCZsbMXB/ROezROxrSOR5LqF5Wabb8I7FmJyVLbnXIJA+tE9BdTtI55FG8L31F41QKbwiTor62czKWzb2uky7Zith4w2P9ixlGFbUln+b1T+VCNcYcBi4D7wFz1tpN0GI2xsxO/7Ml4N1/4svWpn/2//m9/jrw1wH8fJlk19I85TBcCJl7WztV98sDvBtZhksh+Tsew1mHVEv90d1XDF4Hqp/EuBPLxpcVNOYEWtjJpuweUcLy8F+1FD7SPH37ZYfaR5o8tY941D4J6W37dA7pRuuOXOZ+EbN3GYbVLAdeXmOvl2U88fA/zJNoW9rH03gjSG557J0zNE6X6B8O90UkxbuKOy/eswyrZqoA0wVlknOwxmFcksZhUHNJ7WWp/dyTLO9RTJgUOnPmQ5egIFdtnFCLKsxA+c4EJ7DkV2E04zMuSVUVFAArC3iiZfCGam/lH4lv2l80WNcSFKCdUafCHetEiz3FawZZyN/3CPLQuBAz956crd1DytIq3IP1L2dJdMSgneSBWBpZJ5K6yrq67Sc6hsy6M9VuKCBu7Wsu8+9ocQ5nHCaFf8aXKWNMDviHwP/GWvtnTWj/x/7m/6+K2Vr7X1hrX7TWvmgKWXafF1gsf1/tpcECzP6DFO4Q5n7uMKppV+nPOSQbluIND2+o3uK46FD5xCHR1Rv0hMqcaGrHWf5dTxbspCG3Yth5UbtIdjsmTKqp3joXs/iLiNalgP6CsI7dcxO646T0mtsZYle1dHpbus6Ft0P86QBg4cfO9EPXRWNSiWgfe/KBWcZFLaa9y0Ja9peV9tc6BY9+JU37OBTvKx5yOKsSwlgl66XqEBTVrkvXY7ZeSfDwX4GtV5PsvKROR+VmQPlmTO3TkHBpTJSU7cWJhB1qnrcU39xm5yU4+K1H9M5OGM5JstiZiqvHxWkpU9co2W879BbV9iveVY2daj8JU9OkrXJrzLgM3SPqq/aWHDJbgWiIuzrpwqycuwDZNYf+nEvzhMvcT/f2lVh/3uupFqoxxkeL9G9ba39n+sfbxpiF6b9fAHamf74GHPgnvnwZ2Pizvr8/gNIt3RR7R9X4TrQN7WNSGA1mRd6w7hPDmqG/ZPE77Neh2Z1ItJPpYq1eiwizShhpnPKkAcjoe2TXDP1Fgcqapx2yOzHzPzfsXvTIPPSxDrzwhdtkSkN210q035rHHUq+Npy1VK8FxJ6hPy9oQ5TSouoe0Zue2bLk73lkNo18/RmF1w5npCoaVc20/QbpXUPhnnYnfxBTfKB2UlCIyexEzL+r3+vQ9yLMmZ5gvNWIg3/PUTDGnoM3gPWveGx+I8QdWWzPI9mERNMh+KsNckfaHHpug62VGU4895g7Hx2kXO0KWmEtcdLSO2T3e59B1uynJDqherQ4OqUGVUkG8w9Vsu1dTDE8EGJiYYMmJc36M1uW/OMJwzlDZgOqnw6JPQHm2qciEm3Y+nKVKPXP7tZvgL8J3LTW/sf/xL/6R8C/Ov3f/yrw+//En/9lY0zSGHMEOAG8/2f9HbGvHl39gqH8qSZUJob+csRoLpbd1texsv5XJmR2IvyeIcjD9muWzE5M/azaJuGbbXIbkWBcE9Hl8qsxhUcidXh9NbhnrlnaJy3jiuh446IhzFoym6oJP1w5yGA7S+6ez7gs52n9osTEg5qHifU1o7mQufc1xqxcj8jsxiQ7MYm2xqCF+xoZNi5Y+ssWExn6B2KaZ5jWtZZ0M6Z6NWLjzSkzdMdy+HsTtl5x2X3OI9m1gqx9nFdiStuhedonteeQW9PMP9E0pNYStI944FoGb/QYHxvR7qXo3y3xaHMGtzDhztoczvKAdjtDlLRT/pVh9kMd2caqC1H7pC8CS2jxewoGnpTEUnUC3e6dUE7g/F2PuXfUuss9RqS+vGHvuSSVG5JYrn4rxXhGmVb5hy7ZnQhrDO746XbUp6lR3wD+ZeAzY8wn0z/7D4H/C/D3jTF/DVgF/gUAa+11Y8zfB26gjsG/Ya39M+92Xl8o7qAoq3N/wZk27F2iBHjjiPSO3JfFt9IEGX3Io6ql+pFD85R2gNiD4adFrBuSquuYTDYdJSk3ZL8GqH06oH00TfXKFA3uiOGZW3WwrqV12sJuSqXFjCW1p5t/sq7bfJTSw9FfdMg89uguCxRhrD44v6vfq3IroHlCt+FEW3ra+nnDzFVI9COGFZfMlpXtuaMaNMhDf9kQ+0lmrlsaZ6Vxdcfarby+YVyJmVQjylfVTOkc0c5d+UwmwERpzJFaHYDNToHB0pD/2fl3+Hn9GHuDLPVmjrjrQyEk/akPxjAq6RIW+9C4HLE1zjGqWdyBwQ3UkmofFwl77qMJJrI8/E2PzGMXd6hTT/RsC9YCajeuf8Vh6S1lGiTaKskA6mddZm5E4DydFeXPXajW2p/zP153Anzt/8fX/J+A/9NT/QTAuCw7brKu4jxVV6sqvyoM+n5w19QDnuzEmEhW3PzamOFsch+sm18Xrz/ViNn+zTH+9QwL7wQkWgEPfz2N3zUketKtBtmpwzKviU52S2SQxEKPODZEoUv6s7RSVk5aineUZxXk5HlyJtOM0ZWA1rEEk5ICLFrHdBy3j/hTRbtUS42zDsW7lr3nofaRQ6oVk9qd0DqVwoTTD3JeHK4gr5Zb+WaME8YEWYelH3a489dyVD9wmBQ8RtXp+zcTU7lqaJ422MNDasUeL1ZW+aPHZ0gnAjo7Of6ra6+R+SCD8+UGXz1xh5vNOXaaeQZzCUazMV7foXU6JtF2KH/qKvzjgQI2kg1Lb8ko9vNxTJBz2XrFoXhDF9RkS6bBmetj6lGSZNvihDFbXw/J3Uiw9aoGAtGUrNK6EFP52KH1V3qU/7vcU62RZ2Iy5Q3Zbzg3LsgPNJy1bL1hKd0fs/mGy3BWDs/+kmo9HCRmOZoQb3RJNZ+JLIPX+4xmHOx6GmMhTDkM5pPkVsx+nGWyJXBE7P2S5DGoTfE1zRRfOXqXWqVL5XZI7UqP3Ir49P1FCUx2X4mYlPVQueNYPqaqXKBB0dI9KhHypKTG9syNCYX70s6Wr2tRdQ641M+nKDyAzkn1bGPPTgPTYDgbs/26Zeclh70XYza+UuTY39VFKdmSya+wGmJTEf0DhsVXN4gmLr1Rkn+8co7eIEWjnSV/2yfaSzKYt7S28vTDBEkv5MWDq1SvBsy/o/f28PcC8ivy47tDQ39Z0Lkwo00jKFjKn7ToLbjkVtRaMiHkHg8ZzFt6ywmCnOzsw6pD+QOfOAEzn02HEXVL6d6E2ru67Zt3itTPfo7MfbEPubWY8p2Q3COX6mdybfodh73zKdJbsqW4gWXh7YjeIUnakm21f7pn9Ab3Fl3q53wO/hcuXt9SvqHLU39eNpNxWXS/8r2QvUtP3JtM+aEKKfMGkNj1+OG9U4wDj/UvK657XHkSQyMKXemaoF+TguH+X5VtOE7FLP/VB+TP1aEY2oyd4QABAABJREFUMFiMCQtKOeku+UyKhs5RXU6WfzRmVLP0lwAL87/QB59oOSz+bEB+1ZLeckhtuxTuQXZFwImNL2SYFAz5x2OsC+OiS+Ujj5lrEZtvLzFT7TIc+UxCl0krycw/SpPdkAPUWHAyIYczdX5z8WPee3iYjS95jPMOu88bHn8jQfu4AB/V6yEmBMw0tO2gQ+kWbHytMo3qjGmdcOgvGTbfyJJsSdBduRWRbOprOkeVPrN30TD33oC9F2Me/aq/75tKthRm/DSvZ2Khmmiq+/yCFsXGFzQazK5rcjRY0OSoddxl53mNNsM05FcnjGox2Xs+zXOaMbtj2Hw1hRtIiMH0chB7ULonZnyQdsitOmLdj1VfNp+LSLQsTqALT9RJEFlDZsOhdSpLZssST9E11lGuqizVUJ3v0D5lwYGrnx5mqdCBnsfC2R3IB/g9Q+M5y3DWTq3Vhr2LKZyJISjF+ENL/YLs2ZOipX5e0NDK7ZA4YSndnyjwYSyvf7Jtuf9XVS4Nag6jqmHzTcWsN+5UCOtpwhsFMo8UZNE5olq6dmmbSrnP37v2Av/pp1/FthLUrkwnXCG4Q6m8Zj7r0z7iMa7FeAOzP5zoL6oOD9NGgXUJS+GRWlyJjk6n5kmX3JZoLOldQ7Kjhv/ecxmqHzqkdh2ZLbvQOgkzn32OXKhOKGtzXJ1IHNxS3eUNNPr0u7Iw5x5rlBemYeHdEWtfTeB3HQqPYqK8bvXDmm75vSXVoH4fgoKdmtCC/dlyqm6n7RJBgnP3p+S5XYF6vY5Lt5lhdGkwnY7p+yX6MX5f4bVYaJ+IqTdyxPkQM3ZY+Bm0/28HOfK7ITtX5sgVhmQ2LbUPIL2jv2tUi0nvxkQZS3rdZfeSxpyjonatSUE77/ZLLuWblu5yQg/srNJVgqwhse0rGXArZvEXQ/IPlGqY2nNY/BHMfhxTWJGewQlgcn7AxqMq2cSE5w6tkUwFzB6tMypJcxolLUEhpr8U8+hXsgQZyKw7hBlxFryBThsTSQoZZiGzoR5ufi2UE+L6iCBvSbRkrX6imgozOoWc0DJcDEnv6PTye4bt35j82YvjyRr557j+nvoV5GC8FFD7YZL2uYD84wgTGnrLijsv3VOIWeOsFi3AyreS5FdgvDwhShpKn/qkGvL/x74hyAtoG7sQJbULdQ8kGCxZRlPPTpiV5aNye0z+cYx1DMOyQ2rXcPn1O1w4ug7I8WqNFnx32cUdWfa+EJBsWmwm4vSBLapzHYyF7Vel36yfSxKlLEHgkW5E7F3S3xXkDPlHDr0lh/wD4XkO/GAiB8CcKNGxp/TAMP1LBPmoqqnUcNZh5tqY9Lbei+5Bh62X03gDy3hGzZX6BTXUO4ccrCfY2+G5Orm5Hp1Rkhs/PsHobpHBZJrKMjYU7hkSLYc4E+9rWE2stJbs5oRkN9oXmWc3tXD7S8Jj9uY9egsua19LU7luaZxNU7ob0zoDg1lhhWY/Dtj5RsChP1ByC2Zq+nv36ZDTz8RCBVj6I8XeuB2PUUniXmOlZuoecCjeG7DwjtQ71gNvpESQ4icJdl6NmP2gqzFiTovhCfFv5nqA9WD7W4GsIW150P2uZVQT2Kt+Lknp4z3pWQ9o57y2tcBnd5c5OrfHsZdW6R2KWf5xoLoNWPwjiVcq821u3lui2clw+YV7xNUJrVPQX7SkDnWJY8UJicekr3UCtcUmJQlHtl9KMp6NyG78EmcUJSDZEEd0NI1sLz6IGc1Ytl9KYmLZRhJtS//8SGifKaMLK/miN4DOVweExZAHny5RSI84Xtmj/PI2S5c2cY1l7/WQ0XxI86WAo19/SO1Ak+TRDnECshsxqb2A5okkraMenWNTkU3eMJizxEn1TqOUIbcZ4Q5h93nxpwazDlFWgRKDBUPnoEfuapKdF328gdyt/SWlWz/N65lYqN4ABlWH6mcBuceS8AUFCZn3LrpUrwZsvpFl/csexgqfU74p+keqEbPwU4fmmRzZrZjq1QC/p9jw3Jph95JP6abBNH1mro3IbqipH6WUCVC6H+ANLA//xVnyKzGl23rjMn+Sxwxdtrp5PEcxlQ//JTXtMTDJKQxj+H6Vy6cekc+OuLE9j40cgoUJr3zxJkdmGrjXckyKhmP/w4jlH/SJpp4hJ4Ty7ZDcekxm2zL7jgQiqbpGjkHeklu3jMoOiZblwPf28HvqR+bWJVZ2x7LslH+RJLsd6r0Zw6Qc0z0kSFww8EnPDLn80j0279f45PEyW2sVmoM0k9DlO5eEV8+Uhngmpt7McXlhnUkxZuurCqEbVwyFlYgwJzJ4disi0VY+QKIbU1gN2X7JIdWwZDcMyRb439xj8ccibWfXBe4wsdwSUUrv42QmYuuNz9FlKvZ1zGy/5FO7MpB6/OOJjq35kO1XpB/1uwo3eNLVjX3DcNahu+zsj1pHFUEZJrUIE00/7IqhdMuw/VIKN5h69RNq5tfPJAjyAqlN8hqpOpFE1IU7LoM7JVqjNDjw289dIU5amhcjor/Y0AQthE8+Pka7maWYHVKa6eF4MZ0gxY1Hi1gD1asBOy9kuPdXhFFPNnV8to57tI877L4aUXgwpLAiHM/cBzG5VchuBWR2Y3qHoX+0xHDGxR3KAFi4a2gf9Smsjmm+MWYw6wmPvhoRF0IyG4ZjL69CZHjjwEN6QZLichvXi/nmc9cIQpc3DzzgRnMevzRi/KDAwWyD+WqbtBvgL/fJ3vep3BS5z4ksqR2det0DrujZR2L2LnjsPueRaEkcPcnrNJh8v8qw4qjMygv8W74T4A91uUo0Del1F392+FRr5JlYqFgtuuUf9ln9Tpq9y5ad5xNYA8ldl/KtmEHNwe9B/bya8+5Eo87cusQapfty8nUOO3SOgt9wiVKG3Zc0fu0cm8anV2TimxRlHszsxOQfR3hjQb78ri5a5esCsBXvws6VObyZEf/g3ZewDvxHX/kdRhOfV756ncSrDVI7DsaLyfgBX1i6Tzo74bMbBylcSZJdt9TP+9Q+HpJddacOT7GwrCOriDNy2H4py/bLDu3DLt4wpvrZkJ3nE9TPGxZ+HrHzvMewJrSjdWS3Bti9mIK2v8+lsg54ez7jMjzYmYHQ8KhXoT7IkvRDvnb4Dj9bPcalhXXWByVWNyssVjrECctHewfY/WSOH948jevKGmMdw9yHoeDCBQnFsUzLJ5FZsuuWwaGQ1lH5qIoPZQsazqlESDV0cV39jkPzhMt45pcRmpmffI4a/k+SkzffzJJZNyy9JU9Nfi1i/p2Q5ilZoSd5RY+n6pbOQTWK20cdshuGncspohTTS5Tq20TbktpySTbBLo0YHIgYl/XEBzkrwEJGqcrWwMaXmNo82MeP1z5oSy11M4sJHHIrDv/H3/0XGHaT/Pz9swxGCY5/8wHz1TaPtmZY7VewHxTJrniawpy1DBZi2sdSUuGPpR1d+EWfcdlSuWqofQiDpSn3agzdJY/Vb6Up34lIdAyDWZdkS3N4kAqscE9HuxNAZl2MJ28wjRq6ptIi3E1z8tQGtVQPz42ot3L8aOUEw06Kdz86yc5/dRjTSLDz00VSOw7Nn88z/05E9nqS6n+VxespTXFYcWmd0o2/9CDAmbIEnnRlwoxh8YeG8r2QcGHMoDa1mszEJNqG/OqEZDcm91CE7NonMYMl0Qj9wdMd/c8Ewz+1eMCe/e6/y2Beacyxr93POpDooiTjGbWaMltTD0/W0DyrcN3ap/IM7T7n4Y10CfDGgudmNg3DeQmH59+ZYF3D3kWfIGtJtJUm0jsoJX3lljKSwhSMDgTM/sxj9wVLZtPRFOhxRHfJ3beOZLbU8I+TMPNZwM4L/r41GGOpfiqtbJhSJ2JUkTYz9sUSSO0a+ociFn8Mw6rCKiZFLYjMlqASwxn1lmNXduzknsO4GlP7CEHZbsZsvW7J33fxe+oJV68PGZd8xgWX3a+PmZnpMflBlSAnnYL1JJZpXFB4buHR1KPWsOQ2pU7b+ssjeJAlva0LqjdS6y+3qsFLuh7TPuLuf0+vr89uklcZh1G0fOc4ZDYNpXsBG296VG5YEl31s5unXaKE5d7/4c8Pm3gmdlR3DI0vjhnNCiARpdQKArWG/J7ke7VPxqQaEcMZTYJSuw6lezFbr7mMiw6JjoQgyU5E7MmtipGGIFlX6t/uc74SqafDhHHFkN6d2olboYJmizF+3aN4d8Dce5p1JzqWQU159daB/GrMYF5InfKdkM03PbIbips0kdT3QcZh7SsO/QWXxq8PCLLQPRFOd3NDdiNm4aeG3qIrIYzR950ULY3zUvm7Y0vv9YEGDZ5l9pOA7JrDJGcI8jG9RYfUrstwQc5Xf2BZ/WaKMO3QPgG5qyny//c8vReHFB9oSFH9RL3R5J7LeC5i98WY4gOZFfcu+LSOeVS+l5HweTcmzEtZlV+RLLB1Su26KCFBd5ixHPqDJr2D6tvGCZUI/WXlhKXqMeOSy8IvwineR8TG8q2Iys3P2Y56+jf+XbFHezG7lzwqNyOaJ5Qo7ffsNCxMrk5vAL3TEwrXE8IdXglpHfPIP47oL2hUmtqTYHn362MKH6XoHZDSHQvZ7Wh/pyo8nLD9cpL0ti5Z4+IU8LAeMaxKNNw5ZghyMaWbwvlUbou/H+SkwRyVDf5ALs69r45J3U2RbE0dqg9Deou64DmhnJfJlpyvxYchraMe47IlqMQc/MOYcdFlkjfkV0N6ix5Rctpc39a4eFJUbafJnCWzG7L5mkeyYfZbUzgwmtHtOvtY+oX8SkxvWWBdv+5JpP7QYbBgKd+E9jEpxAZHJ6RXEkwKAkos/mzM+pcSpPYMlVsTOod88ushk7wr8FoI/kAR8NaVPy3MWKxvqX6sKMnsqhIZE23V/X5X78vWay6JtuHG//Xf+3zsqNZTbdU6KahEcTrqLN2Pp5aLCe4YvJ4oHO4E0g8TdI9FWigld9/Z6Hdldy4+DGQafJTCRPLuhxnVdcMZh9ZZS/OsZe9iktqnAVHKMJhTO2HxDx7TOSwbR+HRaF+0ndmLSHRh8zWH3gEHZ6IpUn5N4AcTA22fxZ+P6B6OyW5G+N2QcUltsGFViyFMaZEOKy6V2wGpuqH2rsPaV0QWGZcNOy/4BHlDcSVgVLMEGbEHDvxoyKSozaV5FtqHfNyxmYLidMFM7wqI5vccCquavfeW5JJIrfnMXNXuXb49IUpq0pd7rNp/+R+7Ap/19F6sfsfXqdS2bLypAUHnoMckZ+gc0wOz8UW1Ep8YBE0sJf+w5nDoewFxAjLbOt2Wf6A0wXHJIb1l4OnaqM/GjporL9vlf/PfZ1KOyT90mLkxYlLwaB2XU9Pr67/ze6p5co8M2Z2I/qxLomepX7Rk17VwRlUI8jFYQ3bjl0Gxw1lL6baUQO4IWqdUp01KSuvIbURESQeslD7rXzYs/zimfcijfzBm7l1BynaeVy+XGEYnR5TfTtI9BOVbGifm1mLaxxwSbabqeHnrQRCH4aze76AcUbzhkd6NRWvxFd8OUneBRBsy7VmaZ+Tbz2yr3TaYl8XbepCsT8MdjDQNfg/ckcoaMa+sRD49DTqemBCH85bMuqR6ia7FH8aMCy7DqmHuoyH1M6n9JEUTT7lWGUEshjXV99Y1+0kszvSBzq3FOKF+j0lJp2Dtk4D6OV+GzbIuquU7IY9/O2TlX/kPPx87apxQE7v2kd7gzVdThCmH8p2QZFNvUv5xxLCmS8+T0IZgOj+OE/L49Je0mx77+33m3522nYZ66meuWdrHHLyBpX4RKjfsPgbdG0oQvPUq7F1yGJUdKlcNjdMewwVL7pFDqh7SPuJx4PsiJ6f3LCf/HxNKdyfT0Ad5oUq3esx9ECiMNoZE21HQQxJmPlMLrXYlxm+6U+tMTPeQxe/Y/XQU6ygYwhpD+U5IdjOQemzK0grySjCZuRHsh1RESSisRArkOBYTZgzjimWSVz38pLNiLPs5WqVbMJqF/rKmTdJK6GRonEpRvTqg/mJE96gestId6SAGc/o5iw8DLdBIUs3845j8asy45FA/L32tiaB8N6K36MnCk1CXJ71jMbHFX00+1Rp5JnbU9PwB+/Lz/xadQ57qz2XD4MSEhT/xxCgdaKfIrwWMSx6Tqdd+VFHNlmzCpMjUSCZO/tZfmFB6Jzn1+KsfOqoYBotigBYfxiS6Ee1DPiaWZtQfWExkqV9wSW9pJu1MU61NBOmGUgCb56D6sWX7VXnv1W+c1mdZXZJ6S+KFegO5T6OE2mDDWUPlVkjruEeyYffTlUczhklZwuUoIc0CyNxnQnFjnwDSCo+kex3NGNLbltxWyCSnGnFYdegvW5J1KezDzC9DdvMPxPfavSRAhdc3jKsxhbvqz7ojaSIGc9PTqQZYmHs/YOd5n9FcRHrTJbNt2XtZqdLW6IKYaKjeHFcsR/9Bl/WvFPYTX9yRTsUwK33u3PvRvuDGH1re/of/u8/Ljgqr33IJsgpU8Psw+2OfxlmJlXsH9X+DOV9Rkl1NkXpHYlJ1vRn5VfUUe88PFTT2oyTWSLZmjXac8YywldlNS/egI1ZqWbtB66TCg7vLqtHGZUOUVk925rq2skHNwQktpZuwd8kw964uR82zltidGvjmZAdJNi1z7wcUHkW0j7p0DkuO547g8bcNxQehhNIJgz/QzxTNBPgdKN2B0r0JWNmeF96d6Ocb6qLSPurQPGsp3o/pLxlaR3156s+p1CncF5e1eH/IuCQIhN8xpFoaPsx9MMYd6xjPrDsM56Rv6C/qd6x9MmJUg6W3RuRWLe1jPpltS+mGw/KPe3gjy4E/hv6yRrh+W5YhdwR+x/D4mwVSdZEFo4QYA53jKmvKNy31cx7uUBjQUfHpluAzsVDdkWBZk5KKfHessK/ZKyH9AxHz74xFJ3akWx3WZONIbzq0T0ckepbWCahfjsl+kqZ1SrtymNEum96RJ6p6NaZ7BBrnprtfWpE6rTMw92FM64Sn+Jqh2mO1Dw3u0LL5RlLEloImX9YV4Tr2hB3yuwrr3X7dEtYm5Fdkm2mc9dl5yaF/QIa4/GpE+2JAes2lddwjyGoEGyVgOGvwtxMEebWAtl9KTD1feghwJBZ3h4bhQsTSW1qwT5r847Ih/xAGy3IbpHYNu5czOJHafZlttdesC4N5n9onIcmmOhDz706kOd0zUzKNR3rbsvN8CqZtqEnRMKoZ7v6VNO0jDt1lj9n39UBl1wzZNdlMwowGAc1zluZpqH0aM1hU0FrvSChk+li91yfxS0/zeiYWapRUflNmEyZnZZTPrqEYx2sOnSMJhcZmVYTPXB8xrMpOMvOxQ/O0bpWFuy6DhZiT//k6Mzciap+M6R7WhKt5Rk9w5Toc/b3hNDRNg4DKZ0rcC1NK68huxsxctQwWdNTOXglwJ1IvhZlpeohr6S86pLccMhsaNZauO8z9wJ8a/aaOgZ4h/8BlNGOJPcPyH6lhrwQUO43smZDZssy9FxPk9bNktnQEj2YkrDGB6sziw5ja+46CzQ6GzNwUnjy7IXzlgT/RJTPISz6ZWxPSvfWVIX5P/c8wZWie8EjvarEEOZfGpYjKrQmJjqW77FG+PVFg72Ed2dbA0k+H5B4JOhf7mgqu/JomhMm23Q9Pm5Rg9n19Jk+yTjM7ESYyTIravfOrmiJ+rkC+Tqg3ejhnYCPFuKQfvnRPb3LjvHZav28Zlyw7l1P7yqnGBUvtE+WGRkmofgKd5xfwBjG7l5PEiSmduakdMErAzgsZkg1Lf8mQni4IE6m94o5Fhm6eUZSiO4btF3z6i4bWCYfEVA+78HZI/2BEatcynFPWqC5uZh/Ua0I5OzsnQ3Cgv+BgjTiqJlZyyqQArRMJBnOGccGhfEPAiCeXqvSehh1+VxHsWAF5h1VD6brH3nmf3GZE66Sh/pylddxnVIPDf2dtynt1SLQtpR+l6S8aEh3oHmJqSVcjv3HGxRk6bL+YoHlGDP+NLyb3swcmBeW17j6XxhvqZ4uSwgr5rV9C2/y+0EuJFuxdVrJ184wEP63jHsWbLtmNeL9z4fUVGfpUa+Sfy8r7p3w9uTkWHljm3p/CdifQOu4wLhpyKyItxy7kHqtEsEZImOUfxuTvdpmULPPvDkh0Y+pnXPpznvSmdSEhsdNm/jQXalLU5aV3GPoHYja/G5J6pc6hv/CQ8WtdrIHNNw3tE3qQ3LH+mWhLw7p30Wf2XSNuwIomMU/q55kbI4YzCgsezBnmfuHgd9RK2n7ZIb0XM6o4DGdkz062LKX70f4xmF2T4qh7XOFow5pSWDLbhu1Xpz9LBPNvNaheDXAnMeGxEYundghyUHhoufXvLBL7kuT1lg31FyKSLRjOQuWmJdES18uZqDzKrguOkdpTKMaT3vH8O32WfhKR7EaMqtOsVyue6njaekrWNfiwRtGevYNK8GsfVz92UjDUPh4T5rQJmBBK9yMlpnye9KgmgvyjASa2NE/KV9M9pMvMcMFOVfhKMKl9MqD2SczMZz3GZcPmGy73/8XCdH6eYP1L6t0pXRkqN0P2LjqU7oV0jlv4zTrjkiV8ucurr90iKEXMnt7FTwd0e2mu31nmcLXBZCmA2pjY0wg1TGl3zK1HOJH0rI0LhvSOQ5iW7nJSeMK3ShH7yhFIdPXhZLZ+GY6WXxlTeBQQ+6KIJDq6cGV2IvZejveJMJk1yerS22ofJdqW9LaD39XDsvtqmfUveax92SMOHOrdLN4Adl/Q1Gnx5wGdgy7ZTcvCW7KqOKEe8MZl6RBK90XWHs6qvTaYF+Atu67fceW7GXaf89i74OGOZIIsrEbESbGlQP1eE0FmL2ThFwMSHU30SncgtWdxh5atV5IkWhZiiNKGrVccqdR+/vip1sgzsVCx0D2UYVR29ln3hYeW+XcjvL5GiIOaQ6pp2Xw9w+ZrDnf/5QwmhmAmpHxTiJnH3wJvoMIeBxJNzZULD6B5ysPvOsR2GhMeG955cIRMrc9uo4DrxjgP0xSv+dz57ABfOHOHU0vb+Me6tE+qfSUuqkf9vDKlZq7qw3UCqYysC90DOt7SeypLSvcCRhWLP7CktwxOYGgfTdFf8Il9aL4YsPKr6pfuXvaYfduQaFk6B5XLGmZ0Y3YmUH/OUvskwB3rstg7qB0snhuTKw0oZocM5y1LP9HFb+MLvvhVVUN/mt7nBLqsHvu7Y4KsHix3ouREN9CFKkoaRlXDeNpKyq1JwzCeicltROxddNULnbKlchuhqDQnfDbfyJBd15RrVDGMKwZ3Aos/G4DRLj2q6GQkhq3vHnyqJfJMLFQTQ29ZzfjhwYDcip7Q4YzGqd5Ax93eRcU0Fh5A+TPVXvM/dtWktlC86SlBetq4TnQtxfsaNaZ3lM7cG6TkIhj6ZK+k4cMi+V+kSf0wT/6hJkezH8C7Pz7H/Z0qhcyIzIaDO7J0Dxpa50Nyj2HrDbV9wpSmXMmmFmyQV104mHc4+Mcha18RzK1zyGHxxw2cAHZfjdh9VQEV5Q99/I7L+pccopRaW+2TGmaUHgQs/TSkfN2Qascs/zCiedJn7wWp5mNfR6gdu/huRG+UJJif0D4iaWNmUz+f37f0Dut9zGxNBxyXMiQ6luYJl/6yvFZBVu9vlILFn3RxAkOyPsE6el9MYFj9zZhESwnZ0vUadp/zidIaT1evBkq46VtGs/KseWPL2tcUGjIpwpG/t8PozS7jsnrBT7VGnoWGf7Z6wB79l/894gT0jge4PZc4aTn4hzE7l4XEGdUUTqZcJRhXLQs/l9o/TDv05/XMdY5HHP2dgMbpJNbTDDxV1+3W62sMa2Lt2rFrKD0IWPuKgGZOqCnWuGJJbxs6Fyc4foT/MEVmQ22g2vsdHv5WQYERS6LfDRYlWEl0LTsvQe6xw6TA/g42qsWktxy8oWrY3pIa+DAdFEw5rk4AnSNw8Acjdp9L74dm+F1NkrwR9I5EzFzR7zopyqWQudBkNtfjzso8mbsJZm6EjAsue5ct1U9U23ePQO3jmL3nXEZzIW7PZeazX46Sw9RUTL6t3X/rC5bsim705dvhNKcKFn/eo3k6y7go1X73kGHuw4BJXjqNJ4szvfskoj3GH0hf4Y0sma2AzTeSJJvyw/UOwP2/8TmR+T0Z7cUeHPp9SG87zFxxaJwRacM6hnE5JjvVok7KIix3DnrsnffweyLeuWNLZt1l8zUhe7JbkUTSWTO1ZEt1PpkiIIsPA7rL3j4+XaNHmeqGsxYmDvHEVUuqoDd+55UC6W3D1muqvyYlqYyGNeWl1q48mbMrTnJSsiy9FeNEmvR0DusDHpcM9UuC/Q5mNfkazqqvuPqNFKOq6CVBTi0sN5BR0W8qf7R1EjonQ+zxPtVcn6QX4rQ9RrWY1nG5AdI7+niD3JT3dN7F70D1Axe/Lwhy7KqMiBMQFPVzBRmDdSzVTydEvt7nMCXOwObrOZVqxyIaFyRmaR33SdVDwgw4E8XCxz5kduTs3b3k0j4OzdOGnReTlO7GU/km+67iP+/1TCxUUOBrog2t4z4znwW0TulWWb4dM1gQ96l9THVg6ZYa3MWHgQr+X1G9Fmam40xXt+InSnN3pAsCBsp3pCZyQvB7IdVPumQ2pqPOOxrXPpm1nzm1BoFDlI848I/rdA9POaJlKN51aJ1RO2b+nYgoowvIJG9oXojIP3LoXxwxc9Wy86JHeifGGav2nn2nCYAJDIWHEck2JHqSCeZWpaOdvSJmf2rK46rcCKmf9ck/UgvMAM7Y4dvHb/Lg0yXWOwXRCod6oKyjFlLrlDz16R1D8X5MkFOt6w6nirBtQ/FBTH4tIrkngcmoasiueHQO+WqPdfR+DSvulNlqyD52hf1ZVKjFw1+XaLx4P94XYdfPGfZesKT2VEs7EwlsBrO6bO28HrH008FTrY9nYqHGCTj83z5iXJHSp7/gUbkOOJpd59agdxiK92Lcod78wcUh7aM+xXsSAGc2tTukdlWPJqd5poXViGRHtMBEG9a/HcnyMbJsvZKmfSKH34eZ62Ni3+yrj5IHetz/xSHcrkv+jk/jUpnqp/Jfle6KVbXwi4jYt7SOeSTahkTf0jkKbl/Jze5mksZZjSo7Rx1Zs5+z7L1QJtkU3HY441B4FNI+KhneYFG1585lj8yOYjPHMzGrv6LSwFiFUsx+EONMDFujAmZ+RBi5VD4zZNd0WSrdj/B7MFkISNUt1U8GKlVcWHrLMliUtSfRUilkjdqCk0pMZlP9WmNlcrSualsQwDe7Lq5W8Z6+dlRWpyb/OCZMm30tLo4a/5Mi0zIAUg2dinMfRThDh/UvZZ5qjfxTodH/eb1MCP3LS1K+9wx+fyqVa0H7dAjJmMP/g2H7JZ9EW0kdyVtpgrzKgkkxZu6DiEHNpf5CjNtzSLSmsjdX3vlk0zKagdLHCQYLOrIKKxHdgy75x3IWJNq63XaOgolllEu0DOkdfUiNsy7jcow70aJqGY/0turO5kGLCXXpwpEAJrMF6Z1YD4Ary0v1Y6GHuoeFLGqftozLHkHeMvdBTOeAS5jVWHnrVZdkA9Gb9xyyOxGNUy69Az6ZF/dYSAR8cOsIfm5C/26J0XGBK3JrMb0F6WnnfuTROmGwjrokYU7tr8o1Q/uEJXqlQft6haAS4bXkJu0ehvyKJchC9WOVJuOqGKzJJjRPSwiU2QlItRxaR72pGkpC9Ed/0cPtC0HfPirnBRaSrZhJziG/GhGkHeJiyOgpl+AzsaO6I6if9/F7CnyNEoa5DyfM3AiovefC2GHneZ/y7VgJyB094f1DoVRT22prjSqGw78bUrqliU6UsvSWHYKcpXE5IrMpQ5k7VhT5YE6O1kQ7pL/g7wMWUhdazBZ7FC/WSXQFSgjThurVkOJdlRfprWk6sqt/n111yGzH1D6RHVg6WjXVTaxAYpBN20wVWe3TEZl1fZBRWh6vJxz+RBeCYsRw1jJzPaK/rBt69VpIog0JL6Ley3DyyBbmbhYnUn2IVc81sxfv/28ngPbpSEqmoboVoxnx/PtXK4Lr3pYoeuGdEckGdI5q5Nw9KAiG11UdOy6rhTV7ZcSj3zI8/rahcz4gsxez84KhdTxBZtUltWvYu2xwItmq4wSMiw79RU3gkq2I5LpPsv45EqWYWLrJ2SsB3lCjzb3zCTbe8JQmt+YxOBhSvN7EHUlP2T0EzlBsJK+v2nH2ypjH30gQZqVkihJKEknvGlLbHrVP+uw871G5FZJoG0Zf6jJzbYwTWhoXhFoclwxR7NAapuh9UCXViJl/b0T7JDTOeFOF0BOYmqV3WMc/RlHum28aeodiorREz06oW36iLeBCf1k93jih8WOqbsmvhRTuOngDQ+Gh4nycwJJ57DH7oWXj1wJMpFv/1isu6W/usNfMM7lf4O76LNYFvy369bgsLUKQkTs0zGjxFm+79A4rG2rvkiG1q/BfJzDUPtXDm1uF1W8lCdNSTrXPaJo1mHdkHb8fk3ts6JwL6C8kOPS7kH3okr/p0zglH1TvoPQI/eWYymfqsyZazlRXMQ2iyBpax31MaAjznyMARZgxOJFhMOsxLjoMzw2pXpuQ2TT0jkW4I1j6vmH35YpEyAndhOfel+Esvx7hTiyNM0kWfx4xLkJmY0T1U+kxYx8qN2KitIffg41/aUJq17LwXybZu5hkNOOT2jVUPws58xu3GQ4S2J+XGS2EjEsOD38tsZ/k4Uxv55OidpzZ9xR6NilBmILFn8krZI3EMKOKVPmt05YDf2JJ1tXdWPpxd8rJkj36CT+/e9AhvzqZLnLY/GpE8mGK7IZ26PSOodHOEgUOsWdxvBh3JEVSdsXTQ7lnSXZiuU1Xov12nDs09A5pjFx/IRIHa8Wy+bpLYSWkfcqSf8T0MiZARJADLKR32QeeZR76jGYcWid8esdDBosxqfq0jzwTMiobCvck0bSO7gzuRMEXk5LYVqPqVNm29Tm69TsTKN2OaZ2E7Tdiym+lWP+ST6oZU7zhkluP2XnRoXdQu4MTGObei5Vj+jBgUHXYfMNlsGBpnvSY+yigcTZN94D6nCaGYcXQOp4ksxVjN1M0z1nCjMafzZMOQRb2Lnp8ur5EPHEVFHbbo3M05ujvjenPia0fJfVB9o+EDBZUWnjT9L44IewiaLy49Zpm4MmO+qyNUx7Vz4YEOcPKd/NS57sqE/yuxptzH06IE46mWT3LzPseM59FDOYlug6/0CZoJnH8GGbHpD/OyC6C2najGaWc1M+49A6HeMOY8t1IrSxXKKSgEHPwD6Voql+0+B1D/bxHbsVhsKDjeu9iUkKgjkDH7sjSOO0S5ASbmOSFQfdaLom2Q+OFkOw6mJFLfi0kzGoiVliZ0LwckmzHEGtymGhpF+0vqcf9VGvkn8vK+6d8xb7Gd+7EMHPFpX/AsPSTgMY5w+xHPTpHHLKPjYgcC4aZGxGjkqY/u88lcCcw86kl91htro03PH0odyPFqs9a0g2FQ7SPOySaAknsXvKI0pa5DyZMKjHPffcmwdjjC2fuSLRSgETLYeP1NHFCl4xx2VC7GlC65pFb02Qo1YrxO0/CF6SKCnKQfyge0zjvEKVUK65/Kc3g4pDq1YjiSkicgNxGrEToAXSXfLZe9ancnELOTkDnsCZNk3LMoJEhVRsST1xspHZSetcqLn1TmlSserWVT1wGNY9BVXzUZF30v+Jtl+2XPLZfdJm5qtpzUtLYNf/I0jmq08MdQee4sgoSPY1FU7uG9LouTxwayJj4ccjS9x1SjZjiHYfhjJy1bgD9BZ/Zn3n7XYDcKkQpQ+zr4tw99DnaUU0Mo5JD7WPVStbAw9+Q4mjr1RzpaeJx/RV5qNqHXUoPlKnZPxjROgXFe30yuzHpHQWFPYkuHMxL3e4NdOR4Azj4J13GZVj82Ri/a1j7mk/mUId37x3BGMv1//ocqbrFuho3umOZ4pyJwe9YYt8wLkqO15936S1K8JHetlSuQXfZZf69gOxWxN4lwR2sA6NZy3gmxn+YonHKpX5GGaaDWUcC5aSmTaldSQKDrKF0SyfOpKj3xUmHjHoJ/HRAIh2Q2bZEvmbzT8x1M9ei/b5yf0EuAidQD9mZCGZhIlj6WUD3oDaA/MOpGzjSrqfoSqh9pBZWkJYTIdG1DJcDBouG7C+yJFuW1gmP3UsOjbMuvYOqRUu3YFRRcEi6HrL5ukvnqPJoraO+avGhWLNP83om2lNYXYZiz9A5quMy6mkilKqrLnTHlsqHHul6xLigW76JYPY9Q+eI4cFfzOH1DaP5CK/nULivr7EepLYtk4KDO2EaAJYlu2HZeln4xjAf8eLCY64n5tl7VKFzTBCF0k01s2NvChsu6EOMEg7Va+F0AqSdJ9GR78kbSjXVW9LIsXhnOpE6FZLa9jCRbs1BTpOtREfGwNGMPrjNr4dkHiTAqr7cfNNgIkv1zB7PVddpB2k++tkpyhfbdN6ZxcSWdCvGnRjKd0f0lpLUL+ghT3S0Kw8WJMTuz8nHFaZV6+5cTijqPWNwJrqUppoRqT2PwYJE5eDoZ21YJlOs5/IfO9TPK3kwTOnh9Tv6IKOkHsrmOUvhnmrw1jGf1J5Ozq1X3f1LVX8kKs3TvJ6JHdUfChGz95xD4YElv6IQg0Rbo8nBrI6y3gHoz4qD9OA3RfhrnVC/NVU3zL833m/+Y6C34JJoM7WfTC0QSUPhdpdUMybRhujFLiYd8YuHR9mr5yEbMnN+F3d+QPcwtI85NM/KohF7WvitYw6br7my0Ex+mc1UfBhP2aaGxhfks2qf1GIp3tSekNmUeNobCnQW+xLQeCPtmKnHCfIrchKYSNRn5sd4Tsz57AbHs7tYz9L4tIbfE/l685shQc6wezFNb8lh5pqlei2cJgAqjzWc7ojD+YjqZ5baJ0P83pQdldI/Yw9GFbljU3XttP1lDSDyq8rSCrJa8LlVS/X95v5nmNmTnnawNLURfQCpprohTmDlaNgRdt6Z6KToL0kG+TSvZ2KhhmnDZArf3X3RsnfZkFvVAijfUXT55lcjFn4RYj2ofmqZfc9QPyez3GBJl5zdS0nKt6bJdznNz0c1KxfqHRX/QQ7Wv1Fk7zmHr/61dzHG8o2zN6gU+9iRi1P3Wc63SKUCnGM9al/ewD3VVcBaYJjkpQUtPFADO/9YU5y5DwNax7Vrp5oxs3+SoHlOFwUTSbM5cy1i7sMhlVsTEU/a6hYEedWHuy9qAVtXcsDdyw6Xf/Mazx9aJemF/Oe33uRK8wBxbULtY4s7sjiRJXtH6PRRlenubxlUXfoHNX+vvxQqxOygYflH2gX3LqRpnQtV0gRT8cxsTPegHmi/J1bAk4S+9lF/n4Q9mtHodPOrFekOTsOg6pKuC7FevDNlg50RayuYGg+9kSUo6mHefUne/8Kj8KnWyDOxUEHHU24tJrfi4A5la1j4WZfuAZf0lkPxus/OCz69A4Lbxr7B72tOXbqpwK44wdTwZqleHTOck/o82TCynWwqhmY4FzOejfijh2cJA5cwdqm3cni5gCOX1tnsF+itFHHdmK1WnoVSh+ybu5hDfcKsMqeilCJqwrT4p91lj0RbkLAgK9lhesuhcN+hfDcgSkp7sP7FNLuXEsTTlJLciqVyYwq4Tcf4XVmoZz8OyWzC2w+PciDdpDNKEd4qcPPmMoUPU+xdVHtsWDWCFq+HjGshhZWYvQsuo6p6x9aBykcew1lNk3Yveex8MaB8Z0z+3pQi7cnzVPlMraswrfZbbj1Wq8qDyq0xpfshiY5l5qYkiiALuTvSQGNQc3AmChwWch0yO5bIlyfOOobSDYViHPn9kPxDaB99uurz2ahR0a7yREiRbMqJ+fA3cqR39LQ/Iej5fUP/zT6FH2TUWjrlUHhk99lOJtSFae3LCZIt6F8eUvlxiuzWtJcXWLAOufse7o0C0Rs9fvbWBaxnya04bGYzBHlLMjDYvSLJHjRHOZrPRaSrAwrXzX6mVbquUW+U9JkUdPlYfHvI3vk0reNTKZ+FrZd92UI2DLlVUU8KD6B1JsZEGscmmobqhy5h2tBftoQZj/xqRGs3xe+Gl7BDF7M0IXszKTnepgBmQc7BhobmKZ/kLoQplQyxp925fyhiuKCw4jAN1rHUfuaz9YpKAr8/jcC8GJN/4E67JNNc09zUAn4ooF5P4Q8szTfGpP/A3ycfNk+6VG7G1C+qbRgnZGIczBnmPhgzmvHBUenWPCtcZfEObL6W3L/wPs3rmdhRTQyZ9ak6ytMxOaw5lG+yn//0y/gdi/+ZRobJpv5sMDutr6qW6rXJ9PKgoj73YRp/YCl91qR50mVQdXFH091iYsn8LEfpluzPGN3cE031a02oG3J/GdLrHsm3CgQFQ/lOTGY7YutLMeOZmOKDcD9l5cFfTDIuK+HaRNB4Tkfc7BW1rIazhuIUJVS+If8/ViPU+iXlsno9w/wvumx8GY7/vSGFD1Mktz3crQQYKDyUsCO1F5Delb8/0bIkG9rNY1+60plrAalNl9n3RJqZvTKmeE9C8icToWFVC7b6oUP10zHecPrzJDQabh91WP5Dl/ZZDQ7o+TRPugxntPD9qUa4cE+twcpnhsyOZVK2NE8npZSawkJyq5DadYgTetATnal45Slez8RCjR0RQZINGB6a0D2s22X7uI6gzJYl1YD2EYdky1D7OKC/YCjfCfQNHLV1co+mMIYpREymuZDhjMP2mxWsIxTiwe+PaJ1wiNKGURV6S4balRivb+kdMPQP61G3DlK8J2D2o4AwM6Xk+VI9zXzoUr1iGNRcWmfjfZarddQWezJP9/rSgqbqltHlgRRaHbtfB+YeS/U0/wvZQIoPYta+nqf6ocN4JknxQUiYkdrK76rt9MSHNC4b+gsO9Uv6nd2JxTp2fwgwXAoJMpqvr34rodhxAyZQNyW7KaxQkDU0zopfIL8XFO/LgjOsOBz+vZAoAeQCJkVdbGEqfl4W4rJ0Lya7FRAlFG4cJVSvP4nvdMdTbcQbfUa1mPxa+Pny9bsBHPjBhNZFtWa8nhAw2XVLfynW7TYhoYmOe1/6yKrH7gtCUHbPj4mmT3jsQutiQJAzTPKOklUO62hO7Voe/praUpMCeL3p13hqPAd5S3LHJcyK/5/oaHfuLnu4Qym3RjOG0r3JNI4HRjUj6vPjkOP/fZ/sunbTZNNSvOWR3rOU7gjDY1bSdI+o6R3kdOPuL1pSLXELgjyUrjUJs7og9uddJnmHKBcz/05nqjEVush6KpWym3J9dk6HiiXKWUZlw9rXXBbecugelTglt6INILOpr927bGmfVA97XIHMdkTnkMvST0dM8nrghjU9AK0TCQqPJsz9SYLlH8un9iR1r39gKsRJGR79uiO9xqIus9tvaIHuvGioPx8rwfudLNWPDWtfN5RvfZ5cqKF85LV3XQqPYgorEcmWpb8stlPvgKYnmd2IZAMWf9BgXLZ0jkpIMvMLn9z1JFEC9p7XvLr0qa8GeSgVfeWapfggYrBgmH9bt9PsutpNk4KcmVHaiAN6M1JNlxJYN9GZ6i5rliAvesnm64J7tc5Iv+l3oX3Eo3skK9vMQDXs4t+5TX9RUsM4AXPvx2Q2DdmtJ6kuluqnsn8ryxV2X6kwmQvUMnKhv6gg3fu/ncfvWebes1NGgZRkzdOG8k3wyyO83tRuXdY/Yw/cgfitONLa+h1Lds2w8HNNmprnpdAaVRyCPGy+ntLYuWYIC9OFZKFzJMG4YNh+wadxXvie5lmJTqKkwMPlqy6xB+XbEammJbkrvUB2w5B74FK6rbhOb2RZ/qHd3yD+vNczsVCjlKFyQ3Zjb2hpHXdpndTuFmQNYVoK/b1zwlDuvlImvW3IPxJOMU7IwOYEMPOJYTj/y9Tj7gHN3mNft3S1WgTo6i8ZhrNKAwRINmOitAQZT7SuW6/LqhFkZW2e/XCKPgf1CMcO47KZXqbUyD703z/GHSsfYOVfP0V6x1K6J+HGzgv6GfoLDu1jjggjnlBFoxmznzKYv5Fg7iOh3MOUbB2H/2iEE+rnj32oXo1l4546jmd+P4MT6tTJrmt0vPOKHrLIN/sUxNyGjtxRxSG7EROVQgazCtJN7/4SLtE7GjJzxaFzHFrntWCDvH7G3KrqzNyqQ+xaMlsWY5XtOq4YOodd+otmXzATpnTZ7S8aUjuQXR+xd8HbL1n+vNczceu3DuxdMCz9ZMzD33LxOqIgm8iw88WA7D2fSVHFd5BTqTDMaPHl1mPVYBVxlBrfHJG6mmZY0xuU2dEIM7spEuCoInVRmNKYMLPp4PcsWMtg3qH2STjlgToM5zSvTzbF9GyftGS2HJwxjGsRQV5kk/YpgdNiD6XlmSV6BxX3mGzpgRhEsmFMqhGpuoIoig8jevMuybal8M0tcv/7NK3TeYazDomOpbfgq1e7HhN7hkffTZF/KAGL9WTNDnMaYvgD6B205B4Z4eK/HZD/NElm25BsB2QetHj0WzV6Z8b0Xowpvu3RPQSTxZDUowTZrZD+kgYkJhY6qfSZR5ADi8VvTx+knJ1agVRyZe4lmLluaR/Rnpfd0IM2LsnRmnusS2l61047JOoZ9w+kOfCDMbFnePgUa+SZcKGmlg7Yl174t6mfF5wr1YppHXOZFGWcq10NqJ/1GZctyZYW2mhG0C2ARN0l2TLkH8cUrzdpny3hBrr09BdcjTSXhY+xnp7u9M40mrsV059XflOckF/IRBpvts4oaCK/GmvadA6WfxzSOubjDSyleyO2XksTpqFyM2Yw62hXTup77H4pIH0/QZRROaLJjyHKWGpXYvpzLulGzLhgyD8OaZ7ySe/GDKsOhccR9XPuFNigC052zVB4HLJ3XmjzSTWi+r6rY/mYjHJ+X0Ll7JoeqKAwzUItWop3maKPDL1D8mgVHmqH3bvgk9oTnmjh5wO2X8kwWIyZ+USLdlhTyypMyQgJU3/UnGzr1tOpNskbmudjZj526C0bUg05UQfzorJ0D8vhGye06+fWJ/zk+3/j8+FCdSfsH9FRChqnXSZlS1AQgKF9yN9nwLsj6B00LPy8z+KPDbmHHnMfRWQ3Y4YzhpVfnyHIOCTaIZOcrCGTvMF6CrgNU/LujGpTLehf7TCY07zbxDCal/41yCsbNMwwRXkb0ttmP1ka4N7/1CV2ZUp8Yp1O74X0DhgSvZjk4wSJjnxLw6qhv6g6tPqppXnSJbMXUz+vBTCY9XAmGvVa75dOgNmPA+bfkzq/f8DSXfQw0ZSz9b5LqhXTPWzIbJh9tn9uxaF7ecSkKEgaCNWz80ZE5wTEST1wTiTcztbLvnqsLgznYlon04yq4v8PZyXCnrkR7McLjeb087ROOvu06dGM8J7DmsEEhr03A1EGS9LYDucsqWZM/pF+X7+nkmrtq0937X8mFmrkS9hhjerVwcEQZ2yYf1tPvIkFQgsKYvQf/OMerZMZ/IESRMKUs5/27HeZ8poStE+plYNRrlT3t7tUbka0TwkDOSkYRtdLzFyzU4qzpXJFfT9nAss/HFC6rd20ezRmXBbpbu+yvu/Cn8pqnd6LBbQ9ElM/l6DwwNI9KAXVcM4y+5bPuBqTXwEMdA4KOzQq6zIY+9A+LjGxOzIMFgUMzmxZ6ud8Nl8T7SXMRYxm9eDMXIX+gmHrVYfcmiVKP8GiqybNX0mRbFl2XtXYOEzDwo8dytd1mdr4gk6vIKsdd+ZGRLKjE6xzVBzW7qWxwteOKQTD7+n0Ld50xcLa0mWwe1THe35tTOluTOGBw6F/KBJK+U7MeMZy8E/GYDXfH87C7EcDCo/ip16Bz8RCdce/DHIYzVhq77okOmppREk5UHd+Y6SL0AnYeSFH/vGE4YyHNzCMC7KQhFlZmftLht6Sg9fVseWOoHTTMLlVoL/gKltp2jQv3QJvFJN7bKlfNOTXAlqXJzihpXMkTZAV7Cu95eBEktMlWtJeto85ZHZimmem2HKrBTEpGMZlS/2sT+E+ZHZCircN9ZdCYk/tnMyWfPWH/qDBYN7gne3s7zSpHYfB3DSdZS0m/0gWj7lfKBsgu2H3+5bpbV30ZGmJGCyozZTZiUl0LYtvycg4nlGN6A9jxhVD8a7EOtGUZxolDM2TDuUbuqBWrhqWf0+XtvSmw87LWtDjGVmBGmflI4s9jakX3hmw/WKa+gVDshmz8YaH34fGaYfKNdh4PYWxEg9VbsSsfylD+4RDsvE5uvWDIAephj7AVEv5m7UrFusoYjH3doa5DyJKtyUve/yNBGEKEi3twlLpqIGdW9N0Jf/Y0j03wR1LmJLeNmKEuspNxWqosPucR/egwYSGYc0jcz9B7BkaF1QCWA/m3x3hDmG4EGFChZGZCMYFh+DAWGz6iciBcQJmPxI/IMwYGmd8ukfBr3sUHlpmPhUjNErD2rcqMgleKUq8XNQIuHw3ZFQR6tJYGM9G7H13ROab27RPQfmWZftXxsxeGZHZkY07TOtD331eMet+X4bAvTcDDv/BCL8rR0GyqXIoLIe4A4lFEp2YZEvHtDOBztcHZB928YYaDBz404jMXsih73Xw+5byTYVvjGrC/Tz+RoZUXYt8VFF4BhYW3p3QW5JrYPeSoffciCCjzoEJBaB7mtczsVBjX5aS3cuG/FqIO4qpfRIwrDm0z0ZsfMEl2Ywxsd2neoAW6KiqG6jaU+LcE+s4dkLZhTO7Ee1Tv/z7orRl8EaP5W+t4F1oYy52cF5oK4N+wWFwKJymjUhrmdo1rH47SexBakcg2/yq3uBxGdI3U3SfG5PelayOGOpnPVJt/QxRUpOg4l2F3/aXDOldy6QkreuTPu3sRzGxr9+ju+SR6KhPWn8+wuu4RGOXcmrIt7/+IY1fH5C4l6Z1VOwAv2cJk2YKOVO7KMjovar+3OfBbyUZzgk+HCUVtDbzrrfPomqe9IiSULsa4Pct2Z9n2fximeyG3KzNkz7twz73fzuPdSTSHpe1O2bXDOOZCG+k1lj7bEh+RdLI+tkEYVY9a3NogGkk6B42pHYNw8OTfYrLn/d6JhaqdWBUdnEO9BlUXZqnEkyKLpMCFG65FO9KF9qb9ySqThhyK2oc+z21rVINEZcz25ZUWyPB3OMxYdrw+LuWoBDBV5o0vjIifbIFwJ1ry4SfFQnu5ZncKpB/LLXQ/E8cYncan5M1dI+HpLeFwUw2JCK2RvVwdkOq/eTDJKm6fOxPQhWaJwVwW/jFkGTL0HhOan6xRQ1RKcQJoXlBALPOIZcoZffr6vxaSPTFNmbisPDcFrnSkMYww/s7h4gfZhktBtPbsxro44pCiCufGWY+3JOsLqeBR/WKwe9qJ090LXHCMCkZ0o2YzLbUXYKm+WS3Q9J7whuBdAJP/r11dUFSqiD4vYjZT4b4HYedF3T65O95uEOJqN2RxR1KF7Dwt5N4PXVfYg8O/UOVak/zeiYWqhNC84wh83ZOi20nVj/SkfxsWJtiatxfmuikJNeosnM6oHNYYRXpuvJKU03L6r8ZkfqtbQgNJhvSXS0QDz16awXi9QzpLZfqVYHWyjdh64uaFsW+drzgOy28oWzLnVMh2bXprHpR07LyvQmNi8pjrdwU3a5yK2JwMCSe7mp+H3ZeTKvk8BRWNpwXJmf2LR9vCId/P6S4EuL1Laldh+xmTOdEzOq3XQa9JF965Tp73SzDYYIodtjZK/Dml69hJs60gR/ROewynLXsPacFGOVT1C8oqSTICmJ88B/tUr0W0luWlSZVl2Fvd6r07x3QYtx8zWP7VT2EncMOzVMu/ePCJ1WuweLbIzrH9PV7F316S0lKtzSaNYFKs6BgKN8NlE0wUNtvkneofRxTvhuS3rEMZj38pxROPxN91FzlgL305X+bIC2fUGY3oj+nOJrYN+QfRwQZgz+07F5yWPzphERzxMqvFAmzlpmrajLvXhbsISjEzJ3Yoz9O0O+mMNtJolJIalVZAN5Q3KMTf2vC5usiiLgj6C9LghamDU5kyeyKXGLiqdfH1YUju2HpLwvcltp1dIznDf0jEek1lQbyXE0DLaZWFhASx07HLGFePP4nnionUPtq7t0+j7+RhQtdxv0Ezm4CdwTVzyz13+5zuNrg7sYsnh+R/0FW3yzWRChV1w6a2VTorzXavd2x6CxLb01Y/VZCOKFZS3YNio9ChjMuhUcjGmfTFB5O2HkhSelexCSviRXoZq+Wl4744Te7zP7NNEHeZe+CIb0rfLs7EitgVHQVhb4gLUJ23ZDdjpjkBKLIbEkxtvJvfV7ie3zYeNMhShg6x+Hxd7STZrdigizsXnJpnDMEacO4FrH6rQTrXynK7/9WSP0vjPBGFm9oCA+N8NsOe5/N4n6/TO6jNPPvWmo/9/F7ULo7Ib0Xknvg0TksMnSUUGtp+YcT0k1FpTfPq0WU3YqnEZIOOy/JCeoNLfmHliP/SIFgrVOWmRsBxRuiOz/Z9YOcoXJzvP97ztwYCZg7MpRuw+Jb6hBESbXUSvfVFVj/chbrWb546B5fOX1bkkUX8v/Lxxyt1VlvFzEGcj/MSuRcMEQpQ+/UBFBTvnUagrQhvzIktxExmIrI28cS+7ts7eOY/rIAvIN5hyAvWkz9fJIgrxSVScHQORWRaGvhjyuWxgWVJakf5Okc8hiVDbk12apBFJvIn46dN9WPLd5T/9nrxyS6irnvHTB4/c8RKUV5Sg7dw4baRzGZFY8ga6ifl7/exOBODIN5B78lopw3gsFSxNYrPslPM9OUvZD5WptJLSKzaUi2RX7uz7kSTeRh7SsJgpxCdEcVUeXm3x8z/45l+6Uk7SMexQcx6Q1Xl7mjypYyEVKn7zi0Tmq+/0ScXflMXCzh2AU6K98a6cgte6Qa2vH9xhB/aEm01AOd5BQ/VHgQk39k2XzNJb0rHGP2cp0rOwf4yc8uEGbAPdGjmupz58NDhKFLHDjTsDGNJIezkHmQwJ1YorSiI+uXLONKkuYpxcNPSpqMjaoxldsR698Q9sjvCywRZsTXjxNaxLnNiN4BS+G2S/2SZfjtjkjTW/p9nVAPY/eQQo6DjKFwT3eIIOOQ2Yv3R9mjGUPztMEdC6QWpjWGzq0+3RJ5JhZq7OkpLN+KGdRcDv3+Ht7QTjM9Jfk79I+7jF/ukWwaFn4R4QQWd6BjeFyx9A8qHibxn1Qo3PQUqegpPiazK59OdkOjzO6yarvhvMxne+eThNMWV5RAvcaZmNn3u4xnYrpLLu5kigAPoXo1UsTifVGWM3sRmS3LqKrvl38IK38hxeLbQ3YvO+x9a8Soath7vkR/wSGzG1O+I1KJ33LoHBHCsvqJbvy9QzFvLDyk3UuRONJluBjyn17+uzxfWOXF128T38pRuJIkKGgROAFUP4tI71oa5xUCl2zHJFqKYAftsn5XOtPKZ9KnpjY8smsOYUqKsbVfDSXQea3F5hcM619xyK5Ns6jethT+YV4qqA0F0fUXdfuf/Shm+2WXwYIChwdLGvn2FhxJM7MCNOcew/ZLSVonjUKCt1ycydOtkWdiobqBms97lwxBAda/VSVKqt+390pEULCsfCdP/k+z0yAzh1FFobzWwOHvjbC+xalMWP/XJowr0HpOx3KUMozzYjBldiK8saV3LKJ0LyK1Zyg8kqSwfsFM7cuW+hmPo/9wSP9ARiS9pqV7wGHrVR9vAPVzLqMZPWDZx4beoithSF8AsvYpS+WaZfvFNLNXYuK+R3bd0nhOoRXbr1rq51yNc/eEpcw/1Bh2XHaIyiF/9KMXCcceo14SvzziP9/4Mg+GNaqJPu6ZLtnNSGl7S2pHjYoOvQNQvK1J36Rg9l2mtU8UeBb7EqhHKWgfg+o1eZ+eANhKHyZJbzpU/maOylWB39L1mO4Bw7jk0DiraJ/+nAIvngT6eqOYRFsTxdgXCDnIG7qXx7SPGSIflr7vKEGwL4Ng4dGIw39/Wz3wp3g9EwuVWMFhlWuWIGv348UHc4bypyLDjRZDBovT+B0Pwpx8UrnNiOapFNX3XHLvpTm/uEn6UoPkhoQjlVtjorSMg09SAfP3prqCKYKndG+I3zPkNpUrVfs0YO+5DMOqo1TkixIjuyMdtcnmdJq2q6QWiT70TxGrZbHGQm/RJXfXp3PEULqpaVN21VV8ZcMyc23M4i9CynfGZLYtyWbM0h+6ZNYNppHAX09wcWmDj+8e4nZnlg93DzCspxmXFKwWZizFe2rKP4nKqV5XfGWQt1MbiMekYFj8aV9O2ikguH7WFdd/0zB7ZcKophjOZHOCE0j4HWSfnDSG/CNdzDqXxoxLurhp5/Rkf1lRyFp+JSbIQO6qfFHGKrZoNDuVEI5h77k0K781p7zVp3g9Ews1TkDpvlKJnUizb+tC/3Ao+sleTO0dYb3dseK1Yxc65wKipG6QnaNahJ+9c5z2oxKTIyOCrDCIYQr68w7twx5bb6iJ3z4m5OTgSECQ88DC2lcckp2YjS949A7qaBvWvKmGQDt89wj7ZUn7iIJ3o4R8TN2zE5yJBM3RdCjRPWTpnZ1QeKha0ERaVMm6IdWM2X4pybDisXM5pXzXZUfHctOS2nEo3IOPPzzO3EKLe/fnaVytcfAPDFHCMPuepm2DeSOm/p7gccOyS+Srvzop6KQaLMY8+vUMsWupXgumKEhNwtyJZe+ioBfJuuHx1zP0Fw17r4Z0jk0fwLScwn7P4m8lqF0dMaoYeqcCKjcGRAmJXbyhAuaymxrbjo+OmXt/TJiLxC3Y08NoQkuQt5Ruf44U/u54quN0IbM5RXFP4NTfHOD3pDfNbeooN7Ge9OW3QnJ3fDa+GitYa1NSuUTLkFt1mKu1aTwfTSEM8uwUH4XEhZDSg7GmRRZqv/DYeSHB/Htj0tsOsWfwBrKfpLdlfhOOUhZpr2/oHtHP7Q11E45SYjpl7ifU8H4kRfviz7okWobUakK81JEs0tZA9bMJ3lAQjO5BQ/9ATKoZExQEOdv5gjio/WUdjfPZLgcO7RF7lsHsVCBiNCxJ7amV5vcV9NA6o/dpMKcUvcWfhBTvyDYeJ2H1my6zH2gzeCJO752e4A1lkMyuSai++ANHWoghDJYUIdQ6bal9bHnwFxUNZAYuj7+eJUrr64zV93DHVk6Ga0map5I4I+W29hc1TjUxFO8pW/ZpXs/EQrVGwtoopYZz65RGoo+/WaBxyqdzAh591yNKQ+tsSO8QjEvCj+fuT/30RX2v0n1ZWba3S+Jvnu1TeBSz/Rp0Fz0O/L7D7sUUo+Oj/V0iSlnqF5KClB1wcIcqC8Zl5YCCcpFSjSk6cSZiVFH9119Q+IPf1ZsfJRUpHiXg0a/kya1L9GFi6QK86QVxVPHoHPIoPQhIdKB4Wy2o2Y/EyJp5X7+vCeHg+U3u/vExnq8+Jll32HtZePf6ean2nUhH83hqhMyuqcRwJ3qYdp4XXz/RVp+4dNOw85Kcv4m2xqz56wmSdcuJ/65F64yleTFi76JD84zeCycw9I8HeEPD7iVD6aZD+XaEzYbqkmwpT2r7NdEBewccki21oQYLlsyGQ+V2SGbL0jkkEXaiZ0X7forXM7FQMdA9LDhE96BD8a7aKOlty2DBTm0nap3kHnqUbyhuJtG3TAqW/pIkgI+/7jIqqzWy/LsuxQNtXj38kO3XLMf/Tp/+Qcvucx759YjUnRTz745Vi00DHQqPQ0r3on1nQKI7jdfxxY+KXXUgMJbynZi5Dyf4XUv3kL7W68sMl9vQ9xCQwjB+vkfrpOq7ZDsmuxPhD2NGFcPalz3alycEBYk52oddivemAhOrBy+KHf6Tv/Zf8r1bFwCoveMSpUU8xMgi3TmiG3fjOYlFBgvqdLTOWJJtiWti31B/MWI4Zyje0UUpuyErTOlBSJQ27L1QIvvYYeGnDt7QYBdGip7cNZQ+UYgbU4ZCf86h9jOfMC0LkTu25B66ZDf1fdO7IcNZmFSjaYCxAjRy68L7tI/KjvM0r2diocau6rruQf04o6phNCsYbe0T5XZGCR1TmW1L4dFICJ/LhqAoinLpFhz5RxOCrGH35ZjtF11auznefvssxTsua1/LCwG+ahlWHIZHJjRPJcmvaoQ6KcVsvSwARG4jItFTGkuQsySaUPtYFxTQbtc+4tA56NM9DDOfKWkvvx6Rqis+0Z9iGsMs+FdzLP84xMTQ/LUB/VmXrVdcrKcUk9zNBOXb4f5x7g10LIcZKY4ia/hf/cFfI+rJ0RolpW3YvZTAG2qE6g01ODnwpxFh2jD7vk6q4m3DsMaUK2vxm7KbjGo6PZQ1YBnnXdyxxtVxUiPXRBuc9ZRG2Q39vcm6BN0wlf2VVbcGeXEZEl3JCUcVQ/eA4GjJHZc4oT51nFB3I0oZSvciRnNPh/R5JjxTbqCjJcxIZSRbtGHvkkNqx1B8FOIGDmFDF4cgm8Idw8ylHXbqBfjVPr4fcn+zjBlEJGcHpA4E5JyYye0qifaUhpybWpQzUPhMdSPGkF+LCLIuxQeCrmW2dempXFdbKtWQmsmZqB51QoWmZbZlrGueMRTvWnqLLv0DMTOf6madbkQM51z6h0M20ypRnJs5ukdjUnsaNoRppZFktl0RVO7baXx5RLIZ8ujXPSahR/pQl+FKnv6CEqLVJ9bPEWQUIBemNWXqnA+I7qiGtI6hdCdm5xVLakcX0iglNOXeReHYn4QLe0OVJX7PkuhbnDAmSiq0YjinQLmZuyHNkx7D6tSSYizWcchsxTTOS/iSaoA71Nf4XbXgEh05fwfz+j7pvZgg41D9AJ6m5/9M7KhBXt761M40bvKgWEwzn1kyOzHrX1HacnovJrVr6R3SbtF9aw7bTNC9NkMxOcLxYry2Qxwbur00rVaW/Gqs4K71iN4RgSUKDyN6h2IGSzG7lwyN07Jpdw86+9OWwZxh7zLTNBXZjYezhnFFbTHlP+lnEl4nVJutLzitM7GqSQcw+7YU+rMfj5mUdCSPqtKQ+n3w2w47r0ckOqL6uSMFMzz8y2oNFVIj+rsZKicaFB7aKSvfUL4bS5uQtdQ+iZl7P6a4EjL/I1eLYccyWFIU+fG/NxZdpSGxzPZLDvmHUPskoHFB94PBgi5l3SMKEvYGqvej5DRYwwNvIJ1tqmFZeHtCsmmknZ1REk3tkyEmVC1ffBCTaghat/tGSPN8TOVmTG4jpnHWYfcF6SGe5vVM7KjEAkM84U75fcPe6wHlKx7JliX3yCHZViSNOxHyJihI+JGq69/t3T5EKaEgg+oPfZonEkRpw/ZrMYU7hs5hD79phfSe0wKqfRwzyRv8QYw3jGmU5D0K8hJe9GNF5gRTnr4TGszEMJwTftEdgXXdqYVZALe59xVxPqoY5t8bsfq/iBjfUOrzTi3J7Hsx26+K9Z+/vs2dv16jfN0wKTtCrk9ZpljI3k0wODNmEPhUFtvU71fg1ZjUhhwQmc0x3QNp/C5sfMGQe+SQ24xF1W6DnaavmBC2X87IEJg3zF8f0DyTxrrQOuqTrEP1WogTqISZzIVsfNEjtSMVmTuxZLdUmq1/KUF2XfHlk4JL92jEYN5l5qqMfqvfTkkPu+lM62JLbsVSuOYzKcEkpxij/COh6U38ORJO41hKD0YKcpjXyC5/0wf7hGtqqZ+XczL/WAFfYQqSHXFVN76qIyVKatL08Nd8mhcj/K7VDndMvP/KDUvpXkSYt7hjw/hfa9I5YugtONTPehz8kzHjCgQFS+V6VzK3g3rkG2ccRlVpXjnXZfYjy8w17SxRSvVfelt+9t6yYbAUESccar+TIvamIW1Ny+6LsmDvPeey99os6W2HKAXLP4oZLEekGnovYl8OgERmQiExprFTILfqkF5zSbbl6Xr03ZSC1yZQuDe1xZxyCLL6mYK8ofbpkMGiJb0Tc+T3enhDePyNDJktQ+u0ZTgnf9Sw4rJ33qd2pYvX9Cjck6RyUtAlrHVcP6c7NKSalr2LroYAD1z8niZXia5l4ech1Y/1sXp9Hfv9JT3o3kCuX68/jUTK/lJV9ucukX8+K++f7uWMDdsvpEnXY7yeUDOpqUUhs6lxXJiLpZQ6lyDICuzVOuaw84LD4o9V75kIdi87zH4AOFD9pDvFHSo5ZTDnkKqHJJd7lO7GTL5fZeZapMtT25K8tU7usSX3CHafz1O+3SdKTZ2jn0WUb4pcUvqdLPVzhsGsxCkm0qVCoGCYfy9g6cdQP5tgWHUo34qlPCoZ8g+caZSk6HZhWukpjdMKEZvkDWtfSgvZ41jiezkWs22YTAHAA/Ynbsmm6u3Yk2mut6RAjNKdX6J+Hn0nxcLbIe0TDqvfymMdKN2JiRJQum1I7yh+cvfNEAysf7VAam8Kl9iMdQHrWYKiLpZxQm0xE8BoxhGLK6Ej3O8rF3VYk5ai+DDCHVryK3Yaf6k2WphFF9iiyIZPtUb+eS2+f5qXsSrgu8suURqWfxgzrJr9bPn0tgAIreM+hZWI8j3doKO0pfppTPuI3rBxRW2UmbceU7zu8ejX8upNfiDyXaJj2Xk+SeqHedpHHLqXxzTOucS+Llm9lw6p31mWUTD2HBIty/Yblu6Sy6iiRd864UhVP1A3ovgwYjgfE2bldh3MesS+2b+E7F02tI84jGfUCkp2YoazhigbM/dBQOegRo+F+6rXE50pSdDXJfOTv3WB5I5Hsq0F4QRQP5ekdD+iv6T3zx0L7tY5OrVSXw0Z1Bxmr1h2Lvukt1WbjmcUS94/MSH2wESWzvGYzAPB5XJrQhY1LliapzSEwcLsh6KjFB5aknWVJ5ntmPxDSLak5B+VHPoHNKwZl6Ud7h1SJChA6V5Ab9GhckPDA78ruePTvJ6JGjV2ReMbV/Tkpdf7ZEt5gczyhtZxpZ6Myobusqsn9xQs/SRi95JU4iJLaydY/81D6usNDd1TAfEbA7rvF6WTTAIYFt8esjNIgwO73x2TfT/NJCeferKpFBUZ1hRc1j0qKl1628Fvgz8Avx8TZlzqZ1zyD8CZqPHfXzRgzXRUKhFGejdmdxYGSxBmJdFLr7us/EZMdkp1Lj4MGMwnxMbPyCka+5bBnGH5rRH1MynFB00X16ToktnS+5cIYPaKpX1Yk7hxwSVOqGTJrVrSjYjH37VU3/HoLboUrruYSBfF1J5OA29o6ExTSsrXtYjHFdlFGjVLnAqZueLSPS6lWnLPYVKOqX2oUIncRkyyDTuvx8y+begv6uEs3gWw7F7yMSE0zjqktyy9w5BsPN0aeSZ2VBwoPIpYeDsi0TZsv1Zk75KO3ERXO824ZOR4XNDtu3hPKXZxQnrVzKZD7rE840FWjfpEB/K3fewHRUp35fuJE9q1dp5Pk+iqvDj437gM5y2jGanto7Tkge5YvczMpmHmU8Pc+088WqqRd15UmklhJaZ3KJYvfgiDgyHeSMTl9F7MYN6y+4J4/JkNiYn9nmLHD/zhL8ez3WVlvU5Kht7hmMGcYbAgRE/7SJLeIdWjw6qD13MYVzQznxT1d/UWXBJdIdPDtI5aRY4bOgc95n/sMimqI7Hfsx1Bdk3dleKjkMy2nRorNVAJ05JHOoFh6YdSoJU/c3BGUn3NvacTJsjD9nfHpOsRXtehe0hYovT2dOHfGkr7+8FQ3YK2JdlQJNJTLpH//78SbWFs1r/s7Auci3flYuwtO+TWQtonLdWrEelNw+DEmO4RmLmuozBKSy8wmDeE2ZhJSaohd2z3d8itVx1KdydMqhFhVrtjsq0oxPq5hBDqLfn7c+sxUUr1YH41JrcRMSnKuuwNLJmdkERXM28Tq1dZviaPe5SE5LYnS3JJ483CAzBzI7yRHhRJ8ORu7S55VG5A58KExiX5qYazMYe+p3CHpbfEM7CubvCJtsWZaJeyUy5s6V5M8ZEWV5AxpOvyb03yQkpGGQliOocd0nsx47L+OywQ6/vkH4+pn/EYlww7z/sUH0hjIU6XShHrGOpnPSo3hrKt9w3NUw6jpYBEG8o/SbHyHZfyDRFXxt/qYI1KmDjpEmRh87U02y8maJ4xUwfu50jmNyk4gortOMx9GOAEsnfsXhKgbFjzOPCnEcMZIXaIDYmWYVRWGIITSBicalj8joQUvQPTKdL1COsYjv7ekO6BBPk7Los/n9Kfqw75+w7u0FJ6oO/vTiBVF4InuxWz9dWQ3eektBqfHNI6DTvP+wxmHfrzSvzzB0/SUKSr9QaqczEwqDr6Xh9kSHTUdRhVJY8b1SzZnYhxyZC5n6D6ocPMVYvfdVj7akJ60qrP5rcDIl81e/HhhFHV0D1sJCxx1BnZfF0q/s7JWInZqxK8JHqKcRzOGkaz8RQQoV5smDHkNmMmJcPWK2liHwaLMUFeZdbuqxG5rYj5nzXAgeYJCdV3XsowmDfk1jRIqb3t7fv4F39mGcwaKtcM5b+VY+6jocjbp5Lk1i3uEGofyz2be2zJr36OZv0mlCreurBz2Se7GZPZckhvT01pBw0bX/AYLBgKKzEn/l/hPnrculogyZalfGeki9nQMj45ZFw29BZcohQ8+pU0zTNqWk9yuiQkOlYqf2RVHlcU8b37XBJ3LM9R6ZOE2Pzf3yV5O03plsaJQX6Kz7Gw/s2YdFMN/OyqdlVh1RUOFuYt7lB19c5lh/yKEOle1xD5+u9jT3/WPO0wrkYUHqh10z7skv8sSe3TAZMC9JYSjCsxhQeWwoMhJobSbcW1W0dUkzCl+X+YER82VbcUHsbkVh2y64ZEVxoKb2DZfslMj3ch3J3AkFvVQ7bwlkOQdth+o6JL6o1I6qiKSpw4AYVHMa3TOnlSrZjWcZfhnPgKreMuq19P0TsIOBrLumNLb9Fj/t0IN7Bsv/J0a+TZWKgWKjdHeH048P0uOy+LjzRY0C/sjiWbKzyM2f2VEcPZJJ0jqt+sM815yht6i0mSTbVPCu+m91Hc2U1Nj7xjPWrfWaP4r61RvK+R36Hv9eic0CUuVVcIcFDQ5cadqN5LtODRb9YoPIz3gybyq0JXYmHx+9KQxr6hfVbagex2THYD4oQhvaWJE0B2QyCx9K50C50jquUSXbWqEm3Irrj7NuhkSyKTzTcypPf0cxYeyK1791/3ptAMw6imPm3lVkj1Wjh1HFg6v9Vl92sTtn9lTPW7a4AGHn5Xv0d2w1C9FtI55JDaUyxR76As451DCt7tHNWp1Tou0U/hIdOkQUOird/XH8QE00ti+YZ60eU7IdaH6lVLb1kP9RMY3bjg0jztkN76HIlSoozl0a+mGNWUhDL7gYr8J6p6v2sp3xrTOeyQ/CzDzosOqV32Z9O5xzHDmvqIo6qa0t5ArSB3Ymn8xoDJFzt4XsSDe/M8+PAATiiBRfNsjtSuw8w1aVQHc4bsY+2YxkoPGuSZIioVCz6cM3QPGCZFCWW23jAUH44o3Q1Ir7tktgSaGM2YfWdm45tDinemI9O+YtyfBFH4PUuyoV1qOKv62gl1WUs1Y9onrBCSOe10saudMrGeILsd70vllJLn8fhbhvbFgP6X+5yZ3eYbZ2/gejHPVx7TOSklU+WWQtTcoaV9yFOfNG8orASkdw3dIzHJlqV5xlK6/ctTyx9ospRbi2i/NGLnBZ+Zq5bWUdW3Cv0wuGNlGxTvwOaXLHFSo2flq0L8l+pk16b4pad4PRML1R0aiU/uwvKPJvSWHDLbIYmWA44CteoXkvg9TXi8vsENLKldtXFGFYdk03Dgv75NoqXbfuzD6KU+zdcmFLIjvnroLr29LOk1j+ya0j7csdCHXh8efz1B78yEVEN9xicoyZlrmm3PfjSmfsafRp2PyK3/Mm4ovemwdyHN7iWf0VysHNIUVG5GtI5rl8m/nVGgwzkd87lVSG27JFrC7LTOSC9avWrpHY5J9GNSu1Ly51adKQkbkm1Fvre+MCK/ArvPG3qLDss/DrGOIX6zzcLxXdLlIf/lS/8t9xtVaokebxx5wM+2jrFwfJfRUkCYEjY90VUgR5TVTH7ztQSxB4W7mjQV76o0CXMSkY+Luu3vXXSY+2PxBgazDpMStM8H+P1pqJszXZhjS+UTh8y6Q2ElAke/f/S9GfJrIcPa5+gyZUK9Yb0Dht1LSbw+rP4FTXAGC5b0jqV7XEa0zgkJUqShHOENLOOymu6Nb5+gd1DyvMGCIWglsaFhKd/mH1+9QO6uT5TS2HXug/E+DGKwaDGI0YSF3rIoLYNFcZcqN4ZsvZLEiZQE3TyVxAksYdZy8E9lo7SOToHCPYfiHXUamidd8o+nprpZtZFKt6QO635lwKimUsI6kNzTB9Zb1ocae9J4ZrYDwvT0fYpg74LHqGo49b9dF31leqveO+9T/PUNLs5tsN0oYK8V+Hf+4/81rZ08/+hvfYGP//YFdnaKuMZSuOmTqgfk12Rdbpz2qL3nMFiISTZUuzuhGv7jsoLMYlci56Ag4Ef5tthWg0VZhYr3Ysofqy3fX9LPNVg05NfG+/abtW9ZxRP1VbZtvu7toyz/vNczsVCtow8hv6oP9QnzyLr6YKMkFK+7ZLbl1izdi0k15fJsn2Ifdd464ZBd0w6WqkNq0+Po4R2uriyx9IculRshzkQs/8bpJMmmoXDfkn0s8UT3sPzxk4IQjplNh93LDq0TaeWsNixbX44kWUsY/J7B606UYJKC/lJMbj2iccHSOqvWVW9J8OHUnvSX9csxrVOQ/VmW3IpDnLT7nIJJyeB3LZXbAWFKUsHG2SROCJWbIUt/2iBKWoYLIRt/6Tjt4w65VQWPvfzbV3m8XebdKyfx72TIbFipmHoug8WYSQmeP7aCYyydUyHjik/3gIM3Elt1XFKvOMhPe9YzhvLtmJnroeLLH8Ow5lD9NKZxbhqe4UP5pmHvEuy8pg6ACRWo4Q0M1asRvcUkqVYMsQTfybZOq2QDSnfE9nqa1zOxUGNfOBon1C8bJ5QanXtsmRSmWsY0dA8pUrF91GHvoqF3OGLh5xFR2tA8YxjPhnRPypveOh+Sf2WXh2s1MjdS9JYUg5OqQ2FFIuUwC+WbXfy+ZflHEekdo9v0Q2kGJgU97cbC1qtJka5/5O5jFbPrlvv/YpbxbKRMpV2HxmmXZMMhuecQFHQaBAVZQhpnXbFVd+V4jROQW9GDkduIGM4qcGNUcsnshlNdrC5J9bMeW1+skGgrOjLISfQxLsPD33Z56+4JZio9bDoiWQd/oPfy6O+MSbRkr5lN9Vi9NQd+TP2c5vGxa1h4e0zsyWVRvB8rhbvNtAUnQXWcUE+1eVKopdxjGR2HNUN6y2HmikOYMURp5VIlWxKot486dA64DBYs9ec0weqcDqncDugccT5fO6o31lHZX3CYua7m8c5LsPtyhDcQVCG7GROmlf1kYhno8g9cxiWHIAOLPw3AtZhMCNUxyR2X7iBF/tOkdtyMGu1OYNl+yaVzVKLe9a8WlN10VOGeoxldxKyntL3yLcve8zGTM0PcsWJ+hrNiYtWfE+s0f1eUwcyW3vTKzQh3AuWb2i39jsqVuQ8CxrMRkwIk2iGDMyM6xyX06M27xGktrs4Rh8df088YZA2tU7D48yHtUzGphqXwUDQUYyE83+MvvfI+jmvZ28sz/yMPd6z3EuD+v5BgVI352r/0Pn/04UVwofhpgqAYTwXSMUHeneoQ9PMHeaheH6pt5oud0F8WnXq4HFG+rk2j+qndV0UlO7HgaDldBK0xpFpTtZkjF4QTamCR3HWJUgZ3KL7A07yeiYUaJqFzFCo3AurnfEys7KLUtke6KVBE+5hD9aqleH9E/rFqq2RDi9Ybwt7FBInimHK5R/6DNM7EkP3jHP2lmPobE/IrsYx8SUPtY2VDOaHkcnFCwLBxWW2YZDvGHRnWvhPTW3LIrbjM/04Cr6/Lgd9/wlEydA5rx7cOtE+qr9hddvftGv0lQ+f1IY2XpPHU1Ao2X0+SvZ4ivT3F6CTBBIbq1TFhTojySSFm5maA3zdsvpHGCdXFKN4dcPj3O1RuBgTtJP/g2mUAcp+m2H7NUvugQ3YjZlQ1JBsupjLmD++enaYKWhZ/sEfxliGzFzMqOeyd9yCGuQ9lY8mtWrZeSuMEYkwN5g3uQBCP1KZL74BcAZO8YVjTdHD9mzHZ7Uj480XD4h+uMS44VD8L6R2K6R5ySNZFqZ65Zmkf9pgULXMffp4a/rFiyrde8yjfC4lS2u1qnyowIvZgVItUW55JESbNNMhAt+LqtQmTksU+zOIYaJ8LCHKW1teGVD4zpO8l1eBv64PuLrtMSmo+Twrg9TX3zmxqd+sc9Ih9S+VDjyil9thwRoIVgb0kV5u5boWnzGlqZCLYfUGoIMHXpgzRBym8poc7MAT5mCAv2Fp+RdrM2IfukYjUjsPupSSZdbFY/Z6hPyvWfmbTMvs+lK+2uPs/96lfLNA+4pHY81iabWGM+q1u32H7jSKN84a//D/5EQe/sMq5g5tEoUt60yO95rH+jSqJnmX9q1pkyaY+h85Bj+4hxXo6kboQu89J2aVYH90PDn2vxXBWxOg4rTbWsb8TsfOC3rfcumXt15fpLxh2L3sU7qlHG6W1W9fPi+M6qUa0jj+dxP+ZWKhxQmz+SSVmMm0au2PYetUVUW8z5uAfxZTuBUpfXlR/Mp5S8Fa/5RG7ajSPflrFL0z44pc/g80UoxlNVIZVdREKjwIyOzHJhp1ecth3WfaXp/jHyFK5LtWSiZhGrttpqC/7eaHdZUdIx5FEHbNXIrKPHXIr0h+EaUN6RyPM9LbsxPmHDmHWcuK/bTMqSwye2pPaKvY0nBjVLPn7DiZSGy63KT6BiS3NiyW8us9oRpe0RNMQ/c05gmaS0YzQjr0DlnBpzD9eP8edO4sU/BFxz5deNi17SqoZ4XVdmmf1sGa2DOOKFq0z1sClvySStLxmUFiN2LtkaFwo4k4MjYuWhZ/KTLl3IcXsRyHuxNBbMhQeR4xm4/1o+OZFnWL5FSEoe8sOXsfdj1P/817PBB81M3fAnvmVf5fhrJlmFhk6R2PcsWb6sa8jNcjoNproKmUPINGLaZz2mLkR0jno0TkRk7+vuJjsxQa1bJ8Yw+peGedGDr8Lc+8P2HotI+FIQbuzCSX87S8ZhgshxVsekwIs/WRI83RqX0+Q24xonPFkM/lGkvSudtNJHsp3BW/bfN0lysb4LYe5DyKipMO4IFZ+mJSteVK2lG+olky2Lbm1kGHNo/BgSPt4WotuM55G4jiy3TQjrAvdJY9xWTbu0u2Y1kmHmevyOK38moFkBBOHVGVEMPGIuj75u2od+T3lZ1U/Eow4vaNdNdGV1STMSG+KowWb7MSMiw7FBwHjsi5WzZOiDg5rYh34fcveJUjWZacprArEHPu68HWPRyR3ZdnJr8ZMcioj4oRq91v/53/v88FHfeJYTLRlChtVxFHyBobxjC42o7JD6f6EzLb84/0lh1QzYu23xBRd/Y5u1qUbujBlNg296xUe7syQ9gLiR1km5Zj0nmX3cobMthbp3AdD7JQ40jhvyT+ylK8K3lt8EPPoV+R4DdNQ+6RP+4jGlhuvp3ACWV/Su7GQRHmHcdGlcg2ckSGzadg779FbdOgcVWkQZg2zV8bMXFUKi2bnhtYJnygJo1pSSTBHI3a+M2ZUcRiXdSNvnJWW1HqQ6MriPaw65B9ZOodcVr/jkKn1SeUmVBbbBIGLWU+Ru++R2Z566U9aZQw8p9jLKGmmfWk9cLnH6nlmtmPCHPt//85fG7L1pu4GxsqAGRQs7ROW7ddjln4irUOybdn4qoYHsafPpHLVIfalBQ7T05H4lhRspfufoxp1UpIRLtGzJFsKXMhs6mlMb4lWF6Vg6+Uk7RO6oKS3LVHSYe6PEkQpizt0aFzQjphsS+3vDQzmQYa7Pz5KWAr5jS+8T+OsZuhhShenjTfTcoXOWhJtHeXDOcnwtr4cEWUFE8bCyndyJNqWVEPAtXEtVI1l5L8fLMi92l80pPYcMjsR45o+kERHfdHBoqV5KsHeZeWE1j4W6HdYU2+2P6+PJL3m4mykGFfUHhtXtNDCjKH4ICT/OOLI700oPgppnoHehTELJ3a5vLCO+TTPOPCx2ykyG7p0hmlD8f6A6hW95+lth85hpUqnG5buAW0UYcrsK8vGJUvnWEz5TkzmT/KYwCFVl5PXxNIwZDYdnLFD67hHflXDlMoVl/zKFFd0RBigRNcw/15Auh7jTnSKJzoaKjzN65lYqImWZfPVFFg5K+OkLkp+39I9HhKlLKOaAGAmgszGk8jIkM5hh+yG7CaVq7qE9ZYcCg9jZq5FpHYMUcJSfc9jNtGV8/KEO7WcaEpUfBThBFr8uY2I2Y+0Sy/80KVw26V5IaJ3WAu7ccFSvNMns2WpvedSfBAxKiuFeVLQvD7ZsgyWIvrzLktvKe/pyW249pH0rcRq43QPCiKx8G7IYNEyKeoh8UZyBiTaMCnE5B45GjdaKe+HVYe9Cyl2L3r827/6h5QqPbYbBXpBkjBnia8UWXor/mV2bNUwKSboLTukdh36B+UTswb2pju7N1S96g713mS2DMmGw94FQ/OMUD2r33Uo3Ysk1D4UMy5ZSrdEsO4dMAxnHNon5XYFmP1QP7sJFV8/LAtf3z6mvqvfebo18ucuVGNMyhjzvjHmU2PMdWPMfzT984ox5vvGmLvTf5b/ia/5G8aYe8aY28aYb/15f0eUnpLyDjq0j6dZeiucKudlfZj9QJedJwyn/rKK/o0vJpm5GWKNoX7B0DqtGqx0P6K/4LD9kkP3RMTchzF7L0X8g5XL/KWv/4JJQc3q9K4WfHfZJdkQS7V+wWX3OQ/rCBs0LoNfGZF/ID/U7IfQO5wlu60LzmDWob8IRBJiFB8E9A5A4a4yWBunPVrfGuCOoPDAMqo4bL84pWYPVFKMy9Cbnzo/Q9j+gpKo5z6cUPt0ROGBklaSDUkbt17z6C8YOi+Mmf3SBv/P229yemaH04vbXFtbJHGyw+jkiK3XVCbsvhTjvtKk/tf7DA5GgmesuQQ5w9JPR8x8JlHMuOwwXA7xRpbZK+H+UCG7ISMgDpSuOYRJs796MpuG0r0J1pOF3AmgdkUPde9QzKgsYY0TQPdYRPewBglhRg+vO/5n1/AfA1+11j4HXAK+bYx5FfgPgB9aa08AP5z+/xhjzgJ/GTgHfBv4z4wxf2YPwjrT8WFPTXnr6rga/b+p+89gS7M0Ow979meOt/dc79J7U5mVZburuqfd9FjMjARAQ4AAJEqkKJEMiEaGUoQoMcSQfvCHxFAoIMgRIkGYwXAMeqZ7pr0pl+WyKiu9vXm9O95+buvHOnl7QiQxSXIQkXUiKqoyK8295+xv73e/71rPmlABnriG3rkR+fWE/JqU7vmdGBNKdQ6w+P2AdF190vYh3SZLj8Qi3XrNYfYnDvVGnn9y5xLewFB8qDor1YLozZaCJh5FAkZ8HI5dkvqaZv9Bhu6STGz7vz4gKBjahzxGE5ZMQzu9zcZMXo8ZVoVdV/NbGPLyd3PkdhK6i+ojTl1LyOwoOt3vQvm+zH5R3pDbtuRWFNDWn/LoLqSwDoQXerRPaBfObcD8z0a4qZhekKK3k+dy+Ql3Nmb46+evcry2RzY/IiokDOdCJo/WKWVGRJHL5OE6bgDlRwmplmXnxQxh3hCWDDPvdZj/gVIQvX580DpLt+yYx6+vsX3E0YV2V2q17kIKv61Fu3/RkupIFTZxXaytztKY1Ljqkts0Y/u2oX0hwImfaZ3++QvV6tUd/9Af/2OB3wD+3vjn/x7wm+P//g3gH1prR9baR8B94JV/5hcRwuSnIa0zEd3lhFFJX9ZgUvbh5kkov5+mfVhUZXdoZfvIWNZ/IyQoQ+NkivLDmM6VIe3TEfl11ZKFNcUwdpYd/gcvvMN0pUt+TYtI9GoI75XI7Bo23nTIbVp2rvgMpgzd5QR3CK2jHos/CkhcmP9PUge8JQwMK4byHXD6LmHOYVSRj8oN9HVWb5iDCU9hTUMF64pvJcit1FP9Q5GmZ0WdLukm7L6kEwQLqWt58msSqgxmDPvn0uqdujH/3i/8Pv/3T9/k8Mw+/8mP3iRIXBYrLRZP7DCz1GBvv0gQuwzrGVofTZLbEG1wWNP3YWLd8DuHc+xeUc9z53KawoYQ8HsXJa72O5aJWyHlh4ksPq0xLG3u54j23JbGrqOqofZZl+6i9qjsntyt3UOWiXEea/la6qBE+G+9UAGMMa4x5hqwA3zXWvseMGOt3QQY/3t6/MsXgNU/89vXxj/3//9n/ivGmA+MMR+Eox5h3iG/4jH7rmXvkiFbV0PZiSyptox9xVUV5YoyNyJ23FTGZpzVTmzqKfASmr/UZ+OrAqn5HQuvtvj2xlk2b07TXdY0KVM3LPy4j9sXICFdH5v1HukC5AQGYy2ZPcvGvxpQWFNorxPoyKrehN6CdApJWsFiTzGV+xcgMw7KCEqqLXO7EenGGGlZdpj/2YjevObifl078WDWMpwyTNwckV91DjIInAgGU5qIVW8rEXC62uFYeZ///Y9+k3xuxNqPlyg9cOj8R0tsdYocK++xvTLBv3TpbXa3y5Ru++TXfw5Ic4fS1o6qcgMMJxxI5JAoriZiUw0Y29Yhux/TnVObK7tj5Zo9FGoBLiluMrOr3TdTT2gfzROUpCtunDYkKZFoGmehel9KcvNsl/5ns0tba2PgkjGmAvyeMeb8P+OX/5c9Iv+FQsRa+3eBvwuQPrRo9y+ohTGsOsy+l9Afx0tOXXUICzDzXojfCcHK29M4DZn62HyWk8I9zDskxYBseUg2HTBKRxyt7bN2qkzOTai/PcvsrZgwL7JKfsMSln1Kjy2tY4KEDSbV+J66pnHg9ssO0WRI4WqR4STj/iHM/+kej//KNKmOYGGHvmVZ/4rP9AexRqgJDCvyTmV3BJ/Y+YJh6l3LzjcCsrczxJn0gSY0v+4caAzA0jyeknTxtEthTRec2pOY7oKgZdYYfmn+Jr/z8DLpiQH+tyq4niXT1ORtcLPCRz+sMtWy/L/slyg89PTAjidwT4mBOBpyVO/p605Seij6kzrenVAPdG8xYafgEZQTqjfVwdh70WXmJ9A8achv6KLbOGeZuWppnHCY+iTC6zswMEx9NOLxb3kUHrpkdgxbL2fJ7Vg6h/45qKestU3gR6j23DbGzAGM/70z/mVrwNKf+W2LwMY/6881oeHY/3tdMegpQ2/apbsE8z9Ukz3Vtqx/2SMq+GMFzlhi11Ntm4yRlN5AQNr4dpH2jRpR5HBzbY7OzQl2V6uk2qppnVAflN9L6Cx49KcFVEj8nyNmOgsujZOqkad+6jOasAczfb8PT35rmolbMdkdCbhXv+GS2zA0j2khpZuWOGsY1RKGY3HwzFsO9YuW3K0MYVl/ntfTcR9n9HcXNiKlYT8IaB92STfktcrtiWkVZ2TnsB78dPc4nSclRq2MzIcTIrnEWeWuFjaV6VS64zGcVvurcUbfw96LcpcOJ0SX7s26lB9HmNiwf15DFr+jTcAJZfVOXIWdPbW+5Ndli566FutSZKH0QKFxqQ70pxUOl9u0rP9CisyGyoDOCWXNJi5kt/+CLlPGmKnxTooxJgt8HbgN/CHwt8a/7G8BfzD+7z8EftsYkzbGHAFOAFf/vL/n5r87Q+mRLiGjCamjWsd0M83tJpQeQlhwiAsJqbaiaJzIMqrI2ej1VG8NX+mS2dVRnnq3yOS300x+YsmuenSXE3pziqvpz1vCnHxOS3+8x3BSMLE4rVSP/rw8TU449tbfkWFuMGXIbWkEOyo5dJfkAfJbjjJCx9LA/rRKjvyqpkqtSwGpbsLUB5qh1z4V6Dbx9WuxIorsn/OxjnrGw5oUUu7QsHdRf3eUE107U0+4f32R/JpL9SOP9lHpcre/mDCasLSPCvQwrDgH0Zwz73dItRxapywLP04OVkDia7LWPOYx+YkVhTqtutIdA3rLj2Mmr2tqldmHxR+ETH00YlRL6C4o87W4ETGsqRbvz1uy+zH9OUthI2DpeyOivMqo3BOPubcGslF7f3E76hzwQ2PMp8D7qEb9FvB/Ar5hjLkHfGP8Y6y1N4B/DNwEvgP8a+PS4b/y5fctpds+nUOygVhHnYCJWxEmhp0rslM/BZbFGajeC8d4RPnY3YFgBvZx/sAzHxbUChlOOJQeJ+Q2lTPaOQp+SxS/YBwX5Lc1pn3KmvI7hsYZCXyjrHa9/FZCbzk+MLY9dZuGBUP1tmzGhSfQ+ELA6KUuo4qhczzGCcHf9mkc98aTIPnqN7/gMZxUgnRvKWH2vXgci2NZ+tMW5fvqeCQpS/W2HpzMLix/u41J4NjvqM4rrsbqw84aTOjg9XRDTzWhc1T6geyW4dFvFvAGss60l7wxrEI3+eyeeqrNkw6zb/dI71v2LzhjVGVCe9ll+1XoHlcASHvZZ+vVNBOfSqMQ5Sx75yRgCXNy2G6+7pLZM+xcSbN/NkNmX8OUVFuDFjf4OXr+v/VCtdZ+aq29bK29aK09b63998c/v2+t/Zq19sT43/U/83v+A2vtMWvtKWvtt/+8vyNOG0bVsQNy0TnIK33yywouqN7SpSRTt+Qfj+EORYVp9eclNl76fpfiY5i9mlBaDQmLugAkPuS3Y3Zfgv5cwt4lTYQyddXDwykx8wdzVqG9bT0IT6V2fs+S35AwJMwacmsuW2/IfuH3NTVKUkiwsqsZdvpxmmQtJ1LJDYfS4xGptqF6P6I3ZyivhBQ2IjL7hmgyxA0scz+z7J8TX8ofWB7/pTKprjDqTihF/aiiS+DDv1wkuxex+XpOyS7TLr1pT5igLYdUUxaeYU1gs1FZR7WJoXIvxuszdiRIPO31LamOlGAmhN0reUZVw9zbEb0FaXAxEoaXb3jsnzdUHowkr/xKIIjcJ6p/e/PmAAhSu27HwwNL8LUWpccx+Q3pVuMMVG92STc+R8LpxBPdOSgbSKC7qAWW2fFwR5Lyzbwfk24lBGVdTDAikMz9TCqnR38pT5wytI64dBZ9MntWxJQh9GZcZt5jHM6rSVXi6RIxec2MWaeG9K5LeSUiyonoV36g0WPi6njqzxmivKV0x6Nx2hER715MULL4PZEIo4wgxOl9YW7y2wkP/7JPfkNBa6Nawv4Zn5W/kjD56YiJqz6llYhRWSVCf8ahO6f8p70LLvvnZZ3O7shVax2I5kZ0FnwYt9eGtXF8USiorzQFkiymWobuIVlh5t4K2b3kMnFHCSheT+/JcGJ8/BpY/EGPOAXTH43YesUnt6Xe79Pkv1TbUrueMKr4orfcSZPbSUg8mH9rKIp2Rn/c7sviKMRpS3S7RFBwSHWUYpPZg/0LBQbTz3b0PxeQNBOjp3EpoXTXISoYMPLTu0PL/uWEwZpH5X6M31bPbueKEj1M7FK9qy6BO7LUXx4R385Idb4P5ZWI5jEx93ObOqqjvBJIBlVzYOF1A7Vt+pMu+TVdTFqnE7wOcsIWXV3WnliaJwzFFQHGvKHCxUYll6Csm3S6lTCsSTzdPOFy7B/22X41R1iyTH2oNpO3k6J1REOF7rxHd1GTH+tYRrMR+Qf+OKpI74F1daJYB6a+l6Y3byg9TlRvzxqmPhHoONc1RAXIbSj1urgWsPbVFN3lhLCQovZZzM6LKTL70q9idUELCtKhPvqNHNZL6G37xFnLqKqLZumJ/vxsPVYeV0NdGneolJSJm7B/ViHIYSkht+FQeDwGfbQcSo/0ftXPuyTTI5zdFOV7huLK50iU4vXHvdNNiSTy65KQjaq6pR/7x8E4gU78z8G0AFsm0Vw9sx9SfhQSlAxz30pRfKInHAOdRY9U0yrT9FRIbjehdjPCHepI9/qQbqprMPmxRqvFVV39U02HxR8GZLck2PC7lsYpM07uE9eqfs7QPaoSPKiqUd8+pPZU95Bkejsv53ACOPTHQzrLDu1TMYVVQchqN0N6c+LeWwPzPwko3PfFPfUtw5qMf3HG0DppZWZsxPSORPRnHPpzKgcaJ0Q5jArQPRxJAH4MBlM+Ux8lpFpy1qYbEWFRCzAoSnpoHQF23RHE+YS5tzXOTjeU/jeckFgmzuhBzm06xBnB6yY/jajegmHV0DoVU3pkqX2qVD+/q/Ijv6ZFWlyNqV2H6ttplr4bMZwwf6GXqX/ur6BiKDyG2g1lFtWu98ivW2Y+iHADy9ZrWY00S/pw8+tQWLcUVi39OWgdThHmnTG2Uosp8aF6L6J1Mqa3ZPC7Dn7dI04ZVn9JO1mcUjFfehJpUjOhD2D1F10GswmZHXj414zyQ9OG+gsJbmCY+1mbUdkh3YT8uqF8SwCw7JbwkibWzm2sSpjuIf3cxhtZvD6UbrtEWdh60/Lklx1SbdWUlbsJQdk7qCNrn1mSlB5arydEfKppMLFV9OZ+wqg6ri/37IFo5fg/ColyGlN25x0aJ10Sz5LZg91L6t1mdu1YR6uF3llyGU7CxKcOu5fGPdQvNLCL8k6l2hII+QOFE6f3NWrdfcEjtxuRpGH2bUO6E9NZVncCK9mgG4xTVVJ6OEYVJSI+DUZ7ltdzsVBNpMb41usu1Y/32b2cJ8xLyzmaMGOynqV2I6RxSnBYd2Txhj//LltHXSZuqh9oEoXD9msuuQ2X/Lod5wSoh1q95jKsic+58OMhJlHqdH/W4owMqaZD5Zbe7NyDFKm2hgDZdZepjyPq54rqNAwtzXORsqaOQFiAza9Il2liSDUciivitHoDexCHbpJx5+JTh8yWy+RnQ8JSQvuwOhSa7MiBOvHZGJi7Gsl0dz2kcdrXr2nEHP6Wbv5+T9GRE7cj2ssZnEjTIROrTkw3RHYZTitnwImgsBnh9xTeEWUUoTQqG3Ib0H51wJcX75P+LKc8gzmHqfdc/K7wSElqPEWbj3ECkWpaRxzWviZXhDW6R6Sb4jXkt0Lahx3ax0R2SXUTCqsicT/L6/lYqAlM3IlxRoaV35qiuyRvzezVEcOapXYjpj9nqJ/ysa4lv27wRgmtoy7puvw9uS3LzkuqEa2ri0f5cTBWyKvOGtZ0WSk/Ckm1DOk9I+7orDemXhuc0DA6qpzPdEMc0+7JgMq9hKXvdWgd8eguG7Zf1/x+8U9FysuMlf5e21VyyliGs/uNEdkt4d2zewmprqX5QqgLT1YP1P6ZDKYSjEHDSgMcTBp2XpJm1CTQPqRM1tYhX/lOE5Ynv+LQPJEhymrnXP+yx6jsEJQNmT31fWff65LdFlfVCZWGnXiGxjmVRcOaIb8muszUNUXKD6cMuU+zfOfeWZwA9l9UZ8QbWYy1P4+tvwm1Dx2efDPF4T8eqaX4iUPxiejbud2YOKUJ4M6VFFHWMv8z+c960y7DqhKrn+X1XCxU2Wt1cYiyloUfhyQe7J1PU/tUwQa1z2JGNe145QchYc7B7+rIDPO6UMy+q5ieyZe2Ka3ErPyKT2bfMv+TPiZWC8XvWRonU/RPjshvJlTvDsZ5qUZHWgMmf5BmdH7AC//ydXqv93FzEfvnDQ9/q8BoAgHVtp1xYJmrXqADk9cjJq9pB018mLgdk72VobcsXej+OZcwayjc9RlVZS3GandNuj7tsXV4MCPDXHZH6q2nsDInhOx+ItF3BDNvafcKx6qr2XdEdomyMPNOC3cI268UcAPGeB3t9MMJQ/XGmG/Q0d/pjsRPza8bwjN9+nMJ/vU8/YWEyfcdardiNr8RMayoFu4cUh+5N6/c2MaJNNaH/Rdjtr8ibsKo7NI8ra6OE0IwG4klMI5p94aWwsozrpF/Duvuv/7LKo68fTJm9mpMZ9EHR4OA5kmHxe/12T/nkm7IIDesuXQXnINUjSinD2DjazGJZ2h/fxa/GysWp2QYTikxrnPIoXVC5rbc3TTZvYj2EfVS+jMWFga8+NvXKf6NdU4vbPHp3jyF/JBSYUD1wh5ROSE+02V0SrjHqY9HpJtqvwymE3pzLvsXlA5YeiLh9GhSN+DikwRvyMHt3R39PKu0cy6g8pmH3zFjaK60t0FZk6nclnqc2T1dfECTOOsKXhxnYPdrAb0Zh1HZoT+fsPqLZRJfteVTlVT3cEK6lZBuWdrHDNsv+/h9exA5n91RVE8hP8TrC0Zx6I9CevOGoOBQ/cBn+xe0S5pInZHKvUSp1p6yqPIrHm7DH/MJDNWblkwj0cl41WPvoupzJxa7tX7pc3TrNwks/KhH+Y7LxhddBtOG2mcx7SN6EnsLGWbflczOCVX/pMepKTPvq4kcVDTdetrfrJ9Nifk0J9tIcVX9zsmPxXkyCWy9msI6QkAGtZjFySaNUY5h5HHj9hKNOxP4v1elfa/KfqPAK5fukSQO7I/t140hQVmWl4UfJWrItwz+IKF+2mVUhuoNMQNSnYTuCblo+7PqC8e+asvso5QGCmXLaNKSbo77xA9UN0+9t09uVzlQ21+w5LcS/C5EaUUdWQPFj9M0zyUMp9WtcGJR9cK8TIKjimH52yGtYyonpj9Qv3g4oY6GG4rPWj9vCd+ZIF0XS2HjzbSO8nG/tnTDFxNrRz3q5gnpF4KyQt6ivMXvGdJty9QncmB0ll2ceJwr61o9vNsJvRmH8p3Pk13ah9FkmurdgIWfKP57+1WH/IZh4nbMzkuGx7/laAJkdWkpbETk1xOaRz0GUw7uSCNTdyh3Jla1aX7NIcwpB+nI77VpnNYi1XRG/NX6xYRzZ1d5slHj7u4UG6s1ivc8TKT6s/jQwV3NcPWjE2SzgWC5ecvOyyXcgfA2o7L4A+mGpXnMZeb9gMoDQdBSTUN/0qVyTW2nVFOa1OF8zO6L2mUHEw5RMaFyG3onA5onYfeNiMP/tM/Kb0yy+QWPUdVh6n2HzpLei2wjweuPJ2NAZselOKaoxCk19NvH5frM7SQMJ7XTpeuW9a84TH2UMJgRbjK9L7RlfrnN5KehjHh5S3YXtn9BVGzFdKpV1h836nNb0jM8FfOYyBBlLfWzDv2ay/zbQ3qLybi/LKa/ExqaJ51x2fJsa+S5WKhxWh7+nRdTrH3NJb8Tk90yai3NKm258MA7SJGzxrDzkj9O4NPFxeupezD3zhBnBJUHEcMp9TF3r8D2GwnrXytj7JhWN3aqJimwuZiNdonUkxTuuyUW/kSZTqmmjjMnsnC4z0uX79NbKx7Uu0FZnYHugqE/60gnm9WDsPEln1QrJtWC2q2IUVUZAZUHCc1LIcOKQ+Ghi/UtcUayw+JdiXAmf+oz9XFC9rFP86Rs3am2oXk5IMo8DX+A9S/r4Zz5IFQu17jedUYKtMg0EryuoX4lon5OEUKT15Ui7fblb8ruiOIdZ1SjB5+ViTPj9lRgaJ6PKNzzmXurj3Wh/OCp21RCltxuTHEtprSSMPOe2K9PAWlB2bB3PkN+TayDzqJDf94y91bMcDlgMKMT8llez8VCFYl4TEy+rnqocyKm8NhlWIPK/YDyQ2XUD6YMTmwpPBHR2A20KMOidoCNL2aYvTpi/bdD8qtjIfQ9h9JtYWsqd3Szbp+w9F7vM7zYh9jQfFwRZj0NvVlXdW+iH/e/3iWOHG7tzlD7WH711lGZ3Aprqh/9jqJ7yo8j8hsJJjJsvebjhJbV34gJyuo49GYcZn/okuoqj6n2kcORP+iR29asvHUiIdWzdBdcZt8PxB1IpEed+qlPUDbUzwu1k9nR/L8/JWhaqqX3p3YjIUlBbitk7t0Iv+EdZB3EaUPncMLETbXJ3BEUH3ZJNxOqV3YJFkI6Cy5eTxLEzKYnfuw3cmT2JZhJPFGpnQj2z3n0Zl32zzlk90ImrwfUPotpHXHHIBFLdzlhMKMcrIkbwvnk7wo5H5aebY08FwvVGSnQK7+tQC+/l2DTMZ2TEUnK0p1PSfC7I9Lc0+jD+hkJMda+IgqKOzQHPnGznpHgYSwaibI67t1ATstU06Fa7mFWs/gNj/SeS+GJpkD9WUumjsa6hyP+9rkfsDxbJ/1HZfy+PdBY7p93GUwaRe8c1ffSOOHRm3MYLY9wAmieT8jfSR9AHoIK1M8rTSQs6vvYeKOA31O/9NC3Y5rHNHoMSi69xfHsPif9ZvdkSKptaC95TNzWThaUpMgqrsXj+E1JI3vzPqtfd5n8WHhHv2sJig65DYfenHPgjt2/WKR50mGvUSR3L0Vv0dI8Df2FhPIDuWZHUxGt4+D/y9t4fSisqOWngGGpzHpzPttXUiS+/GDZ/YT+/DhbYMuy8APYfs2O5YMqgUoPPkd91CQFxYfQmdeT31lyKd5MUbwnXlPzlC4Ou5fUkpp+ex+/px1YNZ9zkJrcn3LYfC1NZk80uaCiJz811q/25tyD2/NEts/Rl1YprMBoOqZzRLvzwtgflWpaqgstflA/zfp+mf1XIjrLDrWb4YEOtfQ40TBgWz8eTko0XbqWZuJ2TO1DlRHFFanfn5YUiS+eU3ZPCc5hXgqpvQs+k9cjqncjtl51mHs7xhtqWJDb06g2ylvSLX2vrWMO2V19Da0j7hhLbiU+X9THG2WlThtVDe0jYzdvLEt6lFd7yxvA0nSdVEeRk9VbOsL3XjC0/1abVG2IWRhgjCX5WoPkGw36c1r8UUGfw95lDRd2XoIkLcetdSwLP7bkdxJ60w4z72o6FZR169/50rOFoT4fSJ/pJbv8r/5bDJZCTOhQ+UxHdpySADq3m+D11f55mr7cn5cnvLiW4I4S4rRu+7svwsxV/d69Fy2TH0mR7gY6xkcTlslrlvYhR2C1pQHVUp/eO5N446z6+sshsz/w2P5mgJ+JCHezTHzskNuLCQoOvXlnPMuPya+5eF1Rq3Mb+gAAiqsJ3sCyc8UhuyuxdXFlQJzx2HkxDQ5MfTTi0V92mfmZ/sxUU7VqZo+D1lKUN+Q3YwqPuzTOlnBitYL2Linmp/DEMqoZhjVL9ZYWXedIQn5dl7uwaJm4aenOyyPlDfV5Z+sxjRPCZQ4nIElZgoUAbydF6YF2/52XYeJknUJ6xCD02d6q4DQ1HImLMct/aPC7EY1TaRJPJ2L9lEucVS9ZZkZD51jMwg/HmVdp6Xf9rqW7LFDzh//xv/3nIn2eC/WUEwEWJt/1MImOOSeE7F7MsKpdYetVl8lPE9rHtEi8LhTWn44dDdkdQ2E9we84hFl9GLVr46b04YjaBy6UDNGERpCFVTGunEzIF+ce8gdLFbymizs0FO6kaB0DO3SJXIuJDPuvRIzu+TixwLvlB5DddRhMqg4r32Osspd2IPENzRMu+XWk5WzE1M/mqF+QamniM8tgyqdy3aGzzEFPtPRYnYg4jbKvrEtvxqW7UFaIW9Oy+2pCas898Cjl1/VAB0X1Vsv3dMmK8hKAx+O0QmPBjgAHetOu8gd2VcqEOUNyNGbymkI72sehcLTJqYkdPv0nZ4myUNuxJK4SwPPHWqx/uYLf8RguhuQe++S39X34XW0ivXl1BgqPXfbOS2TjDTSkKD2OSDddtl+38B8/wxr557Ly/mu+4rSK+rBgJHez0DkMG19TnKR1pKe0jmHqIzWw3UBHmtdVfRtlobPsMPdOSKqb0DkCk+/XGUwZCo883FB6gdyDFCZSSG9+3VL8nSIbgzLnTq9y4tUVorwWIi+0+dtvfJfMjSylew619z2VESG4gb5Gv6f2jiwV47Bdi0DByy79xZg4o+9v+yVfYIerkN3U7H/raxFxGqp3Y9J11dDWGdeGjxJaRzx2X4vpLerBy9QTdl+GzIbLaCYiu2MxsWzf0x9KkOwN9GtLKzHle8n4SLak6/q5oCxLS1DR4q5fkNu3czwmnQnp/fdaRK92SOaH9PppPlhborCRCFJXGOOSJmM6qyWmPpIwyOm6RFlLd06Rn9ZAf0G07fym6lgnBCcyzFwdjS9hPt0Fh6O/92zX/udioQIMZjVPTnWgei8g8SG77pHdsfRmHPLbCVu/qsSUOKOdJSiqMZ7Zk+HO68Hm69r13IHh9v+8gIllUusuCE4hZ6V85fsvWHZfNNz/+yd5XJ9g7VuHGUxZXv36DbLpgP/Le1+XZHDsgc9tKe4mtyUuwM5LDu7QMJw0+OO4oMGUwt38rsXrOgRlfVCH//M9Wkcd/L7FG1q2X3bIrKWksHJ1bFYexNR/cYDfcQgKhmFN70FuU8ic5gmH/BOdKAvfNfQWYPlPAtwAOosuo5qhdUJQjOZf67LzBVGkJ67rcrV7Sb93OC0ua6qlrzHKGtKzfRbKLXqPy8xXW7x8ZIV45FL9vTxBQQG8mX2JU8pLLXILXfYvSmdx7HeHUmotW2rXLVHOkmqMKYmJTpw4o2b/2ldTBGVLaSUe88Qyz7Q+npuFmtnVSNTrWcKCS3ZbLKjBtG6zu5cN+c8ydI7G+F3tYEF5nMDh6fdFeWlL98+5hCVL4Xpab5CRWa23KBVVfkN2j8yeg981dJchuFsiTkHuRJN3Vw4zDHzchkfruEPxiThNzXMKGguKuoR4XcPETeF3OssKNotylvplqZOirMyB3WXLcL5InLG0l12aZzTHf9r77c07BHmH/rTD5B9l8DvixaabUvbPvtuhtJJQu6mMqCQte8jMhzE7L6Uxkb7//EZC5bYh9sG8Wyaz4ZLb0o6basVM3FQbbeFHGqVGOQHowhKYa0XuPprl5Zfvslxo8N7146SfpNl7wZCtCx7hBpbe5QHNrSLZb5WY/0lEZ8mw+0KO/rxl4Scxu1fkgtAuHOBElskP2zJkdg1zbyuVcOMr427Glc4zrY/nYqGaMTez+FgXg/ayYl3ql0QTCXPSYdY+C6l+5tA5E1B8IjF1dns8kQpgdKVLmJc4pPBk3AfNaCAwOjKkcluXnnQ7UTRNLTkgDlRuS8E1GvnYxOF/ff472OkRwYkBm2+q7Dj1d3u0TyRCJkbyVXWW9BYWVxMST7tuZtOlu2zwBrK5VG9Cf8anelsdjsmPDP0F0flSHQmwn0Iqopxh5oMBpYfyGnWXYDSRJk4Z+pMO+XVLfl1qqM6Cx7Cm8aUbysjYPi4A2dxbfVEAq4rRaZz0iTKGdMfSOOnhBjBxO6R6iwPOlZOJufrgMLfqMzh9dVhm30voLKgefunf+ZBzy5tkNnxyuzGr33RZ/uMWmaYw8c1jHrPvJuTXHLa+HNM+lGJUNtz/ayWmPh7h9WHnikfzFGS2PRZ+3CN+XHi2NfK83PrP/uq/edBwD4tqocRpNbqtC6MJ7VBJxlK6p5Tk5gmZ00qPE3oLSv542vjONNRfbJy1ChRLwXAqYfKa6mBvAJOfjnj0Wx75Jwoma1xMSE33CbZzOEOFjG38jYBycUB3kKaYG9L6aJJUS9zWzJ4mV61zESYw5Fdd8psJu1dElo5TurlP3Bny+FczTL+vzsWwBrUbspEMx5SSOCsx8mDGktk3dE5ElG57pNqW3G5Mb1oR5l5fjfrOYbWA/I5DdsvSuBhTfKBSaecLMdVPXbJ7gpSNJnShKj9MGEyJhmKSpxc4XcasB4W1mK0vGo5fWGOzU6Szn2dqtsV+o0D5rQyNF2Lcjsuh7wSsfSXF3DsRvWmPKC9F286LPm4AtZshrcM+3WXL0vdDVr8mW03tulpW+a2YUVG94ihruP4ffU5AvtaVcHo4aQQq60JmP6H8KD7g7MdZS27bMPOOPETdRXGihvMxuy8r4ym3O6ZQd9T0HtaUcjyqCONYWHXoLBuCiuh1rSMp/IbD9Icj2uOG/aibJrPlUlwRgobHOVJexN84dZXD5TqTL28zmrQH3NHEh9yKR/Ghy3DKUr7fZ+5nqsm8gQYI9dMZKrchKDq0T8TkNi351QGV212Js7fHeHRHPczBjGXyqkt/VubC1b+iCVyqY2leSGi+OWTunRivZ5h9NyTVseRWPbye2luV6x5RVrC4wYzSRxZ+OmRUVqAvDrQuB/QWnuaTSqPQXXQp3XN4uD1JMTPizbN3Sbkxtp6iuwT5xx7FFUPzeIrDr6+yd95n/5JQnZ0lT+2ommX7JV/vy6ahcSLF5DWh5sO8Ib8V051zDyzxT9t5f97rudhRS8UFe/Hr/zP2z3pk9yz1CwluXynM6aYo1KOSoXnG4rcc0o2fJztP3Oiyc6VwkE+VaSiHNLuXkGonNI+LFZWkf54Q6A3sQRbA9IcBK3/JJbfqErzQI2ylyWx6TH+kRvTGG2pZ5S7v4zqWqXyXtX96WAnNs3K05sblx9S1kJVfdinf0aWps2wYTcd4HYeJG5bBlDNeELpxmxh6yzHuwCGzZyg9ig88TFuvSxRefhiPLdGavvl9feCNM5DZl8Rv6lpEqhXRnU/Rn3XG7Sj1VYOioXtY/qZ0y9I66lC7EdFZ9OjPKX0w1RB+Z/0rDkkxYuaHHtYx7L6cUPvYobAe4kRWqYp9fd/Wk8CkfTIms+ky80HA2td8Ug1DYd2yd0neqdxuxN4Ffyxc187uDSHMSeo3nEp49G/9O5+PHTVOO7LStqFyb4TX0+10VBF/dFQ2TH7YJrfhMHldOU1RXjzUxpkC6TFSXdZoh9x2Mo7bdg/0m51jStwLCxJjaPIE3QWfzJaL34Hjs7u4RfmNmsc8sZgWhpz9yj3+8uFrpLyI3f/kEJWHEbFv1PJ5pK5CdtvSn/bIbjkEZV32wnLCif90wPSHCa1jDsOapX5Owg7QWLd63WH+p5L8hXmRW/bOe+SfCOywd8Gl8LBNf8ZQWglkojsOR39fSPfchogvT76RZvuLqnsrDxJqnwlF7sS6gXcXoX1Yutgwr4sfSIYY5S3ZzQHu0LD8+7LDDKYMy99O2H/BsvNiiubxlEySFQXVzb4bk6knzP3EMPVpxM6VFNaxVO4LbJyuyxW8d8EXNTGjB1MAZStRS0kdiWd5PRcLNcpBdk/OybDgMfWxZflPBTiQL9zw5FfLEuMechVF6ULhsch4ccrgBpa9V0Ux2b0sYUtQkl5zMGvJbLp4I0v3C/2D46Z1OqZ5UjVt57ClPcrwH77yO5x77SFf/OsfMfxah3xhSC3dp+z1af5klrBgWP01UaRLT7Tr7l2RV30wqTHq1CchccrgdR12XirgdxPyazrel78birqyYaneUbkyKjnkdix7VxKaR92xTVkfqhPCk1+pkm4IUdmfM6Qbhp0ryiFonFdWVHbXULorrureBcP2axIlW2csRsmqrVY/r1HtU9FNf9bgdwz1cwWivE6gp9K77pzH8nf0ng4nDI1zGg+3TsDOix6Jqx0yuzWg/DChekOq/crDCL+rjWYwnZDfUPfmqYeq9BCGNWkzwtznaKG6A2gf8jV/LjvUzzrsvJghvyXgROfIuOGc12y8c8jQeXFIaSXShxlbEtdQvKtjfv5nMb152UH6M0qtzm1ZUp2Y1PUcuS3Bv2ofOxRW1aMsPDFs3Jvi3/zhv8ALlTVia/C8GPtWlZ9+5wX+s//dr47txhan45Hdi6if8YjT0oH6PfVb3SFkdod4Aztm91v2z0n1VFiPWX/Tpz9rCHPQm3XYP+fSPuIQ+4ZDf5SAIzF4Zl8L62lXYvKzIRtvZBSvEyqHIPGlc2idEpQ3qKjeTbUMNmWZ/akenPxWQuGJHtxUUw9J4wsjCiswnIkZLEb05g2TH2rsWrsZ4Q5EmOnNeVgXFn7cY+4tS/NsQvk+VO7q4fR7ltaJPLsvqgypn/IUAnxtiDvSlGxYU6mT30gYTuhEyG3q/ck0PkcKf+v83FqsnUgWje6Cw8RN2YSdSD1FJ9LttfRBhlFFboDWccndTKx2TGfRJVPXBaq4qtt/5yjsXPIZTicEZcPcD+t0l+VKLa4q26nw2CW14/E7/+TLXP2PL2N/VsUdweKPhqp7d0RlqX1i2HzDo/QopnEhxh1Cd0Fe/tqNEY9/vUDnsCGYSOjNaRzaOR3SnXfJ7Aug0XhBPdFUS/8k4+nVzPtDNr/oUlyP2X011gPWgvU3MwI+jCyF9RjraIDx1DUbFqRBrV/UA1P70GXvknar7oLQnemmZVTV4p/5kxRh0XD6/7oPRlSZ9lHV/nHaoXU6pnXEYzBtmH1nxIP/icqS7KYr6/hZ5UntXNERP3EdGpcV8Jvbtmy/kpFDQ+HbBBXJHAFyGyKGR3ndM57l9VwsVCeCyv2R+J1oFzKJkkJ2rozJe1lDb8Gwd0kuTSe0eCMhE0sPLaOavEgTt3VUFdZjSiuKTHQi7TLZPUtmx6G4FvPk1yeYuJUwmjDUzxucQBeX6Y8SDv+jbUnwykoF6c2laB9NCCqGwVxCf8YweS1h54rDzNvOAWihci9mWPOZ/0lA6ZElu+mQamliVbztM5iTBrX8MMZvuQynEgqbMX7PkmpaJq9H9GdSuAMBxor3PHqLal8t/nhAbkPq+p2XHIK8jl0ngiO/W6ewEdNdBMy49l7U3F/QYX0vreOMYROxeFQ1y+O/Os3id5QPldtUPkJ/ymHiEwnBa59FbL+cxn+UoXlcvdX8ZszsezH9YwFez9A6JtXW0h+bse1ayrOFHwe0j6u087pSdWXqSk5pHvNINwybX3s2NvpzsVATD9rLaUpPIsKCof+NLiaBwqqodIf/sE/7GOQ3VZtGGSiuCVoWFg2tYwZnpDekP63ddzghx2e6mRyMTOO0oXo3pnnMpbSSjONrLPkncnu2jrg0j7v0j01IGtg2zLyrjKvsjuq6OJcwnLT0p1zMob4ue1XtXO3DMh0+/g2P3I7SoQfTSg7sLSfk1wzpdsL+Bdkwqjd0NHcX5TTYP+exd8mw8JPRQU+5diMm3bLsXM5q1t7QYm+d4GBEeftfK7J3wSXOQO1jXebm3wqIijGNcxpX5jbk5yo8MTROeYRFQ+WuYoFGJWcs61PdiNGxn64rLENjZ/Va003L/jmXvYse2Ucah0YZtQybRz1KjyyZZsLUx11wzBiYAaXHlvYRfd5ByVD7LKB7OmDynWfTRT0fCzUjcURnwSPOQnKnQOsYtI+pfnn8l3I4kSLQM/uGyRsCwQ5r+vKtq2PejuvyzrLejPZh/X8nFN7GuvLij6o6hqyRzz3KSZSdqau/2jzu0z2qGnBUVmBC71BE9XbCzFsO1pf+cu7vp+nPaYRbvSMluxNBetclzIud6oR6AGbfhnQzwToGr2dI70tss3/GZfZqiDuSOMVvG9a+miKzq5qvs+BijUoalTOW2asx5Xuq7b0+TL7ncfhbHZUgS0/1DHIZ5DYcdl6USDvxoHI/JL2vFlhQNJTuO/QW9HfFmbEoZs7Sm3FJtxUg15+zzLyv1O8oq1184rbMiAs/jQ6cBU89UNYxRAWf5lGVWsXVhGHNUH6g9znKGoYTHrPf15TqWV7PhczPHUBcjfCGHgNP5q+JG8nYr69jY+FnA4a1FO0ll53LHu5QRf/2yy7V22qbpDr6IFsnROAzYxt2ugFTHyeMylp4iz+K2HjDU1N6G5pz4A0dWidjatfUlPebDsNpKYay2w6Fhw6tI6ols1vQOxTjjjziYoQZawImP7LEKSmauvPueIeSYHvja5rqZHcMs1eHDKZS1E8LIdRZEvU6zjqExYT5n0i0km7qFp34grJFWcPeF0LSG75SDScslVtq1bWP5nFCw+E/bLL6SxU6S2nKt8xBMFyqJanf7iVNiYY1PWzu0FB+qCmfEwBWEOXGOYv3oezm+VV9jUFJ7IVUG+qnXfKr0Djpk2pZwIwzYg2to4bBRJrBtDoK/Rk9AKOKobvk4HcQeMLRCfUsr+djR/UgteHTXZS7Mr9uKd/rEpQFVjAJ7F7MkmqqHZSua3dpHfYoPtYTHOahccZQP+2SajkMp9XMjrOahbeOOWTG2afbV3wST1OgwoaietyRlETWUUZUZsdQfOBQfKy/K92Ur37x+wNqn0Us/EAI8uonLvNvDdTPzGvRjCa0Ow2nBAobTBum3tbt2W9bgtKYLrglGl6cNiRpS+0zS25DrKinJOjuomr29lEYVWD5Dxzy66o9o2qE31fQgxvqfXr03ymrJRUpVihJWUoryUH+wFMeaWHVYBJD5V5MZ9lh6lpE70RAklZNXViRXSW7m1BajZSenYxtQkaZW0FFteioKpdr4hsmbo3EpN3R6VK5Kwfq4p82sA7SYMQaVsy8H3++7NJODBO3LOVHCfl1S2/R8Og3i0QZzbTzG3Y8D3corapma56xdF8e0F1S+6O0klB6IHGGE0JQ00Tn6RHqt6F1xKE/l5BuSd8ap2DvQopUS/rQiZuW4lo0jnTUB9M6oVox8TTZ2j+f0bFqLf3ZhEzDsvtCligvyVtQFO4yLFhm3lMf0x1A6yQsfTcmLEq8YR1JB4tPLEERsIbtV7UD125Fim/cGCfCTBombliGp4bkVtqi4KWgcs1n6wuQXzPsXJbuIU5rEee2QwZTY+phPKbROBI9ewMO8gMGkw7DCYvXj5n+sU92W5LAJC1SdeO0Q2ajT+ewYerjSGPZaatUv4YyWtMNTbyiHAwnfWGQ5t3xvx26RyPaJ0vktu0BjHg0YWkvezQvfI70qCYSSa9x0qGwEZGu6/jxewpN6M+qpbN70Wf7ZYdUN2HiU8Oh/6+Le75Ff0apze3j6tVFOcuhf2qZuRrDmH/avhAw+VlEbsPBRFItDRZiojEdeTjhsHcZ+lPeAerbCS1L3wvldXJ0cYuyhrWv+bSXPNINh+YJR0diXfqCdAuGkzD3jnJSc7sxs+8NmH03Jso6TNwSt1/6TCitaLBRWLPMvq32W26tL8V+RdjHZBxaPPGTNPsvVhlNJpQfxPhdTXasK6hxds8y9ZHQkyu/7DPzfkJQhv3zikzvzxqW/7hD50hCble5VlFW/qjenC+s5LyldcQ9sIxU7iX0D+WZ/+kIYxX1GRVl0Es3E4JazGDKMDg+UvZUM8Zvj7W5M8K5p/ZdCit9gpL6qXFG5Z3fsfj1z9FlSgnL+rCsMfRnLe3j46d+aPF7utGXVhIqdyCzp+lO699oM+jJf5TqqdYZTDpMfZzw5Bdd2oc8Bq/01CC/nVK7JhrTVsZAr+yuZTihI9q64Ia6FGX2dCRvv5zCGf38spHbToiKGsfO/3RIlLMM5iP8vsh5w0mYvB7z5JfU8ln/ikPjVIbVbwpO1jzuaTFuWYKyZfWrafyuPtjOskviwfpXSrSPq+k/+65ogd3DapAPpg1+22FYddh/wZLfUj92/5IsJfsXpegqPVB0phNAWJEovfwwYfu1ItWbht1/STyuUU1j5fZhgdmiQkJhXaPYzqExVumYR/1Mmt2LPpVz+ziBobsgEMbyHyfkNy25e2msZ1n9mkfzNAQVYYeU4B2z+UZRQXa+Wod+Rw/J7Hufo/aUdaD0yJLbjVj7uvp3iWcPNJomshSfiCq9d9nSOJmmczKmvl5herJNnNci8bqypEQZQ2FFGZzZd/NM3I5wRyhgoSX/vB4IQ+VeQPExVO4NsUbHfe0T7WRP9azle+oNVu7LLVp45NE4Ayu/ksbvGJb+RN2EwYyiyfdecEnvubTPB6T3hJHMrrsMxwhN7TaG6k2lNHcOi9c6qlpKqzHWUTRmd8HQXfBoXFLIrwAXuukX1yPKdw0bb7pEeUvxoUPjrBJdJm7HY6mkGurZDZdUG7a+HJPd062bD8q4Q3mqoqxqWTcQWrN+TgKgcDpk4d++R+UXNznyV+9R+tI23UEa70iX8OUOUcEymPTY+UKMNZDZMVKBGUvxkRZlcS2iftojU1fioIk1HFF0fMDm65+jGtUa9RL7kx75VYfCisFEhsr9iN6cIzTjeRXsiz9IKK5HpHdc3FLA4bIyLjqLLkHVUrsV0Tnk4PfkNB3VLHsXPLyhjns3sPh9y3BaJcfexTRBxdBdTDP5sebknSPq31oHyvfHqYAedOdceY1GcPwfNMmtj6dMJzxifxyzOJKWdvGHA6ofKorHiZV52ltUC2r+Jz0S39I5rN/jdzW9yuyZgzqzuJoweSNiMGmofOYxcVM17fzPRqSblie/KH3qxGcQ5RLirPgIfkdiHBMpgW9UtRrH7iZkV33ckUbS6YYuVmHREmchv65aOixbwsURdiJgcrbN9W+dpjXI8PHNI0Sxy2jgE4Uup2d2KFzcZ1g1mJxkiEFZEyiT6N7gjmD/rEdQ1q4dVHSxyu4ldBcderM+pYfPtkaei/bUU5bocFKN0Ew9AeMotDfUz8//TLtt5ZZL4utCEAJB7GJCTT2mP7Csf9mhfEez/4k7MZuvO4wmI/pLULnpMapKn1m5k4h/NA6mHdYcOocs1hW+cfdFh8yu3vDq3YjNBVexOFuWMAfNs2W8gWrKdCsh3YgYVdOAJly7l7OqQ31wAh3b+VWpoZLLOaY/SNg/r1wAa5QRlWkk7J9zGU3G7Kcd3KHHaCLBG8pDlfjw+Nd9Mtv6/oOqbCSpho548iqjChsxYU5TPK9r6M8poLh8Xzts5bbSWwbTkF/Vztc5pJIos2PwVtI0roTE36rhFCHzuxWCI4bif14iOewT/UqT29vThCOPdAkq76Q1Bk8gKmhXjzMqNfpTDiYxAhEXhGXqjFtUu6/GeO3P0Y7qhDpWc1vKf091JNMLiprFJx6YxFK6r7yp5vkIEmAjg2MsxQdSsbujBHdkSHUtrXMxG29q+pJd96h+6iqc64lqufo5+YU6h0Wqc4eWwhNB1eL0eKJioXM8ojvnUliRICVOQbZuSbdjpt6X/C7MOWy8kT4AoHXnXdmVc+oANL4QMJw0NL8wUgBvx7D1ujhQQUnff+vkz6dq1c+cMVbccuLvd4ky6kK4Izj9H67qqH/EAVvUJIx1A2as4BK07fh/1qJ6JznArOc3QwZVSfiijNpVzQuJ7N0pS+LL64SF8qcpeou6iNbPw+zVkO0rKfVs9/OEIw/bThFnLHFGFzrrqPwysRivw4pcq5m6bvhxBnbfCAWwm4Gp99yDTerPXSP/vBbff51XkhLaPNXVQmsvewfcosGUpjuNUz6FjZj8Grgdl8JGgpkf8tHHx6jd1I2zddgjs6tsqPINl1TDoXnKUrsVk/hjAUdXkNz8ukaks+9YhvMR3kA3WSeSiONpWMTMW/rvwax2NAzsvqgkk+0vlMnsa0fK7lpGEwnlxyGzb7XI7mqaA1D+II07hPz1jKjUPSjfgVHJZfLTmNalgInPhP/x2+ICmFjts/YxhZi5Q6jejbj7ry8LSz6tXbX4CKY+iSg/0BQpzkD9tIeJLXsvlg8E10HRsPZVRf4Mp9Q2a52wWGPZetVl+Tsh5XuSSGb39f9zG8qMmn03Ue85BeWHEYQOtp3iy1duEhYTik9iwoIcDemm+qQLP7ak20/jQC2do/KZZR/rUltcUbzQoW8Hz7RGnouF+hSssPorlsq9hLCk43gwZZi+Fh4ocKK0oXZ9QPWWdpFjs7vYTMKTb6ZItccs1aWExkmXoAJzbyuhuT/pjNlOmu44sSzMWGgfcZn4SEqjxNduMPlpiHXE4I+y6rFaA0lKScwm1oWodUox4c3TstKkGg6bX/TY+mKZ1nGIsvp7S48jRjVL9+hTza0WWlAyeIOE2ts+O6/HFB64uCPLxpseuU11IfbPi+efGadtlx7qeA7zEpmMJgz9mkvruEPrGLq1ryb05wXEcAP9OcOaugHCuDsU1i3ZbYfiAw+vZ9i9nMYkWvitoy6ZfSm1Yh+CgkP5UcLkpzHbr7jgWn7plU/4YHMJyiG9GWWedhctxVW1/GLfMJiUxXpUNuQ2HVItmLgVS7s7ZRj+cpvW0dQzrZHnYqE6IbSOQ+VTn/60UDv5rZjRZEL9lM+oKkBYmDPsvJyjtBLQ/e0WO90C/8PXfqo472lNYqqf6cgurFl2X9SbP5gZx08aIW+GFZUKfgfKD3RDrtwemwmHltZRIdUnbsZMvdMAIKjGUmyNe7uZuqV036V9RCyp4ZQuP5U7avonaeF8gqKhN+MqAqiuDzROP71cWTa/4NE6Bbkn0rb6fUt+TWLx3jxMXVPbLd1OaB3VoKCwnpDdNTiR+qOS+2nak97XR5rdMsRZoeAnboUMZhJMIi3qxO2YoGgYzKjUMlZtqvYxWXkKawn184Cj9zHM671vHXHxuga/OKIe5Ig+qTD1/TSTn/YprCfEWWkICmuW+jlRqd1ASCYzRkzVz7pU70SkG5b875eUMfYsa+QveM39N3qFxTEn08CoxkFs4vK3xf0MJkT88Ea6wa79QorRyKexX+B7W6ep3pZ/h7FiPfGEk4kyaunUbsQUVjXerL8Yyf2Y0ULZP+/Kl5Rol0yP7cuJj5RUh0u4gSX/xCXMq6xonFYPMChDdHhIUJZXfv1LDtndiExdzKuJzyztMxG5fQlcwoKUT9bV3z1zNSG3CROfjS+UBkoPegwnVdvltgzDCYdh1RD7muikOmJPKcIx1sMVyH7SPhcqsc83ZBoJhccO/RmH1hGf4iOHoBpT+7hJZ8mlP28pPZAzwCSMJ1iG7qLD3iVDbsPQ/UaXL//td2mdtsz+i495/bc/pn80xLtR4ObuDLkN4Ye6SxmCosEdGTpLrkbSQ71X1kg2aB0t2PyaZeMNlyQl+F3rm71nWiPPxUJ1Qjjyhz1GFd3GJ26oh7r1apqgZKjcFMJn53UZy1Idw/Jkg8mpDk+2Jki3YnovDmT4S7RbKX36aSyNMDbuALJPfAqbMdYb8/R3oXlMPvzBtHCQUZ6xNC1h7WsOcUqLxO9aGr84ILtr2HsxYeJWTOm9LEvfDehPKyiscdIn3dKlq3XCoXDfOzDS1a4ZJu6EpBtyaHo9wTRKK0PVtB8ErH2tSJSzY7TmUz2pgiB6c+ILJL5szvUzkvYNpgz1Mw7+rkf5LvTmDFu/GBJnNTR5SqRO113WvlHFjrHr7dcHjCbUMotTEnAPpyz+4S6DOYvvx/yTd18mKUTc+vgQu8MCV04/Ij7fxQCNL45UyuTNwZ+ZeJDqisSy94pcAHFaaYKjsqGwGTH9oaV1QnTt1MfP5ut/LhaqSWD16wWivOR3iadv/KnecjA9di+2RLyzDuS9gP7Ix4YOW696lN7OKuupKFObnKaW6Y8C6md0w26+GOgGekisJOvpaHRHY/dAqNHpxK2E5llL65hiEhtnx7rXU5aomcJEmmrtn3WJU/DoXzBM/3CLzJ56tFHGMHW1QWFFM22vb8nuagcZTHh0jsUMJ5XOYh1oLyuCZ+9CCr8rGEbnkDlozcVp6QjyG3pQw7w5EHd4fbk6hTYytI9JYb/0+y5ByVJ/QYToVEvKLnekMinVtaRvZcluW+qnXAaLEUd+6wHpUy2G9Qxez9DdLmAiQ3o9xdyZHT5dW+Cjj44TNNRtsT2Pma+v0TqhGrh5SnLL3oxL+0yEMzB0lgydEzG5DTkLVn7FoXXEwQnU+hvVPk98VFfUkcyuPO29eXnvpXjXsVZY0+06LBr6izGr7TKl3JDKxymKj9W/y+wZpj5WO0uoScP2v6ywsWHFJfs4xagq0e7ETSjdU982uy/lT37d4vUMnSWHpe/GBNVE4N6BAhlSDYfqdRdvaJn8tEvlfoLftRRupbj7r86w96oCMgbThjv/4zJh0bD0vYDKPYWWJemxM3ToEC6N6BxJaJzVe5Ddkz/MenpAyw8TwuK4JHBg4mZE46wcnMrSEqdLi9UymojJ7lnm3tJ4ef+sR6plqNxw8DuWvRftuAWlC9neJcPs+4ESuo+EVObarDSrjEYeR37XMvfWiNIdj/xSB+taLBB3fJzAkFvxaD+okFvzeHh/lsLpBkHVkmoLVZ9qW9I7LnZmRH7TUr6tHpQTwZHfj4izosfEaYhKz1akPhe+/uysSCnt47q0FB94Bwp3vzOmLadkDW6etcyd3mG3VYC7eby+Ib+pncbvSphrHQjyApe1jqo3WXyitLj9cx6j6ph7NB9QfSdF/XJMak9a0aCsXmhQgtyOHQszIvbPaQwYZQxhUQkgxUfj59yMBcOebrWtI668T1/I4ETjAAwXKvfErurNKsLGG6j3mG5KHdafNcy9NaJ+Jq0j+bbQQ7ktnSJhaezqXHUISzA8PqT2ozTpTkKYFX7TDaRQinK6JKUb44tWQ9raVEcN9+zuOB79C11Oze5w/c4StasejbMWpkd4jzICwM3F/Ntf/WP+zp036T8usfydiP60j0ks+xcNXt/gXWoyW+rQ/PuLtI9A+QFjsbfq396CpbBqcIeW0YSYYon/tIsCd/79zwkpxQ30IZXvgt/WzTjVVNhrcTWWjnJftViq7tDsZwl6KbzBz7ORhlMJ7sAyrOjDCsoSWnhDxhcZw6jsULkX446EaczcS+MNoPa+S2ZXbajyg4Tq3YBUW7VXb16iDHfEQSbSxK2YdMNhOCkwbn/WijxdU5pg+VHMxptKWnZCjTUr9yMwsPEVPQQ4qjO9viJ04pTaR6u/mGI4CUE5YfeyeqVR1mA9jVnT+/Lkh3nLqf/pHaKcoXlcJUimobZbVLAsf2+osey+pbwSasHMyaOUHvvzgwocnqzjmRiv7jGYMlRvGpy1DMFkzMVfuU15qcWf7J7Dvlshve+w8usuO29GZPdjKneEWYo/rLDXzSto7Y6w6r1FedQG0yqpeouWoCKRduuUGFujMgfchT/v9Vws1INkt7KhsCIhgzeEwkZIZ9lV0kisJI7hUojjJMzONVj4YY+wIG94bt1h/6WY3J5Md04oWNpwUhes9lFFfYc5CY/3XnAIqonQlY4WXHY/we8lPPmmR39e5LtUy9A5LrRQ+YaLsbB/1qX42FK7kYhHHwghPv8TKaL2zyszVaWEZTgpqERvxh1PpiQL9PqGwoaCibuHNGxwB4awnOiI3RZMrXNSx2PthgQrUdYw/1ZM/6vnyDQSpq6F+IOnwAkltrQOC+e483pM86hP65hDnLWkdjw5dhMJaB7tTvDx7cNMfmKp3Yxof73H3OUtqgstPl5bZPT+BDt/97BMeTuWhe9DZs3HRKp9q7ek+O/erhKWRKkp3x1LJ6uG2nWJeiq3nqbfqHVWvReOuw3Ptkaei1k/RuPJxIfeEjiBS1gy5LaVqJepK5u+ftaB0NDZLGJnuzS/mWdUi5l+Vy2p3KrHqKgYytKTiPVfcJj60NI8qTesO+/QeCkg8yRFft0yqjgUNmLWftGS2fIYTrgkKZh7SwksraMOtZsx9dilcTGifMsjKMKopribTF01WeVOQpgz1E+5lO5bUt2YzS+6lB7o28vu6BjefSVh4fsQ5hLq52Qv2b3sk2qJK5DblrU7aGn/yOxbZq4O6S5m6CzL/hFn7AFNpb8ckV2T5O9pmnPiC2wcFlVaVG+6mMTSPWQprAg5bx15sGwtIN7IcfjCJsdf2ONeawqvXWD1wRR4lsJdn+DlLpmP0jQnlJEV5WSxWfk1aQ6EDrIHuQDGSgXXn/Ep7Ca0jjtKAZ+R4Gb+p5at1yzuyCPVlmvhWV7PxY7qjuwBLLd0H3L7MSaC7VdSVO7q1+xcMWR2NLf/T7/5d+huF4SQueHQ/I0e/TlDfykiW4+xLmx8SZDd9hGH0bGheqGbCbV3fFIdlRrWhY03DbknHukG9A9FjGoxYc6h+KhH9V5Me9llOJNQvuXhBLK2ZLcdpq4lYjflDP0ZJV4PTg8JyobenEt+Ve2u/qwhW48pro7IbrhsvmEoPRqQ2TdEGX14lfuBTpXxxCzxofIwItVN2Ho1h0ksmT1L9U5M9bbUTk4sG0xu29J8MSAoWQYzum/kN1WieF3w+wlOZDn6e0OMHQPmDid4A6Dpkz/aYrdT4PvvXmDl4TTl/ICZtx2cvjgF9mGe7pz8UoNZMQOigqwqGkUbuSwm1RkZTBkGNY/q3Zj+lCOKtpFtB4RcMlbjW0UMfZ5u/Z4+sOyupbdgqJ/yqDwMyW4rQjso6fgwidor/9uHv0nuiYczUl+R2wVlcm55bL8sxVVuQ2CyxIf5P/DpLClsrDOGLGS3LYf/Pw/J7MqnZR3R6pa+q3n5xpeKeH0JjguPHLyBEu7mfxri9yQrTO8rNyDOQG4D3E1Nwqp3Q5IUVO/ooVn7qsPupYwQmfcMzRM53IGMgtaDxqkUw0lL87jLqKI+6bDsqm00I5t3upNQP+vSOiGXAEB/XladyocpSg91DBfWVbN7PXUP9s+7DGsOG2/kcAeW4mpEflWIToB+L8NgpUhmxyG169J8Z4bGGUN+VUkzXtdQfhSQbqD6NYSlP+0xrGnwkIwTAtN1S29e5UFQMhpjlyUd7B5SCkx2R99LZteQX3fIbyS0TjzbGnkuFqp1x2/4/QB3qJ5f/ZQ/vghppNhb0JuU6liWCw3CkiXdhNxegjswZPZjjvzjXaq3RRaR9cKhejsZU04Suks/R1waC+3XD5FuKjlPIbwBrSMecUpH5/55n9pNTZoaZy3eAHqzPlFGCvZUy7L98lgtNGuo3NEcf1QWMnL/gjvexVSa5Dcl1MjWY0Y1YW3ckY6/yU8tE7ckjqndiKlfVAR79eY44eWiFFVRRiDeyv0YExmKj2XkG04+BVJYCk/6Er/0LaOZiFFZjX830PeUaorOPfczQ/5qdnxBU2fFuvp6n6ZWuyNY/3J6HH0kgcnexZx2/QchUUbj5Cgnm0nzzDi/9VrI4o8HZPcTUk0FG7sj8RgSTzVs66jzzHrU52KhmhiiiYhhzSMsQvuEarrugrLkw4Juv24IzVOWj//+BebeEmGkP+3QPxYQ5R02vj6FN7TktxNGVe1snUOCPbROOJQeqJAPSpb8dszuJYfeAhz93Q79ecvGmz5OoFt8qqW6eesVl85hw/xPNVrde1kqpcKqFOupphbhxM2EzmFF1gRFw+SnlqCSsPCjoYC5LwzozUln0J9yye7otEjXtSAbpxx2XvTozxqCokPp3tjZmdL3n2ppIRRWDaXHMY2TLvmNsQQyHjOnOkJSRgWfzrGYUdXgN1ycKy06h2TEKz0eT+WMnBDpphhXcVZTsFRbQ5PaZyOcSAOR/LoGBNU7yUGbbPGHEVHWoX8kpHlStMTKHZj6yBJWE9b+xYjm0YwwPgZm3pMbt3VU9Oq5H+6T2Zc67llez8dCTaD2nkdQFPp78Qcx2V0BIRqXYryBxplRRqJfY4VolATQUvospRCwRTvGTo5I12H/jK9bfwzle8L35LYTpj9M2HhDuPV03bDx5RKZHUPtMzE8E1/lgd+T5TosWNa/osWQf+LSn1F21cSthOGJEaWVhFFJ+VD9aS2o7qIzRvNkRHjeVLvKelC/aOkc1i3YOtCfT0i1ofzAMjgUEhQMhS3ZmEdlM7YnA1ZwiOZxYTL7M4bhlDhO3UVD47QhceHB3zTUrjl4PU25pv5ujvy6cJH9KbGjwrLm9E4kFVZmV2CLzJ6l8NjQPiR4b+IZKg8C/H5yEPGZeAr1aB92yT0SY2DyQ4dULyHIG+Z/AN79rBRw0+pGhDnnoLsTFWD9GzVZUz5P7CnrSAySbicSRh/16c2qgT31jouJLd5AtI0kZZh5p0XriIM3UKJIdi/BCS0Tn+mY3Hw9S1gExqrz4npEUFQ92Z9xWPvlhKO/N6Q/pxFnbjuhuBqz9TrMvh+Q2TNMv9ugedwhv+6QahnSey6dZe2AJubAHlO+mmbvksaBcVoGwP6M4BDuQHiciRuqH43VcWymRpTu6zjsHAG/4zCqwN4ly8T7OlU23hxTXiIpm6KcpXc8pHxfI9P28YTiE8vkpxGJp+AKN1AsT2o9RZjXAzOsGdqHPJKUWnRxRn3c7LY8VfsvGIorUm11Fl3y2xFxBgpbEeV7EgxtvpYmvRfQOeRot70RyelQlLbVGxNaorQhKItF5Xflos1tqMHfXTYyaWYhuz2O+Wypvn2W13OxUJ+G4JY+3SO7o+lJPK59jNU3PJgQlDdTT3j4l0tjjAwMpyV0bh/y6M9I9FF8IrOfdcDrCDjxlA81+emIhT91uPfXU8TFmOyOYfsXIoKCw8x70DieorCRsPXGBNW7kqfFGQ0fFn/Qwzqq5dyRJd0wGKsd6GnMt4mh/CghXhySqctD1DytRYJVfTz1R2m1t/YN8z+JqN2IGU1HTFyXRaR3VCr4iVsRhfUYkxiW/yQgs+7j99VAL6w4tE5IJN1ZdPG7Glb0DkcHkfCdown9eaWfuENY/MFILaQY2icjGpdicpu6yDZPSCBuHe30gwmPzhH1W1NtePxrWSr3E7LbhtZhodejjBWTKwODoyP6sw69RXUcSisJfkfetNJKgN9W7RqWxZV1Qhkmk/SzrZHnoo/qdxJyW5Zb/1YNnJDaVY/iE0t/WgQUrGR76YbaLROfOexfFHjBJEqoG1V0JFlHNpPMHoRpKD4RojzdtLSOOATFFFHWkF/RDhcWYOJ9n9ZJRQjFGd1SZ67GBEVHCqSWRn2Pfy1HZk+7l9+TV6p+Vpjw/BNd9rK7llHRMP+7KZwwwuu71C9ayrflXSo/SCisDtm7lCW/Zmgd8RU3+a5q1rBgSW97lO/D3gUP60F+zdKf8Uk8S3dOUeJOAMUn0Dw5ljhiGE0Y8o8ciqsKskjSDlFWkUWdIwlJKkN+I6G74JBf8TCRbuudw0JWDqYN2bGeNdOMGe17yvMqGeK5Eb3ZDLmdhPYhh/LDhKDsEBbE57dumsRVt6VzKiS3Kbx65W6iu0dBn5c3FC/WupDa1yX2WV7PxY4aFhwG04bCQ4+570v3md2LFCo20IXBDaD5UkDrkNKb82uGzrGEKC9jHGOrc+OUT351bCP5QF4lJ4bm2YSooDe9uBYTVISsjPKqRU00NqflLHE+IU4ZmifHO0xKvck4Y+kejqXWXzAy3HljvE4B9l5KDqLGm8ddBjXVs5lth96SdAu9OYeNL+Wo3NaMvz9ncQJhccKCVTxjR/z8p5eedFtNfq8nN6w3MMx8IANf7VP93f0x0jK3Zdk/LyeC1xfIo7ARk2THU7KqSpTS42QMmtNDXb8SUVzRVC2zbwnzDumWpXFO7tKl3/HGxBhN8rpLylJwhwZ/kIwfXMvUtRGFez57L1pGFUi3YupnpVHN1sWm7c8m+B25E3i2u9TzsVCdSLN9vyOrSFiAta+6lFYiwqLmw4kPmScp0i1LdtcyeX1I6b4ayv0ZqfWHU4pZjDOGzI5D66gYU34Hah852nmsnKBRPlGbKjLsX9BFyutbUk1D8YHLsOow+56wQKkO7L4MCz9OmH3bqB1Wt9RfC1n4ScT0hzG9hYTah2pmDydFbBlVDfNvDzAJHPpWn+LqmNLc4SCFpXpnnEwybuInKY1hKzcUyDv5mVimg0lD+ZEE5M4I+pMupScxlXs9xcgHCmkLyobstqG0onFrWDD0p13mfwidI1DYisdSPAVfpDr6p/DAp7v4c8dA+5DD/gU9qH4XGic9Mnvji9eebNjVeyHFJ3rInwYOb72mXnJ+1aG4atm+4uP1NZ1rHRHLKr+u3nXts0jshmdZI/8c198zv+KUnvooJ/Nd8YnFiQy7l5WmgdGuldmBzrL0mDsvZnDCsfa0L5X7oe9EdA6JZmISfUjuCNonJITIbas5X7mXMPOueoVBNSa/bmgddcjtJeMGti52o7JLqm3J1BMWfpjQWfJoHXXIb0hIUvkoRXde3YrJj7VzZ3eFYcw0Y3I7CTsvZklSsHs5P+ZQjfn8scHEUjKZWEKS8l2o3lY0kPUM3WXDzosO5YcB2T3L/gVD4YmsJGHJsP2yw9ZrBbI7umzNXB0xmBIUrnHCpbccUX4Y059Xp+BpfzSzb8GRRTpO6SEY1uzB5cY6MPP+iIWfxEJ2Ho8prsqlkNk3YzynoXnMF4zN1wQuv/a0qQ+9ywMGk4bZ9wJ9vpvjP+uwwkDyGwkYDuDNf97ruVioTgyrfyUis2epn3ZJt2NKD8ai4EFCph4zqlqcyB4EeUVZOSuHNUtvSbrQxvEUYTWmcVoC67m3+1Tuhcy+DZPXLPWzamxn6jH1M+Ygr3Q0oV197ZcTcpuannSWFNld+2yoBJT9QNYWR6zVwWJ8gKjJ7UQMJwy7Xw1Y+asJ5V/ZZO2rLnsXzYFKfzgJey/oVuz3ZMJrH3XGWfeG4kNonIO9iyJsJx5gwQkMXiegfVi8pqBk2LvgHgDQMD+nrkQFl5kPYlId7YJTV132z7skrn5ceGLZveTRXTAMJzUObr08wg20AAczkN9OcANLnHXozmn6NvGJCIPq8eozC4s6gbrzDmHOMJpMiPKGOCNP/+I/Fo5y+6UUh/6oybDqMpy2B77+ziGHYfUZvdI8JwvVOmBjh968IdUWeaR+UX3UnZcctq/4ZHfkAxosh3gDNclx1PaZ+hBaxx36C5Yjv6cUjuGUIaj4bH7RozvnUD/jMFoMsI7aX8UVTUdmr8YkvsTIpq9MAFDrZFQ2PPytNPXz0D6SxesDVlSR8i0XJ4L2ccujvyLBd/GTNK+ffMgg9Dl+eVVa2XU9XJOfxaTH9GW/a6ncVLkyczWm84U++e2YyY8tpUeA4SCiPLdlWf1GkemPIvJrls6xRBOex1BcS+geUu3cPAXbL7nsXvbw2wK8WUcPycQNXRp7i4bsK3sMFmM43oOpEdVah84hy3DSEpbGvdJpQ3vJo3VK9ht3jJx8yufPrw859K0uvQVD7VbIzisKvXC+uUd+K6bwxDKsuvQWZSbcv1SWC3akyWL/SEiqCbtfDcg0n21HfS6E07mZJTv97/1t5n8E7WU9ZamOJcwZ8tsJzWMO0x+F7Fzx8TvgfX0P/mmNTNMSFNQaapxR3KITCTbbW7A4gcTUpccx7UMu2T11ANwACqtS9lhHjfb2YUNmf/z7ojHMYWqcqIc6BKOKZfK6pT8pX1X3VMDsQoOtxzXwLKktD+vDV756jUHs8+6PzuG3DcVVQR1GFYdMXdBc64ktkGno++ws6/IzmlR4Wfuo5I/5DdXVWA4WTX9WvKjsjnSlT1/eEGbe67HxZn6cxgft04J1zByWns5zEs5NbPLTJ8cYdtPkSkOO1vaZyXT4wdsXSNcd/CsNoveqgByl+c2A3lyKxpnxaTYR4dWVe9o66hIWVZZh1Lrz27on+F1L/YJUW8MpdQd2LxvK98cywElhk278h58T4bQ1Cgbbel3FvNfX7bx2K6Bx0iEqWuKMQ/VuQuVBROezGoln8Dsx7hiXU7ltyO6o0BeeG+beCSk9jtl5WS2b3ryQPNYo0dnryR7SPmLI1BVF43cMuU0tbidSrmdQttQ+Cyg9gtE4WTrKQ/Fmiu37k5jAoXjLp3rH4nUMP/jBJd7+2Tm8niGzb0k3E/qzDs1zEf1ph8y+eFOVe/Jzjcr6+70BzFzVhbB2XZmooA8/u5/QPQTNUwoSCwsShUQFS1iScDuzZ1n/cp7eyYDi4zFz6omH33T5S4vX2bs1yU6jyHffv8hksUemMGJ0t8T1G8u8s36Y3OE2x7/+kPZWkf6JgOIT6XXrZ9OkO+q51j61TP1M0ezbLzuk66LPTNwZEZSUyNdbNMy81yYsGrJbDk5kqV2XFDIqxRIHHTZj7cKzrZHnYqGCRnPlO4b+giXOqrZb+4pPktYOUz/tsvWabNTTHyZ4A8v2q5pguYEdy8yUqTScTlj8QchwwmXzS8LMNM6MhReeYXhuMM5qSqjegMlPY4prkVyeLpQfRaQbEViFgjmh8j+DknBBfsdiHUvu6zu4tRH5de3M3QVNy6LJEK9vGE4LOrz9snJKy7c9Kg/FR+0txmy8kaZ6J2FUswymoHtIgLPWcV2wwoJaaO5QuJ/CClRvSaRTuS3VfPGhuALNEw6dwzpqMysp9l9M8IbJgQ399568wOKFLVjJ4fYc2t+aI7pfJLdpcAcO5u0y3ncr3P/eUUxkcNMx3SX1ldvHEvbOK18htxsRZWH+hw3irKV+UWjQ5vEUtVsKQkvXYfUbJV2UjHrc9TMuUc5w7Hc0+Uo3dM+Y+jR6pvXxXCxUN9BNvHMUFn4YjNHeahBPXkuoXxKGp3rDHMjC8jsRpYeqLeOUxoV+F7ZfdcjsOvTmfLoLDqm6S+1mzKE/aB34/Od+L0X7dETiGvYvSdiy8aaDM3Z1rvyaQ/10muITyDYSSo8tvQVdnFrHZG9JX2zSG6XI5Ua6rPQUWNufsxA4BIeHUA4Jc+POwzGNLneueGNgmrIFmscdCqvj92GoxecEMJhyyG2oH1pYjymuqbUV5gxhQVFGavTrNf1RRGbXsvuqau7SXZfdyypRTvydNQY/m6T+J/Ok9w0LPxSxxW8J3zP/U7X/TDIWuQSGeOhSuxGRalsWfqSWWWbX0jqsDACzsoE7NEx+JC1CYSOmvezRPiWeVn7DsvdyQlCRYXHpuz36s5aVXxLeyB2pU/G5wk4mnkZ4xUfQPJ6SsierD70/7bL8R2oUD6cMSUrxMcOqS5QZ24DTym6q3Aso3dcH3lk2TF4PD4IR9l4sKZZmM2Yw6bD8R1C51yPVcJRJdU+I77AEbtdh+oMOpUcB/UkHb5Aw/VFCqq0Ho34OHGMJbpRxv1Ohf3JEbicht6EHximFpLIhmXsZgrLKmdympj/5dUthU5BeN5BUMd0SRTBdHy8+o7FyYSum/DCmccqlP+XQOaRFHxYlGUx1LJ2jP4dXDKcME9dcSg8l6/M7urDtv7lAnOIAGLz9sn8QLtebdxhMiLGKgUPf7pI/2sLLRmx80aV9FHYvO4r6qRrqL0Wa/v3qGSoX9/AHCcOabvBhHooPPPIb0hWX7qjjMHk9YvuV/FjsY2hfDCS2ccVnfZbXc7FQvaF6eJ1DY8Z8H/LrhuIjjf423vB04waO/pMBhVXLsOrQuKg5dqpjWf+6pXksRfuYZsomhvYhmfLKD7TQ4zQ0Twj12Dri0VvM4g2gecJjOGnI1BMmbibktgx7lwp0F1IScp9xcUIr89wAiicbdG9Vqd7Sh1v7WYqt1xw6RxKifEIycslnR0Q5SQNBO+5gWh6pnRcdGmct9RekEuvNOFTuQO/lAfmNRNaUtZi9iy7NYy4Tt2IGM+IQ9GcNpYeiQZNA8eHYb1Z0hM0xIsI8jdlxBzCoOcy/NVJuwHTC1CdqtFceSBu6fyUhyku2+Og38kzkBjgPsgdlw9NYn+JaQnrLAwPtIw7dd6aon3HJb8Z0F/S5CY6h+tMdWcoPfj7Nym3JjJl9kKI/n5Df0I+f5fVcLNSgYCQHy1iqtxPKK0oqsR74XWfs5rRMfhKy8ms5grJm/IVHEqrEvsav/TlLet+Q3TUc+oP62EJt2f6iRMPphhLswpya+p0ll9qNkKAkocbuJYdR0VB5EJFpWva/PiS7DZl9iDIO2Z2AX//rPyOfDjCHe/TmHaxj6C1JL4qxlO65lGo9Fkpt/O6Yr78S0zifUH6UMKjp6M9tOngdl96svr/dr4+ofi8jmt5ewmDSoXoroXo/ojvvsvwnA7yuPEq7b0Rkd8wBVytOi8I3+95Il8yubt25LXvgIqifTpPq6AHuLHniJ/z3mwwWYvLzHX7zS1fxLzWIl4acrW7hnW0zcVOfSeU21C/HdBYd0g1D40ooluq+LDLdBZewLD9/5V7MYC6W3iArZ23zuCtNwfGYOC3/W+m+vu/mmc/RQgWIipbclm7ArcM+3WXdavNrlvyaLCd+JyQsJnSOxzCWzIVF7cjuACY/TdT871jWfnmC/Lol3UooPHRxBzK4VW9pxtw9pA7B9ss+7kgi6alPEryhZeeyx2DCUP1xhjijlpUb6OtZG1bYuTZDvJGjsJZgYktmTyzT0n0lhswWO3y2Mk921zI4GtA65jLzjoYIfj/BCQ0zVweUHmgR9ecTaj8W+TrVMvRmle3qjRIST5rbx/+6ot6Lj2DybY/WCwFByRBlpZrKbxjCoqsAiKJENO1jKmdSbUvlfqAp2PyQ3qIlf3mf5uMK58+vkCSG37txCWsNufyIdzYOY66W5ZD4zNCfMcz8zMEdQnY3wdvzx1hOaShSbRkq919MaB53qd5wGFUcvLGPv/RYpOnybYW7tY84FDaFqqzc+Rwd/U4IE58orDfdtMxc7VC9qYtF4qkVlGpbegsZatcc3I5D8LUWQclQfqBWlvUE0K3eNHSXBAZ2InTbzOtImn1HdmjNr2Xd8DtasN1DwivGaSXfDadkWrOuhgq7lx22Xk/z0e+f5+Rrjyk9dBhMOjTP/Px4t0b9V4DMvQz5rZj573jE4xOhtxyTuIawlNA6mhEnq2PxehIVhwVJ6vrzlsKaEDj9SekHpn83I/zOpYTOEcjdT9Gft1Tuqcfrdy175z0qtzsHav3RZExnWWLqzS/6AER9j2RpSH2zjNt3uP32EYJ7JWzfw/1BheHdMp17FaIrHbK7ctvGWbGv+vMSCVXuatTq9g2FRy7NU/oMl7+tDkb7qB7u4YRh4Y+2idJa7KOqLl0z7wdYV39GcfVzdOuXGEOXgv60w+o3inQXdLR1DotKEmcklnADizc05P+wdDBKjbLmACVjEqWA7LyoiMcob8ltWLJ1S+OksOqH/rAukvLlcZLKtAxnTqgcTxKFshVWddu1LuTXlFWV+PDknx4hu6tRY/GRUqe9kf7+7pmAbphiOBWz86LHoKaFH2UdSvdcmscdpj9Q3ec/7Rd/FtM6IUFxWIS5d2LaR4XlydYtjXOWVFvR7sUH7pgyoofCGmgec6hfTIhzlvt/rYgTQu1WTO1DyfEaVyIKK5q2Tf3MJ381S/GOT/meWne5LUP1E5fajRETn0H5lIYD3UUFdrhDPbC165Y4pbm+mvuafpUeQJy17J/1DyZhi98PyO5Ydt+YpnNYVqLZd0YMJh12XkzRWXCVOD31OYrviX0p9XtLamXkNyyz7wcKmV01pNrOOLFE8T6VuwnZ/Zj+ciQjXkFtpfZxLWyv6zDzgRygy98e4Pc1/enPJ7SOODz47SqjKix8fxzlc8AVMPSmHXK7cp+OJsw4jlEz7MnrI7K7ltl3euxdMrROWtoXA8K81EatU/DyqUdkvIjiA5fMLgfWFutAZl825c6i3Augmnn3skO6bsZJzDLvZXYVjGYSi9827J9JUX6U0HlxiIlg/mfCrLuBbu/V6w6Tn2g0m9m39KfUA40yhtINn7BkaJ2JGUyacdy8gnkZC8Enbo/YeSlN+6ghjF2i1bxELjWH7Lal+Fi7aqo7tkzn9KC4gXJncxuG+Z90CGox+xdg+5U0hU3BNeK0HsDtV9J0DsHUtZAkDcFywN6VZ6tRnwvhtEF6ShAELMwbOodTZHbkqnQHymiKMtoFyzddRmXD0d8JWP+yLlTeMKF601C+16N5qkB6P8DEHv35DPWzEnSYxJLfVOskv2bZek1x6gLnQucoFFYMo5JGuH4PhhMaJJgInnxTQDbn/7BH+GAeJxNRKIzwfqVNr5Ml2stwc2eWizMbPDwbktn0yW5JdBLmjXasaz32Xp7AGugsS9OZbkjxZB1hcKKsWlBRWshLZwyUqNcM03+a0tdUcMnU7VixNaZKV57G42jHi3JK/iuuJAyrhvJtFyeyB8G5a191mLymzITBVIr0voWqYabYpXe1zHDSV3bWlKFyP6ZVkjFRDyCE5YTSfX1vnRlLf77Akd8N2H5FIRmtoz7x1xuU/7CCN1S93VkyRHmHzJ4l9XYKJ4LHz7BGnosdNXEVhjD7ro54J4LiI6G5WyfGmaCHVbN5HR1HmYZl9WspqrctYVk49P0Lho1fKNKbN3SXM8QZS+OUgzuSiSyz6xCUxlbggqzGiSdXJUB6PLJ0R9A+5NA4rRoyt2EprGtnMwnc25jGDB2MA6/Nr5D1Q1LpCH96QG87z2e7s6SrQ0ZHhpz9m7e0CF1wgpitL03QXVACYG5Lou3eotwB3lCa0dxOQuWBToR0OyZTt1InZaE/4zBxc0Sc0sO79Zqs5INJQeEG05Ibjs73OfHGY7rHQ7oLDuE4VcUaGJVkvS48cTCJZTDh0Dily4/14NG1BZonHPxeTGFLF9f9cy4mUl0/nBoLzAsxo4qhf3JE8ZF6wLsvptUSvGvoHE6Y+H8U6BwWPbBzSF9blNFn0Dn8OWNPObF2l0HNYfr9hP68gsfqlxPSdSnfc5vihGY3DYOaQ5Qee/MPOxTvuzixaii/Ywkq2pX9tiEoJ4wmE6Y+jhhVLYv/6CGlR9Kr9uaheltik+H4SMztxBQ2I0qPhR+PcoZRzVA/p85E40JCvjCk8Ngl7vh878Nz7Lw7R5IY4tghte/i/3GF5EGBI3/PcO/vnMHE2rG3X86Tbkr5VdiMtbCmRN4bTOuhePqw9Kckn9s/54GFpT+JsZ6VduFKmv0LDu5Q1GhrYDQpi0mU1wSpmB/yuD7Bb1z5GO8X9umekA8Lo5TD1lGHyv2IzpJDlBcgorPsEF/pkDnSITjbJ8w57Fx2KD+MD2Iwg7LSpouPoXTTp7BmmfpRiqgA7ZMxXhc6p0OCsmH6fehPeowWQjJ75sCC3jwhCeXS9wIyjWfzojwXRz/IuwPquWU3hYApPHSZ/Cxg87UU1Tshg5rH7kuWVNMhzhiWv92nt5iRdjQDo6pD60TC8X80oHE6x9xbfZons2pqn/dwYsvuN4/gBjpeS4+0mxdWtdO4oSjOJtGFBSCzp+lXmDeMpiMy2x7dQg6/ainc9xRUe2REzliy13J0j0S0U/Kub72WpvgkISxbCmu6hGx/0WKCBHfkEJQNg8MBE1flRBjWpPYaTDp4PdWeUx+HRDmHvRd8pt9X+ye3bdm/ZLGuEbTjRsLOlcy4Q6EhRPNWlTif8EFumeZqheJCm0HZ5+LCBh/ePsLMjww7VzxZvOf0ewaLMfRS2L7LxDWXwaTem8ZJl8Ka/s7l74R0532ydUv9jMa51kfyxycuft8y+0OX9iEOPGzpdV/6CKPNxu8IhJf4GqY8y+u52FGTQsJwSoEQYXEci7ijW3Rn0WfidsJg0sOJLPlVh9HyiFTLquZcG1K9F+F3tOgWf5jQPpqlP2PYu5ijc0hi5cq9mOyWeFBh1tBbjik/HDGc1DE8qklaV1gdt4aMxe/oGPZ/Y5fw6IDMpsf8G2t4mymiw8I6hqf7TE23GWwVyG5bFr9rKDw2LH4/wBto0aX3HMVIlgS1XfhJghOrrZTe8Jm4OcB6P29NWUcdgd68obPk0T7kHtTu8mQJP5luW/Yupth4M0Pv8oD8esJgSsTu2nVLZstl6/oMubku/X4aY+DDW0dYWNpn5zVL+dUdwq+1OPel+3in22Rnu9DzcAKH6r0RcUafSZzRTmpi2L2cIk7D5q+IxodRh8Qb6FJsHRhWHKKcpXPIoXVSJdWoIkG4O7R0zwb4vYjGyRSlx8+2oz4XC9Vpq2ZyIsPCj0Myu3YcKKZ2VJTWRWrvki4bc9/x6c8b7v3NNDsv5+gseMpSKsOwIiL0zPsjopzm/umGJcw59Gd1LDbOW2rXHLqLKar3RFwxsbLkG+f03xM3LeVHMYOaQ+k/KFB+O0Nuy/Lw8TQc7vMLx+8pK3Xk0rg+SfmWS6YV051zye/ENE6ncUIYzOpS5g7VBsruJjSPKsG69Fh4ybDkk9tRu6uwolDbxhnDod/fJb+jQInyPfFNE19TovIDsQxm3h9SuZsw+4dpwrxu/I2zcjH4fTSjv1eC9SypVASJwXMSrG/ZaxSJY4demCYMPPwfl/HrDtlNh0d/KSXxzIYhu6OLYPGxlot1DcWP00xeFz16/wsSEslCNO4IdAy1mxH5NQVn5DftGHVpqL3ts/6GLDqNM8/W8H8ujn4nERE6Tht6cz6dw/r52asRzeMe2Z0Eb2iY/ES4nN6cIxvurkd/1pLfGHvqreq63E7CcNKndjOkddjHDYSziYsxScohu+XQm1d7pbOsHap2I2HvoqH2iaE3b3CHY25qz/Lo17NEUyHptRS5Byn6Jy0/vHMS5/wAs5emuALNMzFh3pODM+dIwvZJQH7TZf+cS/lhQnY3YveSkh8iDGFunLqXl6c+txOze8nDukI7ti7UwEpl1D4Gc29FdBZdvBseraMw/7MhD/66S+6hyHrtY5ZUy2H6A+1SM1f79OfScN/B71q2igX8ussK0+SeeKRebRNELg/fXyJTN3g9S7wEUTGR5HIWRssjStfSbL6eJtXSxXbi9ojerE93Vsd29b0UmabiJP2eJd1Uze+MxqndDnSXDbkNiX6ejnedWGPtZ1ojf9GL7r/JK/F0KWof1QQnU5cr9MmvapeMsob9FyzdRU2O/K4AXt3lpy0toR+9vqW/YFn/kkeqFYt3Op4KpfeVnNyfNUzciQSoGOPXpz+KCPKGQ3/cp7so1XlvOaFzeYg3xpoX7igIwhvCwrc8vPU05R9nqNwyDCcM5Tsu/eWY1nHYf0G78upXfaKskebSqv1kElj8flsDh3Nj7lbWoXnMoXnMY/J6rGFAF7rzciyUHgFjjFGcNowmdOuun05Tvu6T3xSKKL/mEJSkog8LDu0jGQYT6nREGbH3szuG6jWX3JbFfLuK804ZY1WHBxWFAJfua1m4Q8is/FwjEGc09KifSbN/weANwQk13u5Pyb0wnJRIaDiZ0Djt0zquh3Hmaqg4y8cJnUOW/ryhuyg00rO8nouFao3oIul9tS+GExYntuQfeVRvJdQvJmS3HEjGLNVxql+6rhRovzuGfuW0yKq3YeW/a2mcNuR2E/ZeMOS3YyZu6/Y6rLgHkeLuENrLygx49Js5pq6FzF4dUXzoUH0rTewrijLVVBR7UFaT/akgpHsYiqtW2KG2Q/U2VG+IFJ3bMowqDo2THv7A0jrmk99I2PxiiSinYzVOq7bL1EWdbpxQPRqn1abbeUkx6pW7UFgdUHoSM3EzJs5IaJP4elDjjEyOSS4huzUm650Uaqh7KNbla3x+uiOlrpRWRCoMK8KlZ/ZUF3s9XXz87jhxumDIbetrym8LQe8NtciG0xG9JZFtOssOXlfSwcpt/TnV2+pbjyou6V31vCduiHbthM++Rp6Pheqob9lbTsZx4sIsJh4UngyYvirV0ODFPlFOoLPsrqV3KqA/Z+ku6UMbVRRk2zlkqL3rs/S9EfWzQox3ll22XnMoriQSqjwRCjyoqF+baegD3n7VZ+eKMIveYKzxvCUHgBtYnBG0lzwmbowpdw1dFKxjOPytgfqVngyHGBhOyIe/d1FY9M6yQ3/OMvdWj3RD9as3lFi6sOKMZ+DmIO0Eq7ZPUBrX5Fccwcm2HPZejRjMWNwxpKN8x1D70CHVsWTqAZOfJJRWErJbLgs/sqTrckD05wyjuZD6GZ+gbCjf8th5BbL1mNKKpJNxRp2XOK1/Eh9mPgjpTzuMaoZUQ1HtlZsecdqOHzyp/U2kiZ8Spc34UhpqenhID175zlNl2efoMvWUeVR8pJZNUEkOMIutEzl68w5+1+DdyxFlVCYALPyRS2ZfyRztkzFBWfK2VFNN8PUvZahdt6Sa4xQOX0kgu5dcdr8YHYT8YsZM/zIs/HhI6VEslDc6grHQOuIT5oxYSY4udu2j0ne6Q4mYH/16Fndoabygeq38KMLviaXqjNQ7ze5aolLM5hfz7F+U+7M3J3dBbzlh6uOQzL76wXFK7aHF76vEya4rt1S+Mijd8g94VgZFnffm9eCsfTlL87jLcMKRS/WES243IczqwjXxoUf3WEx2R19/dsuhfsYjyhhqN4by/qN5fuWBMgR2LvsKnYugei+kccbQemXI7DtWrNdQOVt+T06D7Zd9uouasj35hmDFYcGS29XD4ndFjnmW13OxUKUVVTsqKMHsuxKD5LbsuLbT/3dC8fPTTXn6N34jJLubEEzEZDZ/rmwvbMiyG1QThhOG/E5E7EsD2vzSkNFsSO6Rz6hmqTyMFRd5eES6CRtfzDCqOOO05gh3KHpL82JIf8YhLCRkdxMmbsD8T0PSe2qYdw4ZZj5IKK6GlG+5TNwJaZz0xotNX3vjhHbV6nVRsVNt+aCcSL6i8h1D84Q/ThEx5HcSsvsJvRnFhZcfJKQ6qtvjlPqf3sAw9WlIZsdQWBXSZ1SzDJdC+ksxE7dG5NegdiNkNBaPJ74CinNPXPoz2oG7xyKW/8ETvIGlfShN46zGxqUH0unmt2L83lgh1rHsXRA/Kns7Q/2sy/TVFgCDCU0O8ysefk+dnMy+VQrNnmHpByM6Sw6ZvbHG4hnt0s/FQjWJDH2DSfUJ0/WIVMuQ3Y9pnlZ/FCTUTdf1RlgPcjcyuIFl4ppD9a7iGju/1MW64sk/paUEBUFvrYHFf+RT/VjE5tylOq3D0pDWfpTGCWRMMzHsvGbZPy+talA0FO9oNyk+cugsKe5x9Rse5YeJ8DZ3YlqHXepnUpRWIhonhLIJKqJV+z1LumEZ1fS1pTramXJ78ivVbowYzIxZqEXDcD5k/6xLZ8GlcVGAje6Cws1MrAc3vyav/ObrHnFGuoTcXkxYTChf9zn59/o0j6UIi4b9s/5BKK/Xt0QZ6C+JoxWUdNG6+28s6T0rCmgx/fFQrteypTejIUZuXyzW7I5Ml6XHCZV7CWu/WKE779C8ENGfkjlx4lZI5W5Cf1YsrvxGwv65DIW1cRBx0x5QFv+813OxUK2jJ723nDBxK6J+Js3su302vuSKBfVEQWSNS7o9B8VxeyOEvYsOwylD45RLqpdQ+YM83iDBHVmKDx0K6wmpbqwxYQrWv6xgiN6cS/ofVvH6Y/hEAhO3Rj9XOm27+B0JpgfTUgm1j0LrtIjNcRYmr+lrsa48+6m2FdveFyB3WJPQufaJJjKJbw4a3N1lXfy2X3JwAth4M02qCTMf9Em1LcW7PtMfhmQalspn2tGmPgkYTJsDWkv7yoipT0MKT/RA5LYsW6+6zP10jPI8nifxDJk9pZbEKUvjtIPf149Ld13Cspi06aZyEponHDqHlFfQPJ5m6mPL9FXI70jo3JtxyW0KpVm/ILJfb1a26e65EcV7HlFO5dT6l8dRQRGEZZ1M/RnpVPvzdhxO8TnaUaOsBMxL343pzgtosPl6DmcEE7cjohy0LgWUbvtjVKQWQnEtYeKWHceXC34WFA2NUx6pzlhRVBVppX1Ut8wka5n6WPDe7qJDfke38/6M4fGv+fIDHeMgOHfn5bHu0iD091X1SEdTMb05R7tjayzWfhBQeqgQ4PbxcfBCRaotXZwEzBhOWsr3E6p3EgprurgUH2vIsXs5R/uIoXo3YueKT6Yh4cfOix7751IKkUhg4UcBU99P0Zvx6C3C9AfqRBQfaeTpdzRTj3JCdZrEMv2+NKl7F8fJ2FlI7+q4yu0k9KfHeV4N4T7rLyQ4sbono5ICgvNbEqTn1w25DfWH/a40C95uikxdIXaptoWlAamOhgDZbfWnUy3Fu4vwPSS7+znaUb2B1PzWMTQuJhSfJOS3EkoPoTvn0riQkCmNCPPjzKYxjDazH5JuxGR2YfKzECdQbGFuU3PoOA21m0OcEPJr+laza/Ip7byiP8sdJMSpp0nTClLTjV5W5czRDsFsyOCNLv1fazP3P3pIcETZplFeqvzYB6xh9Rs6ZvtziY7wvJywnWWH4aSgC+WHI1IN2U3aRxzc0fhkuCLxjBNqt9t6Vc7QjS+6zF4diZC3oY6Fgnd9hjXN2jO7hvpZh9ZxmPqwTTCR0LiYCFCWUTKKE+qhTVLjMAmLXLpNyK0rKM4bqrxyAugvxphgrCYLtItPXo9pnNQm0Ju3lB4ljCqG7jJEeUt2S73RKDt+7z7K0ThlyO6O+93jOPvKXQhLug90j/4FK/yNMa4x5mNjzLfGP54wxnzXGHNv/O/qn/m1/64x5r4x5o4x5pt/3p+deGIp9WZdsuu6qTZPqFGd34nxGw6pt4tkd3UUl1Ziooxh+6U0rSM+vUXLqKIPIygphrt2Q/CGvfOZg5DbidsRw5mE1uURJFrIO1d8Zt5XnHdxVSEVjVOG+BsNeLWF4yQsLNT5hSP3+F+e+xM+ebjIi0ee4AwMo6mY4PSAsGRJfEuUTwjzHEgKrQu7L/jStdYsXheax9JjNpZ2aSeE7tGY7IYrjOZgbIVuGZzAUH4A+2fT+F3YfdEcYBr700I4lp5EVO8GVG8lZHcMrZNFqjcM2XWXwZRHYVWiZxPrEla7oTaRWkUCBVtv3I+taHBgPcHZKncMw6pKk/ZR2HrFIdWSgNoN9J7WL8cCvR0e0ltW8HB2T3bwsAiHvjOgfUytw6Xv6VRrHx0Di5uKTPoLXajA3wZu/Zkf/6+A71trTwDfH/8YY8xZ4LeBc8AvAf83Y8w/WyJjVeRHWUnBJq8PmLwuPaY1hvy6Ek7cocAM7UOuhBkNWUemP0zozjtjuJgCJ1ItyejM2FYSp6E765LZdkitpfAGP7f1to6mcAei8gVn+7z69Rt021nM22Wi96s0fjLL995+gf/jP/ir5G+nefAPTwJw7PQGjpOQpGUaLN+Rymj2HZHyctvqxSa+tAOD2YT+vGHv5YTyPRidHzCYMuSfqBGe25HAuXlqzCXtQOX+iO7hhOrtkKgWMqoYISBTur2vfdVQPyOKdm4noXFGXq/qXelI4zSkepa9V2PCoqU772ken4XeAhSfxLSP6QQorOnByW9YMjsO9csx+a2Y3JYenMpdXbSCqsa6W19KOHp8iyQfk4QOSTHi0mv3WP9bAYNJQ7oOnUPSBWMsnQWX+Z8N8HoSqJtEQ5S/sIVqjFkEfhX4f/6Zn/4N4O+N//vvAb/5Z37+H1prR9baR8B94JV/5jr1ZM8I82p9rP1Cljhl8HuWrdcdGq8G7Fz22X9BFONM3ZKty+TWOuJIrZNXOeDEhv6coXHaZ1QW98h6UsEPJ82B12jq44Ty/eTg5+MMdBalLG4GOWxPc/s4q50vsyMuarop7Wj1pmE53yAceuTWldCSqScYq91r+kP1Hp1ALtPmCYfZd8UvOPGfDujPGU79b/aZvC6d6PzbQ8KcIc6qfusdjhhNWp58I01cjNk/5+Pv+kx9ErJ7WUjydCvBxIYoJ+BG66jDzPtaWIknDPlgRmWG13IprKqWT1J6P4qPJavMbTgHl7vsnqVzSP9tIkPzqEf7mHbPoGjoHI9wj3VJH+ngN1we3p8l99Cn/FGa9FqKampAPjdidKFPcT0mzBrM3JDZ9yLcAB7/emac7SXo8P75v1hRyv8Z+F8AxT/zczPW2k0Aa+2mMWZ6/PMLwLt/5tetjX/uv/Ll9+RuHE5pNLrwkyGrX1UKQX7VUHrLpT9pSVLyS1nXMJgQ78kNtNBzmyrocxvqow6rulgFJX2QcdaS24DSSsSw6tI6qihy6+iCMffWgP1zGfJXs9xaO8LMdS3Q7LY+8OxewtbrolYrtwp+ePsUF46uc6N5mMPfCokzDu0lb5wJath7MWHqA4fF78c0jzu0l9VfXfnlAvl1y+YvL9BblBtz/1xmHLRmyYSG/GPtfNaF4opH96tdkid5OoueCCg9XZrSdT0MU9dCBpMezaNqVSXuOFWvr1jJubdjtl53yW7LOzZxM2Ew6ZJualxaWFdAhxtarOOS27E4sUuUA69vyOxa9l9KKM11OFKt8+CPjpEGTOwSFi2FNRjMwrXdBZobJUxsaB1R/3fqDzOsv2mYuAlzb2un7y67uANNIZ/l9efuqMaYXwN2rLUfPtOf+F9OZf8v9CCMMf+KMeYDY8wHfa9Pb0EK/SQFm69nKKzKk+MNLMOKcjlr15NxNLmyj7y+plf9WWlOUx0dTVHW0DijiVB2V3GL+TW1SfqTP4/hyexZKvflOV/9ehbr6ZKRX5NCyw2soi73Eja/4FC5pR3JCeXerE12eFSfILvl0DqSYlhx6RwRC6s/5ZBf1d+1+4LPYEq339Zhn4lbCcX1iOy+xe2PCSMeFNYT/n/U/VeQpemZ3wf+3s8db9PbqizfVV3tHRpoAI0BMMAMZoYUZ0aUWQ51IylCig0ZbmgV2lDsjUJ7sRerCK20oVhKS4oSyaEZx8EMYQamG432rrqqy2eldyeP95959+J/KjEK7Q6LKzK2+4voqOqszDyZ33m/932e//M3pdsiiMy+N2Jc0sAjcQ32fo6gLQw2cyRWUpTlxIgtVR8RZpkY+8qfILeXUNyIye/FHDznMv2hJbsvZlT7tIffl6rUiS2DKYfjJwx+V04t2WNlTplEwHwSQHbLJYodbrxxBmesezFeDMltS2DpjA2tbhpSCfl1l7l3h3g95dhWPtX7tP0NqWtP/ec/xxtC0Hy0RfUoR/8XgV83xjwA/h7wNWPM3wEOjDELk0W3ABxOPn8bWPlzX78M7P6vVq61/5219jlr7XNpckx9IkDbOpM58U6EO5DWp7usBdk+5UqAt5XQWgtORq9T12PGRegtWgqbMVHGOUlM8XsJYdEyKglZ8PsybOjPG6avDeTpdCEiLCQMq4q8SdcF4o8LhsOnfNyR1KPe8GH3KmOJ3zz1ARenD5m+FtI6B97Ikjk0FLbG+F1JYsQbRROZ0+rqTQKtUz6DKWHE2cOE6qdjWudk9TOcNgynfLyekZNLCZZ+EjFYCRnOSEcm3DfB6zj0lgy95QzdVY0rM4e6b0fPGtqrLvVLHnPvxRy8yAmgbyKlt0RZjV5HVflj7b4igk7rlEft+ZjCpux+xkWVI1HkknmsSW9J9p5Tb/ikWgnNywnuky3CRhoniPH6lp0vp2VOEes1rQNBTbGb9/6vL500nv9CFqq19j+11i5ba0+jJunPrLX/JvCHwO9MPu13gD+Y/P0Pgb9qjEkZY9aA88Dbf9FrRBkpKEv3YwobkiofPeGT3ZeHUlgSxljYEnGkPyf6mxPKX7NxQbQ1vyO20mDa0Fmzk+xU8QYWfzYkt2PozzuMSg6ZI8v+S1msB3OvO+Q3lEKdpBOOryqZJN1ISLWgN+/h9Q2DaYdMLSEuR5iux+G4wN3fvUDjvE/xPhw9Kf+n+qUUx08nrP3jNqOSdumNb6dYfN0y/WGXxnmHsACNpyM6pw2tsw6jikf5TsLM+9L3l261WfhZn/YpD78LtSd8Uns+mQNlro5Khr2vJKQahuI9NSomVuM4nNL9Kt7jRNWaeAa/LRudKAPNKwn+wFK6n5DfnDRQdTlOJz6UHkSYsZxrgrZhMJ/g113Cgc9/dvm7JGkFobkjifXcvkPvMIcJDenrGdrnpJBIPJFs3IElv5Uw+35E0JoMY7bsidX6/+aF+hdc/xfgG8aYO8A3Jv+PtfY68LvADeBPgX/PWhv/Rd/IxPqFRyXxLd2RasrhlNWoc1fO08dPaOF2ro7we3Jxzh4I2O+uckKk6D4lnHM4bak/Jue69d8IaF6JKN+JaF/QLjfzwQjrQu0pEa9X/55LdtMjv2kYziQErYjWxZjjp7TDJoEyrK6c3+bxpx7wR3cexwk1W++sTX4XC97AUrjncvCFEv1F6C0nOGM4etLl6Jk87kROUv5QY9Z0beIIkzYcvCxVav1qifqVDINZM9Hn6x5hRWf0hha3606UqYb+gox/rQMLb0h73zonLu5oVuNdvwfp45jynYTMrkvsq8TprupYHpUd0scqi4ZlF7+rkyzxILfpEBUSqq8F/MPD5yistAHV0MfPxnh92f6YSJlehXWHwbwl3RKi4A0thy9aalc9uiua7B29GD+yFOWfi+Fvrf0x8OPJ34+BX/r/8nn/BfBf/PN8785pw+qf9ugvprGOji4TQ/u0y2BOCROZA2mKcjdTGBszWEiYf3sMRo1X85Ilu+OQ+TRN+W6MN7TsfdFVnuenYB2X5jlD8Y7y4gdzqUnNqaYrU1P911s05Dcdjp5KUbgnlCHVijm+7MmtOgzY+HSeqQ8d2mtyqRsXYepGxLDsan5tIFVLiFOiK6Ya+n2mPh2y86UMo5mYsKT4oPSRUYM4a0nVxO53Q0tvgmGGBXkbmESN0WBOmVOpuiZoU9cjusseJlYC9LjoQgKluyK7+C0lmXRXDMdXfeKACcNeDWriw/SHMbUnZIKWPUoYVB2KdyHdSsjUZXfUSQzDacO7d0+TKw3oXhmTqw5wegHTPzA0zrvMvJ/QeEwE66gQU7vqUb2RcPAiXPybbXa+XsHvqaad/bn7yMTpz4YUJZoQmM9mpKGxk7S6BfFQF19PaK9oijP9gaV5TvzP5R8m9OYDLQwrF+bBjIgSvXl3oqVX5Ld1NfM2fcFHmQOjRm1KwLpibzyK6wneQDcyaGkG7sSaobPWIzrMsLE7xfzPDLmdAZDBGks6kvdVlDUT8zZDb2kywuwYOUePYfurGfJbltP/8Jijl2formqk6vck78ZAmJNVpEkm5c6xJCypuj3xJUgCKN0Xftxb8BhV9H2wlqBjGK6NSNdSWE8sr/olNXFHTxsy++LMRilzci+a51wKm5KzeAOJEYOmYTjj4ozEwyjdccjtxaSPA3q/PKYy2yEThJifl+jOSwHgDYVZp0LDqT+0DKeUP+sNLDtfr2h66EN2dzJ27T/iGvmXuQAf9ZLdjWVYdQgLCe5oQnE7EpbXuODh9UX2OHzOkKSUQHf4tKdOtS378OGUUkh6Cw69FUv7QkS6Jkgru/8wBVBRjJnjhPxORH8ppnojwetrWoKF6o2Iccme6PH3vxLz177zI3LZEc7MkMzNNN0lh86pNKOylK1mMgmMAzH/B3MThtCx3sDOKVnfWFcBtp/+jYq8XdvCddNHmrgVN5TqUrkzpnojIk4Zjp4MCHM6gqeuj4VW3IkZlrU4+3MyhkgfSlfWvhBTuJYizqgM6ax4jIvQWZFZcuVOOFEFmAmtso8TQunemOyuId2wzL0tm8xRVc2QiWRydvCCQ/o3DwjHHpkgpP3DeXK7EiZaV/5ei68lhHlwRwn9WYfcfkLhvnwL8nsx3bMx47ImYI+6Aj8TCzUJIJi4IacW+hQ29Au3zyRM3YjJHKhTzu/FVD6V/NZEGjVmDhQeFqe1SEB+pPlNuS97Pcv+KzqOnFjzeSwcPQuDGY+F1ySdyO8ota+74rD3ssvUx+qgWxfgxcfv8Y/Wn6LVzFJ8LSON0vQkj+piSFjSUMEJJ9zaY6h8Yph7eyhu5o5l4c1oQmyxJJ5l9Q/1xgctTa9yhxI39mf1lvRnPEYl54QGV76dMJgz9Gf9CYjvUrkzJs4IPM/uT1SeBYMznIxZnxnIEXDWUNiYDC6alsZFn8KWZDWFDUv9Uob8TsL+iym8gVS/w6pD7BvilKV5JaJ4X8e5We2T8iLYSzEY+8J96wqzm3s7xp8YwgXtiQuKgd68w2DOkG4qmC2z7dJbSSZRk58j9pQFUpOku8o/yjGsKkY8aDkMqpr6MLGicUcC9JNAnW/iQX9uIoXY0W41Kkh+PS4ZOmdg9Y9FxAizmu/ndxIK92TZM5wMDhJXpmTWqFuOsuIGpM61ORrkae4VyX4quKW3YE4svYs31YnLVlGUv8yxGOy7X0oT+4bG1YTtr3pMXR9x+JJcX8Z5h/KdhN6CzG5bp11aZx1K679wNKk9o5oUA/Wrer3O6QnuGsDB8ymK92RL2Tml3TbxjTi7dUv5z9K0z3EihkwCOHhRMN7UG/ukGlq8nTOw/xXtoO0z0Lish25UUXlVuuFRfzqm/VhEudjn6E+WSR07NDfKPPvVmxw+55M+Tqg9ITfq1poYZvsvpib5qVDYSOjNO5NxLPgth+y+w3Dqc8SeciLJHGLf0Ljg0F1NsK6ocK2LYuEPq9L2d06p3jGxJlLuSJOTVEOs+KAJqU5CqpmckF2aZ+UI0lsyTF/TGe3EAuCdsRbu8VfGdBdcWXc3NJ7d/4JLITNk47BKdkPeAVHaULkjkeCoItGf5uTyAXBCLf6ghTybnrJMv+sw+55l74spptYajIuw/2X5tFpfC7tzMaSwoVn8qCrT31TNoT8vf9PUsWG4EDG+NCBzZJm6HpJ4OkVGlYTZDxIOnwnIHsQETWheADeE0m3dY78nKCpoOBQ2DLvfWgREaSzehYUfi1ea2zHEGQ1WvL4lfTigs5Zg0jFe28X/f1UlbFyO+T9/4x9x83iWmQ8ijp9Q6dQ5nRC0xYhzxxOG2lBaMXekXT9Ow2hOFkvu+BHXyL/oRff/y2VdLbjG42IhLfzMEtQd0s0YZyQryEzN0rokCtzeF90TsnNhR4vGHemGjEvanZoXDcPZSTCvhfLdkNK9hNaaR3dJCwAAA3uvxlR+HmgK5ENnSRyaX/vGW7R6GVLXspTuJ3L1s9A645KpiYf5MNUaIP9AAbqdNU3UvIEEe/UnLLWnVK4MXpvGieD0H4gJlt+0HLwEuXWfMG/I7cV4vcko9dgyfU3KheGMpXjTo/DzDLn9iCireMqgbSnfNOy9LMPdMKcGLnMklWjzkpSjnTVD5U3NXeIA5n9axxlD8b48/9unHWY+GhLmIbPvMqrKdrN9LkdSiMkWh1Suy/llVNbv+1/d+RqtuxV2XnUIWjplZt9VyRSnOLmfoymNcp1JmJt1wBk4pI90Ij7K9ZlYqO4Qwhc6TH1oKK4LCll4c8Q47ypNpKbdMagLcF94Y5L63LQ0z7gMZyztNeGRQXtCutg3rPwg1uItQpSTCVu6rtFi+Y4mVv15Q/6OT/MxkZyDiWFC+sjw+58+iefFjK/2qT0lG/LEh95KTKqZkDuI6a4YBvPKW9p5tYgTWfyW/E+LG9LoVz82zLwnCtzcOyPCvJhPR0/rgSnfNJqEFeHoSQ8cmP5ItuLNs5LK+G2DO+moa1clNCx/qiO9vwBezyG/qQfWm7jDrP7eAXNvJycpMzabJrunAcDmr1dpXpYoMX2c4Lfh+HKaweMDpm5EWJcJSSjBa3gM+gHtM4b8Xkx+y5LddKnXCiTliLXfGzJYjAWfjYR9u0OYf2cgEoxvye+prOieD6X/umV0Aj0aHfWzsVCtB/N/K30CXndXDFHWPcmW7ywp7300GzOsOLRPCY4ZVVUTjqdiZj5UczOYs+R2FfA7LLviYN6I2fuimdRywkwH0w7WldbJHUPQcHBHKvzL91Wren7MTL7HVLlLWIlZfW6H3/73f8DjT2yw+euW1pp3knxtQumI/L4l6OhhOXxWROywYE5iK2tPio86KklZ63ehcdlSvS5icZy1tM7LbS9xpVdyYiX0tc9bumf05vbmHYqbI8Zf6jDz4UTwWDLUH9dQpPhgROupGcKMM4l0Tzh+bor6kwl+X/V86thh7+tSKozLULof4t/L4Ix1OpkEerMuUTXETE6O/rRLmBPacGrpmPmFBvsvZfU+uhBmHZrnXMYV2H41Q29ZTLXaVUGNwaHH3DsjestwfNnVqfQI12cjC3VmxV76y/8h+b2I9qo0N8Npi9c3lO8kNC7oefL68qpvPBHjt13ZmY/FsnfH0F2LKd52Ferbs1KNTrr8oCNf+VTDMPvuiMNnU9LtT8HUJzHNcy5+F835SxNrmk7COOdw9PURNnZw6x6VS3USC2vlOvPpDh8eL/Gfn/sj/p0f/XVOnz7kQumQD46W6bw1I1lHC+Z+3iIsp9l/IUVuTzWt9RRuNq7AzIcR+y+5hOUYr+my+DMN8ra+7lC85zAuKCb9IbEjtyPtfZgXvJRq6B44kaZ8gxlLflPGb4OliKl3XKo3+sRZj50vpyhs6GRwxqqhpz7UMMGJLYWtEcePpektK5K9dU6K0dZzI5wgxmxmSDUM/ctDTD0gycUs/4nG0t5QcnRvAO3TGp6Mi4bZD8Zsf80nt6WHyBtAqpUQpTVA+PS//GdnoX4mFmpmfsWe+9f+I/K7Ma2zLukjFdn9OTl0NK7A4msRx5d9BvNyTUk8aepz2/LadEJ1l04Ixc2E/ZfEGe2uwsLPQjZ+xaOw7jCcsqSahuKDGOsaRiXp802sHc/rTwLHJoC0SfRwxL7GlvXHLc78kGjkQmyYW2zS7qcZbebxBobSbUg3xVbKb4ucnDo25PbVoBw/nTD9nkNn1TB1PaZ+SQpZjI7NYVWjR7dvSLUmA4uMobipxRulZc/TPqvJU/Yw4uA5n/m3xuy+4pPb1kKoPZ+Q25A8OyxOoL+6NoCgJaZZ+W7IwfM+09ciGuc8KncjaldFUyzdU5duXajciTh6yuOL3/mIU5lj/uYbX8YvD4lqGdL7rlL9aobFn/aoPZnF78PxE6qdm5csJpaqVZxdOVwfPqfgYhLD+t/4jz8fob0mRvS0aUcWMoXJfDunOmnle2Ma5/2TAt3rawetfKKFlDmy5HeSE0JwHEDmQNBHdhd68z6zb4v1NHVd/95ZVqqcdZR64g6Mkj5G2vG6yxOFpFVXPi7pwfH6hsW/G5C9nWL2Jz7efz/FoJ2mfMuQ39Dxe/Skx/KPxpgIZj6Qi3Z7zaF5AfIP1KgNFyPGeVmjm0RNR+1pTvz18ztygBlOKSR4nHdwh4kILFUdp/15w8ELPpXbCUdPBxQ2NIGLU4J/vJ6+X+VTS/mWhipB21C6nzAuCGkp30k4fkzBcenDEdUbsbxQrxryuzGdCxFHT3jMfBDx4f/jCf7e3WchgXQ6pPyJg9/VRHBcsex8NSdIcNUw/3PV2IUHyicIC+JwKE9hkjdw01C9/mhr5LOxUFGX6A1k0zguM1FDQuX2mO2vBRS2VaxPfaTAr6Al+5hRmYmTiUNhY+IiF0nsN/uB5MXHT1sOXxY5oz8j3LS/IIflwYyhfllHa3EronFZ4H3xQXJyd0xsiXL2BNhvnvWY+TCkdU7z7enXfAaz0uo/zHFtXJTD3bAs7DBVF34a5iaGvQ2X2jMKLJt6r8G4YOR52rRY3xLmDcX7GqEu/mxAlIXdr3gMTo8xsfRl7kAgfH92EsYxsXxsnYfFn4ZyIxxC65yDG1py+xFhQUrYOKNU6sNnhX4MZywPfi1LlJGZRvrIcPSsQ/ljTxzd9Rbe0OL/oMTqn8DoWpn+ghpJb2RJH6kH8PqTWnZWHgQPFcaSvujBH07JozbVTDh66S/kK51cn4lZv0XHtjsSyB+nH7rcwajiUbmp8sQJFbUTNJW3VL7ukNuxJ8dbbSGh+pFDlJbjSG5bWvfyXb2xpfWY7iKU72rkNyo5ZI4Vs+N34eB5l+o1GdeW7vQYlguk2gm7r4jfChpJ9ufg8BlflolGuGCqDvndEe3TGVJ1S+Y4oXneJcxZZVplYPoj5ZqW7scknkvq2NC+OiZKV6XiPJjwHLYcel/qkn4nT/YooXEuTX43JntoaK8GjEswPj9g7p+k5EjoaxTbOT2Z2G0bjq8E5PYUKpE9kHQ6c6BdfzBryG9a5armxP3d/I2Exe95tE9pl3RiISfewNJ6LGHj16dwx5DbS2id9sgcicNauaFFGbTEIEv8ycKfMgznY1KHOkHy2wn5bcnPK9eFYdevuCx9P2bzEdbIZ2JHTQLhhs2Lmmh4XWF43kBp0aOy0kF6p2Oy+wn1JxMKd9V95g5ieiuG2XcT/IYS7zqr5oQcHacMzXPirx495dJblopy5+uimfWnXaKsZfGnHbK7Ygv15w3rv55nOA2ZWkx+U9175U44aejkDh2nOdEuRDmoPZ5R5GXaUHtKu5wTGQqbCUEDenMO+e2E+iVXmVRXRpz77yNSTYs7MpTvhkoVXE2o/HEOr2epX3IlqMs79GcUSjbzcUT+vQwmeWjFOREDtgWwW1c7a357xLig1JKFn4/J78UkvmQs3RWhH9Yx1K56uE3tWdVPIylFzwkcLmyNSe+7BB1tFCaReVv2MCF9pNSaoG3prop8M/NBhDNWjWvGeg+9vvzCjr4zImg4+APxZUt3RB56lOszsVCdSFS51LFm2sbC6AsdBrPSiud3BaEU7rikGzHZXZlUpJqWvZddsruaIZfvaOzphAprKN1PGE5rmOBOgoGtmchT2g6Vj1z6C4bF12Lu/5U83hDGOYeZj2JSDYWp7b7i0XpiTH/R0p/2OPyKvBKDjqV0T0z2ODBUbygTSk1KRByonDARHL4IU5+OsY4ht6eMpVHZUH0zYFwJyBxP6uJFj3RNlFvQtrQAAQAASURBVL44UK2eORTw31t0qL8Qkt03HD3lneSKJu5DM2DVgP0Fddtx2rL5rfSEjWXY+BWPYdklvxfjjEVqNrHFWEv1Zow7kNAuTum1vZ7cTA6fS+GOxWst3Y9oranBOr5imHu7o7ST0xOy9f2EUdkht2fpzwux8Aa6V+4YUp9kJKPOiNbYXXFItT5nbn5+xzKYVW3m9S2ZnxSYuqZueFQQLzN7mDAuusSBKHj5vYjZd7UDBi3Zge+/ID5n7kALPrunN7o/r+9bvC/Hu7AiPLK/oimPRflMrfNyXOlcCindVYziwvc9yTFKcqEOWpbC5hhjf5Gc7PeiE2OGgxfcEweQOGOZ+zkMKx7jEgxmfOJA2/Cwajh6wmNUMOR2J3lUBdWmmbqUDoKbDPNv9Fj+rktvKSEOLMVbLvXHDYOFmFFVtMRRVYa5rfPiNhTua9dPAph/Q3/25lx6y+oJihsR1pjJwytu7M6vxpTvRYTlhPSh6Im53YTSeshgSto1Y2Hu3ZgH3ymw8EZM8b7sPo8f14QrzOoejcpQvjNW/9HXcV+9NSK/q6Dk4XRC+vjRatTPxEJNHCY6ICMrGAvleyHt0w6z7yvUYTitN6153qH4YGKkO+XhhvZErdmfcUk1JbPuLmp2HXTkHfVQBTCYlbtcbt0jU08of+JJX/5As3G/q0WfXffpLRpSTWhecJh5B0r3I+Z/oMnRzlcDugsuo4qmQLWraYYV5dqnGkIQMvuysGmeU908rliaZx2y+5O8pZzVaw4srQtQvh+S24vJHcS0zkha0l1Vk7b+l7PUnhCFLtVUE1O+CdVrDqPJuDVVh/6sdyJ6zNQTWW660LjoULo3ZlxSVkHmyDKsuqSbSrLuL078Xnd8uosubscht5fg96C35NBdlMna9CehiDnzLss/Hqp0Ssvczu/KUNgdaTOIs5aN77j0Fg2l9YjMoWXnK2kOXggo301IHzkcPe0/0hr5TCxU60Nh3ShKZlZsoOPLPqOqZeeV9EldFeY0ADCx0jVGJSNoZQjd0wm1L4b0VhLiQPLfMC/IJ3NoKWxY8psKRDCxbnh3Xtbp3QUdiUs/bMEEKsLIuDbdSPD6MJh1qF31yNQixgVDblsWl7ld6eDT9WTC7IpIfJQhOlIzl2rqSE7VhV9maolsx4/UPO1/NaZ0CxoXfNyxZfdLkikrpAIG04bSLdV02183dNciyjcFoQUTmly2JhisdR5m3x2x8PMBw7JogoXNhPyWpT/vE2V1avWW0Y48pXHv9Ac6ntPH8oYqPBCRpLukefzxk5bpjxN2/o2QYXViHz8fkG4kOJFUB1EGMoeG3pIWbGEdMrsuQQvql5RIXbw3kQtNMrEWf/pozOnPxEJ1h+hIbVv6L/bprkDldkTQFBHa68sOZ1g10kpN+CSz7/cp31ONU7xjWPiBh9eV2W6UUbSi35H9TO2VsTifgRq1oyd8xmWo3gwZVWHvXx2z8Z0S3hD6S+KQ7r/gE6Wlgx/MWkZVy9Zfj+gvJgx/uU3/N9p0V9RBHz0jLRPINudh1I6xluazOv7yW3r97pJu+3Dakt9OqL7nEU7M07qLLnE1JHWsutBY6J+K5Hw361C+7jD7hkuYn5hC5BTVk9kfkqoLzHeHMQfPZfD7mrsfP/kwetJh+UdjOo+FLL4eErQMw2nZ0ndWnImNeiKt1IRbu/STHqm6pXjX4fhxF/96FhNDYTsm1Y6pPeEynDLMvzlSfzHxBcvua3IWZS2F7ZjSg4hR2dBeM8y9PT5JETx8LvtIa+QzsVCTSaBWZ82QeTdLfGbA4TMCodtnJy7KFSjfizGxunYB61ly23KMG8wbmufF4olTmum3TzvSJuVg7oc++y86tM9K5Rnl1cA8+HVNoxZ+N6C4rmHA7Fuqw8anRjQvgJ0diVI3PSYaePz1b/6YSm6A78aMzsiqMnMgS6Gtb7iKPF8wNK5YBtMOpQ8CUnUobI4YPD3AHcHCT5u4I6liTQyDORG3h1WD8eX/mq5bBjOW9J5HEsDcWz2ytYT6VZ0uQUvxmK0zDnsv505s2e//lTT9BbmzdFZdopkxtact/ScGciy0sPFtnUSVWwm9OaliixvxJGZI9bh1YftrObyBuvvlHw20YgzsvuJy8KxP+lgCy/0XU3SXHFqXpFLI1PTf8o/H1C+6NC54tB4PyR5Ytr6uFOqp60IoHuX6TCxU6whDTddU75V+kiG3a+mejRUb2Vf92F106c17ZI60u5U2Qra+meP4CyHLP+jjDiY59WXR54aLMYW7LumGpXXWoXxL3WzntKF8U43O1PsKd9j94sR+8SBhOOVQ/9YAO3RJlockXZ9kdsQrl+7wH730ff7HT1+gP/b5S6c/Jl/uM/NXN8XFPBcz9ZGhcSEgylucUAwqJ3poGKFYShPDYClH0Ibsvhz6UnXtYmHJMv/HAYNZS35zQLqmB6lyO2L3lRztU2poUi0NF2Y+GFJ8oMZxOGMZXBzhDg3JzJjx5T4z397GT0fYaojjWmwlhMRQ/Vi1evuUtFlOCI2LLvmNX4yih9MTyXOg3NKdL2fIHFoSV6VC5tDSn7c0L06ojg7M/VwE9ShtOPhiQn/Ox58obdM7PoknOY7XtwzLk57kEa7PxEKFiaaoIO/T5mOqo079Ycxg2pHn/I1IcufbklfMvRMTpxy8Pkz93Gf3lSyJL0A+1dAsfOHH/AJj7IjQkWoKDnNiS3ZXhmSLr0dg5MIc5gzx1xucXzjk6cvrpNIhpBJcP+GtH17hv/30FaLdLO07Ff7e7Wfp91PcP5zCHRvmfia4aDAjwvLK98d4/XiSSGjYfymgdF36r4PnfQZzlqNnzMT4FzJHCWFZVu3uwLD/hdykSbLUHtcJk6oLshtPXKvrl1I0Lxq6Twx55Zc/JtgMCKciZmfaFPMDdl9fJv1uDn87IPNuFmvh1577AH8gPqwT6kQbzon3mq1JnWAdmHnfnqyQ6ieq85uXNBFsXJKV5uk/HshnoKLyrPaE1A5h3pDb8E4mVK3zqrnHRTkdYjTuPX7uX7Dt5L/MyyTq+OWSYpl/Q1r4g+fkhpJ4cPicR2bPY+9LWYZThjiQOVriis0+mk4YTSlqO31smf54QqXrQP2ydO/5bcuoMplwTct52YnVxAQtOVEPpw39uyX6YcCN/XmiaHKLtjMkZwZEoUtSinjpCzcZ1jKkP84SNtIELSbGuYbKzQS/bxlVPAYzHt7A4ITagQZz2gmnP4pJ1wz5TcPxVUlvhlMOmS1PsFQyGUUuGLqnNKfvrSST2CCdKGFWVMeFL+zi1AJ+8qMnyG/B4vcd7N+dof9hlaAp7kDhAZTvRRAbfrR1nuPHDdFfO2ZckqgwvS+JdZgV/6ByR5KY9uVQiooEMscR8z8XcuJEMK4kHD6bpbeopO78nk5AY6H3xR65HZ0y7ljGvyBsubSu92lchKXvfY7CJkysWinOWqauD3FHCcs/iklN5BxhzlC8b0k1BCrPvTOSc/OU2Djl2zLqdUKRWfI7EfXLaqIydUk+/K6hfXoCnwyE86WbMX5rAq10YPbdBCeCdM1h9/0FgiDi0sIhp/6BYe6tBGc9Q7KTIV/ps96uQjqmvxyz9MNfLMT8XszRM6qPj69oeub1ODHddSLDuGxonxYX8+FUyh0KAYgzlvozim4sbMeyML8Hs+90ZLZ2SaTq2fcSUi3LYCHmwb05SrcNC6/HFLYiegvuicW7iWHm45A4kBM3ruVUpUF2z+D9j1PMvRdP7s9EnhNaynfHDCsuvUWHxe+Jz+qOLa3TPqOCJCdBC5yRmahNVRp0lxRF70SQ/2mWUVmnTH/W0FkTgTu7J1+u2fe69FcinOhzdPSbWLvH1MeW9X9Xu2p33mMwazVZGlv8fkJ+JxagXPROyCcgKMiZaKfyOwmtNX8imzaCqjahuG4ZTuK4c/tSdDYuiFxS2FGSX/0xl6BlGVwZqG7+QYV7f3qGUcVl70uG8XRE6WKdQT9Fq5/h1NIxmV2Xo6dlOtw9BbmtPunaxEMglkRaFDtlKuW2tBvmt2MZjY0thXUx98clxUie+zsR2afqjIpqcrrLhuOrBYLmmOUfjmmfcmhccKm9OsLrODhD4cX1yxqDKl1G0pAkgO2velLfAun8mOs3VqjeHBGlBO/5fTHPTALNi7IXCnOGYdVOakrYf2mS4F3V98TC1CcTsk4emJyGU9ck5R6VxRNIfMNoypKqOVTuRIL/Ajh6Osfij8zna4QKcnxrnXHIvZ3h8BmfOKXjJNUQK2hYVhCYRo0uJlJNGweG+iWX4QxUb8S01xwqd8cnbiUgmGVYNRQ/9WXtWFSaSGEjoXHBJ8w4hHlY+klf8dx9j+kPlfUUtCXEcxf6VBdbVLMDbALRtRLHP1g86Vr7q1p4d/713MS/Sez9VF0lhxMrMCNOGebfjAmz8v/vrOrP8doQDIyLLgcvZOh9VKV1Tgv84XX0TI7+nE9/JZblUCYkTlvO/u6A+hVD7/EhB8/7MiObNCnpY0uSSuSm/UyXV0/foXjb48F39Hml9UQBb59GE0+DhFQjIgkM6bqhtyguQW5XpZl1YPoDS39RizjK2ROqX+IZMrVIhOhZkbtLd/rMvZMwmE9+oXWbxBWN82KyPcr1mViocQpmPpiEQ8xM4mlaiUaBI6u05sftxG5b7PTSurQ7mZqO9vyWImYqt2PqFxXV3V9IiCf8zsQXCz7Ky6Pe74vm54Sw/6o8kLa/mpUt5HserTVBYKd++x7Tb7nYxOG5uS0K/hBvM02UT+gvyNgiXTOk98WGql4zzH4QUtjWSPX4S2OOn4Cpj/v0Z0WiqT3uUb9qKW5EgtJOJ5jjgKAFh8+pE07XhI+GOSEexc0xJlIjs/aPI3JbDtPFHuXzdY6vZpn+MCH/YZriunDlxIfK7ZjhlKF4x+XweQiCmB9vnKN7OqF4R2ky+19SiBxGRJXGeUd+VWX5XXl9DQK8niaGpfWEoJuQ3TUTpYKofIXNMWFBbLdx0bD8Z8kkrCNHf9oR37evEql7OqF8T1Hww+nPkVzaG8rCBiugOOhobOmMzUl41/zPLM3z8uBMPNVDrfOariT+ZDrUVJHeuhLhjqB028ENpVRN1RWOUP1EO02qpQeit2ypvusJm60mIitPG7KHlvazQwaRT+fbXb6wdp8f3bvAjR+dp/BASSkmgePnI7yeOuLKnZD2GWWr9mddCd3eTZGkEpr/J8Xy7H5NuGNhXbtJfnvCnPrUTOxyxLd1R2q6cnva2RsXghMpdpySrmtnY4r6YZFxyTAqOlRvhTQvqOseF3XSuEOVF3ZuRBw7lP5RnlTNobsqRW9+XQkqXl8Pfm7X8uDXfKK8pmqDWcPUtRH9BcP8WwPapx0GVZfuF/pStl5zWHhjxGDWV5bqvBhsD+U8mZoawNn3kgkxHZJUgt+NNXG7+zkipSSeYKXy3YQoA90VNU75Te2yzbMu9cs6Ywub8sZvnVdqitc1OKHoZJ1Vh+6CQ3bDo/ggoT8v65/9F1x6KxPnuhW5dvRnVNed+qdDgskRn/iW6g3oL0ccPxvz8vn79MKAaCPPhwdLhH2f8bSO+PS+y7m/06F43cfvW+qXoXHRJ7urpqlzWgskLCizavCjGaKcoXDbZVw0uGNLf1rz/OmPpPHa/XpC5XZM7Sl9j+ZFxUWKN2oIJs7cW99wRRBJx1w9t033wphsLWZQ9chvCgZSiknE6NU2M//qJrYVUPyjvKQuqyHWs8y92Z7kYsGDX/Un3v6G1X8q4H84pYSZvS+lmH8rZP03UuS3FOkz9/tpMvuW5kXLYEaq2OG06tLK7Zj0sZq05jmltYzKDqv/YJvOGlQ/dGmd8emejmlc+hx1/QoU09w5uy9hWpTlhEllYuU6pY/FbhoXDYV1h1T7IfdS5gqJC9kjme52VhzCsubf7lCQkRMKeC7di2k+JQOHw2cyBF3xAhZeg9ZZsaHOnNvnvZ0VdvYqVB47Jvl5hZnXfIg1QcvtWra+VSLKyhE6cyiDjMG8pbcq6Kl1xiVd+0XgmnVg+pMRub2E4xdDsrWY2i/LItNYSO/4jArSEvWXEk7/YQd3JGUrDhxfldVO5bohexix+rsuu397TdFBv5bQXZ7Ytj8WMX66y+avW4rZIbfuLpLbUKBF4kP5I5+5tywbv1okzliinCG/4dCfEzbbXfQYFxPSdb12dlcK3+yuAnc7qw77X5DsJnPgEGYNgxmDO5Cj4qDq0DwvUk/QEX+gdQ52f3WZoGUmyl3L/BuGM797/Ehr5DOxUJ2xjsM4A9mjiMyx7Ln9iSue17dkdgW5FO+JSTUu6mn1+mKjj3OGxTeG1C9rkadrlqkPHKZuROAo+iZd17F++LxDZsNX1Hdb1pTZfXFai+uWoDLksJMnkxrj+Am9t6blNRWA33Zwh0oaGZclJal+0iZdl2bfCQ0z7zr0lqTjH8wqhLiwFVPYjNn85YD+nMPM6z47X3WY/l6aUcHQuOiQ31b3f+G/q1FYd6g9WSDM69/ilE6PwdzE5TDn0DqlADcSIDYUXjmk9nLE01fWmSr1MKmYceTi9FycMScWj05kye0MMZHA98GcndxrS/2KdtXSHe36nRUd4Rg9TMfPyInFHRj6qzELbw40aOkr2Wb2bRiX9Tp+T/3BuGzJbxmGU1B8EFN/NlLCddGw/ctTj7RGPhNSlMRTHqrfgc1vulQ/MQxn9PQXHgjY7q+F+EceUU4jxXHRkt2fjPpmof61Id53UwTtyc42ZxgXLFFaTs5RysjmZiSfqaCl13RDcSmPvhLiZ0KOTgfQSnH+whE3rq0y/a4QgeweYFRrHf1rfZrPJ3C3iHm8za35HNNvwfbXA5FnpsSUciKFmAVNh+6y1KyF+9A9ZenPa5EkngByE8tyyDrQeHYav6sjeVRRUMTOqw7T78sELegmBO2YwZRIzCt/Ytj5qkOcOJDA1v9wjqBnqRYMx896nH58D+eqZednywxnExZ+ZnnwnaymcQtjqq/JujN9pNOtdD+kPydZSpyR7DxqGwqbMZ01kX4KG2poj57ICPkwCoZrnTOUb4k4NM5LBiN6pmHxZyNaawHFT8UpTjcSUu3PUTPlxNA+49BdgcXXJO2VwwZMXevhxLD0p/JFso4stvObsqIZlwyz7yRkr2UYTDsTB+dfBP3m9ycRMhZmP0iIU9IwhQU98VFaokKn5XFh4ZCL53fxC2OqqR7pQx2XvWVL75e7NL86ZPtbCfPlDv2jHP7ZDhdmjphZaeAPRCop3U+YujYiVVcTNJhyWPzjLaI0E9WrIXMge8jCZjJxNhGEFeaESw6rkrHUn40o3YX2aY/iXUlK4pTIN42LAUlgiLKGw399QGa1Q/PGFPiWTCPW7+9A6shj961FNg6mGC2NxYK6IpioeA/m/qlPqp1I8VtgItnxaZ8RZ7XyqXbT7qoSADN7yvgaTBvaK/6Jtm1ckIzHRDLBMInM14ZlDQvijBrC7oohu58wnEk4virO8COtkX95y++f7zIxzL8V0151T8amXt/QOpvF7wh6Kt8LSR/J5aR8b3wy2z++4jKcsiz8tIE7lIX5YD7BCSd1by1mMGNonnGJzwxFoJgMREYVw9ELMeVzda7fWuZUvs7feOr7vPVnVxicHlN7OcI526Va6GGM5eqlLYyx+KURjmP5+MES4Xdn6M1pce1/OSHOuFjXYI12iwf/xgr5bUvzSkLz+RGDObGq+nPy1xrnNZXyBrLpAbHGsus+YU5qzaAjq5724yFxxtJ6YkzQtnSeHPH1tduMxy5BwzD7Y5+jpzxa5xxSLQtGlumOGxPs+RoGlNWI+n0x0byhpfqpuvM4JQjK6xminGEwIx3b0k/GOo18qNyKcWKNd8dl/bzeEFprLuOyUmZ6Cw7pQzMpC2LyG1rs7hC6Kw7VTwypY8Pse4NHWh+fiYUap2T/3bioRI6gaxlNJ6RrapyytYiwCM0zPnFGT35nJaB8N2ZcMhTXxXa/89dKlO+NiNMw/YEWybhgOL7ikWporp97N4PfkZ9of8FMtEOG/+TiP+X5K4Kg/ss3v03xyWN++5l3WVo5JggiEmtYnmkwiHyC/6xIfJCBt0tM/ThFZy2ht2yZ+TCk+r5LZ9llXBQGjDOZ0y8abD6i+GGK0p2JjmgEh0/5jKrmRE7TPgdT14esfm9EqmnpL6lO7M2L4LzwQ5eFN2LKHwaMi4bUeoo/2zhP7vU8fleyFetYgqakIfNvRnLAbqWoPndIcXNI+tAh8eTFWthMGFRd9l9W3pU3gDAj1ey4MKEaLkZEGYkg89tKRvTbk+yufd3X/HaisfSRw9RHitCM05IPbf6yR+PJhMyB/P17V4Yn1Mb9FzOPtEY+EwtVmUWGmQ9GuCMJx8o3BHz7PcQ0mpUpWaphyRwoWsck9kRUl/iSGW/9UorE1eKfvhbTuTI+yfh0Jz4AUzfGtC6gFJbdCJuN+W8evMrHu4uszR5zerlGf+Tzg+0LHDbzdLeK+E7CgwezbP9khcblPCY0E/GgYe4tGTxsf012RKOKoXwvIc6oXu4tqeM/9Q8c8nvauRqXJFVe/W6L6Y9CwoJwWXdg6C0EdJe0EGfeU0hvpqZSorvksP+Cy7igqKHh6pjBYZZRWYiHOwSsat3Ek1wn1dLx0X5tjtZp5RtEmcmDZBRKUbjv0LkQU74zprekejW/ZemsGpZ+aOjNuax8b8CobChuaHY9+55k59UbEb15hyQwjKYSvJFl5r1E6oklifzy912SQFq27I00Ry8kpGuGTO1zNOt/CDG1zgZ0TsvNJFtTAG6UhsrNhLm3oLPq0ryg48fE0kg1z3qk6zHtcyL+lu5ot+ycFkNq9feV9JHblca9uG7pLvgUHsCg6pDZHZC/GbD1yTyVQp+MF3LQKvDs4haOgeggiz87oPt78yz80GW4IKlJ6S5g7YkjXZxSzI07smT3rIwuzC+YYQCdZY/D5wTGp4/l4Hf/t4ocX/FP3POye5LY9OcnBsOemrLussgslVsRuV1D9VZMEigZxmRjMkdqOodTyDCtDLPvy3azcclhbqXB/Jsjeovy608CqXi9oSW/BcXNiOItl52vKmS4u6qH3x1D/bJLnDE0L2QUE3/KI/Hh6ClHZOkzHpmaxJm5bYcw4xClncmE0ZJq6mFLHVvqF+XxVfnEYVxS+fEo12djoTqaSGWPYrL7MrFtXHBpPC0P+/0vWgYzDl7fUroDrQuWmQ969OcN3ecHxGnZxuy/KOJEthZz+g/7FDdC9r7gMi7bidGsSgHhiRMn6F8q0D0bEbQdHGO59sEaw37AG29cZvz9aUq3HdJv5CluRZJFf+jSXRF3AGM4vuKy83UpOkcXBxS2xSeN03qTvIEGF0FLP3cSWNpnE8xEIp7fVDOZBJaFnw9pn4XuithdJpGphTUa/aYachAsbEWEGYPfdPFqPk+d2aR5yVL7QkT2wNJZk+1jHBgytYQwn7BabHD0dIryPeGjhc1EQ4WzLt1VqD3uyUpoU0nYMx9Y0k07sakUNGcS+QUM5nSEz78ZEadlytafNZTuilN7/OQkFCOZWIrmlTPVnxdTbTCnxVlctzTPfZ6aKQPDqmXnqw5eT4YGJgEnG0m/X3MxsZVlekWA/PpfypI5tMz+cYqjp0RWmX8rYTBjiDIOR8/m2PiWx8wHCaljJY5MfxLTPSWm0LgAc28ohXnxz1Rz7W5MsfTjhOprKVI1h8K2pLyJz4Q/aeisKepyNC0TMRNDZtulP+My9YO0bBrzqksPvxKSPUhoXpJsObelXTa344Cjj/VWwB0odmjnK2mmP9YOF2XkmNc4l6ZzVs3IqCxJzvFliQzjjCVzZLh5OIeZG4IrwrkzlhxnMOXQWXVgYUQ3TBFlIVUPSVyNOKO0HtrxTERhwxLl9XZYR1O++mUtj/ymYDKTQOVmTPVTNUyNi/4JacYbysEvSis+ffadDnFakUFY8RWCDgxntUgTz1C/8uhL5DOxUP2OlVS66YioEehGB3czZPc0Ts0cTwi3PhTvOHgDQ/FBSJgVrirz3ZGglEWHzilL5tChs+Li9zTDDloRqWPnZFcNs4buokvtSQ0Vqu95HDzv0p81EvnNOphYwL6CvERoaVx0SdUcBjOCk6avyfi2vSa4bOp6KGHdBwHtNYd0zZwcg/kNQWhRWmYZcQB+H7pnIoK23FRy+wmDeUvueprmY3YiidYo9WG0+v5LAfkNh95yQnK9yMwfpcneCRjMWSo3DLPvRycS6MwHGbb+5DTZfUtYVAZqb9nSPpfQPzem8oEnZ+odS+XWmMIG9JZjhovhJJIH0nWpIbqLLvVLirUMmsJIAbrLloU3x/RWDO4Qtn+pILLOsVwRG4+ptLGOBasTZ/ZdGWU8yvWZWKg4OiKGsxH1y1KHxoGapt6iHDu6C4o7TDWFa46qMe1TPo2rlsGCmDgPvpOh8MAy8+GIzKFiEcO86keA2uMp/K7sd7L7SshLNcVUsp74pPlNGaQlrgza/nz4mNfT96l+KtlGlNfX9mdcEg+sK7b89tdc+tOO/OxbljBvGc5aukuajnXWEvwu7L/gUr4tTDJ/X7OX9kU1Jtld+aCaxEzSqsX+6q6q+cztajSb25KKIUobmbi5mm715lzm3k7oLcpDILcr28vtX43pLQjHnXsTgj3/5PeWiNIjXU+ofuxQvO4TZcQ3CFoPrTAtKz/sU7kZKwIoI3v1/KaRa+J7EX5fO66duCWW7idyzB6LlF7YECtsWJHpxSMukf//X4lrGF4ZUPrUY+Fn3Ymhro55dwRBw1C5E9JbVK2TOTAU7rtEOYMJtZBMDNUb4A0Tjp5OKZPpA8PSa0NGVb1OcTPGHVmK6zJ8UMdrGE6pBhtXFBbRnYwNw7xLdm/iXRpqZy/eg86KS5yR15K8sJSAV7oDO99KmH1bU6/8pkN/QaVCnLbkd8SIuvA36wxnhG+GeRGTvb5KjOpHLtaRBbl1DJXrmqln9+UCnd9Qtz8qG7qnYqq3QjG9zkD+AToxSob2OTHM8tsJndPyOrAOlN8LcMeCoA6+IHl16tiQuPJMqD/mUnvCId1MCNqy8cntJzSuCIoaVg3NcxnqlzU0CJqqPaOcoMT+jEv9MeGp1Zsh6bp4AlHOEjQ11Ahz4ur2F/n8qVDz72RINS0PfjVHfic5AeS75yJm3xux8RvC+Py+JXOU0L4SyiZmpUf60JBqSA3ZXnPpXBmTqWk33n05TdDQ99r+VkLnDNQfV/PijGR2NvuewiEKdz38nvA/byA//TBvmH0/ZFRSRz+YUc3VX0hwRoaDVyOSQqTXvwT5Wz6NS4JqQN6ktScd/Jaj02HOsPfqtHZgI6XnzAcjhtNQXI9Pont6szL7HZcEcY1Khrl3+wzmFOjgDiFVdzl43qf2lGX+rZjhtGEwHzMuW6Y/tJTuh/K72nKovRDjDYSmdFdlF1T9yGFc0iDBicU3DVpCNMY5mVeMSjKoc/sKMDOxEAivr9KlcVUSHCxMf9hnXJTGP7flsP1Vj968Q5SGU9/t405S+1JNxS1l9yRBepTrM7FQ3bFl/ucdusuG8m15nZpE8/zg2CXxHdy2J4tJI9eS6jtyRg4+zKsTnTGka4Jvyu8HklJcGTDzsTC/4bSheNOncl3Oy15XY9XuWkz7tMu4NOGBju0kDDhiNBOT20vY+I5D7/ERgzmNXjPHCe7AMJpJKH8YUPw4YFwy5LYk00jVZUlkIig8GBA0DZXbCaNqQtCBUVnQUdCxE8eTgMSX3GZUNRQfiHRdvaWwisGlIdaD/RezZHdF+C5uRmT2LUEDzv6DAY0LHsOFmNItF6+rBO6jJ30OX47pno0487sxcdrQPCtnw96SHnp3YE7EhP1ZjTtHFaM82N2E+lWRndM1Q/numCgrW3eA1ppH8a5ya0vrMfsvZSnfjWhcSDGuwPybCaWNiObVhMF8mvYZqTaijOyMBjMyZnuU6zNhjZ5eXLFr/9Z/RPWmwiScSIZl3lCNVvuMFlec1jTExLIbT3wZ3RorJlWcAnupSzjyyH+Upv/MAO9uRiTnCCq3LMe/MqTwmrLtvb5KhpkPQ/qzHk40IcDM6+NB5xc2i1FGseiHz0NhXUQVhYbJKijKyr68ec5l/s0hO19Ja+o1ub3lOxHjovaF5jnFsFtXTZw1kN+NaJ7zGVVFxDl6Keax/7pF7fmqmr5TlsoNGWt4Q02RsNoJRxWx7UvrCUfPGKY+1jTLHQnmCgs6cfyBZVTUfSvfC2mcC04kPCaBVDPi8NkU2X1L47KyUP0utC7EFNY14zeT+PmwoMbuYZP3UFJS2I7pLLkUdmJ68y5RWlO4ODAnNp2plmQ5rWdGrP1P8JPv/6efD2v0h2a4cWBItYXDZQ+0sw1mpNDM1MQ5lSGDYfGHddwx7L2a0LjoMLg0xFzusDZzjO155HZlO5d9+pjqddWDUdrg3s1oN2kbCg/UFW/+ikMcwNEz0v4XNiypljBBE1naZ1VLOaHFX+oxLkmgtvCzHu7Y0FvRqBGrxqF5LkWqrhCyYdXiDizN8x6Hz4uk4YTa0frzmphhRARJ1yVtGVUNU++5rP+VKeWcphWs21nT7tNfSPD6ltyBOAzjoqW/FlL+uE6cUkCEE2nxtM9DaT1S0mHBkP7NAzK/ccDgP2yw8FsPmLqu3IT+jMPx5RRRRtS84l15pY7KslkfVSRKXP37m/QXLXFKNfPUjZDy3VBZrQEyyLBKWUw1J1mqocJ8594ZMprS/4d5WPyux+a3P0cmaQ8T9IYVw+4rhsI9wTHW0QKJ078I3E18TbEGqwXS9YTCbY/xY33mZlpkUmOerOxgYiP1ZGz4a2fforOi3bY/p6M5fWwmjC0d/3M/0+cXHjgkgbDIVCM5CfhNHRtphowh9UaB7K4A/N6S9PzVTyxBN2FYEfaYOdZC8rsyFutOzDX8jjRTTgQzHyY4Y6EOndPCXd2xdqj+goJ4/R5MXZesJiyIFYYDqbpD+4w6+/LdhOyeYeptj8OXp0jXHPIbMq2Yf2dA0DAcX/ZoXLZ4f+WIo1aeMHYZhR47rRLHV1zyWyKbj8vKo3JCLfLWmjexIoL5NyPyG4be4wtUr0t+nt8d0zzjs/cFj5mPI3rL8cQlHCo3+7RPOVRuqhnszxv2vpCmckMwnjfU0CWof44AfycUlti8qpRnbyg4xxsIgkkfqzNuXtAUxG8b9l/y5Hv/tRpJ5LD/YIoocfjdD56jeMsl6CYkI5f/6rVvMrraJ30sp+bFn0aU1iOmPw7xO9BZVfc7/XFfRg0bMJyR9CPMQ2EzoXMuYvrDNoOq+ALNy8JTj54RgF95v8bhs+q2vT40z8sguHXeoTfvSqK9Y8ltQ+usixPJg3V0bsiwqolOYTuiP+PiDSSJmf4kkr1Q1WU4rbn7cFoPbZz+RYpK7QkxmOpXda+K9zXQyB7H7L+YEfljMebbX/qAUehhE0PgxmSCkEwQ8vQ3P6V13sGJLJWbMUHLTIgtsPiDOm5o6azC8RWfcQk2fkvwV5i3bH5TBiHewHD4rIfNJEQ5y9zbIRvfzhJnxRVwxjD7vgyQR2XDaGmMdSY6uUddI/+S1t4/1xXm5PmUOnLJ7chcLLctXHP6Wsy4qG4ztyPPeOvCmb9bw/3VYwIvwjgWExp++8wHTL3hk5kk/RE64Fiy72YZzBp6sy7Nsz67X3RpnvNxhzqqq9ccdl7NUr4n1abX1WSoswaH3xyDA/f/SpH2eU1n/JbDwp8dEZZipq6P2f3mLHPvJBMFgI7EwWJE9XpMnILZ94dUbg/lQ5Doc7wBuHspvL6oho3zPk5k8XqWzL5Db84lcQ3jvFynnVBNWtAwJ76r7fMydJj5IJqk5mlcWbk9YufLDvmthHHZkprv8717F2ke51mZaZDyIjpDpUP//NZZ8htKiN79RkKUt4ymZSp8939XgUR4cZwWQpJZDyZhaYb5txLyezGFjUSlzrGH3zZsfcOleF99ReZI/lTHV3ycUOWT0/Uw0UMz4c9T1z+U7c7se6qJUjWH5lNjctuG9imXhTeU7Bp01Bj4PTj84jSD16YZ//05Mp9kyOy7/A8/+CpxylB72tBddsnf81haPVbi84ElexyTbiTEpYjMUULmOKH2lGEwbcjuCt8zkZhKuf2EoGFI3U0z9a5LOB9SuS7c0+9B4+kpSjc9jq8ExBmoX3RJNS2NJ2P6i5biLY/jx4U1Ns6nGMwFQhQmdWP1ZoQ3MLQviuScrqthS/wJqWYNIRxz0HxM5JLK7ZDyvZhRRbVhdlenj/XEuuquiFy99Usp3JFh7d+/xZNfVWrvr5y/weJind3Xllm/vkjaj+g+KOFnQp74d64xKrvkb/vktmD2HS2e9JEhzhiSVEJYsPQXhCWPSwpJPr7i0llyaTxmqNwak91RPV26Y+gvqMl6qCzurcYEHctwRopipd+Il/so12dioZoECusiRuS31DSVPgroL8iW++jpgFFZrie9JR093tAynJlMXkLtuE4oCUvptmrCccnSHyufyg2hdtUjcQ3ZBz71ywpVK9zXwqjcHlL/1cGJWYXSkGWA4UQQ7Pr0lgyFrZjig1ix4p4WzLgoLHJUNuTXPaqfiKP5sOsel4S9Jt5k2hTDwbMuo0oygd+kFUs3ZIgrG0sZoXl9KN10ye/E1K76HD4nxKBxzseZQEleN2b+rZip68JQo5xl7YUtIuvQDVNYC//k1uPs3Z0RZXDfof7pFMFSj3RmTMEfMpjR6LZ1QQEXw2lLfjdRAPCOdsqgYWg+GdI6L5y7fDthMGdxhzKAK98PcYfQPqdooIc9RnfFUv1AfNqZDyPcsSW3Y8juqix4lOszsVABMg3ZcAvDkwmC1zUcPWsYldVkBB3ZM/pdO4mZcShsKuy2sJWQPjKMzgxP0juSFPQGKaKcpX1GiyXdiidpfJK/GCvTr6OnMvjXcrSeHNN9cojXU1TNaEq8zspNQVxR+uGxrB1+9xWP/BayidyRr1XrrEOqkZDds7TPqZ7tLulrRiWHhdcjvL7BZjSKTTUt9Us+3WUhEccvqYt2x5bhrPgNnWXJYoKGlAOZY+2yC6/12Hk14PiKR5g1zL8V4cwNWc03+GBzhTt7s8SRSzz0KH/ikGpYFt8YUHjgMDzO0Nkt8AdvPUNnLeHwRUvQVmDy4uvRSWr26NLgBFY697cjCuvCW3vzDrltcQScMfRnPIZzCZk9Rwu7bcke6ZuERUNuzzKYcokDg4ksnbMJ6ebn6OiPU9Cfdk6S8vI7CWFRY7dU3UwIuprZj6piV7kj2WynGzFz70YMKw753YT0nTSDWXEC3IEhWc+x9JOIdE1Tl/6UFKeJD8PzambGFUv5vtzrlr7rUno7TVjQDhg0OfG5SjUsrXNKoPYGagJP/1FPb2LdcPjFiO5pkZcPXknonBbRpnwvwhsoa2AwB41LvsLD6lIewMQvqqxx5NKfuLhjURJn34HcfkL7YkR+R6TrzCSvdTiTsPvlnBhQZfmWbvwVy2NL+/zw1kWiVsBUuUvwSZb8pwG9FUFm938jxaiCpkmrTb753DWSYkSq5lK8b2meDWid8UWG9phEBalE230lQ6qhMiksSAM2mFMd3VuQkYadiCdHZcWzl+6KKzysKpq99oTDzIc9UjWH+mOfo1m/E8oNLnNoaD8pEzC/bYmywjmb51y6zwwYVuUoXXtagRLNi7D3RZfjyx5RBg6fm4zkDAyrEKcsp5/fpnXaw8QKKevPiydZvpsw9aMUAMU70F3wsJ460XFRAWut8yopwoJCItyRyoTWWUN31dA6C/XLOZGjY5h6xyO36TCYS5j/qUNuVwv68FmPTE1lijsCZzRBOnoqH6KMdvuoEtFfjcnUxvJBPbK01hwSH/LrHocv6QHrzzp01hIK6w5LP+nK8nHDUP2dTVaWjlmvV3ni1A5uaUzr7Vmqn8ZU7kS4fcOo7JDdF62xutzkuJ5no1vF6Xikj6B0p0+YN2qiMmqgxmWdWEfPJxQ2EupXLeV7sUx+U+AOoPvLXbyh0Iv8tsSC+d2E6Q9kZHHwnEoLJ9T7vP01cQrn3/ocpaKIlGHEMv8oUDrxeUvxvkPrvNhMqVsZhlMSlk1/aMn+xgEmNoTViP5jI37pX3sb6yv+3B1CcqWLOdWn8T8vTyZIhuMnFK3YXjN0l0SyLj5IqNwe0p8TKWM4NcH4PHmK+l1L0JDFT/OSvo871Ahw6hN7YoiRasijaVxS2VG/ItJyqi446fA5SY+DtkaXhc2YOFCNWdyMMRam3vIo3nJxBhH1J4TnZg+kZC1sJQR1PSzjEsT5hHQjYf/FPO2z6qY3f3gK10noNrLcq0+RupYlcS2jokN7xWM4m+B3RYY2CZQzQ/xUxJnCMWZ2SJKC/lJaWQKrcjzprup+9uccMnsu/XmHtd8fikziQOZQR3f++3mqn4Zsv+rKWPh0RHfBoX4V+ityXumsOKIp9vUQSpn6aIr9z8RCTSZkh86yR5SVucTMe8IJy7fFvll4Y0Sc0d87v9nBMZbxdExmaoCp+7xztMraY3uU/vebuN+oER5miGMHDCcGCXFaUmm/q3IjU1PXf/hshsWfDUWETrQQcjuGTF3g/EN2krHiH+S3LaVb0q0X1gWtWQ+Crsa5829ayYYvu2RrMSvfH1P9RKPSUXkiNV5zyUzINPVLLp3T6titgc5aDr8zcS1Z0dHYXXLI7kL1hhxecg88RkUH64HfMtSfjzj3zftsH1UoVHu4Pyrj9/QzeyOVNrktKUobE8KyYyxBEPEnn1whCR1mPpy4tkyypPZf0hQNxMp3Yk2t9l/OYh2RZepXNbQYTBsaF3zSNUf+qQ3Rokq3YfVPEnFtz4ZKVXwwZjAtMvvUjfCR1shnYqG6Y5i6EeEOrYIKjuyEtKw6rXUeNr8ZkKrLNSTlh+wdl/iVFz4knxnB7IhmL0Ojn+H6/SUa96ssXzhkfrpFnDIM5sWn9DqCT7qnY0YVy+HzjmLL65a9l9Mnbifp2iTT6ozHYFYBEEHTkHhKZx6VNerNNEQiGcxop3v4xg4rDpkjmH03ZPtVh+MrKY6vGio3LZVbMflduaiMqmrIRPy2RDkdpc1zDn5XC7Swod1v8SdtGk/EymBtTngG1UnT+GSHZy+tc9jLE7UCOjWZoVpX2LOJ1fTFKXkZYKF1JWK7XqbXSfPKpTvQ9Wmvpth/yTCejsEoddtviyBd/QRm3n8YrCHOgDsSlDUuSUFrIiV4z74rZtjoix28gWWcd2lfDpn5mUIz+rO+woldxfo8yvWZWKhx1tI479FfNAzLDoXNkDCvnTDoWIr3tMNFGRhPxfSHKZLDNHfaMxzfniJ9PUNyrUScOJw7dUD+VIvVQoODepEk0Mg1Ton1U1i3ZLe1m+U35QDYWzBkDrQrbH4roHVRBl/Zw4TK7Yj25ZBRVbtk/3RIf9Gy+xUYllQ+pI/l1Dz7ToI7UBMRtC17L3vkN7RLT39sOX5S+G7rrEOU0ZF6/IR21aApzqkzkWxUb0b0FhWMkakl7L9c5NQfyVUlzD7UL1n6c5bxQZb3bp/m+OMZ3GKI6ckEubQutUR/duI7FcDhS4qT9Osu2fQIUwt488dXmP+pwQ0t5U8NbscluyOMtr8U4/cE2Q2n/ZPI9sJ2RG4voT/j0LokyZA7VplU/40+QctQ/oMcxQdDOisOmS1/0nRJIzaY0eYxnPkcMfzdvtJDZt8PGU5D40JA9dOY/oKajPrXhrSeGcFjHU5d3Cf4WYGZd6D+P61QuaGnOfEtFqj3M3hOwvWjeYIg4iv/xjtkLjSF7c2P8UYSyVVuRXSX5fMfZ+TykfgQ5WNZKf6kTpQyHD3pkdnwWf6zkNI9yGz4pGuGle8LvPb6gNHA4vA5OQTOvTMUcXpbO2SY179nd+REnfiW5T9LSDWEatRe0IMRZY3MidOw89VJWskIjp4Vi6xxSdaObqj/zxwZomrEN178mFRhRHC+DXspSnccSneE1fYXlVtQuQGlu5bqRy7RpT7lJ2uUM0OClR5TH1sOXoKDL6hmnvnA0r4Y0z7liDUV6HcYTojso7Lh4FllcPWWoHjLY/RkX4EZNcvM72YYTkvgd/R0lqCjphig+ukIrJ1g12PFiD7C9ZlYqNYRg/7oCZ8kZRlNwd4rcuBL12Omvp+m+HGKpWqLnVoZd6SbkNuPZPF4NiFcGeEay3S2T+/jKs2jPPbjIn/0ztOEoce5v9vh1O869GcVJNs+5ZGuC2oq3UkYlgWkF+94jCqG2rMVopwCvQYrEeOivEaz+7IC33/BJfY1/h1MG4rrmsDUr1r8Wp/aU0p6sY6scYKWob9k5S/lwv4XXEr3x5Tvxky9J0HgYFaTn/k3+rhDI7l4VtqvTEPqg9bFiRXQGHrPDcjP9Hh96wyVQp9+I0P6yFHI8SQAIrurEiK3F9K8AMXf3CUaeNSb6rrnSh1aZ6S9L93SzzGsOGR23ZPd0xvJEnQwo2FE5U4kJ2sfVn44YjhtKbyWoXqtTeeU+AzlmwL+h1PiTXRO6dTafzFFd9kl8Q3d5YDe3KMtwc+ESZrfSzh41sVYyUyqN0YcP57i6GmL3/VJH4mNf/i9ZWzV0luE6Y8T9l/yiM71CYKIpUqLK+U9vvu958keGMq3fWpPJXilMWvTx6z/2mnN2btMDBYsmWNLf9qh/YUB2Y8z4mneTeguiY1fuR3TW3Qo3vKIUwnjgmHunQ6D2QLzb4YMpj1GFUFFgznDzPsJccrhwW9WqX4y0cUPVXYELXX8gxmdIE4MzfMB5Xshw6pDds9OItShdTaDO+HK2sk0a5xzyO3H9JYdhhVFrWeyI7q1HCQQzMd4NU3hxnk9jK0LsPSTkDjt0P732xQN9MYBf/25N3i9dpbDTp7U75UZP5EwPB9ihy6Vj1wx/duG/rx2/HHBMJhVaZJ4Br8bE2eUhL3/YpqVHw5pnk8xWMzJf+tBTOwLU22ds5TviGaZ+Kqb89tKxh5WftGs/bOuz8SOmvgOxXUYzUou0Z/3KWzHrP3hgHEpkY+TB14XWB0Qnxmy+0uWqRcOcN0E1014deY2D7pTTF2zTH8yVMz4NYPdSxMnDqNKIvlJqFRjJ1LcDg5Uf5TG71hhhJccCluxLM+PI/JbRjbhKYMTWza+XcA6sP01nzgwDGYthXtSmlpXBhKZ/QkgnjWkOiK6xClD+c4Iv4PEjJMjtD/jMayKSDKYkw1mYXtM0FI8TvG+pfa04KHYlz4MC4XNmFdX7vDXXniDldM1Lk4f4g0Mw+WQw1ciOmdirGfZ/O2Ena/CcOzTaOU4Uz7mb//oy2z9bBnzZxXc0BLUHaZ+GgDQX1AJMpzWYh3Oqtkr3ZUXQeLBzpdTIkR3RXqJMq54wyOVT4OqQ/OiQ3/ecP5v1UhcWYH6fStnxbRD66zDzMcDBVc8wvWZWKhRWoV4dtNjOGWwjqG96tJeS+P1xeFM1yUhLuQHrMzWwVgGY59xK0Vvs8jfvvkC9//gLPUrht0vphWFk5KVevNvreAO5Ynv9eWP1J94hSL1Lt3TEPQSMoeWKKVAtaOnU6QbCfltS3ErZPCNDtazTF+PKKwrISS3rSbF74hVP6oY+ou/yDFtnfYmKSXQPJei/Zx8SYsPYqKspXnRUFpP6M8FyoRKGQ6eSxEW4dSfqKnMb8p8I9WMcScYc2fF5bWds6RMxPZelTv1aeK05Tee/YBzZ/aZO18je7ZFZaqD9S0pPyKbG/L2++dJ1TQ4iTIymRuXJdpz+qptSw9ipj+KIVGkUeeUOUFfOudjZj+ImPkw5PiyS/rIsP1LLv21kP6MJwz5C2OCBgxnEj79P5QYFwzZQzkX1i875HfH5HYs67+eYvaDz5FJmjOJN/cGgjg6K1I1tk87mET+9J0VR3rwH06xeX2B+dU6jb0ixdmuOuvrBZxQ5JbSPU2n0k15Qo3KUgkcX5VDnYmhuJ4wWIhOjHHLn8LWN0XmVeCuPi97IO+Aw6cDKn8/x2ghYuerDkHXMi4J183tK/MpCYTBTl2TatNEWsSDGUN+e+J88lFaJcC0g98zTF1LGFQNo5J4pd5wInzbt3i9UF5WVbmTtE95YCBoODhjaG2U+J/+7i9Rrnb5zup11l7Y4ru3r1DvZ9hfn6JzmKfVyWJyEZ0bVcIPK3hdB6wguoU3Bxw/LmrlcNbidzRg6c06RBmVJ4Mph/LdZNI4GmbfNAwqLsOqS5wVbziz7+C2XNprGrle/L+PcGKYfRtytwI6ZxKitKH2pJZb7WoKDOQ3HHZf/hyZpMVpWd30Fyzdy2O8oTDI9LH0Uf05w8xHIflNzcuzuw6dn8wx/aaH872Kkp/PDskeJZTWxydKgNpTEqmV74TMvRuRORSOGuWgt+iQ2fHIHBiCtojH2S2PMC87xdn3I5nqng8YVRWY1jznMvO6x8w70LgoJ2ljYe9V1VxRVs1Qd8nFb8vErbgeE3R0TB4/qWSXcXFi99OX9DrKaXdPXEvtxUgj5VW491tZ4kCdf9AU3uxEwlY7ZxKChjyk+JMqv7/+BL4bszjV4lSpwWOPbePVPeJmQOquHFgK64omd0dw7n8esv1qhlRdgxW/bajeSPA7oujVL6vkGVbFWvM7MJyPGJUcgq6mUVHGMvtuwmjKsvxnMemapXwvYf0vF3DGOlHcMZQ/FT2ydFcqBXdkyR6JDvkwHfyfdX0mmil3IE18HED6fY/ES8Qor8g+UZaUPomrhOnmeRd3xCS8NqHriDc6LFtqTwRkDkTYDdpgjeX4qo/XE6lkVJ54kfYt6WNoXNYuPPc2NM+KSH38hGH3Sy7ZPb2p09diuosuUzcidn4rJHUzQ3HdYl2H5sWE4g0fJ0qofiKfgKCp38s6smH3W9pB85uGwazw3O6CJCuDGQdnJMw28V28vq9Q4ophGHAiaCzdV3DG7Dt9hrMpivccGaLVRVoZ/V6Z8FOPztN5nPWQOHDInzZY4zEu6fW9nizQE89QezIrAnRssZO8KEVWSmrdOuOQrovoMpiRUnX6XZfWOUsSuCd5U43zDnPvxLTO+AyrkKlb5t6O2fmKQ+bQYfb9MaOSy7Aih5pU09K8AFiXKC/5zqNcn4kdNfGg/uqQxhMxUUpHfViUujNzaCndFSs+TkPhek02P3WRqEcVNTLZfUt+P6byKUxfG53EkPfnDcOn+sRpKTiTAAZzYjiN80pJDnOG4ysikYQ5w6nvDmBxSPpY0xO/K/b+0VMe3oM02QPL8TPilZZum5NwsMGsQ37TMvPRkKWfiGhZugXjqswc3AEsvNFnOB8z9emYxkWRP+QnpdTATE3ivuJGQvEeZA4gt62wuFTbUnsiy+HTMukNWlrIey87pNoJW98skN+J2fqaT+2qd8IHHZwZ03pqzPHzMa0zjkjLJUPvbIh1Db1FcUpTbfl0NS45LLw5VMhxS6RpvyPyTPrY4HdkgZk9ipQCPu9S/XSE15fUfVxwKN0W+Wf9LzskvppJJ9JIONWUucj86zrxHuX6TCxUE4O3maZyzaXxmMJfx+VEYjwP0o2Y0ZSO6MZzMzJLKxv6c7JRH1Yt3RXoTyt7qnkuwPlCg7k3Ye7dkPl/mMIdKdMpt5tQuW6oPyGWefO8bsHMR8Iea8/FtM5kWPwHSrYziXxPrSuJcn5LyR/OUDe7dH9M4YE8RgczltoLMceX04RFya/jlGHmXag9qYek/liG4k2XB99x8fqG4ctdSvcSUi3VslPXeqSO5dzSXzS0Hosl20g/9DsQJhulFQYRFqF6XbXkqT9qcPC8R5KWJ8C4ABiY+zOPYMfH7TmMZmJSx4aVP22Q2vXpLelhnLquk6pyN2J0dsjmN1Okj0SuCYuqm92hpbek9GmTQGdJ8ZQmsYyLHuljabkS19A+Z4l9WPiJQ+Oi+KuJb+ivCoMdTStXISx8jo5+J4agpbTh0h1L+5RL6bamGIdPp+iuivnfuGIxiQIW4pQcPXqL+to4I45q5kC7wOC9Cv2ZiQfqsW5Gb8GQORIAnT7UlMjvy3i2s6RyovqhS6ojO53OaU2esjvyu1LWqmXt90dsfy2DiWDrmwFxJmH2zYlM5bpDmLMcP+YSp0Xk8PtmQm5J6M85kIBNJySBC3dyDKYN09eG7L6S5t5vZrGOJdVw8LpqODqnIXOkDIDeEqSPZZ0+80HCYFZZVe0L4I7KWEeeUvVLLt5QiytxDfltCPMO/YWE8v2IB3+5QumuVeRPonFwuiYL+uI7now26gnWc7COuvbBtMPST2M6S84kKwvGiUOcNidCvTAvz9XsniN1Q0EbTNCxpI9DokxAmIPclmFcElnokdbIv6S19891WQcGswnpmo7zOA1TnwxP0uq8AaSbKvRTzYTiZiw2zqIWUpRVedA+q/pzMOVQvpNQvhdiPSEK0x8NiHKWbC1m7p0epftiRoU5+aqmWpbuWsyoaugsezTPSBFqEhhNTYja6zJOOHg+gzOGcUU8UL/tkP6dfRGDl4QIBB1NdXI7liit+vv4cUP2QNlXa//QMlwQ07+/YAnzHnFgOfN7Q0p3DMPphPSxpfbFEHckOYcTQ/ZA6tjElWe+O4TcQUjmSKdM8T7sfwGwiuuMsobqjS5xoNPJHUxselIWYzWUSCa7t04n1aP95YjswVg+rgcT6XgFdr/okjlWE9lbsrQu6h6FeW0Q1lO54XfshKswiffMGWpXAwpbkrfk9hMGMyrvHuX6TCxUJSwLIB5OW/orMVHWVQKdC73TEf1ph8pNS3tNqR6DWVHRequ6OalWzPzPx5hEsuLjq4bjx33685Z0zbLx7TTl29CbdalfydI859I+C7kDAeOjkqHyiZzx3IGlsC0zCL9jcULD/FsxlZs9YbOh/svsK5nPbxvqf7pIWLRMXQsZVuSksvBzlRNh3kgjtCc8cjhtaa35lD/xqH4aU3igkW7QMuy/lCXxDelDByeG6ts+YVHWP63zAtTjlCVZHkqdGkF7xac/J9SkcUVwEcDc28qJHc6kdXSPRdZun3ImbDBzMgDJbVsZDacgyitNZfOXU/RWJ07WVcF1p/50zDhv6C0prjOzrwW8/PfvC9ori7cwmDOU7iXMvD8x1SibiWVmQmldKeFe39D66ucIR40DTUJAHfjp34sI8y7DqnaP4k3vJI1v8XUB5n5XQD1onq0n3iPdECnXHRjy20r/6C3D4uuRRoFzypvy+hNnvCWX4h1hmO01HZXjkmFYMTQuQeOJmMqthHHeof5YTkZfgcoVjEjB/UWJ3Eyk49XvagJWe8IjaIsd744EKTkhzLxvaZ2b2GeWNHzorlr5p07Ef2FRjWD9+YigIZZXqu6QuJBqGLIfZ+jPGzpnYgazhup12WKe+UcDlSsHWmC91ZjNb00ILiGU7sUUNi29syHDqmH6WsTST9TAVW7oRMrtWMZTMRbI7Bnyu7KDl/+VMNDSHd384qYiNzd+5wzNCxo3j8oO6SNL55RDa01Tq7l31Jz1Z1ya51x6Z0OKGwn+7ewjrZHPxEK1jnBKO7kZ218PaJ/Wj5Y5UsNU3ByRPbA0zqcYTcn3NGjLB2rqxkgBDhccmuecidOJGgFvIJrg4dO+VK3bltV/OiZ7lNBbUX1W3IrI7mlxRbmJaVgsY4WpD5QYEubUvDSuCNsdVrWgcw9c0ocO828pLihKa+Hl9hKF4O4J7A7zGuk+9Dqd/siKC9uUjXlUiAmzhv2XLUFHeKfXhcr7HuOSyNWJJ+w1ysDUJyH9pZjcpk6eMK/ve/9fyRIWpPPqrDos/Ugq1ocWP/UrLvUrYIYOnfMR+y+5tFcEN2E1Ks3txWS3XKJSzHDWnmQMpI6lbOjPSlzYn3PoLKkW9rvCgd2hGkEn0uAlTql0O3wmjXXh+Itjgpbl0n/dZVwwmoA9wvWZWKhKeU6w5qGdopg2UQaqN0dkDy3br6boLRjitMEGwkiDtnigG98KYOJz/7A27C9apj/UzTx+NmH2w1CmuwuG3S8HDKYcgia0zsP211w6a5Db1dcVN8ROcoeqgyvXFEgbp2Q35IzFlMociDIYtEUEKT4Qs2pUkUIz8ScTqL7gqe6KwRtppwuzekiaZxTcMP+6NEWFey5HXxvj9xTmlj1MKK7rBIhydhKgBvsveWR3XDKHWhxhXs1hflMx7LPvjsgcWkxsKW5GRGkobMVEaUvhARTvuqz9nnxby/dkG9RbEppgYpUU1Q9dCvdhNK2pWhKoFEu1YPc7Eem6JdVUedSfk31Rb0nkaXcEjUtK925ejqRE6MDUGwHWMfTWCgymxT9+lOuzsVATOP1PQjAWbyS8sbdkWPlBh+a5FFiYfS9m7r0JNnlTDJ9UOya3HzP9kWX7G5J6+F1JRYKWhGy91YTiLZfWKX/CZhLEFOa1M82/pQhIvyO4KcooOS97EDH9ScjcO6FMHgzUnotpX0jof6UrQ7Czk4jLabkPZmoRxQcRcU7mYJlawqgC7dMuQdcy925Ia81hOKXa8OilWBkFdZU/3WXxWUvvpBhVDIOKMMjekjyyivdVx7sjLZrqLU3Pmpe0MB6aVtQv+Wx8W41oqhkymFZj6A4TSnd/wXsdTHmSbbuS5aSaSttrn5IkqHxnrODkjozYRmVLbstQuh9SfT1gOCVWVaptCcsJBy8pWjO/JX+pqU9ivB6kDzy5bk9p4la6H7L722PyO5bO6c8RjuqGluGUR6puOHpGc/7K7Zhb/26K9hk5cwyqLnGgWXKU1pTp+LLH0b85IF2PcYYOuW3ho1Ea5t8aEKUN+XWH0RRM3Rgyqihu3AlVY3pD2P0KZHct/cWY/rkxeCo1rGfY/qpHd9Gjt+TgvNAkNT3AX+wR7WbxevKSajzGiTapveoxLrr4DYfpjxQcNn0tYvG1LsOKYVD1KGwl5LeVSHL270VMfRIrwfkwEs9hMpQYVaWF94YJhQc6TluPxVTu/MLkuLMsUD9oOfTnLVEhob+ok0gxmob95zOS0rQsg2mPcUlQmXXh8AXVwt0Fl6CdkN+JWXg9Ir8fk/iWvS+mTkqs0l1Y/cGQ7ilL/TEfv6eFa2IYFQ1zPzewINsiJxQSc/S0HP2K61ZCzEti/rfO+uTezhJmDaW7j0ac/kzgqNaBMKdO2W/p6W2cdwl2ZRBR3Ig4eM7DuhKMZWqWxFUXG/y4wOGzasLiQHPl1leGOFEab6h6L90wDKd8ln4SUntSOJ4zhsrtiMKWoXnOMP2+Q/1xw9mnt9morTKsyAP/+Lf6jIc+tNMYx5IMPEglDKc8/C4U7+s4tq589IN2hDvySTVD3LFLf9rFGUmWParIwz/x9TBu/EpKNjgZ6Kz65PasLHR8SB9C+d6Y/ZcC0sdyfal85NBeVcmQuBJBBlYliDuG3qLL4utDNr+VwkSo2zTQPRPj3Hbprlhyu1C9MeLoqRTpujNxnjEcP+6Rqlv6V35h1BadHRAPXAZG6ENnJXViAFJ/fBI9FCqx2zow9/tpalctrcsxmW01klFWTarfhdw7WdoXIgp3PaY+GdOb9wlzj7ZGPhMLNcxK0Zg9sMy81WD3l6r4XUFGTgSHT3uU7lr8QcLxCwn2rsJiRxUZVqSPdfOiGUNpPWRcyCh9A6WMhEWIcg7WcUgfMgk7MBw8K1JH0ILOr3bxLNz7YBknsBy+LOa9+aSAqSQk6YRgaoB/TTKLh5ffF70vTqvWLj2YGF3M+XgDif+CrkvroqV4e+JHcNYhNfGaemiJE7SZxJIrOt0bWkYVj+lrMc7YkviadPk9jWszR7LDGVVlMd4+K0OO+mMppj6WcS5MTH79hNweDGYEwh9fSZHfTQjaCp31Bpb2rGr04rpeP05bZiodWkGGl1fWeatyir987j3+9vUXiW/lKD11RHTVofVJVbXpnEMvLcbV6h9DlJYpc39OVMhUO6E/63D69xO2vm5J/ACvp4X+KNdn4ug3sTLerYH7v11RJIyF1X+8S+YoYe69kMxxRH/GIbPpM3U9JLeXEDQNsx8kJ0QHGcSqw3UicS0TH3LblvyGXivKQfn2gM5aQtCWK97gwojAjxgfZSfWjVD5SLE/zpMtVh/f45vPXiP1RoH+ogjEUc5S3Irozzrkty2z70cELdj6hkswAbub51wKG3qDMnsOTmzpLrngQOlBSGFDg4pUQ7Y+vUXREFf/uA4Wjp42jAoOO686uEMNG8p3xnJdGcv1JLtrqdwaUlhX7dm4GtNbcOhN9GZBC/yGx6jkMH3Nkm4Idx7nDcOKS6olL4PiXTh6QZGRAKVbhtVig/F2jrf3Vul3Uvxg/5IyYV/ZYjD26Q1SPPHKHRoXBMlljpTeHaUN9StqUAenFGXkjqSyPXwmoHhPBHMTQ+7wc2TpY+zEMrEg3igWSutj9r65QHvNOXnq+gvqfgdVj/plQ28tpnbVxe8lBC1D5XaENdC+oC53WHEYPNOnfUbNinUtlZsJh88Ju/O7lqnr8vgPvJiVP7GUb4otVHwQsf/lBM9NmEr3+Nk/flrarr6IGV7f0Djn0T0Ty658oIlLquZw9KKaiChvqT1l6KwlVG7HYrpnoHojYvtrvmrvGQHmrfMT12qg/lRFEuSm/l02OdJ3bXzHxx3rIZSXVsL6r6cBSDUSgrorknXDkt8L5WVaVB0cpcz/4p7HKZHFnRB6y4bqhw57X1dcpTeE9b95AYBBP4VTC9h9b4FSoU9nlKKYGeJ5MWk3YvUrm5N4IUP1usbS1RsJpdsw/XOPcVF5XlFWwcYgjvC49Iu4+n/W9Zk4+hNfP/DMhyMaF1MUNyK2vxqcGPgePRlMPOJ19PWWjJycl1yGVcvRsw7ZHQHsmUNLel++SMUHMfH7WaY+DbHGEHRcvFGMO5K79GAGTOwyNd3k+F4VJhr+JIDev9fixUqNt++d5qOb51n8JKJ53sPriUI4mBZlbebjCVP/jM/iGwOijMeB41PcijDWI3OU0Ft06KzomPbb0DznMfVRgnVV60XzY7xdORbmdpVHVbkTMywLE/b7Ohkal8HrGI6ftOQfOLgjS2/WJdWY+AKkxfl8mBXbWfbpLyVMv+tSe0LxRp0VJV/3nxnibKRpPuYw825COyf9Uu6eT+2ZhIXXLGHOobDu0Cq4ZJoO1U9jdqolZn/m0SsbBs+MOCrnuf/eCtmS6lSvL5EmBqY/jmhc9Citx9Qvuqx8f8zOV1IkvmCzzKE52cH/WddnYkd1xjD33pCD51MMp9VYWVdTj1N/3KZ7NsIbyEyhP6dmq31aobbpmiHKaTJUXE/orEFYVGc8KjnMfDhi/0WP9mlPbKOMQ/YwIV1PGJ4aM5g1zOS6vPDcbWpfCnFCSDyLMZaXyvcJ1tP4XeG3QVNTr4e27Z1Tht6cz6igcez+8xn6s0pv6SxLW3/0rHbR3H6C15XIbzBrGU49lEMbqq8FpBqCoNpnDFM3Io6vuBQ3Q9H0phWhU7qlWf/ia78IeoszYoWNy8J65U0lMzJvaJn60NA6B1PXLOO8TqtUAzIfZUgfGzIHiqH0+hp7po8sXs+Q+CrFyndCCjcCsruW48ddKh96eMPJ4nIsh908USkizE0ejlUpNYbTCYNpl9n3h3SWXTl3rwakmlC+Lf+ph/KUR1oj//KW3z/HZaB9KqUuviXoZO0PezTPuQxnM5RueLSuhqrR7ipwwkQKa/CGltJNF3dkGEw7zL6XkNtRzTSYMdQvp8jsa9rUW1Q2QOFOi8Zjhtmf+ERZy6d3lnj/pxcpfhRIDvzMEbVagf/6o6+S3xYuCzra2uegdTVk6tOQzBE0v9MjU4849Sdjpm6EIhkHOiEKD2DxNdlcPqT55fcjcjsi13SXHUr3NOES71UoghNahqfG1K4GjEuW4bSlfSk6UQK0Tnsn1ppRFuqXoXtxLDPgtKF6M8Z6EgS2zwqqSjdiory+Jk5PWEtWuLLX16DDGakECRoaIYc5w8HzalyjnKF4Xw/p4TOKQjfGkk2NJfupCRaLsvbEl6pxGeqXUgxmdXKMi4aF19p0Tuv3rT0tG6BHuT4TC9WJoHlJEEyqqRylcTlFqm45eMGn9dSYc38nwok0Fi1uxqTrsk9Xgp1Gi+X7IUFLdWrjqZg4I6C+enOkm51WcvPxMxXlHE0b/I7ha098SrQ0ovvcgNmv7bBcaJIrDkl/lGVcMBw/ZfH6CYWdmNyWofqux85XPDpfGOBey5O936S7HDCsuPTnLb0l7aCt84pqL2wmzL0dM30tonlGo9x0I6a/kDCsaszrhALdSw+G7L3sce6/jxTrOFSZc/Z3I5pPhjhjS/vJEb3VmFQzoXIzJr9lqL7tc/xsTGFHuVlBW1aPQVMdfG/eI0pPmE1t1bOjiu53b9ky874yEwwaJPTnDTjaraOcoL84EJS4+HpMd8Elkxuz+2Ca4MBjOGMmTDfD0vfrzLwL2R1DlDEsvBHhtycP2fk8lU8T2lfHZA5EOn+U67ORM7W8Yhf+xn+A31UD0VtOCFqi6nlDMchB9odhQeFciS+Y0B2rKVLEjhSmJlJHHOZ0nHr9yQjQNScfBzGn9r5oSCohz59/wPEwR5w4dEYB9d0SXssjqoYUPlWNPHqmR+n7Wborhty2pbtqqH6aUH/MYeajmP6MRHFeX7E8CsoQljusaOrUWZVbYH5L1LnhlOx/+pNo8cQTQSdzKIQicyjHQGdsmH9zzOEzosodXzXE2YTKJ+IWpI6lRSrsRDTO+fSXxLhKfPQaTT3I+y8pETpOiS/Qm/fwB4LMcru6j+maIT/JieqsJcy8D61zDuOyTHqtC4P5GG9miOfH5L5boLsqw5D9L1rK16XGDVoSO2b3LY3HJR5MH+rn7C1pYY9Llvv/yX/8+ciZ8gaynfS7+uFzO458lWYdenOaLLmhpb8aEaftCROofE/hsP05w+w7GoXKB0pq0jitNzpOa+cdTkPzoqChwbyYS2f+YER6PcV6c4oHe1MAvDi/iYkcEt/itjxFQU5Zpv8gQ2/RMPNRJKPhrPxL45Slecaltyi+afMxdd0nTngz0rDHKWG4uR0Flw2nzYSvMNnpetBfsGQOdQq4Q5G9Hxq01a5qLHr4gow6bDbGGjVSXl8MqP0XfIazMjVL/ElETgucyHL4tM/8myGpSSr03svqpVtrsoMcTqsZSzUt7TVZtuc3HKwxOCMo3JObTPluQn7D5W889X3CewX6c4bq9ZjGRYfTf6RSxx2rnMjvWHKHSvUO8wlRXl64QQv6ayHTH32OatTEEVUvTulpc4dKKw4LutHpmqX+uMXtuJRuazjQX5SUobgeM64ktM665Lc0JEgC0fVmPoqI01pk2QPL4usjMgeGo6d8ivc1Sdr8eprhUkhts4zrJey+v8D3fvIUXtshfeRQvCP7R29oOL4i9lOYkTv2w/ouKsbkdxLSNQX+JqWI/qLqNusaZt8dsPtyhnFJY83OKYe4EIvAUtb0KNVQI+V3VAocPp2iemNElJXDNkau0ouvd/Hbmti5DY+pT4cnnX7lVkx2T5jxYEaMrcGsJbcvd778jmVUdukuGToXIqavqT52Qnm8hnmoXu9OqIQ68tuPh4zKeiiLWxFRxshI+GJILSwQzY5FSZxzGS6FeN2QcVnvmddXI1e76svUuOfQW44Zz0d4Q8vsa5KyPMr1mVioJmGS1gxJoGPdJJbhnJxTwoIhs++Q39LxaBKICzGHL0/qsaZz8n0qdyPCgnbR9qpHb1E1Eqiwj7Kw+scNorTe3PFiiFcImV5twoMs1kiRmtuWy55JNPrM7llm39dN9YYWb2BYeDPCHVlKNzVzd0fSZKU3Akyk+J/usmX71QxBR+iGE1rKd2JK1z2csfDjKA39L3dPwoUTT56s218LcEcyIvO7Inus/3pOs/4ETv+TMRvfTtM6p/s2zksAGOYfsvmFlNSecJXvVBb1b+bDkNk33BOGV5QTGpE5gt0vFxhOG+pPJaQa4DU8Wk+NKd2D7rwr7NOD/B2f/+cHX+TqmR0W3ojpLVsyWz5b38wx8/6IUdkw/XFE6e5kkpuoBKh+5JDaE7Ya9JLPl5GvsdA5rwCwKPuLaOzstkumJvpfpqbpxtGvDTUVuuET1F2mPgnxJiTx1lnD1m/EkwxPAeLOGFprPsMpSSXcEey/UhFTKgtO2yNqBRyvVyjdkTNI/fIkHrFsaL4ylJ9TwdCfcRjOJex/QdBY/aJP7iBhVAXrGsZlIQ/pGifHuROJ1OF3LeMydJcVv5g7kB5++uMBo6rB/TiPO1YZ0LpgaZ/W1CxO2ZM32xrFsOd2leRXv5Qi/wBMbEjXE7rLCtjtLyb0zoS4oaWwGVG6NxFBTkF/PiHKOYLWli39BbH7o5xhWIW5twf0zoUsf8/SOaU6d+lPRGI3VqdEcTMm+HINx0/4ZGORo6eUGzX7XjiZqKWo3A3pLLsEnYRMTb9PmDf0lg3lW9qtw6yRIPARrs/EQo0yENRcjp8Qjle9EdK8nDAuWrqLLnH6F0FfZ/9vstm2jogoD60YAUaLIfkbKfEEKobmM2MKW9oFTaJjeeajEYk7STd+bMjzL9xmaa2G35Qx2vFVh8y+dtLqzZj0jQzLP+wreHbCxgoaE3n2rlwAo4yap1FVEE2qmdBfle1P5aZsH8OcIVUXfuiNLN1Fh/SRYf+lDNk9S5ISB7S4mZDdc6jc0jQrXTccvhJRWhdvwLqTIOGBfAP6CzKHSB/reB9MKS3GP/ZIXN0fJ9L3tp6VH5cjeK/8qbr0cVmJhIUNS3c5xdyPXeqXPJKUFv/xZZfOhUiW84Ghu+AyjjwWp1o4B6lJI2fYflVElDgQpbCwEzPOO7hDWPzZSESgmwmtsw4Lr4+IfZVSj3J9JhaqN9BOsfh6xLhkqF/2ya+7rP1hjzAP6SPoz1ncQcTxk3nm3xoxLqv2WvruIVFW4PTsax7FB/EkwQ4yDwLGeS3q7spELPe4IrqTMwMuLB/w1rVzHL03hxMbwst9cjuqi4dVhzBrmH1/zP3fTJM5TuiuWMYl1W+DaUOYkR/T0k8i2hcizvyDtkD7NYeZNxWt3pt3qN7Q10rEp3/PHujniVNSL8SBAPPjKyKOJD705h0yRwn5Wz77X9JAo31OPNHcnriuYSmhv5TQn/OYf3NEnDJk9uUMGHQt5TsR3SWH6q2IhdcV3tueAPxhYZLz2rcEXcvMDzcZVh1GFWUGFO65TF2PCVqw/D1D+4xRx75iiSKXxyr7FO9NFBK7lqlrEDTVQG2/KvezMKcg5PqlFKmm5fB5Jo1cCuuqSX6U65EWqjHmgTHmmjHmQ2PMu5OPVY0x3zfG3Jn8Wflzn/+fGmPuGmNuGWN++Z/1/RNXWU8bv6q6UC4nlsPn8vSXY8r3Q2bftxw+m6e3YGieDTSRyhge/NYsM+/3TiQaR88o17OwIYOGsGAYzAu2wqqjda62mKu2aQ4zZLY8Fl+LyOxb8q9nMfHEV6AInVMOUdbl9B+GdFZcKjcgCdQgpZoCzuffiunNe2Q3Pdb/UpHh1CQaaNroAdwQSWT+LfERZj4KKa4njEoTaKoFrdMpcjtqJuO0/XOcUSkBuuciytcd8tsjTn13xMav+oxKYlyVbxi8rkN7zTCY9XEi/fz5bX3t1jcNpfWYoyc89r7kUb05pLQeExbVcOZ3EqxRybL+10/Jnt1XoxkWJvlWHjTPuGBUmrgDw5OLO7z5d58myhrCrISIQTeht6yHN78lw45RRc3iuCS1RNByiPJ2EjMvtcO/sIU6uV611j715/Cu/yPwQ2vteeCHk//HGHMZ+KvAFeBbwH9jjPkLqQdOLAFcdstjOG2oPaVjNnOUkH/g0l6dwCgXJiz004ooTzzDuGQ5ejp3QmzJHBiOnzQ0HhMLPnNkye4ZyvcSipshzr96RDYVUu9m6f9wdpK455Lfj0m1tGONCg65PdkJeYOEwaxPbjfB71ucSKZlcUoPwfbXf8EPcEcGv2dY+LkCzUhgWHUYTln6MxLmuaOE3rxDcSti5oMR1Vtjjp/RCFgLV29cZ1XBEOV7Y4qfqvFqnU1z8EKazL7k3eMi5HdjksAymlPDmHgy4sBC+e6Y3KZL0JahR+JCezVN64yLNUIGDl7UruaOJjmvbUuYUz2c35bxRKphqd6MWPzpWN78MzH3GtO0Hx8TZeHoWTOB0SKmr0Vk9iW2nHtHUu8wB7PvjcntqW+o3FBIM0aL9VGu/y1H/28Af2vy978F/KU/9/G/Z60dWWvXgbvAC3/RN0pciH1D0IHcbsLSTxJy+zF7X4/pnJfcorPiYIshowq4Y4PfT4hThtn3LKlmQphTUrRJ9MTndgyjiiZTTmjBWlqnfY6beVwnYTxWmRC0LfXHHFprAr4z+4rsDrOG4Szsv+Sz/wUd14MZOY34HeGEXh+8nnYTryeMNGjAzldEjhnMT/ijgZSwxsLh0ynBOdMeBy+kWP9XHKbf0y4z+0FIbtcy88GIwVzCyh8cUH8shd+zNK6q3HBCNTRB+6FRmWHqY0vlI5fWmoM3UKy6icVNjTMyeived0g11UAOpy3jSoI3hPSRIygrZwjzvzgt2uflpJ05lF/V5q+AGyZyzE7HGGMxrhZ36TYM5hO2vhGQeGayaVi6C57i4HsQ5VxaZwXweyNL57Toj/mtf7FyaQt8zxjznjHm3558bM5auwcw+XN28vElYOvPfe325GP/i8sY828bY941xrwbjXuMi3JK6aw61K56tE+5TP/cY/4niuDpXApZ/ccu2b1JjONpj1QroXnOofbMJB3E17EbtBQ343fUKARty+GzekOKP86Q+m+rVP9pZvKmKKtmMGNpXHAlnutrCpbb1vx78aeW3lpI4oPfkFfpcFp5q4uvRYyLk3n5ggyHc1t6YFINQ+1Jo1ysPck+El9jyCgz0dM/8OguS8d/fMVnXDTEKYf8hsOn/8EU3VVLYSskfeCQqU3e4AXDsCrebeOSy+ELgtDm3x4xnNLIFRTnPv1xRLqRMJjRVK9zGmbfE1llXLJEeUvmwOJ3LWu/36V7yhK0pI16WMIMywav7XL0RAZ3DMazHNfzzH5fOrQ4bSjddshvKoGxdTHG76n+NommcI3zsjDKHIkLceq7Ea1LEXf+rUdz8n3UhfpFa+0zwLeBf88Y8+W/4HP/PxUd/6v93Vr731lrn7PWPuelcvhdBYd5fZEz+osJfs/SXdZYsnjd5+A5j8Gc9D/dUwn1Sy7luwlrvzckXbMUNgVH9VYSxtMRmUPBMu01h8yhYeZjETs6y/Koql+SJ5IzNiev/VA4V7qvpL04ralO+WPFeKePDVPX+pRvS6nauOBLR38UyzjsQF5McdownLLEUyGjkjDMzJ6+X35TO3LluqG/FFO+I5eY9JG0UeOiq/zVj13ibELtiYDyPY2T/Y4hv6UMAyeC4VxM9WMRUfZfTFG6n+D3k5OEwM6Kx+HzOsYTbxLW9phD7Xl5HiRrA4wV6WTvSwWChmiEc2+28Pow/5ayp9yRBgK53YTUeorUrQxR1pzouRJ/om2bN8y+pRPJiXS0Ny84J7bv2X1L6zd6bP+Sz/IPDAs/eDRC6iMtVGvt7uTPQ+D30FF+YIxZAJj8eTj59G1g5c99+TKw+xe+gFFn+NAmezglAvPhc/plTSKSx/zbIc5YBr2Lr1lyezq2D17IKpHjOMHrWwr3ZfrbuGJPwh4eNgjdlYRMLaGwDsOFiEHVZfpaRPGeo9S9WIyiMGsm9a1lVHIZVUXk6JyL2fxWjvplc5JaEmWlNHVCQUZJILgnXTOkHwSMS1qgoy92mL720EtV8uvFH2sqhIH6k/IB6C6JF+D3wBk49K6M2H9Z5O+gJYjKnRj+Vj/SgzzOOWQOxVwaTHn05w1z74yIU5B/oEbSCTUBLN63zL6pCVj+9SydU4b2pZDpj4Wm+G2H9X+lqFHq6ZSMMbZUwnSXHIL2REHRt8rbOhIsJz+wRJ4MkxrYiaF6Qx6xUV5qAvNRAb9tOHzGoXXmX5Bc2hiTM8YUHv4d+CbwCfCHwO9MPu13gD+Y/P0Pgb9qjEkZY9aA88Dbf9FrxCllKzkjGRuk66rF5t6x5LdlEdM+7dA8K0pY85xD+5RqnjilrKLuqqFxwaVxRRDQwo8dcltKU+7PGQqbcjvJ7jm4oaX0YExm16Nyq0/7lEd/UUfgqGxoPaaMp+G0pkupliCadCMmdSDxW+mOfPQ7a5ap6zHDaXvi3ZRqSPLh9yyZQ0thKyFJWYK3CozzDlEWWT2OLMdXXUlBNiLKnxrqVzXeDDqW428PWPpJTO5GitJNQUOphhZH8cEYd6Sm52HaSpwWTPcw8nLzWz7DGYv1YObdlhqtef2OTihUY1SRtaVf9wgLHtGMJDLO2DD1YVPatJI5QTGy+/9v6v471tYsTe/DfusLO+d9crw5h8qhq7u6ezpMDpwhhyINioJkyRBsARJtWJb9nwHaAgwYNmBIgqFEikNKM+MZclJP9/R0d3V1dVeuW7du3RzOuSeHnfP+wvIfz76naNjmXEFDoGoDhRvq3HP2/r71rfW+z/sES+VWMKFZOqTqY/rzsnfPr4v5lmxC+5Q0cIdXDe1jLs1LIXM/0zVKNoRRj+cCxuW/JvaUMeYE2kVBioB/aq39h8aYKvC7wArwGPhb1tr65N/8H4B/GwiB/9Ba+51/1c/IzCzby9/+D9n9ekT2vo/f0QgVhAVaR9DNqKTj1BuIqeP3YhpnPaKE5NX7Lzi4IxEowpxm3rFnCHPQPRFS/cAl0RZ76gmZBSPnZGMh0TRHQbVPgGjrQftcgF/zSB0a0ocK7c3sWOKE3leY5shYuHojYP8FfxI4bGidi5j52SRcbEak7e2vwYk/CNj+Uorp6yGt4yK+FDYiOksuUVKddpQ0R9Mqb2CpXfoMFRiXdeSW7miY4A3FvPL6n2nl82t6gK2jRi63rsZoWJG0xe9YnEjv60liSXZbKdFeX2NtZ6wTY+bDgEQroHE2TZDTQxO7ogK6A9kPpWqW1lkr6/YAyvciGmdcZt8fM5jySNUjtr7mMXXNsvP1iNwDn6nrAT/+zn/8P509Za19aK29OvnvorX2H07+vmat/Ya19vTk1/q/9G/+obX2pLX27F+1SEEXKPINuXuSDA9nJFRzArH0Qaz1KCUycmbH4vdinFC75+KPerpJPUOYsgxnY47/flsGDC0lIhfuiGzsTuS9YVodberAwvE+icZk0eUsvUWZR6hsiMms+cpGPRSEZCJoXo5xh5biQyXb+R0t7u3XJaOufKpyY+p9WTQ2zrrkN0SdS++47D+bwhsKa+wuywv/8IpLfjOie1q0q+6qFlX7hFAPE2tXG85YgrykLFHS0H1xQONyTJCHw1fDSYKzzOdMrGlQlLAMZg2F+x1aZyyjEuQ3x4xKKjOsq+iiIKfr7A0thbX4CPbrznscXk7TXVb+VbKpjWRYEQY+85HUEXM/FX5trO5pes/SOJNgMOWQ/mid3BoEaUPlAw9nBM1TT0ec/nzwUReW7em/8w/w+iIdxwmJ6IbzIW7fUTxOM6a7oNpnOC3fpsH0JNV5BN1l+XX2lyOS++4RChAlwO/I+nvmg5jGGYnMTKxGIMirux1MSxufaIpxhAPH/7DP+n9oKXw3K/fmiUtd7E12qMeW4qMR/dkE/Wk1DIm2JdHTiDRKqt4dl2U8FrvqzsOsJbMtj/8gL3fpzrJzlOuU27TsfzUgdzuB31OtOqwa8hsRreMupfv6dVyyZLd00ugh1g558LzKKGPlHTuYi8mty/80yGkTkNGcJCyDacv82xGHFz3hm0D5dsDW1z3cgSEoKtna68NgPiZZcyYjXGGk40pM5frENaUhGbjf0S47rsRMfajv2Tivz5460Fg1uxMT+4b3/9EXhI8KmuDUr0Y4gRKb848V/pBbd+itKLrmSdDrynfHJNtWjPnLsUaonhZlakeGvLkNLcbSHSZ2h4atn1e+VHZbx7Tft/QWLJXbAam6IbOt7tX6lvxD2HslS+WfZzj8UiAVqwPEULoXk6xrJ+ksJTl8RlOc9KECbMOkVAjT18YYK3Ox+jPxkfdrZtvQeW1AnBB5pjVJEyk8nPALGhEL33XBCAFpHzOU7oeMCjqi6+ddessxSz8YMSpPdFMbltYZwXuZHQdvqPTmVM1SuO/IwmdGR3rnwphxEXI7Ic5YD932a1r4TxZwd0k7XmbHsvpn4SSzQKZ0YXYyFHAVHpHecbAGknWRxJ0ARlU1m9lNISth2jD7XsTxPxrjjLUjN86bp6b5fS5UqE4Ag7IclokFc4QpyO4IzMe69Kc9LRYDm19PTOASw/yPYtrHRbwO8papjyc3foJ6+APLoOKw8OaQ/edSDOZ00UYzETvLEflbCR7/dsTSHxqihEPjuZCZN5XEPEqpYVj+Y4fenBCEwromS8VHEfvPy+MzuzHxn1pyGBct0x+PaJ5IEqwKZqp+GuD1PXpLaiSy+xGd7ZSixSc7X7IOmQORXJrnIE5airel9fd70Dyp91ScpIhkdgy1Cynm3hmy90KKYUUP2hOr88qnGnT05xwGsxLsDWYVsJHY9UnvWw6u+ngDSNYmyoecIb8Z0V51qdzWqDe1Vufx35wnvWcp34vYe0HhdUFO9MT2mZjj/0KeYK1jSQZVZQREKQvzQ6J7GWLfJXbBui69OT0QEkvKjPhpXp+LhWpiwMDJ/6HN3X8rR2rXnXTLajRaF0PM2JA6cKncjNh9xWHqekyQVqdffKhcKWtE5DCRFmh6D3KPByTaCXZeTeENFNvdWXFJ3HPpzzl0VyOqbyRpntRx6PQFcndXY9y+HoCtrzlkdoTf1i/KHyC/aY7q0NozlvSOwn+tAwdXU4Jyzo8wHY/gxSHWGn7x1E3+/E9eYlx0yT2GUXUSZxNo6hQn3CMOrAlVZgR5HdvW40i8J2dByVzaK9KWtU5D8YFm7MWHAY0zCdI1JUxPfRzRm/dpnDMU/94moz9eZlRW0/hEplO7aknW5K5XfBTSnxZntPXrM5Q/kVO314uofuIrt2tg8Xqw8mctHv7NknIFJgSi9J5sP8ePMgxmVLNm9yJGeZfMQUjjokOyrvIst/0FIk5HaWGE/eUs0+86DJZCimsh21/x6ByH0g2P+Tf1tcZC5Qa0VwXzRGntIMOyM5mrw/xbPWpXDH4XWiczbL3u4Y50U7qLLrkt4a2JliH/0CXIGooPI5ItkSUqN8ckDx2mrtuJ9kgu0eOiofBAdugmsvQWHUZFGUSk9xUt6Q2EJBQfRKQfJqlecxi1UlQKPd47WGE0G1J8eR/raYKWrsX051SXxr7GwAs/Dik8MBPbSEPruRH9ebmjmFj/OaE0YI3zmgwlWob2cZUfgym58aUPQ/af9dj8uQTWNZx//SGj0OObf+9tKrcjsHD45YDDF2Ps1BjzXIvDV0L2/u6QxgUB9O7AEYf3FZeNbyaO7DK7q4LBDl4qacQzQU/csSZS/Vkt5swOtE5b2sse3WXD4WWfZM0lVVf50Jv7AtlOwoRkHAjz9Fouw7JLal9Q07ACjTMKVdh/VtaLXl9NhBMaMvuir/WWBTNtfzVLdkPToPpFmH87OurYO8dj6hfEXPK7MP9mi96ipXlauGzhvmH/uQTT10JSdXFKG6+Oya0big+io4Tl5imXMK33XXwwpn1KCyhVE1xzeNWhfDsi8dt74FhcY9mrF3B7DqdKh1z97RtkdmNaJxyCnI74yq2Q2Q8Ctr/qkWxO4KkRLP9zl+ymZCzjgiF2NSGSTHnSlBX1kPXmlaBnYqifS2Ady+KPA8YFw+EgS2eY5FGvyvZXDflf3+Hlcw/5t19/Az8Z0j/IMr3UJBy7nHl1jeCVDvmHIrpECQiqIa2z4hcka4beMWHcUx/HZHfklN08owFLcS06YoGld2WANzg7VHhxE3KbYxIdYehP8/pcHP1M9Om1i75IJKFMF0wMhUf6f0++rvjQMqiYo0Ft4aFl7/mE/P6NyCFMiCjjksEbGkYFl/zGiETXJ74/YevPaofdeb1IoqXvVb6t+rW/GnKIR/GhKHDeboIwre9buh+x+6rco8cFNVGHV5NECdWwsQu9VSENu68ZLqd7NIoZTpcOqL09R+oQ3l9eZr7c5vBZw9S1mG6ojnz7yy6FB6oZrSudlN829Kdc+q/2WPpvfA6uJig90AKpXbZE2ZjkoUv5psqH/pwlu22Y+njA7qtpEm3D7ksJBoshp3Mtal6Waw9XcALDXjPPbr1AYyZDFLqQjOi8O42XsNx7fIzxVMj4rCXRdJh9N6Kz7JPdjenNaWqXbGri1Vl2J6knmuM3z1hSh7IW6i7D1HUJ/vq1JLGvBnLnteRRcNzTvD4XO6qxE1lxyzL73ggQGbd7TF16mNbCc8eWzO4YY8VXHU7HmMgynInJ7MXMvifNUrIuqKl/ZSD7xBTsvpRi/1mH+kWXdC06Ig7Pvt2jcjuieyrg4HnVmMVPvSMfqPZpBZONqpbmKVe81E/N0fGbPlDghDcQKSX2YeGtiOFcSJwPCa1DOjnmXHaX2XcCnMDyCydvcSJfw1ntCWz3pGCY+UCWlOW7IfsvQnpfOKb1IPOzLPWzCT0oLzu0jrmkaoaZnxlmPgg5eCnG71pm348JU4AD2a2YwYz4vckDl/v1Kdb3K/ibCaYv7ZN8J0fhBxnG/5d5lv87j/nv+sq8qqnESRx4lG5pV+/NuxirU0MpNvLFqr0QHWUWPDHFeIK9OqEUDfvPq7ltnbEMpwWPWddS2IgYzH+BalTriFTrBDAueeTXZTVeumXoz6ibjz3tYK0TScK0IbszJs7EDKsO82+JvNKbcxTeMKej3ewmcQLZ0xTWY4JKTJCVJWNuJ1bIbkUYaHrDp3xLWGyYlYvzuGDIbsoh+tgf9RmXrKwuQYynRsxgShr/9J6h8MgyLlvaKx7VD128dEjCCfkbq9f5r//FN+nPePRnDV/K3+PNH13GdWXE8GRHb5x2GE5B599us/x9kVCG05ZUwzLOT6KECkIaQKyk/qzD7ksuzkDvsz/tkGxaDq6kGcwo1TD2DKUXDnhlfp3suxkSTYP9nekjs7VEfcj+cz71846mTQbSNe3a3lD2nuM8YEUfXPp+U5E8Rcv02y7dZUgfimyebAojzu1EtE9Abiti9t2Ycd4hKkRkN8RRGM3IByG3/gWrURMt1UGgeXl71aGzqt0ktw4z73Uo3wvpzxr6izH180nK11wRlK0Ul1hoXg7J7AkPzD9SHM+opGlI4Y5L8b5wyGFZuie/HZKuxRQeKXYytznG6ysZubukY8odQv1ihvKn4sI2z+l9hqmJYdq0mfiWQnYDSg8Cut/uEoUOnhPz3/zgayy8GdBdMmS3LP/x9/82QSnim8fuUPoggTtSA5Joi8DRelzEGYuvaSKxvEws2p6JYfpaqKSVsd5b9VPL7HuWMGvpruikcAJL+lv7lO9FmNhyrFinHaTIb0ZU7oSkGhEmlDvK2q/nGJwdkmwKGjSRRs+luzD14y2mPxQyMJiVId3t/0WOwiO59SXbMdUbltYJb0JflEuf31aGVm/OoX3MpbdoKNzyGZc0FFj8vvqM7soXCEc1MXSPxZRvPnGEE6aYbISs/4rk0Ntfy5M6sOQfx/SPR+Q3LfXzMkk4vCqBHcDcm5KiuCM1W6maPSJq9CeZnwD5zZCDqz6tUymCE0OyH6UJU9BZlZanfcay8mcB+8/7VG9E8sGaspTuWKo3YlonxKLvrsB4JgB88UWPQ6Ljwu0cHBtxY3eeY5e26VxbxBvI7vz5yw/54P4qf/FHL1KYzNtxJnP1tIM/06NxJoff06SqsB5RP++S3tMub12XUVEkaAU7GBI9oRjJuiX2xTtN/WCG/usxcSrig/UVzFqazLweqnHBUHoQsveimtT07RTWQH5baED9gkvpbszwxDQ7X485+1926S1nxYoq67on2zG7LzvMvhfjDSzlOwH7zydYeGtA63iKIGeZ+Sjk4Ir/WT3qyNytddwlfWDJbn6BdtTYg+N/NIZY3XThcUjjjMPBswnO/NddrCegfzCjRZZZ8zm86pKsW7wBzP9UT2/hgfzkW8clBy48DqldnYzyysL3Yt8yWhrTnZMVeKJpmPnTJP2FeOLYIqa+IiNded0/71C5HXL8n/fpzRu2XzcTM1+DiQzJXcFfjQsW64klbx1IpAJOTR+y+fYi7RP6rEEWPnywSr7cxx3p9Ogu6DY0z6lrzv4wi3W1o8eedEujckzhsQw1Guch1VDzhrFHrKnY024qWbNKKbdvOHtmi6lyB+tp/JzdCemciKhd8Jh7O2RcigkKetAj31A/55Jowu4vjenPahZ/99/M0jruqt70JwYXaYfMnpIAm2dg//kE5bsRj34tObGBN9Qu+IRZQW/JlmTSqYYcboYVw+w7T5cx+blYqO4IhlVfTB8DjbMeySbkNmMe/3KRZFMEkanrAc2zOjLciYJhOCUfTxMrgjtKyqRi9Y+b7Lyi3bh0P5Jt+SN5o2buJ+iuancbTlsa50QJVCqeaG2Z3YAwaUgdQrA8onbR496/mSTZ0liwckca9txjS+JKk0TbMvWRofqxGFhOaIge5nhYq/Irv/gOmWfqFL69y5VfvM1zJ9dJeqoBk01FRwY5jTyfUMwz+9KHVe6EuEPLzHtQO++T24rJbRgaFzQlm/pI6dS9eQdvqMZT8fCWUdWy+mcD7n+0zO56lSilRql5yie3rp208++3yG66lG5D9dMxue3RZz//UYr+rMPCDxyW/0KLzT/fpnl+YkPkQPt0yLhgJpOwyaTJfubRal2O/txZnYxwl+UPltmP2fnyFykQzVU6SXZbR3ZvSYnKg2nZjvfmJcOon/fJrwmuGpcFPzkj7TipQyjcbpKqWzqrsPYbJXFAW7D9CxF+F7rzLlOfRJQexFQ+laQlWTOMZkIpJctWEuaUEupkiwPLv+8xLli8tnvU4e4/q5CwwYyh+I8LpOty5otdJU2PSzFOBKP7Bf7k/iVa7QzbeyU+2Z3ng09OMA5dgqJoeFFC8/Ugr+HFqCxHE3doObgiwWP7mKhz3QWH1ktDabtmHRrnZcW58GYHLEcmw5FvyG6p+clsOTjZgDOXNuk8P6Q/b/G/dkiiY+l/MKX5fj+meSqB1xiAI0vIzLZq086SQ29G3IP4gyLZDQevZ3FCy+qfijTtjtVoxh7M/1RTq/5CPFGbqh9I1lWqZPZU6gzLDr2V8KnWyOdjoTpQuukcBcau/tlEdtHW8ZaqqegfVSz5x2Nym5biHQnFchuQ34roz1m2v16hPyvrm9S+sMzhtGXpT12pK/MyaIgScPDcJPzWh+JNj+qnY5L1iWFvW5BOdicm/zimvaw8pvwjmZaF6SfCPl10vxux+SsRpfsiXMeZmMUfxXhdwVjhdgZvLYW7m2TQTfLvfvkNzk7ta8q1MYnEHEqikt0yLPxkQGFNlDs7SesrrEmZ0J+3zP1pgthjsqNrIY2LCZzxZMK3Gsq3yzXsvpolyEHmkzSdcZILKzt4ZzqMQ4/GpZj0rho5b6IY2H29Qmbb4rcMzfN6aK2nB3JUsQwWQ3rLMcMpOWE3TvlCBgpiRvld3U93aMlsO1RuhcJWz6G0lH3JqU0kyDG7/kWy9Ikm/8UwdSOkecqneVZHczCJfPQGluonlv3nk7hjS2/J4A50sfafc8huqwyIUtqR2qdjTGRw+4bGaZfsvrrd7rK6dWcssdtgMaL4KOTwsmJ9UrUYv6cRYXdRXXdvWUz95uV44ownLdCoasjuhhxcSZB6nGDzGw6FtZjsI5ferPTwxbvKgap8apm6Zim+m+K/+suvk/dGeAPxatvHVfPFCR39ey+maZ52WPrzQ9yBoXIrINGNaZxT8O+oZBiX5OA3LmnGvv+Crkvsw/KfQ38lwol0QhQfCtFo/+UcvSBBGDoU00OSc32y+9J+7T/rEXsSLFZuDyg+jCnfMGS3Fa6RPpTUefanskuPPTi87DL1yYiZDwfyjp0wpQZVB79vJ+bEIaX7sQyW11zKt0e4QzV91tNO+zSvz8dCjeVO4o6EE45KYq4zAbuHU4b6BZf0QSCd1AWRHnIbinKMUmLtxxP+pzWQaDqipvliy7dXPDpLHvlHkkkc/4M2C2+GzP7U0Djr4Xd0NGEmprXHFX7bPu7gtw35zZD5N7QoknXN/Ss3hRyMKspYTR46REnDuGypftJnXIyPMkR7885n9Vo+5M1HJzn/v/qU9jGX2Q8k2x5WxXvAiL/aPV2i+Chm50se45zDeCqiezyi/lxIbkPa+GRDyShOoIe99DBk4xeg+KlL/nF45ERiXf3/9ZvzFP4yy85hkVQioL3i0l2NKD6IGc5o0e68liG/3icoKA07TKvxcSblWH9OpOrYFyn64Eqaza+rT2idYpJTZfC68Og3FOf5JDDk0W+5BHntyOO8Ehqf5vW5WKgY4Z9+z9I8qyPECaF9aUyQhSghJn7jrI48Z6QPWb9qqdwJKd8SnpnaN2T2leix+qdtGuflHdo6a2mdjegu65hzxvDotwr0Zz0Or2jHfhLokNkNSDVjim+kcUJ9/fzbAsQVHy6px7BqOXjGwx3LjijMGML8ZEiwAQ9/K035pqiIGBE8wrShczLG9Dz8RMhap0KYhdaqjj8TCRUYzEjXFKYc2qsOmV05maS3PDKbLsVPfdonIdFRIPCwOjEbM7D/nEf+rseoAq0TPkHaECUkRQ/TULmhIYqtJRmHcueb+sART3fTmXB5Y7rLaayR2Yc3kGq2cH9ijXQo68zstuC53qLl5O8PGExLJXv4jCXy5Y6y9Jcx7dMxtQseWKh+4JLdUPlVehAfGdz9Va/PxUK1Dsz/0JlkmRpO/xebFO/1j5ye/a5Ef1FycuxVlJVkAsPjX7FysYvUPdeuqEnqL2aoXlcNm2g6lG46hMUIJ1QocKIld4/8OgynY0leHKhdSrL1yyGDGWWQjqYjHv79z8y8nFBiuOonUmYGWegvaHRa/dgyfW2I3we/o0ST9vmAcV5oQnfJkNl2qJ6s02+kqfcylG/FZPYVoRMnVWJUPtXPUUPFBIaSL0DldojftRQmg4vsnnbtIGdEApk4cGME0PeW5GLoBNqBrSPD4C+9eBvX1a44mDGkGnJT7C1YBlMO9Qsu+c2Y2XdjEi15biW6ykcIczq5Co9DxfV0DGu/kiGzKxORRFu7a2bPsvuyS/JAxsyDOcm4Uw1tRI3TDsPKF+jof6KI7C/IrWPt7y6z/ssZuiciUnXtRgfPOXRXNfuf/kAXPCqFYD9z5hgXYOHNmM5xXaD2MYfmOe3Gfg8SNfmEZncl8c3sWtI1yauTrfioeStcS0pLBSz8EBKPE3hDJh4B4o/WLsuVDwdGlZj+gmTCzdNJooRCFwbTltkfuySb0i15A+1q9fsVZheaLJeaNP5mj/zagPplKzXCYUyQ0WdOtCzFNXmwlh4EpPbNUf6pidVM1i49Mc2dpEwXVbdiYTQVkXts6c26k0YLxr/Q4lf/rTfZG+QppEbk1/R9RkWXcU75WYmOJbstxUTtknskDGyc0fcpPLAEBXj8S0IdsjvaEJ4IHbNbn0Vulu7C8MyQ/BrMvqPre/CiLDfTB5bc5tOtkc/FZAoEW3gdQ3HdMs7J/7N48wmVTsES6UMFxXYXtNPM/cCluyhScfOUy2gqYuvrDqVbUPl0QHc5RctT/KPft3hdjV+t5zCYjUnvOrRzYve0V6X+xIHFHw3Yfz7N/M8C9l7w8du6AUHOUr4lDwLrWMZ51VeJlsPMByGjoo7jzpKH14XZ9wPVsNVYnIOJVNjrO7QW00SxQ3w7x/brwmv7M5DfkOa/vxDTW7I4gWCh1qpPZk80yO4qVK9LnStlA5i2pX3Kih8xK2OK0k2P2nMR7kDQ1ngugFqGdwrHeHh9UcZx01pYg6pD5jCm8WJEblPk6P6czC6iJDSuRpjQkNmT44zXBXeg6RJWDoOdlQT9WU2+6ucNmV0IMlD5SZJE19KfdvEGltSeS7KuB98Jv0A7qnVg9h1p3wdTDs1LcvGwLmCU9SlfUGlygoKdpJqotvK7MlqbedtQvK2hQW8xRZAzzP1sxPTHQ0lTDNL1DzQJewJIRwlZ8YymlE+1+0qaMA17L/okWpNw4DZH8d6D2ZjSHQnuNBHSruX3LTtfUkBDsgFRysHvwtzbUoAqT0AIRupHedq9FO75DuV7Ef1zI9yR6I39hZipDwyFe5qRDysal+5/PSCzL3/+7pJD+dYE4I+gsBaQrElg6PfBbzkMZiwnL2zz81//EHuij5OI8A99HmxOYytjyndVCrVPWnrL0F1wST9MHDnsZSbQ1WDW4PYcKtcccjuhZvldQVHjgiHVjAnyItwk2uJVpGp6MEdlSbgb5+SLNS4Z4oSlu2qJ0qrtn+b1+VioBvZfENPH71hO/c5QNL15S7JuKd8dEyXlQmJlgkKQs3K0K8Hs2228nvTtcULmtlFCjUztYpLGmeTRBemtRAxn7FGuVfm2vq87ADMzovppRGbPkj5UDeoN7MQyUZqiRAeK93QTUjXJXawj8d44J3Pewn25kBxedgkzwm6dsSCo9IGau3ERvGs5gvt5ahdd8teS0v4vyKkk0RNFDyT4w4Hkpk9n2ZBsK8Iy/3gsb/3HEa0T8hJIdKR29XqGyk3LZr3EG7/7PKn0mIU/TJDeNRTfSVF8N0XrmEuyaVn+fkTqUBO90bTq/2RT3Nzeoo79/COx0torHqV7wniH08Khaxdd2isuiY4UxIMZJdv4XbmrhBnJvK0nUrnf1UlgYhliPM3rc7FQjYXUviJjBjOG/RezRylz1oXWMXX7u18Wiz11aJj6SCYIlU8jtl8v4I6UCpJsWgqPdERmdmPGJchvhEcuIzM/U8pIf1aK0O6imTCyLLl30iSbAWFasEnnl7u4IzG7Mjta6IV1jT5zu+FROp47MpSvO9SuCiZqXoqpvRwS+/If7c8ZemfHDJZDRiXDwo/Fcs/sKpCBWDV26zSsfGeA9Sz7zzqMFwKKD2KaJ11GZX2tN4DOkkd/0XLwXFLpKI6htyQj5O6yMwmKEIyX+X5OYWXvleW1uiJvLE3CoPZKQOu4OvLBlCG1N0k2bMQk2sKuraPI9cFiSJA3NE9Ltp5oaoyb2VNNmmhZussxpQfCsBuX9DndkaZUQc7QPm1J1jSBLD4KSdW/QDtq7EHnZMT264KmBrOWVFPHa6ohmbF8lSzNs3rLtSuqbwZVh+6VkQIQNrQoYlcNQvOMAPuDqz79eSEGsa/d0O/pwo4u9ycpKtqpDy+naJ8Q9ln53YymUAO591kHko2A5sWY5kmf1KHef3ZTHIP8A0ku0ttyInwSclF4ZJn7Cw9izbfbqy7dYxNrnkd2kk8gq/KNb6cp37KSTr/r0zin8XF+Tdcp0bJkdyJm34kpPIrI7Ft2viphXu2CjzuE4bTQjOI9/bwoBXPvjZUdsOl8hhAE4GfHYoGtxhQfRWS3peKtn5eCdlTU105dD5l/w4FYqor844n05JScuQvrEfmNEe7Y0J9ymLpmWXxDEZJhSobD7SsjkjWHdENu22HKEU3zKV6fi4WKkc+o3zaMZ0KmPlbD1D4xiRLfjORoXHdZ/os+yZbkFr152Y+X3pYUpTfvkN2Wtr76qeb8uQ3BVsma5uD9uYmH6kguJOXvp5m+FlD51NK+PJafZ0d456jkkNuOaJ6RFmr62phx0VOTl5Ewr35JIWLt0xGjMoyqsfJIZw1OqFl5b97QXXaoXHMJk+InpPd0Ihx8c0R3RZS55llxb8sf13VNBpOI+IJgJ9CxuveSQ/Zxl968S6oeMfWew6isWfsTkzcn0IPsd0KSLUv9nOIzMbD8pwe0zoVUb0bM/m4a6ygTtTsvtph1Ye6dMbXnpBF78r1qFw29lYjusnbV/FYkK82koLL951JkN+UrO6zq+yU6gr+GU4bMvSRhWiPn8u0hw7LyB57m9blYqNbRjHv6xT0yj3wiXxe6fCdm7t0R7VUxcryeoXUyLbvt+EmWVHA0QclvxrRXnaPw2PzG5JjelBCuP2vIr8ciE0+4EKOSobvgqfzYEHM70VYww7ggH9L03sTTacXn4KpHsh2TOrB0lh3KtwzTH8VktlzG5Zj0nsPUJxHGQvOkf5QeWLkZkmpKTpPfDinfCeksuaz+M5dkXfzMzI7c7x7/6hSJTvyZv+h2RHdVp0r5XkT1E8veK0V6i5bHvxETZMVvmHtHLPs4IczWDSwHzyRpnnaofjrCCQVRrf3WNLlHHvvPOpMYeS3w7jEtKr+rRO/l78oIpPXMmN2XfPKPIVnXvSjf0bV1xpqOxT4TZa8ldSDN/7ioPiHZlPlwfyUkt6nv3Tydonkxpnz/C0RKcYdgQsvw92cpPYhpHxek1DrhULuYJNGUS7GJFRo2LmjWnjmI2H7NI1lTBznOaTLVPa6FlN4fE2YNrROSVldvRrROOOw/5xBm1aEm2lZw0LQanlHF4IwtlY+UAjgqGcIs5DeCCZgu1k9QkE1mkDMcPC8q4ep3Arw+1C64pA7U7NVf01bYWXYJ0mJB7bzq0jylh+Pxt11ymzHOpHnsLgqX7KxoN7KexspYNSXjnCOfrp2I/BqkH+nhGpUN47xQECaapfaqS3Y3xu9C82SCdC0mzEzMIQwU7+tB6C0auqcDCg8kXnzycO+96BIl4NQ/jiYIgBQKxUcxYdrBOhqvPpF79xah+HAg26UZwVbDilEgG+C1XDqrEmCaCGbfkrX807w+Hwt1LEB+MGNorzhMfRJiLEx9ogDeKGXYf1HTmtxjyUzCjGH3ZVfxOFVR5cK0oXxrwPT7TKCiNKm6oKJkw+IOY4K86j13JHrauCjl5nBqEovTfSI+Ayy0zsn2ZvsrPslOzLhoyewLlH/CYDIBNM667D+XpL9gSbRVJwYZw+IfeXSPhQynPoveiRMC6PtzhsU3YoWXjZQnVbkT4Y4lXhwVDF5PerKV1x+Ls3ACio9GdBblOP0kf2tUmeC6VrtcbkuLf1SSb+q4YBgVDOU7lvF0ePR91dBBalsp0sOiI6XpgkNQjHF+85D9/2hIdxmalwPql2Qt2TwNjbMOlU8nw5eEnP0OnsnQn5tkz/pQvRmSW+8flS9T1/V+61ekXcuu/TUa+f7rfsW+DLbm3hkR5mDzm2LQx54hfRgzKkHlumFUEmRSvhsxnILFH4eMihMVZ9GSrsVsfCvDsOyw+9qEvdMXcJ7oWg6eVbRkf15gdrJlyW3GhMWY6qcRveMhzcsh6QMr9tSSIbXnESdUt3XnXfyOAhpmPwiIk+qcnWiSYuKITDMuQOpAO747lJy5dFf1bH4j5tifBGQOxEoyEdQvGXlIJbVj9uZEeUx0Ld1TAU4Ij95bZlQyFB7C9mspMvsxlU/7RAmjBbcth8IwB6OKrqviOi3dZRGWG5di9l8EvyGDCr8LnVU5FqYOVQbVXoyoXTb0nxtg0xGHjyp0HhdI7xsy6z6FBw7JuqX6qUyOg4yhdC+mczxmXFHSotczE2smS+uYx8a38hgLpXuqdaO00QTxmMq1p3l9LhZqlFAqSm/eJ9GA1L6LO9Rxufe66r1xwVC5paym3ryL14Otr3n0liwzHwakd3WsRmk78dFXFPig4hIUIxKt6MgD1OtDek8XuXHWgVxAb8YlceBS/cDFCS3ZTXmuRmmNE72umqPMrqV0xzAquQQllRjDlTFeF/qL0q+nDyyV2wNqFz38jubhjfMOmV2HYcnh8EqS+jmX7I5EcbnHECc1O49+qyaX6M2Ycc5gAofhVIzfMrzwd65Tu2IZXxiQ//c22flSlnRN1MPuMc38q59EOAETGbbgs+qnIcNpyG66zP0U0rva3WS3YxjMOOQ3dYoVbnlMP7vHS8fWmJprk3som8zesuC9VC1WjGXa0PjakFFZPgnuwLDypxyNd/NrkD7Q1MwZy7mvP2PYe0FDib0X3UmK4hcMR61ddCa24lpEqinHFG/45DbFJs/sBSSa0kKFGY0RF98I6Cx79OfELY2SwjJ7qxHtUzGdY7IEapzV0Wadidvysc9KjqkfJhmVwQlE4Kj8aI0wYxgVHJI1NWrpAx1VwynD3Pe2GecM0287NJ4PSez4jItQveYwmDIU1gIe/Xpa2O8rafZfBOwEDivLqTrRVn1rjR4cdwi1ix7J36mIyb/skN8KcbsOM+/p5v/gxjnc+QFz1RZ3H87rM65qPJpbEzRmrLxaszuxGPyzDqOCS+VWTPFBxO6XoHI7oPQwJLutn5usW/ZecBnMWhLfOiTlhWS9Meb3qsKNPU0Oky3ZGIFKm/IPU+S27BGLf+8ljUijNHSXP6tFRxWRUFJ1y8JbEaMSzL4XkTq05B5+gY5+McI5Msl1AgHrrWM+o6pA+ealmK2vJgizUDvvkdmx1K4YahcTcpD2Lat/FLHy5xHZnYjT/2SA2zc4geLPB7OWUUUS484Jzd6br4yOnKcz+3qyWydcml85JnlIX/Vbf1GQl3VVl9qEP6HO6YizRsnP/VkROnZfTZBoGOZ/OiJVs1SvG0ZVGTUkWxMJxqLFCazc804YSvciuWkPNRny+vD4b0dkdiU5Th1a3HREtJOmN0pADMmaCCPF9YDB/GTYcNKdhMkZcttjvJ5q186SQ/OUy9RHikofVOW5FaVVCoXHhsQJS7uX4tHteU5l9mmdUkNWeKhgj4PnmBBVDH5bvUH7uB58b8Ls7y47zL4XkN1WDLrf00QqVQuIkqivuBPTPOmR2w4Jck+3Rj43C9V6cgYp3w1xIhSrPTF3KN+LKNx2NXobwNw7I2LfEMyO6R6P6K7EZB87rP+SR+OMsj/3X8gxfc3it5EVzx0o3RF3M//AIfYt03+ZpHQvIshJ6xMnJQkZVAz9EwHDqtACE01sxMcwLlge/t0ZWhcUUWMCh9XvDNVQpFVOzP9khPXg4Jkk47wRvW/LJX0YKgC4aMhuqqY8/Nt98muW7dcVplY/q8FAd9mSvpWisK6SxfnNQ+JaAn+pxwtzG/z6Cx+JGFOyDKoeUUIciCipaznzvmXzawla59S8BDnJWawzyYGdEn8gTMOoZDm7uMfcuX3K+T6zJw+53Zvj7OuPsC4cPA+DpZD8mnbvKCnxYGEtJr2nhd46G+MOFJc+KrhkDmLc4WeBHLsvJWm8ND7isgZ56C54zL0TPNUa+VwsVG+ocWKiK15oZ1k71ZPOsTcjWYczVoHePpagu2zJ3k5SuOuSbDikD2Nm3oPSg5DmKY9RCfxuROdUhDOGzH5IZ1V2jv15y8pfjOgsG7oLegBAmVFRSijDhf/jzpHHam7NEE588v2udv3sYzG7Zt8yPPytJNMfyi1l9FqH/ReSDKcE/DuhonMK6zH7zyfw+or4cQMFpiXezjOqGKrXDMd/ZxeQq3Ppjk6WvZcdxi926Q0TlI41SScD3tlZ4VZzjsyOEIfOsiF14FB4oPANE8olzwkM6R2H+lfGVG9GEuwdSLfk9zRmLd6PCasht6+vsFcrMgo8FnMtfvTReW7/7DjFb+wSp2LMWE6JxYeCmuoXDJ1ljVLzj2OWvx8zfT3EGRl2vxrTmxFDavqDHu7YULofUX43IXv4xZjyrZjusmH/2aezRv9cLNTIh6lrXXZfcfFGFncyCZm+FjAuKlV6XNDOm97TU5qqG+beHRLk9fejkkOYMhxeUXBD6tByeNnHunL4O7jqE5Sio5yn3ZdTwgZTwhw7q1KWFtYU0dO7Mg9wFPQg41qNeINizGBatuZBxpBbm6T6lcD9KE9mV/P/wpqIGZEP3jBWgETKcPiMplqVGyorYh96S4bHvzUnj9VJieEEctALhh79dooodlgtNfDdiPWfrOAOYeoTcWITHcDIycWJ9Ps4Kd+D7M2kzOfGcHjZmyh9de178w4mEXH84jaOG9FqZPnw45M8f+khQSlia7vC+fOb2FyIc6XF3isKePP6QmEK6yGpesThJU9WRrcicg893MBiYsvOl7Mk6yp3wrShP+Ow8ucR9Yuypve7T7dGPhcL1RtCbykjFtI5dfzydXLFlZz2ccYTT1BPfv3WgdrFlHarh5b26Un84QQ/jtJaVKt/olgebwBzP3FEmD6QB1XrjL42yKpD7S+HBFlD9QOHYVkyjexOTOe4uKV2wkpKHjr4PR2dqVZ8NE+3PkxfC6RUaMnzqT9v6ByDzqInfHZyxcdZh8YFmH3bkl+PGVVislsqPVJ7Dv154ZKLyzVs4JArDcgkx3x8Z4X+B1OiBM4auvOC59qnJg9FwlK/hMbOY5kPxx50F12994ZSued/GhIn5ZVVLPWppnpcWthhdfGQ+VMHXC5sgxfz7UufMpXq8neee5fpfA+v69Cbd0h0YOmHYw4vexxc9Zl7d0RmP+Lgqkvq0NJZkSNKbyUicxDTXYH8RsRg1nDwjE/pjtho9ilX4OdioUYJKRqNFbb35MI2zhlSzYhEJya7MwHoS4KIlFEfa5KzFzD7M6EFC28Nmf1geKQR2nrd0yJyIb0fMKxoCpU6NISFiMXvNxguRExfG5He8ijf6tOfN5Tu9EgfWGq/2SezA3vP+4wLqs+itISAqX01X6OikkIKDy2tEz7pesS4HE/0XmJe5TfDScaTpfBA2Ofij0KGFYdEV9LqIAulexFhzjL1SUTnRMQ4mujpY0O7n8LpuiRrk2SYjhq+zI5l7i1FjIdZHf2NM2pa6pdFeAENPVpnZHFUP+dTvGNI7xp6n5bZ6hb56OZxHGPZ+2SWP9m4hOm7JJ2QD7aXOZY6JJcYceL1NWZ+YRPrQONMgnFZJ1b9fJLWSRe/B+2TMuAIslC87WIdmPlQjWyyplKht+hMwju+QPCU31Uqn98SjzPMPsE7DY0zHgfPeNQvQvGRCvT+nG68E4oQ0Z/xGJUMybZ4ps1TSRItZZE+8VEdVSyjsgdGUujO80MqH7k8/N/7VD9w2H82id+DzW9mSR1a9l7K07gakf5JjiCr3TnZ0OzbCTRmDXOq2Sq3R3j9z7Kxdl5zSTScI/6qO5IKduqaFeg9FmzTXvXoLULtgkfprsagyWbI4huBvKa2XQ4fVlheqnFyqkb/IAvTatT68+KDtk9KvrL7JUP1uiFZU62a2bGUPjwgdSgtmnUEh5VuGbZ/zrLwo5Z25RlLsmlo9dMsHz9g46BMsmFwfrfK6Ytb/OlPnsd1Y/7PP/wVbq7Pk/NHHHSztM9ER1aTzkjNX6Jpqd4IGM/KRcaJoPmMft9edQmySsXef8XSW41Y/LNdsttfoBFqkHPYedWXRcxJhW9FCRjOaHoTJS1xShE3g/mIKGHZ/JYaoWRtUjumZEVZe8bSPg6Fx3JHSR1K2pHe01GcbFiSDUPlDTmh+B/mqD2vY9PvfFYzJZsxpRseibYGCNVPNN1qnoHS3ZjekqZPrWMum19PYiwcPqOpUKI5wV1Tk2A1I2+Ava+Lw9o+xRG1z4TmyAO2sB6z/2ySnS8lqN6IGF/u8+UXbrG5U+HO7gy//uKHxCOXzrGY0v2Y4ao08sOSQ/Gu3LWP/6N13ECL/vHfmCX2mUzaYip3ApJtS/GmR/1SgfEkpSX7tX16hxmm0l3izQz91QB3ZHn85grZDYfB7RJuMeBb52+xnG4wn+8oSPkElG+KaJN7bLCeYf1XDdM/8aUKPrBkH/h4o4kjdQyNc+B1HPymQ+fS9GcmzX/F63ORM5WZWbYvvPwf0Dru03l1QOJ2mmRDlujZLaXNxZ7YQK3jerYyu3ZiIyPpbZiVH1WirYYltx3hhJbt11zm3laekTVSAfTnDe5Qu2H9gthAuU1BS+MiR0FgT7r+5hmHmQ9D9l5wKTzUThG7n8E03WXhnKOyOQqw9QbabXLbMXsvGvIT3//WaUuy4TBYDCnc8chvRHj9mNYJn8xeBEZ8h9/8e2/QCtN8f/0sry8/YK1b4d47q4DIJEx2SeuAdZW96gQT1tQISg8jnMDSm3NJtiTUG0wpSKK74MotOyeX6MErXUr5AYePKthkDI4ltZ4gKMRMXYP6r/TBGoKBz9cu3OGN9y9AZEgdOng9yaIT7YlIcCai8qFLcS2gedJnWDVkt4Ux95dDzv7XfTa+VSDIawjg9Qy3/tN/8MXImXJCGOdc8hshS7/jTYLGAvKPNRdOdGO8kWVYFmXOG4hQEaYM8z+s4Q0s09dCqjeDSfy4SBGHlz3m3lYIWv28w94r6q6dsQD1J6NYv6MbPC7CYD46EpwNphwOnhfJ+uCqbH36cyJgN745VAd/dWL16E3C0NoTw92KiCfdeYf8I9EEnRAqnxjy6xYTSD16eMXRODYL/RkXrGTF97ozvL13DGMsHx0ustkqkt0yJBvy8m+eteS2IoZThtl3OvTnDIcvxBMoLKK74LL9mkuqLnXs3pdi2hcDtr/i0FuE3vFwEkseYwycKNVYPbuL03ExfY03/Y4okzO/l8a7mSW56dMYZfj7X3kTUxmJGngiYjQfMKzKOytRU9qJieTVmt63ExUEzLztsvPlAlFKiESUgMHcF2zWj1HNljoYypwsK4a517d0llx6sw5RQkYL7lD14rhkiPJJgqxh/1mP/oxq0IMXxPwfFyS6S3QkJyk8cPDbHEWbt09K784Ezio+iql+5BzFWj6xR89txEczfCeSddDif69A3PmfRSTblun32lQ/7mIiS5SU9NkdyRcrzGn3HecN2b2QRDemeEdNRupQNdrcu0MxsWJYvrrDR9tLLORaDB4WqLezxD8r01uSRY8zkpFE64RHZtey+Y08y9/rkL/vMiobtr9haT43xu9ObMknWuP8LZ+pazD9UUzpuocTKU3v2FSdD9ZX6I0TnLi6xd/40nsEOUl6WqccDp7RiVF4AB/fWuWd+jHOLu3JsGPDxYwd/I7q35n3Y0ZladUKD1DS9ngSOteIxLGNIP/AYemHASt//gXio2JEGRvMWGqXc/IAPSOiBEDnuHBHHFH52qc1bTERbH49R/rQTgwbBGgX7wqzW/pByGBWBGvrSZaRqosnOc4bincN7TMRuW1ZzAwqaoBKD0YihFjFIfbmnUkSIJTvhLIFzzokWp+xnTZ+ocjOV/IEBUN+jaP0FCfQGLZ5zjLz0ZD04w6dJZdxXsZh5bsB3tAymPKJUoLnHu9VSPghdw5m8NsOwcCnd3YsTDiGznHLqKjdsH1cSMHeK3npmFra1RLbPvM/HVF4AIPFiMW/NPRWRHQZVpyJQUWM1zfc25zBeZimdrfKIPB5c/ck49mQw+c0dk7VpMuqPWu5eG6Dx40yDw+qeD2dIpWPJZexrmwnraux8mDa0DodUz/vsveSz8Y3pUQwscarjbMJwswXaNbvjJVcFyyNqV+yE/lvzM6rPrVnYoghtzWifWlM7EH5U9ECC+sRxUcx2Z2AUTWWaYIvYnXnRMzBs774lgPVb41zEzfqsXa7wbQhve1Sv+BMOKRi06//QpLegmFUkl+UE0oq0jkGnWUPZ6yyw29rjp4+jJm+HhCmdNT5A0uyaclthXQWPRGxPzHUzqcYz2bxO9JJZbcMu6/4bH1NO2nlTsDgWICNDe2DHM/Ob/HlX/qYbHGAv+9TvCtYa+YDe1Qbz3wYEyUk2is+kvjO7xhSh4btr0h9m3vo0johP9Tu13qMSkJX8g8dEaHfSZG9Uiex1GP/2iz1Zo50ZUCciPm5L31C76U+8bca2MqY29uzPDe/Qfg4q0CJANonJAwMcmJLVW5JTJmqW9K7DskajIsxs29D7rH6g1RND/Lmr32BdlRjZQ5WfC9J9WND5XZE5aMGzhgqnzgkG4buQhLTFTe0ee4zfmjzpEPzdIL07iTK2zEMzg0p3TIkWsJlDyZlenrPkqpPvFJXY7lDO/KnYlL7hml14bPvB+Q3Q7yBSgVjn1DiLMUH2tGalyU8VECvw/T1UGLAoRZS46xPoqsHT259sP1akiCvWre3ZCGGynVHCym0EBpo+0wvNHnrzkneXD9BcL2kqVxNgrpRUXZE46IezDAt+XH9vEf6IGZUskd1eGY/JnVo6V0csfjDmHA/TW4rprM6cVMpGQZzlm5fgU/h3JhCvs9wN8tvvvw+BW/A10/eo13PkkwHlAt9rv+zS5RuTxS4YyVpV2+FYog1VcK5I50i+Y2YoACZLYdER7+3niIq0/uW1NrTBU19LhZqlJDevnR/zOELMU5gefjbVfIbmqIUH8W0TjlMfaSkZOtYCo9iZj7sUliPMaFuDFa7ZuZWisblmP6CSMPWs+Q2I9KHMa3jvmxnNh3G1Xii34fO8RivK2OH6Y9iDq76dOfdiRu2pXlCo1kTG+qXdOSldt2jCJ/aRZed11QjjgoO7kj1b3/WUL47PEogse7ErOxWRFCQxKO3aFh4o8Ph5QR4lpkTNX5r5Rr5cp/c93LkNuQU0151ZfqQNyz+uE/uMex9SVKaREtCOSe0LL4R4kSC27qLDocvxlR/nODwskdu3aU/7ZBoyanaHUl7n3krR3w7x+xsk94giVcd8r3H59gbFfj+xxcoVHpkUiPiP66ClQV7ZwWGZakvYk++st7A6mGpGAr3HGqX5aua24o5vOxRvquNwoRPNGxPhzp9Lix9vKHFbzqs/7Kkxs1TwioHUw5R0iVMS9hW+4ZCIcr3Qrx+ROtUhvoFgzfQ7ulMKILDqmXqfUe6+wFkNl3GOUvjkmXqo5jessEZGabfcTCxpXHOMP0hpOoBo6LLoOLQXw3JbbpEKalSvYEyoLx9SNekNHDG4pZ2ll38tmTR9QsQZAydYx7Jukaw9QspSvdDDi97pPcVA1k/57L6xwFhJsYJHIYzaaIUJItDGh9O89/2XiHhhwymtXOWbwuO6807lB5EhGmX1hnlR7kDPai9RUN/waF6XaZy3QWXKAXpLRfrqlYuPQxxAsvhFR8nErRVeGDpLsH82yG1zizjc2NOrOyz+c4i194pkR9Dv1ki/1CnX5jR6ZM+EJqx+Ibl8JImaONizMz7Qj1yjyEsRsy+6dBZceQ7uyAd1nA6pnDfoXX66dbI52NH9Q0LbwYU7yhtwx1JfJZoK93D71pi15D9KE3hccQ457D1utSV+TU9ne5wMjt2oXxTOGf1ZqiO10CyI7/7QdXB6yiQq3EemmcM7lDS5fp5n8GUw2DWkNr1aJ7R9042oXJ7xPwbDdpfGlC/aGidkiqheVosKieC7HaA1xPHEwvZnUgq1BDq5z1mPgyY+rhPlDREaUt/xptAWYb2qkfu1QNG7SSV5/cZ1tI4RrvN3DvKs9r7dqBoHd+w/0ISv+NQuqcuO12XR4AzUu3XOqH3FaU04hxWxR04vOSx96KPMxK9L9FR+eCMYVQUz+L0fxVS/4MlEk0jqU4a8hP8+Aj9GE+cDOcMh1dcvAEc//0a0x/ofmZ2RCCf/YkIO4W1+EgV3D0ekd538IaW2Xejp1ojn4+FmrXUzycorIckWkyEaA5hxpDZ0iJyItV6+88Jppp9LwQjL9Ty3YhRWZ6lT8x4w6xhUHWp3AnprYQMSw69BYd0LWZcjsluitbn9XRkhmnnKLs+v26xjiXM6Vh2xpbW8SR3/p0CU99L4YwM6X0jZCCC8j1puPafS4JRgNhweczBcw7luyOCvNSbnWWPtV9Lc/icpC25nYDDKzJna12ISP6XFbKVAcXkkJcv3weE4T7JF1j6Qy2+cVYmG4UH2uWTDcFwQU4niN8TY6t8V5S+ym3Vzs5YEzFrZNnjDQz5xyOy+5FKnJJDqh5zcDWD31fp4AR6UIdT5siLdVTWFMobaFBCDKUHEdvfqDKsOFK1LllqlxJSbXgc5U0l2pbSTYdES/1BZ/ELZI3ujAzZnZidL7t0Tkf0To+p3hoSJaH0MGD/ZdVhyZbVcTO0DKZcvL6heVaQSKomFecTMVuQhdZpaabcgTMJ6lKXPve2pb8AjbM+/cWI7mpMNElEsY7qrvHqiPSuwi5GFUN3xZDad6VCiCfhEDkF1wZpB2ckPX12y+KEhtRmgvItRVtiwR/ExJ52rtyagxPBzisJ0vuWMGOpfOjQWXYxPy2y8RerfPTDs0RvVKTTd0X83n3VZTgbU3tGXNLGRdW71U+6OIGlsyoQfe8lh2RLo+WZ97vUzntMfyzNf2/BkKrr2hQeWna+lKJ+TtOr4RQMKw5+T+Yc2X1REGNfxhoLP6jRO6nZfW9BvNrOiqH4ID4yrxjMqubNbQjz7i4aRlN2kj49UfoOtBnFPk+9Aj8XNSoIR42SMdl1l/6iYedVh+y24g9n384yKmr6M3V9RPOUghVSBzIyGxcm06CiwPMwa0l3DKZl6KxC7MvMIXVoGEwD1qF8y9I+bph5R12zG8j0rHUa3JEhdz1FkJMMJshbZj5QzecNLYNZGFjNyf3mxEGvq2628a0B+bcy4qaOLRihCAdXHYJiROGeS6oh3oJ2K42B61cj/JZD/rEAcb8n+804IdwzTBmwlqkPBFE9+Zyxa9j4+TyxL0De71lqz8glpnHOwR2nCYqW2nlFj7e/PCR+mCJKW+oXDXPvRgwqjqyTsqof2ydVa7pDl6CgSV9uzeHg5QqVD6SyzT2G9jGH4coYTIIwI+Qks6sFGvu6H4X1WD4IeWGryVbE7ssuYXbiXH3iC8SeckdMOsFJhlTfMH0toLdg2Pq5PEHmiX+8obuQIEoq5rFzQp7y1jEk2zHDGWF3JlJ963fUXM29pfpsVNLobljVjc5uW8KUIbelNBMnhOy2Ib1n6c9Zjv+zPZINOPanQ8Y5h+K67HEqN3R8ZjeV+xnkFOnYW4yp/nl6csPU/FijHalyyzL9nrJOc1uKAE82LLs/FzEuWNyhQ2ZHnXT7JCJBB6qRK7eHJDqSIWf3AoxV/W1iQ2/ZMqrGpPcNzQsxw6rh+D8fUb88mb/nHWbeV+BZ5iBm4fd80gcGv6P6futrWtyj4mcu0e7AMP+WsOPchvwByvcCclsB5bsjmU7MG/IbMaUPE8oc2FRdOpy21C+rJg4KKCBjXyqLxTf6DMsuyYZh5l2o3OxTePB0a+RzQUrJTi3b87/6HxElRUWbfbfP5jczjCoRM+9q1t+vujTPW9J7wksTLWi9PCR9K8XsB2N2XkmQOhThun7RkKwbBrOW9L7KgMUfD2mcTSqDKWNxxtKjt07oWU3vCxN0Ao0NUwdMLBMnefUpETgSTUu6HtNZcqVDWo/pLjqEWUg0tKvmN2K2vxmReaQgiuon4PeEYsS+8gcSHf2sUWkSVlxUvWdCMZ/kKM0kCj4mfRASpRzaqx5RUu/JG4j0Xb0ZUbvgisgzrQc/SiozdfMbBq8/scNcU1DucNpQeKRSZFRUSYWFwxdjFn4Eo7z6g1FZn2f+p2M6Sz7dJbnLYD5LsRnndQ/9rv4c5NXRl26pLPO7hlTts88KOkViV5O/ZNPy7j/933wxSClRQpqm7oroedtfyZDdsPgdh+6Sw+FFj1HFEKcVtBUlpXvKXUsxfT1g66u+YsgXBBu5I02tyregtyiQvLcgN+RxyZLdgOFCQGfFOXIuMZHA78Y5w/xbIzonZbtYuS3czx2aidpUF3hUlW9p+5hDblPs/FEFqjfG7L0Mubs+qZol/0gs9u6CGonuZF4/da0vP1dfdd0T5tO4aCg9DKneGFG8H+N3JBdpnElweEm0w9KDiOyO5DEYYaczHwaMS9rlUzVdp9ZxjxN/IEv19L6lfUzE5sy2YK7WSUFlzTNCHpIHMnFzx8KvnRCGMyG9WZ/BjBrCYVXN0/SHfUYlkchVc6IwiRshpdviAZdvQepQPlTeYOKoklTidKoZ0z5p8QdfIFKKiSHM6CgK0g7FhzGdE1C6LUKIO9bXzPxMTiBz7wak9+SFuvW6R3ZTc+tEy0zm8TGdFZfWSZnu+l35UgHk18UVKN7wJz75UHgoQV+QEzbYWU4QFRXk1Zt1FN4AR0ZgzRMe6T3LsOQQFGSFOc5rNOgNI8o3NIXyBvL7Hxe0a7VOOqQPlHO//ktpgrwCJJZ+GLL0lz2K9+Vb2jjt0TqZoD+rmfy4pGYkTnCUjTrOa3cbTcXsvOqy/jdlGFd/NqY3Lwdtv2PZezFFb1m2QUI1xHtINizFB7D/rWDCTVDQRZiWNefOa2KFzf3EYf/LsjUalTWaHc5YekupI6OJ/pz8+J1QBPH0YXwUKxklYeX3NggyKoesC5vfcNh/CaJ8hDP6AtWoGDl7tE4recPvxdI+GdWgUUJ0st6cQ+xC44yUi/VLluW/kAnZyp8HzHwwEt90WourdJejKPXeopHLXmQp3jVHpJY4oWNZM+poIo9QTetE4q82zyuhzxtM3EwmPNThlCG3Bn4bgoKO/M6ySNRPTIh785rYBFnVf1Mfj+Qe3dQIM7sh7Hfj21m6S7L1eRIAvPj9Ot3lmDAt44riA0v9mRivJ/uiwazqSb9rqLzr0zwptcLUjQleuarQN9COGaWEG/fnBBlVbnSovJmg9EBMK7+rU6V9AqxvSbZlYFH5QKNrE4v34HcUjBymBTGV7qpkc0JwIkt3UXagJhR/YfcXloiSMu9YeEuW9OldBzN2OHjmC6RCta5m9FE2prvoEvuG7JZhWBZDaHBqROWaIJ3MrqwN+4uW6seG2qWksuRnfJqnFffTenmIN1AeFCsDOhfHBMWY2v+sx/AbHfiVGs/9O9cZHBuTqln2n9fDsP1VHYWFRzErf+BMLq6IFSIoy9ffCaV1755Qhupw2jL3s7HqrkjQT5AXguB31QSGacNgyrD+Kz7l2xHjvFXAxLzFhJItl+5HdFdlZFy6bXj4t8rk1mVtGaUkJ6987DAuGsKkyCfpXcPyd1r0FqF9KiJMC4MOLvfIbkL+kXILnECQWpixlO/EDGYMh1fz9OcMj37TI3VoaVyQwwzWkL/vsvN6zOw7PYZTanJHFVm6pw60wysJReWK9ZR63TkGM+935RmQlRx6MG2Yfa9HlILt1xL4XUeat12X6s0vECnFGQEGFn6k42LnNdkq9lZixnlD+Z0EYcqQ34jI7YSUb/fJbBvqVz6b8Q/LzkRGHVP6WZLaZdVUYcdncaFOeqWDtYYXFh+T9kP2hnlePveQ/a+P1SS1ILfucviM3sO44FC5NcKJBGabSNY3mW0pRbGQ2vOwRoSQxtkEJp6ICPcMxftqwIK81AdPCN1THwoHdkIlDa5+d0h/xqe7LBw1u2moXhdGO/2x6mQTQurAUD/rUntOtWPrLMy+OyJOQvNCnvSeYeXPY3rLlvq3hsz8formBR3zuQ1NhUwsMvThVaMAja5l6tMQUxgT5A0LPwkp3naJ0pbyvYBEzaW3lCbRVDP0pJyRNafcF7urlnF+QoDZNRO8V/YnpfuBiCc1y8PfTBPkLblNS/l2jDu0JDqahj3VGvnXs/T+R76M4l9aJ1xyWzF+1+Hwkos70FGoSYgA7M6yR/1CBixMfagud1wytM7EjEpG3kgWpj+yjH+tSaI0YmttihPVGsNamp89OkFrkOLm5jzdIIlb9+mekUbIGlj6y4iD54UBdpYSzHygeitKaeeo3BkS5KD6sSFKWHI70VEARm/R0DolMV2UEPHFCWRcFqbBeobuotxQEm09EAdX0wLVPcjsSbnqjuUy2DrmEuRFacTA8vdaeG1X9uiPDY9/PsHM+yOso/q6N+cR5mO+cvI+21/X8R1kDemDgL0XHfqzhtJdJWsnmjIibpzyyF5Ls/j9BluvewR5WP5+QHfeI9GSFVBvWdY8JlbjV3ogP6rsliX/UMMAOzlZDr4c0DkVkV+3DKY8OsehddYS5WJib8Ikm5r4V1Uh2f4CjVAxikPsL8Rktodktyzjksac8z/tULkjZvio5NBZ5UgPPio6ZHZ00VP7Dn5HZUGUMuy9AoNbJV4/9oD0lsf2PzmO23PIvJch84dFAJrDNPH0mGfPruFEqjFrl1UzBTlD4fGI9M6QIOMwzsPc2102v5Ymu20nilHDsCx7oP6iElzcgeq4KCmpCRYO/3YfJxLXNcirYYqSkF+PCTMyu03WOZKL774q1lF+Q4nSo4oWbm81R+m2uvzOyZjqdTlK9xY0zhxWDV9+/hY/eXgSmw1Jb7oEX26z/0KS2fcVRly7bGQrOS/uqPX0ULXOFZl7OyZZt2x+TdfAxDJMLt1WXlRmy8Bqn+68S+9YKIVF2lC+qZo/0TBkHiYwgaF2WZOo8k2Yfh9S25LyTP94l+alkM4JwXHD0tPtqJ8LHDU9u2xX/71/gN+fJGssGWJf9jmJljKasBPXkr6O18GMcM3U4cSxZBJakN63ykLKS2+fOYjpLLsMq/Lzt+5nnvid54aszNfZPCzh38mQOtBuFmQNqYbl8FlNZNyRZTBniF0pWEdVXbOZ9xV72T6mXdTYz8x944RuxKhsmHtnQO1CSoyrAObfbLL2GyXyazriY1cLon0CKjctqVrIsOpx8KyhcgMOXomY/YlKGyeSHqlxRkSQZEM/I1W3HHxjBM0EuBa375Csy0qndUzMpux2zLgg4eM4J8lNnNBOWPnUsvclS3rLxYkklOwtiTmW3VLT9ITcEmYn5RoiqJh4IidChJncTkTtkkv1k4hBVfaX/UVL9bpQhWFV/IT2SU0Lb/xfvyDiPhMLjzORyA8mgsIjETKMhaUfDI6CyYIcdE6FVD+xDKc1/YlSIoY4gdyi917yGecMhfWAYdmhfToizMbUno8m9DTxJm1sqPUyGJSol+hamj83wLrqfqc+NLQvj5Wk4lsqNxXosPDjgMyOYf85h8YZ5yju0UST/Kum0gATbUvpXsTjb6UYzJgjUvVwLkuQi2mcZ4KLhvRnDdVPLP0Zh72XEpJ21w2Hz1lSOx5hShbvo6KhdtHF60PndCQiSt/SWTVcObbFyvldyqsNknXBZfVzrh7wnhrA3LZGmMOKQ29Jph6JljDi4i3hrLH7ZCYv/LV+dRK3njb4XZ0asx+MGVXVlJVv9ynfHRMnDPmtiMYZl7mfjenNqvsPs9JptY8rJj1KCFNNtASBPc3rc7FQYx8W3gwYF5FXfVZvvreo3S3IefRWYqo3xlTuROTvK/Ah0TTULxv6lwc0Tk9IzLWYRGNS/z2bwESQ2XIxkWF2tU7zNBx+fUTjyyMSmYC5Qoew79G6EFL9oE7ykwyjsiW/EdGfNxz/XcjuxeQfweEzIih3F3ycsWW8NKb4MKZ3cXhkFub3DIX1kP6Mw+6XJUycviYSdWZXngNrf0P1q98WVNSfmMD1ZzV1q96ISB+GJJsWdyjShzeyzL0zmpQcUFwLyd93Bc7vRwxnInL+iGeqm9S3SiTrluxuTGbXktuJaZ2NmfvZRHY+1maQ3hcRvXQvmgxK7BH+6XfVNLZPQm5dy8QJLMWHAcVHMam1Bsmaw7hk2Xkty8O/5cqDwZed+86XEoQZTcmSdV0HE0HrpCJBW2cm2Q1PeaB/LhaqsWo0RmXZIubX5S/lDgz9efE2C/cMh1cS7L6sm1m74JE+sEx/aDn23yoYYlyC7oJD95gCctP7lvYvd8l85QDrwEqhwXhpzN+9+h525DLu+9y/KzO0wh2P9V+vMpyJmXs34vCyy/xbfV3wpDr1sBzijiVXGZcM3l6C3rxD8b0U1U8E7BcexdQveHSXofSpdpDuvNyvM4cRYT6mcNNnVNb4NdmQnDizI45n+sDijuNJzqlm6L1Fh8y+nGESTZUvQVbXoT9jGFRcrG/Z6Rf43qNzYFQuNc441K/GDMuG4j2hCsOyw8KbIzonYnJbEX7H0Jt1GBcM9cuqnac/DhlVtMsWHihtbzhl6RyHzpLP9jcj1v/mrOzOb+k6r/6JzNrClBZ87rF4sP0FS247JNG2zHwwku1l2jJ1zR599qd5fS4WqnWgP+XhDlW71K9GBAWJ9RJtGU9Yz9BbjvD6YucMZmNiF9qrDhv/bsC4YEg0tRukDoR7FtbHuNfy1Oo5/t6336A2zFIo9/mjtUu4mRA3EZOd7ZEsjEjWtUimPpKjyrgsXmaQi2mfNES+IbnrEXsTg9q2JcqLBjeYVicbpg29eYfeSqS8paqmUMOpSURl1sEZCg8O8krKdodqvsIJ8WZUMgyqHu1jkn73FwRP9ac9Dp5NUX8uktSjIHGh9SRxIYaHj2cYDX2S5SGZbSlz09suvQXxCyq3A0YlQ5BzqX4svLV6Q5xe8RykD9t+3cHrWbI7qptB9kruaEJxrHlMXdfkzsS6f7ULPsTaKSt3RgQF9RXJmqG7pJzUznICE+rzto851K7A4dUvkKWPE2giVf3EMi4aZt6Wd1PnGEQJ6X6sI5wzva8Ejpn31YAkWpbsmzmKaxHJpqV8N6L4MKb2Qsz2V5I6XnZT/KM3vsKpwgEz+S7TuR6Z7JCvnLxPLjXC/Sg/mUGHjIqabFlftonpXQcTQHfVYEIdmflHsm2cftth+qOAcSmmsB7RW5iYTFxzaD6nidm4pIz6KKMmqXodYbAHDrVLhuYZTeWCHNQvT9JEPMFdo5IhNZF7dJcdpq6PmP+h/l1/4bPmsn3cwQQODB2+duoeznVJp02sEev0xxH9MyMe/6Lodq1jntyyHfEIwpSa0+ymQ7IBy98LGZUNh89K0Jjdi4+ctZMd0fOcICZKWQ5eiUg3RLweF8Utbp5M4ndkwDGqCuuee0fpiks/6FG6Iyhw5n2L2/8CLVQsbP1CTHfZoXt2TJSYKBQPDSY2kyh08Hqi32F0xMu8TCwfrxfj97VrRb7h+B8o+ifRsiTrhuShy/duXOTBjUUe71fIJsfs9AtcndqivxzS/tqAxlmfzH5M/fmQ0g1HZIuynJUTDU2borTcVkykHfPgGZ/iXYdh2aHwSDfT71kyDxJi23ccopQl/8Bh41vekcX6zIfjiT2QtO8A6T3nKJupPz8RAmahsBESZC07ryTZ+YVA0ZHb0D6ljn04HeNMjSAVc+1ggcyuZTAtK51E0+Hgqsv0GwnyD12so5pVNjyWzK58tLKbIp0D9Ob9o5j49L6lN+tgrEo0J1ATu/bbAvgLtz2cQA1tmBGg35+dyNk3QkwEuZ2AwbRPkDPUL2TwB5agKJMQnm6dfk4WKlB9R5EyRIb6lfgIgpIpLpTuhxQ2QhIdET6STUv9mYj22RAngINnfLqLDvtfimich51Xk7gjGFUN5XsRqTpk7iVwQpj/3QTDP5nl7s0l/uLWeV575i5nF/awX20QpgyzP3bpLwjAT9YnsTgZWPhJxPxbA0p3pU/y+hPmekaQV5gyR5lLR7zYFowrMe3nR+QfOVRujgTOz/uM84YwZalcNwynYooPY0YVEbj9jjrs1IGlN+PJUcVA6lGSICcc1G8binfBGRsuLW3j7/s0Hpdpn1JXPiwrNRtjSR9G9OcsK98d0F3QCdVdcmiddEi09dB1j+kYx2rsaiy0T6txCzOWRAOGZfmslj5IyKXvQsD2ayqJ5t8OpcRoaqNpnPaJjg/Y+orP/ktKfhkXVVoV7qvM8PpPtz4+FwvVehrHhSnL7JsuTqAY8xO/11YgQlLN0zjvqkNNQW47VNRO36H6aUiiDTMfDsjd95j5UPP6ZMOS3Y6pn5V9TvVWSOG+oTfr0nxuzPS7DskHKd66dobb7x2jc5Cj9qxCLJyxFk9/ztK4aBksRFjH8PBvJAW4r0y4lXVL+V4oRn4JKndCEj0J7vyeZTAjRWz+4yRBHrZfT+J3PpNiVG9IZrPyvYjs1ojspiC3hZ8OGBfktRUloX0+FFvpsXavqesRqbql9qWAKBdz92CaeHmI23NINA17zyfonIgVomENO19ymXk/ZlxKsPCjJqlDS3YnZvqa9PhhzpI8VJnRPAuFtQi/ozFw86RP7rF2b6zMQsZF3bfEvofX0wi6dcyndUbeXcrYspR+kObYH3dJNCTXCbNKZEzXdIqE6adbI5+PhWoAK+x0WHZY+kFEb8lSe7aA35PMdzAXU3jQo37OJb1v6S541F4LSB06JOsj+nOWx99KSS5ddhhWRFIeTDmU70a0T0W0VzysKxqbd+hz8IJ27kTDJX+mwfPnHqk2jVTwz79lmf9pTOUTA/mQvZccVv9c5rthVjv9cMqw+XMyTjMhPP6NmP0XnKMa0+8YOiuGUVm748wHIelaTLJpyexbDl6QLmzvRZ/OqpxNwpRh74W01LeeFASlT4RyjMqG4fRkiFExzPzIJ1FzODN9wEvH1mQavB4fGbJ1Vg2jmZClHwVyVFlwGU2lJ54DDkHWofppgDM29BdjnEguJs1TLl7fsvKdkFRdgwLrgDeytM5FRx5g82+HhDl7RDDPbAm33fyGofXlIcOqYeNbeSq3FJrh9aA/H9M85dJbjpl7b/xUa+RzsVC9oWrB2BUlrnbBJ9EWQ9wZQ+V2iBMYtr+SJ9FSk9A8Zym9nyBVs+y9nCV1aFj6wZjF7+zR/kZPUY8/f8BgTrVQ6aajzPlpuYdktwzlTxWgVrwHnV6K65uLPHP5ocIkti318/IUGEwbKm8mCMoRB1cTMsW4JwjNxHJVDtMaf+Y/TeAOtNsOp0UPdMeQrBv8vmXjW3p4OqtyO8mtORxc8eXs94wUCqlmfCQefFLmjMrCKJ0AEk2HzomIyp2I/a+PCU4Myfkj3v3pObCw92UhGM2zkF+zlK+7dOf9SXapYevriQlaYUk1QurnfKo3oqNRaHclprAW01sytFd9BtMy4C3eF6+3/ImWzWDW0Dypcas7hOxujNfX+Ll6zWHx9/W5RtNyVMxtxcQJmH5fVEZ3aDh4JvFUa+RzsVDtRHMTZsVndAI9nSvfE6unveJRvCcIZVwUHlm6Y0g1FdJlHVj8zh5RymHzV2Yp/EWWsBBzsFeUq0dObtSJlm7csCQYzEQQTgU4v33Ab52/huvFXFtbxh2JkucE0F10cEdQfzZm8fvicfan5aBiHSjdj8ntRpTvqr6s3ApY+ss+lTsRmR3tjr3jgdJWEobj/0KwjjsUlDZ1fSRieAR+W4ZrgykhHvM/6THOo5C22xGdkzouR5WYyidKhl6cb5DJjvhkf578mna97COX3rwhTsa0jxvap4SmDKYN/QV13N1VlR0HzyRINix7L8qww+/H5NccusviUeR2IvnKZgztk7JiH04ZopTBBGKrWVc83doV1dFzP+vgjiyNMx44kNpX02SsRr5RUojF8vfHRynff9Xrc7FQMfLjjz0RpYtrevebX0vQXxAHsnVKXxonYOdVV5OTkSXRjhhOWx78/Rk2vq2/HxcN1lgyxQHZCw2G01YowcT+cVgV9FL8N7a4eGKLw0ae333/RVYqDezIlXPfhFRdvRGQ3Ykp3nbpLrhUb8m9pHNMVLnerMPW16WiTe8BDqz9WpraRXmedo5b5n+oGrlzzHB4OUnveER2Oya3YXn4N13yjyNSTfn2RymdGBhonM8w+35AkHHoTzvkHzlywE5KKpJftzR6aUZDn8S/KNFbshPrIJi+HpB/pAcqrgb4XYH2iz8OcYeG3GNhv+4ADl8LyK9LGVG74DHz/kDepdOG9v+8TZgVa6p4V2FopXsRfmfS6I4lQBwVXQr3henWLufoTBIFw7QhTE9Me4cqeWJPvN+9F5Mk2l8gwB8Lh89CfzkkfRDjjAVzZPYM/okOQV5Q1bCiQn7+pxGdFYfOisvjX3RINAypfcP0+6L8dU9EVD90uTi7y2y+w9SVfboXxrgvN2h/Zcg3fus9iqcaLGabrDfK/Pq5jynPttlolPAPPXq/1ibIyVaoddzHiSZk7TnL7ssupQcx7lAGwH7Pyq+/oMS9QcUjdWAYrASTVGsNAQprMcX7mgYVb7kTypzBHTiMSg6l93dpntasPUrq85hYaMaoot97PVmzZ9dcahc8at8YEow9nHsZctsB2S0ZPwynoX7Wl8mDC8X3k/h9KR56Mx7DBaW/eH1L+1RM7q64Bbtf0XWvX0jhd0TfG35QIX0g76sopeO6P+1SeDxmOKUsWRNBqhFRWB9LgXFB41dnrB03s2MYVizNkx6xr41inJOiYDDzBcJRrQepQ4fydVeis+M+w4ocSgatFIm29FFOiAjVcyJlREk4/d/1yewpqrFxToB89UMH528cMox87t5aYne9yjcv3qJTz5LOjHhjU9vzw9YUo6HPH33/ZVrtDIPDDMmaIfW9Aks/6OGNNHna+fUx2Z2xeLO3NSYs3VeScnZH0pTsY4f+glQFQd5S/tBjOC2YJrMXM6hK1j2YchgV1cjkNmKymw6jsuHm/3aGwppodqX7Aam6pfr/usFwOsYdaBCQ2w6Yuq6uv/goxsYG526WMA07r/lESZh7NyazLaue4ZRKnMJ6SG/WIbMXKo/gAzU8UcIw8560VUEeSjcceguG+rMRiY4luyEc1e9Zms8oETu3aclvhzRPJnBH4i+U7gqVWftVn0RHCt/u8YjesmWcU9hxlIvFsLJigI2LkpiPKl8gcZ9FlpDdFXX3JtaEKfYN89/zKDyKaZ6bdP9Vh+Z5Hc359ZjDZ3OycJx1Se8b+gsxyVZM+/1pNv6HE/gNB7/h8uH+Ii+eecRXlh5STA9J+iGDP5yFRxnSe4b0jTTFTz1NmBZhOJOkdl5jTHcrxcGzSVIHBn9gKTwcMCo6tM5A67g/IVbDzHuaQiVrahZGFZUz9fOqv4OstFZuAO1LY3rzDpWbYy2wHzu4Y5G/D676tE7B6NWzpHdlZZ5sxoRZl52vCm3Y+aUA148JTw1wAhjNCCc9uOoq8vGUjuTZnzbZfcmlP2/Ze9Fn6qPOUf2dbMs2PdHR3N26sv8p3NFgwgnluTUsOxRu+dTPyn6oedIjyEnD7w3Ehko2tbDDtHi6hbsuYSYmXRPvIrUr5+/c9pjy3WgSMmfIbjzdEvxcLFQzYewsvBVOzGqhdtFlOA2dFYdB1SF14Ey4qprgNJ8N2H8RCmsB45Jl5sPuJFJGM+zifUtuN5J35ycW5w+rvHfjJH/53Wc5/PE8ew+m6M/povYnkuooBfWLLvM/Damf93BCTW9iX8K+3opIM4dXM3QX5Ys1nBEdTpE/2oFSdUvjgngD1jXk1ye5UD3I7AZSgH6cYDBraR/zKd+RN0D+2i6jCvTOjXBHhoMr6s5TdcOg6jAsOsz+xBD5QMsn8UkGjGX6muXUPxmT3dQCfJJw6HctO18tK+LxjtCVMJ8g2VREUuzK5qg379A6o7rX60Xi9k7Y/JtfTzEuKqfWG0J3BXpLkrYkmobuokPzGwOs8xlhvHJLp0zlEzWGYUZ+Xl7fsvdCitgTZ/UJUvA0r8/FQn0i9egsKS9KEwwF7lpHXe/c2/KiSnR0vCd2PZJ1h8bZBMU7sP6LOdJbLsU7qv0SXQH9w6qkzMOqHPrckSHZgrmfGCq34iOv1Pmf9LQb1yZhYQ90If2+TC/CDPhtqS/Ld0YkWzJbyK2rqeiuyBXw4OWIKAG5xxNJSnLy+ZIajR5eSZKuxQLFd8SQ8gZqMtb+zhLpfVj4js/cO2MRmm9JqhHkpOLce0VNXmbbYfrjgOy7GVonHGqX02T2Y6ZuhKT3oHozYFQ2VD8dkdnVUCF24eBqamIuIUw2vyZyutc1pOoadZpIzU7s6wGzE0WuO7Tk1iH/0GFUkNI3ToB/N8NgRsSXIA+DqsvK728SJVWHNq7EpA9Dig9HOlnSqnPHBal8n+b1uVioTigzhe6KdrXspnKhCusR6QPL3pdjdl9Oka7F+G1hhKmarBuVQyXi9RNGfapmqV1wqdwWyNw6JVgkvafF31tUwsrBM4qwcQJonc6QrCneJn2opzxOQJg0R6FfTmCY+XBMb0GhwWHWkjmMKN8ZqdnpQ+6RoiRb56RQTXS0iKOUIbMf0746orPkkt+MZLseCVsNsobMrqLRDy8b2svKcT14Thmr6UOB6nM/g+qnKikOr0waJgONFwLGeUNrVdLmcd6hvxBTP5sk2VQ5lGjL/MFEOgW6KzHZPRHOCw+hP5Pg8KqhfDeiN+8w/aHi38dFS/OUQ+c4FB8FDOasrOJXIfZF6SvdjSdpKNKbHb6+SJCDxkUwgaF+3mdc8pj+SBO2MKWH9Ak7669cI//6lt/Tv6KUmN5BKaZ8W4sk2bSMChKkzb7l4PVlDZnfimgfUxdMDP1FO/n3mgD5PUlZCmsxQdZQuhdSuSGoJdlUU5LeU/21+OOQ2Xd10zsrMnQIco4agEjyiv68OtZER5zR5qkErVMQTUy+BhWXvZdShFmVB7PvDUk2LaWbHsOKIdGRB5bgHofM3STDKUt7Vf50ucdDVv80onMyordkJh2+ob9gqH5smLoWs/DWmGFVkunaRUN/SjbsUQr8gZJa0g8TIuVMLHZ2fiWQAfHQMiw71C8q2+DwikN2N6Z8J2b5+xHbXxXJ5vDliCBjCCuh+MGuNGnW5aikKDyAwZRH6bZ6BQNHjLJgcs38rqV9eSx2loXKJ/aohu/NCq2AiZnHUGa/T/P6XCzUOKE6NffA5eA5c+TLmehZ+qfGlG536ZzUggrTgkP6C3Jhzq/JvWRcUKRP7Moycpw37H97TJB1yByEBEVR5VoXw8mEyrD3ok930Z2EkkHxYcjhM0r6a56UDXn1RigdUzNWdHgGpj6OmXrPEfNowRCmYPpDEZ8PrqRonTKMSqrZ+tMid4S5iXVPBMV7wo2jBBxeyXDwTILiLSEZldshyboWSvOMaj6QVVBmx1C6B6lmzDjnkN0QJhl7wl6HFYeZDwJ6yzHFd1OU7kJ3RSB87jFMf6zBw3gijmwd87G+ZbAo3kTjAiz8hUNn0RV7qhbTPzmmc1L4buc49Ofk0h0lDbl1cQFapxU2UbkVkmpYFr7r4ow1lRtMOySazpHvbWFNdf5wZkLa6XyBcFR3qJpoOKN6KMwaZbm/YEg9TrD5jQLzP7EkW7q5mT3xGN1ADKcwa5j+KNZimLOk7yTpHoPi2yka5xyaJ/2jXaH8sbgCosAhRxNHEFnzpCcS97ydJO9Z9p/3FKzbCkk2I3qXhmx905KuyXcqfSCNVH/aIbsvVWnpnphQQQ5a50MGMyI2RwnlWXWOGbyRYt5x9HmCvJIEBxVXDUYMxQfii+49n6B2KYEztuy/IseYIGOYeXNfUTz2M81Wd8mbhEjI6z/REil7WDU0zvjkNgy1VwLaqw5xEvJ3PZJ70mSVbwHG0LoSMPczTau8mk/5U4fhbEhmy9C5MKY/p5Fq67SlfFverekDS2fRpbPsULvskK7HpGsqM+bfGtFblh9W+5hD/Yolt6aHMbv7BZr1x66IJMf+eCB4pxkT5iyZLdWQ1oH83RaNM648pCZ2P4l2RKKhuqi76FK+G5I61FOrQASofhppIpWA9pUR+Y2I5iX5pVbujBlOiQvg9WH+Jy2ckbJM26diGs8oDMLrWx5/O8Gw6pK5lcJruYzzDpU7kfxUh2I8tZddMnuW7pIEf4kWeF2XwgNI1TXvb1xQTdk85bL7qiepzZyZuOMJ0A9ThvxjS/GRQtKSDWGx1jP4LRe/pxLm0d+ZxRnbifOz8Ep3qEYrt2nJ7umzm0jj59n3+owLUH3Hp3cyoPyL2/SeG0jLdUqWldaAV/NoH3Mp3VWJ012C9LaHO7Y4bYH2yVZMouXgDWIOnxPW6o5VsjmBYevbMZ1FwVkb30qQaIiHa10o3jEEBRF5dl5JPdUa+VwsVCdQI3V4JU1637D35Zi5t2VycOp/6OD3YOdrFcZF3ZTBlGHqesT+cz7Jpiy8S/dDGqe1IzbPOpTuKKtq70WH0VRMcS2i8HGSg2c93K4ag51XEsQJiNIiZ298u0iUEc5pDVQ+0nH8REgYJQSgT39k6S041M4LOA+zMtd1R5basxLUHTyrP+cfakjxJDy3fFNHuddXrTx1LSa/HhOlIUo6LP44pHFen3FU8qVDyhraqw7JRkx2Q6VJdi9kXIzpLYsv6oy1MJ+oct3A0jzpKvzhvR6DaUv9fFoW7aswtdDiby19gPcoJae9VER2JybRjSjdkcdBqhEx81NJUKyjRmrmPVn7+J2QzI6lu+Qy864Urk84GiYCM1IWgt+zlG+KdDSqwsp3O5INtSblXfsp18i/xvX3P+qV3dTTWL0xIrum6PPOpRGPf6FAqh5LF4SocxhoryixpLusP298S1GKcjuWJr8/6whB2HFonJZ197gUY4Cl74+JfaEHzvjJrmV1I0oWJil5g2nhsos/6nH4anjkPJLd1sg0s2vozyoDdVT+zD69eE9NYn8Btr/sMipLHOeOVSpECZh9p037uEtvwSHIWtorDq0Tvsg3JUvu0wNiV+8nzAogzxzGpGqWnVc9Zt+FhR8HOIEWaWfJo37RMPWJMgucUDBZ43xGKdeIluidb3O4VeT/8bu/OnHhNpz9L/qkGhH9KW/iS6VGMUpMlAId2H8uQX/GYe7tGG8QkeiqRm6dcMivSSD4xO7IGSuaJ9XQfUs0DUE+Zuvrefyesm79rqU//3Q16ufCgCK1sGyP/Tv/QIFkWcg9FkUt2TCU74mgsve8izsyR4a7UUJjOL9rj7C47K4sx2HiJp0Gvy+Xjyht8HpKqg6Tht7ixLFvJJlzlDTktgL2XpxIrPf0td7Ee7+76JKqxfQWpP4cXhiQ/TDNcMYSzI4hMrx24T7j2OVRs4rzu1VaJzWhCnKWaGVI+oZYwiaWb+vMuwjnLcLsewF7L/jkNiyle30OnslSuT3CWFlHRr5Uq80zInVndiTs68/rdOlPTWburZhh0QFHsFyqpu67dD9i65uQW2wTxw7DR3m8nnyz5t4eYl3DsOKx95JDdkt0y8ZFS/LQIbcl/Vj9osHrCocOcjoVussW61mWvx/RWfSIPU3hogQMFkNKNzysB52XBpz+vwWE+QSHl1KEX2/BO0WyO5b3/9H/+q80oPhcLNT0/LJd+ff/AcOlMcWPE3SPybG4dQaiTIzXdSjeUVRMkFcgbn5zzO7Lk7S5tCW7pYVbuR2SOhix9dUsiY7q38GsplPd5QliMK/v/4QQ0T8RkNryyexoLt444zOqgNedBPwGYiR5fZE8vAH4fQ0Uoitd7IMsyfMtUn5IvZHl7NIeCSfi03dOkDnTJLaG/nqBRFM/3xnrQXLHltYJh+rNkGFZO3F/PgbDJK1FE6/2KXXtsWdonYtI1F2MhemPQqyjh2ic1wNnXcP+VwPcus/MB5adb0QkdzxMbCi9vMeL04/50T99UVBdJAy7t6jhhN+FzIFG2I3TMksLs7LpHFbUmFkPKjdH7LyWJEpaZj6M2fr5GL/mkdmSVXtYCil97DMuyspz92WHsBAx9b5q1vZJS6IlIWFvwfLgP/mrF+rn4ui3RsKw1T9Urmh6TzPj6Q8t+fsuhXuCenJbluy2pj87X0qSaEJ/KWQ8G5Ldj+mthsS+ob+QIrclBjqT0d64oMlMmIZEwyHRk8ucdeUOPXVdmVD18z79BQ0HvL7glygtfHBUUQPg9xRo5nfB/yhHvDokDF0Ot4rEgcvtT5cByJ1t0F0r4jkxpy5vYh1L9dNIrngjy97L2rlj39Cf1bRo+fsRCz+OcQdCKXoLqg8HM4ZUI8YEhtyGJbWvkmL/BYfWuYj0oYyCB1OGwvUE8z+L6c07LH5X+igudqim+/zxu8/KXGNWpm/dZY19TaRFOioYgozQjMqnEiXaiaLVHcnn9dFvO0JMHNWj5Q+FrXZOxhTuQ/VdD29g8buiQRogs+lN1LWG/CPnaOq48r3RU62Rz8VCNbE6Ymt0U574SO2/KGZPlDYkekpGHpXAb7vMvzWSs4cDhZs+7lDp061jcuQYF1S7+h1LZkfjy2P/eI30vuLOO0sOvSVlKCValkHZIbetkerCj0PaJ8V3Tda1kzTOa4dN71tax12SdWGBqUNFoptrefyaR+5Wgul3HG6+dYLWWonyp4bWepGNRono1IDa3+2R2baM8ob8mh7I3ZcdMnsCv5unfAZVl2RDP9+feG3NvjtmVDRUPtEC83tyqvbbhlO/M5Qg0hMpGUc1vAlh9xXDr/3S24z2Mtz7yTESdZcw7UxOholtJyLLHF4VOTrRlVXPqCQ1bGEjJNmw0rXlLKVrPom2ZeojTdL6cxrApPak3B2VVG4Mq4LGCg9g9V/UCDMyZuscEx49zkPz5BcoC1ULVbYvgxlnYh4G7sCQ6Cqep7OkuMThqRHJQ0PrZILBrCG5J9zw4BmfzGOPUcWy/4JDf95S+9KYVDOeXCBL87UVOsc04sttyy2wsBYyLqpe7S46DOYiWsd9Vr4XMpw2R5YzU9c+M1iLE8oRTe+qEdl9yWVcsqT3DMGLHSWGrEPptm58bs0lvJfHODFh4NL+Vo/uqnZpE09Utnf6NC5aKjdH9GcNcz+uMftmAxPK/GHz53yZU8wYpj8akdsaM6y45B/H3Ps3k6T2XEp3ZSZhjSCz4bTFX+2xmGxMYK7J2DmyDJZCgpyan4OXokksvLi+3XmXOKEuP9E2REk1ZnECqp+I0BLkFFQxKjn4V5pESVh4sz8JpJBTi/V0asUetC6UqN4cE2Ytc+/EVG+GBIWnLzufaqEaY0rGmN83xtw2xtwyxrxqjKkYY/7CGHNv8mv5X/r6/8QYc98Yc8cY8/N/1fe3LuS3dOGmro9oXID+qTHVG/qQfke7XmY/pvpGAutC+7iOjtGyyBtT10PSe0pBkazDUPwwideP6S/EGAv7zxmmPrb05+UjJRDcI7cVi2P6IKR8Q+jB9msJhlXBXs5Y5JTBnKW3rACIMGuZ+mTIuCiHv8oNDS389/OkDiyNV8e4I0g0JrvSAyh+NwvGslhtESV1bDYvxuQ2YP+FLOldh9qlJKOK5eHfnqJ5uYTXVxmSaGrsW70ZsvarPgfPJIk91dirf6IQ486K7DGnboyxrnDLrx+7R2BdKtcchtNy6W6eccg+8hjnJWGef0O4b5TiSPJNDJtfl7t267gyA6Y/DhmW1MCOykI19r4SkfmDIqOSYfObGT0orvJo/Y7SCUclBQ5vve7jBIbdVwxbX3OYumafeqt82h31/w78ubX2HHAVuAX874C/tNaeBv5y8meMMReAfwO4CPwC8J8ZY/6VJpgmgs6Sx9L36vRnfaqfQPl96fS9oY4zf2CxRnlSmV3L3LuRDHt/7JPZNTTOevgD8SqdsSCV3qKlN+dRuS7pRepQx1npjnY/kCY/0dEUKbM1IPaVjxT7ligXMSo4pA/FG8hsyy0ltyP64MEzKfJrgrbGBeGb+XXBVt5ugsGMYfa9EZkdLZr+nOH4dJ1yss+Xv3qD7F7I/Btq1koPA2JfbntTHwsvrl/U7L91WjtT6tDSXXAp3tH3qz2jEWXzpE9nRRqn+Z+OaK/4tE/HTD+/x3c+uMJ//sFXye5HZLYneOh91Y9yiRaS0T4XMiopfG1cgNFURHpPdXNuK6Z9LqRxSuk0jZfGR3r86nueLOjRhtKflXLWRGJetVd8Mnuq8xNNw/S1kKmPINFyjvIC/loWqjGmALwO/FcA1tqxtbYJ/DrwjyZf9o+A35j8/teB/95aO7LWPgLuAy/9VQs11bDc/7tl2qsO9Quqc9IHlsOrDqOq7BjT9ZD6BRcnsmx+Q0dw6U6f/EbI7Dt9zbDdiRb9rJqowvqY7rKh/py0941LsfDIPXnmr3y3Q2/OJcxZ1n85z7ig1LrstiH3wJsErqnjfiKr6M25NM+K3eUNmZguWEZFQ+ukQ3/GULlhGV3p015VrkBuO2JUjbm3OUNsHd7548uM8w71Cy6dY7D3vI+xUD/vYo1C2fyO0ch4R2K/MCsH7vyWZu/JQ+cIH/WGQj02vpWgfjUmudTlG/N3cHsOy7/nsfei5M/uyLL/lXCSByW/0uxOTOGOh9/TKeWOoXDPJbcl4+H9F6F83VWWVhOqbyUoPI4ZTkl1kazrYU0dygLI71n8niG3NQarJszvwvxPe+y85hJktWBHZcPSX/71jVBPAAfAf2OM+cgY818aY7LArLV2Z7J4d4CZydcvAhv/0r/fnPzd/+fiNObfM8a8b4x5Pxz12P0SZHa0E8osN6Z9zOAEhuym1J8b33IVPb4xYvm7YiVtfiNL7YLH3ksZjBWgnmjKQ98dGRpnhItOvS1Nf2bLpXnWMvPvrrH7qmHvxTzdJbmOZDdEFYxyMmBLtrSbP8Fp+3Nib3WOx6K+nXbpz6kb7y0IGgtyVlaUCw52J8WorCzQjd+KuPTiI1YXarTGKQbHApKtiOy2DCK43JHL9pZSr/tzhuyOjsbhs32K9yXzHhfg8S9rF7auKI7dZUtvQYTt7KbBGTrYGwW6YZIoF7HxbeGltRdD7N875NTJXazR9a5+0qV1XGkzTGTo8z/p0V2NqV1ySe07ZDcdctsR5TsxTiQqZaKtRTyYnpQKRiEbT8gnsS/FQHFtTOwp5O7gmSy5dT1U/SkJDx/97b8+hr8HPAf859baZ4Eek2P+/8/r/xcT9v+rarbW/j+ttS9Ya19IuBmcoaF8L5Bc9/Fn8EyiySQhL2LuLRnp7r2Qpr3ikWxYyncipj4J6c9r3hymDemJJXnqxZrMgadjEj2xm/KPY3LrDo/++ASn/2mX7qolu2Up34vwRgqrXfy+maSqqIGoX43pz2pgUD/nUrptOPanilj3uxNyd0N0w9yGxpf59ZhkzWH64xGF+2Ajw6XiNjuNAo0/WiT92OfgSgImuVj+23lO/l6fRNcSVCJRDjNajO799JExRW7DUrwpqlz5dkxvOWbqmuXYn42IJu7RLAwZVSO+8wevUPnQw58dSOIzdNlbr3B/bZbCuvgQm9/IU7kTkd2NSXQUhvGEm5veEw82WbfsP+cRpA3jgtFUzogbG3vQXVEKt3X02U0M5Vvy8K9dSOqIP56it2JpnrMk2xGdYyo7Tv7OX5+H/yawaa19Z/Ln30cLd88YMw8w+XX/X/r65X/p3y8B2/+qH6Aob8P2VzwOn1G+e7L5ZNqkMIf+jMPBsw7FB5OY8ZmJ4nTBpXnK49ifjahdVg0aZPTr4IOqjqaaQ3ZjQPuU4rizezJ42Hu5wPSHeobClNSiURoOnnUYzGgMWnwQM/2e4mbihGFwckTp/khpdmdGYi7l5aHfOCP3EidUzZqqyWO/fQKePb3Oj3ZPU8gOye5GkmRb6C3r5w6rlsFsiiBjyD7yGJV01Gd2BJENq9qpmufF6vL6kF/vU/5U0ND+cykG04b8czWy2SFex8Ebgv/rBxybquPN93FHBrcQALD3msoAHdch3kC7Y6IrLwDrCNkIcob85hh3CM3zkyO+HtE64dM86ZDfkHlx7EsSMyoa+nP6tbMif1i/p7rV6ygDde9Fl/m3QuqXDAfP/jWRUqy1u8CGMebs5K++AdwE/gj4+5O/+/vAv5j8/o+Af8MYkzTGHAdOA+/+q35GlJQ2yesZshvgjHXk9k8EdJZdSvcjOsdh9v2YVENuG1FKAQwmFkcgyHnMfBATZhFrfeI9GmQNpQcxG9/OUrkhB+T2qkgpcWKyQJf0d/mNWNY7h2Kf+33LqKAkE29gSbQt89/1efDb2tFK7yTIb4qD6fcl3chvKL1OeVOWUcnyzOt3edSosvVois4703RWXDL7IVOfjPG6EwLMp2qUMvuhEvRyItt0VpWWAopxTO8aGpcjMLD11ZxA+yWH/OOI4XSM50Z01ouE+Zj+cwMuVna5//ESifdzVK9PDrbYUJq4nYyLGp0OyxL59WccFt8YUrofY30ZfgyrPoOliNl3JT7sz3q0zkW6L8sOydrE9eQ5Q38pJn1gCPJ6gBqvjRhMG1onNdzorDos/WDM+m9ZshufZS/8T16ok9d/APyOMeY68AzwfwL+U+Bbxph7wLcmf8Za+ynwu2gx/znwv7TW/iv3d3cE/XkoPoyZ+rg/Seyw5O75WFc7lTUKk62fdUk05PPUOulMkpGlwelPOQxnQg6vuKTqMfNv9SmuCVJJtBUVE2Y/Y/hghSYka9oRg7Rm8/HEHPdJCMT0tQDrwbAskV3xlkfz3CRCchhz8IyhdkkjxyeJfYdfGTP9ASy8ss1+P09vkMAvjSYJ1jHtFY/dlyS08/rQW1TQWXfBo3FBOaKZAw0xgqwhvya6o3UgeSgw3zpiZM29PVDcemDYXa9CaQwORB2f93aXya07hGk4eAHiwMGrezSeD+kcg/5CTJBxJkYRhnTN0jqu9MHhyhhnJKJ17qHLzmsydjMxFO+4olm2IErrHnodw8w7AvK9wYQje5igsCYILswYSvdjHv98gvL7PnESFn/0dHZ+T7VQrbXXJvXkFWvtb1hrG9bamrX2G9ba05Nf6//S1/9Da+1Ja+1Za+13/srvb6B4X136zmtZFt/QWO2J2VeYtVRuKD7cxFB6EJN7LIaRN5BXfWF9zLhkWPq+dkMngtapNEHGYVg18j3yDF4XJYRsGSq3A4ZlQ+VOgDOemPVGaooGMxovOiHsfFlDhcG8pXk5PFJ5egPL9pd9nEATq9It6KzK2KHwcZLWCYe19Wl8NyJoJZn/naTKlopDYWNiLpaWv6vfUa2slDxZ63TnXdJ7hsxhdIRmpGpqFhNtZZcOq4adL6UJU4bTL62zeOyQ+dkm1dUGl88/ptdJ0Vuy2EsdrAE7cPH6hswjH29gyG461C8ppyBMywmlt2CoX3Bg7EgYmNLumN0UQaVxDohh74UEYU7vvfpJQOVOROOc/GCjpEzv3KGhcU6ohXU0Uk3vGaxj6ByLaZ94Oju/z8VkCjQ2bZyWnqd2KUl+DXa/BHNvdchtSATXPJ1gOKtRapg25NYFBwUZw/ovST25+7LD6Gqf7qKDE8gAIbdp8bsRnWWXdC2m+CCmt2hZ/0WX7G7E7ksiUJhIF/PwkjeZjml+Xrotut/ij0JO/bOA4lpI5eaIzv+buv+MtSxLz/TAZ21zvL3nehs3vMtI7zMrK1msYrHJKjaNWmpJ3WppBGlmBEhCazCSBgLmz0jomT8zwAgjjaRRy3SrRbJFTxZZjlVZ6X1mRGR4c7093p9t1vx4T9ykMCMxGiCByAMkIuPGdWfvtdf6vvd7zYrD5GURYnqzhtZxM2bbG1rnA8xTTX7p8cu4RkSTw8c8Zt6PGVRg+2WP0jXt3FFKD1t7+UEIhESFqXp8ZMvYnZcurHZxXCc2ldBMDMt/VGVUMCxl67QHSba3Jkh4IVfW5okHHoUzNdLJAJuKSO15pPcty39cp3DXMnyxzdTHlv7EOM2vIZeXMGNxcgGDKUPjMVlTTn88INGOSbQM7eNSMSgry7D3vMzSrKMSrrskInbq0FC5GlO4J6QiVY/Jbemk8Dtqzh7m9UgsVDvunv2ORGOt0xHZ3Uh5oL+ROwKihyVDetc5Mtkq3g3GbCgx05N1NSiJK+Jf9mYd/L6l+qQlvS3DsfaSw+43QtJ7hqmPDIePeUSZcad6PWJ0bICxUk8mGuKqdpYk+1372xE7L6bZfN3h8PEkvdmYzvz4EhpL+kB1WHcxojTT5tTkIe/trXDz8hLpDZ+gaIkSenCO/24HEys7dPKyHeuIYqIUzL5ZJxgrGbwu9Muukkq6lmN/PGRUMsSuwevr89unS7hDy0++/wQJL8I78Nm5PcXKXJV0uU/jfonIGszQkRlvxbD5zTKdJUPqzTwHT4k/MCwpfHjmDZfFH4cU3k4LA74nk+XGySTtpS/LDpFpHCpXVCLUT7uUb0glPPemZfrj0XiY4xCmNOcf5Q0msrSPxRz7/Q7dha+QZsoawRrO2JgseeDSWvEYlseOyiGq/TxdIBNJZjKYUCiY19URk92LsHDkxZnZi9l9NSa77nD3N4r4XYHZU2/6DCYt+6+ESiEZGIp3FRK28Hs+mV3VrenDscRiIHx24fd8NT6XxfSfe8uSPpATCFadeZA12ISlsVMgtA7VWo7EXJdEA9yeZuNY6C2kcQN19P0JcQqWfiAjjP68skQbJx1Jk9sxcUK2nI2TSdqrEe1Fl+6sR28+pj8hhlP+PnTen6T82CH/4itvU0wM8L0Im7C43y9TvO7Kp2DfMvten5kPR3hd4daySbc4Q0g1Ivae9RlM6drOvt3DCaB+wY4/35LdcugsypanedwXAagD9TOy1xyUHfafSmCsdGwmGt+3rPqJxT+POHwid2TR+Ze9Ho2F6spZLkqo3inejsdkDC0sd6BOvngn1q47Y8YAs0PpjqCV7JbBxJawIFJFlID9FyxLfyps0e8Y+tOqnUYFw/QnMW7LJUwZUofQWXQZlg3dWfkvlW5HxO6XMhLraOad31Dnm+joZiRb8bjmtUx/HDKYtiT3PI6f3OXOwSTlcgfPi2k9O2CwFBB8o0n19SG1v9tl5zV9bZg17LzosPOKS7Jm6U95tI5/mWPgdR/YTSowd/o97U7JVkzlU03jWieUsbX44x6DH03x23/0Co1/sIx9o8z029qRjbUU1gKyexG7L6TZfN0/ShhsntKIOtGGzqwnS3XGiErRZ/JywKl/3GVYUb0682GfREuKgXRVDeGgouilvZdjooSma/1pad/ipL5X+kDxmPVTPomOSPAP83okFqrXG0czGo6sCr2BjuPWqsi4tbPukeOIdaWHCvIxW193yO5HMm646OM3Xdqr0uMs/FgJICaE+Te6FO5oeJBsWHZeUoZn6/ERreOS8A4rAu39jgzEohT0JxS09uAoHkw4RCnVz0HO0F5wj2rb2hmPmfcjVn+nztonC/xr596m208SBC7p7BDTdwhDBxqq55xcQPvCiP60ZelHUnmmGpbaeTGmgpz8DhqnfDI7higpPmj9nPDV2lnZWbpDybUbp1z83SZhBkxk2HnJ+7LGXbSK21z02fq6iDfla0I1wrRM6oy1uH1LphqR2dfYs34eDi+Ketg4I0O26pMxG99I0Z+Wk0trWd9v5kNZchZuenSXdP8y2+YIr+0uqFFLNBy8vtJuBpMPZ+nj/TWtvX+mV+zryE8dWop3h+y8mCK7o7Fg8ZYWZv3lAYlWijAj+UjjpMPURzHN4w7daSNweyvGbAgbHZagOy0rnlTNMComjoy/6meVIl0/7ZO/qll8smZJVb9MO45dlRPDCY1vkw2lffhdiBOW0jXN4WNfRO65n0KyJXJMb6pE4S78o//qF8j2LZ2f6zK6VSB7pknmd4q0jxm6fpbJD1wy+xGHjxkGZRmZJRsafgQ5ZbymDzUrt44Wr9+VIK47J3RjWNHn+m1xErZ+eU7ssQFjyEqWkOVrLq1ll8krIwYVBUXsvzBm3SfGUY9GZhSJjkOQAazQEUnKJfE2EaR3XaY+CxiUXdoLLoNp8RL6FXGPUocWExoy+wF7z/skqzK/yN9/wF8Vju329J4e5vVI7KjuCLoXBnTnDPXTySMsM7dp6C4pBr38VpJULSb2LZUvQvrHRXiYuK54xO68PDeHRbHl26dCJi/3x7N0CPKyL1ekj6F6wf9SZtIV475+Tg1B/bRL65jqueI9adMVxa5dwoSKkYxS46ieDZfa3xLSMPdmlyBvmLg+JLOnjnjyf8rgtw2V/0o7Unrfkjh0GZYMe8+5ZLeFLkx/HDPKGwprMcU7kqsEGRFRHshmRnkt0jiBjCU2tCNZd4xHz6vW9PrQORWQ3xQDrDtvxshDgs7xkO6qEAAsNC8FWM9QuB8ynIzoLDrUnwnGbimahCW6Y/Z/Qbtod9ojs6vPmXtLZVns64FurRpap8TrLd3UFDBKjeGpedENTajft3jr4dbII7FQwzSc/C+k5W+eHmvTDyNMaMmtqXgPs4Zh0aFwF/afdjn2WwqxhbFdzMkRUdrQXtHCNyOHe99Nc/iMNPytYw6VqwpLS+9bOqcCmY3tyRGkM++QaD4YWYqB7g1igrRI1aPil3Vu6XbMsOiSqmqXjVJQ/D2NWO5/N8OoCHvPKk069gW7JRuCpLyBSpvpT+S8kqyp5nNHsP3LIc2zsiL3+pbcdkTmIJJfVKgQ4ygjY7LCPWn928sOYUachzAFufXxKZCApT8x1E+7dBbl/ty+MBqn7RnctsPkZeGz5Y88BpOQ3u4w+5acX2Z/5OEE2giGZUPtrEv12RC3b0g2AgYVQ/VCEutC7ZzHsCA7+WRDjjXpHffIVj1zqMC0ZFPc4MJtTeLyGzEHz3+Fjn4ngNq5NMmqMopqT0QU7yt6fLg6IPdZiu5yTPmKdsv5N4a0lxIs/ahDZymN34b0/QSDCUuqJo1+etdh9r0hg0mfMGk5/IUBiUaKwaSkJQs/cAjSllFeLP0gLynIqGDGfkuWjW/D7BvSsYfZGL/tEhY0yenNqNzwehAHogHm12OckUNxLcQJFJoRpiC7YyneHZFoJ1QPVpWPmtuO6E1pIuWMIHs1iYmheZxxJqq4DNYbM6VS2oGipEB3E6uBVFzkA++s8Ui6Ca1lj0RTu2zlssXvJkg2YowVc+nwMfkWVK4ERGmf0WSGw0ua0nXnHMo3Q7Zed2CuR/rTDOkNqWS3X0ox986Qw0tJMruW3py+JtmSTsvEX3pXxT70ZjxKd0Laix7pXXEIEh1Le9Fh8gPL2sOskb/uRfiwr96cIBLryvo7yEqS4m0nqVwdkdoXiTqzazl4KkmUgoMns/RmXJJNwTqTn8cU7ol55A1g/6kk/Yp2vMI7aX3vbUvz2SHtJZfmqXG8ZRP8tmHysx7DCdXLuZ1Q8eNpQ/6uIsJXfnsXEPie2RX7v3JtpOlZWtCLO7R0p12q53y8rhZc+jBm9/kkrVVDqm7JbY/oT0P1vHsUtgZSMcz/uKmyxkDqYEhvTrbsg9kQZ6gGLnZFGjn+2w3cvqF0q0t+cyRory74yRvYo0Zq5v1I4WhjwzSvq8/Lr1l6CzGdeY9RAfafTOL1NHRonw7Fwv8Y8m9nINZYNF2LyO5Y2ssJSUxOQepAA4jmqoPXsyQbap6sI8sldwi9aZfm8wNSVUt3JWL3ZWWAPchm+Mtej8RCjT29WUEdko3sPa8L5vUM1YsJkjWNS4O8oXI1IMzoKfY7Y3Z9zhKmHJrHXco3Ruru6/aIhhdmof64vEsTa0kGFeGH7WMKBa58EdI4nWHyspqp6kV/7B8l5n7xXsD+12cofy77n8xhRP2Ux73fEIE6ux8zmNBoMMyKOTUqQuNczKDk0F9QwFhmb8jhY8po7Z8Y0ZuzzHwwILOnSc7hMwWSm2p2ahfS+B3VqDNvO+OodBkUL/04YO27ZUwMN//1BPtPJRkVoXg/IErq2vRnYzL7cu4OM7K27M+orHFHum6ZTSlysVC8FxElLRPXI+b+3AEDw5KaNGMlF99/0qOwJiC/fDvEbxvFmq966jUWtKMbC4fPRUcesYMJQ/6jFEHeMP8TZaaCsN+HeT0SC9UJYTClhL72oovftpz+f+8T5GSZnqrKZaT6mEvpTsDar8mOe1QwNE+p5pu4qrm3ieHwUgJ3oF2x+ot9OovaOZP7Ijon63DsDzo4w3EKXd6w/7RL5bMmg6JzlKE6LDk0LwakD2K2XvMo3hkR+2b8OwuTrbznUT/jsv2qGXNSOUpUGRUtK38S05szLP5AE5jd59I4IyXlzX3fI7du2PpaivayI6fqcxaM8M7ak2rkBpOMMwAg0TJUL7gcXkwwmInoLUaUPknITse3bH1N0FeiI9l5d06duTVQvhUQpcYyl/OGUREK67JHT9XkHDiaFA7dnXWofKad2xqVFM1TYvTvP5mk/EWL2hmPubd75La121sHgnJ81PwlDl2mPlUdbKxsJodlS7IWUroui6Rh5eG6/kfCgCI7uWQn/5N/m5k35eM5cUUQhtezdBd1LEdJCAoxbk+7Z/GmoXg3oD/l0TgtswaA7E7MsCjzWxx19EEOJi9HHD7ukr9naZzTlMh6mkubSDTDIK9jqrsgScWwZBUBmRbHFfR5UVKlxYPuu3RTALc3FC3QWJGt/YZDmLPk7ou9lN3Wz3ZGqi+TVTGkBiVDf2ZMLWxbGuesTIXvxBw+qfypoKD8p+xuRHazj9MZ0TpXlCnFCUfoBrKkbC27DCYldExVYw6eEaHE7SOi+KbKrCBryO5ryuX1hD1bTyZ1nUX5arWPxSTqDtmdsX2Shfk3h+w9K4KNE8kswx2oMW2cMYwmYvy6g9cXW6p+Rjr+UZFxEJpqWBNLLfvZf/ZVMaBwoPKBLAtn34uonxtjd67BGRqCrKVyRbP/0XSI15PPf3vR5/AJyaqzOzHDimX/OUtn2TJxIzryjU82YO9Zh8JdWT5aYOrzUIa5S/JGbZ0U3xOrGX+iacnsjgN7U4bZd9r4XS2GZEtZqE6kuq294nDwUkhrxflSBh0avK6S+aY+GxBmpH8v3NKNXfjJgFQ9pnpBPv9RWkiEiaWg9Xr6vrEP1rPEKcvUJz36FYfqxSy9YwViz9A65jAqWlnKe2rqnACcUH/vTzrk1hXqJkt2yG9GZPdCEh3LweMOMx/0FHvZjqk+HVG74OKORZUPGFNB1jAqxyTrjP0SILMni5/yTbkG9qcdqX+/cHHPtSWrOe3gjHSi5TatmquFsRx+aAm+SqQUY9VMpQ8tOy+55O/BYNJSujUS53MovmeQNRSv+gK/+7L8ya3rRjWPO+TWILuur7cGks2IKKnduHBHvMowa8ltykI8UYf8LZfCRsjkp5aJL2IqXwyY/DygdRxm324T+aqb26tZGmdlc96bEf3NRKqvC/djCl/4jEqWY3/YpT/pkKzKRbA3b+nMJyjeUc0YpbWQdl5M0Z13SB3Kpid3H4pX/HH2qyHMCR1I1hxGyyMqHzvc+edSuAORSprHPXZfjRmV7ZFKoT9jOLyk9zv1SUjsQ3dJ+GZuK6JfccjfV7ffmfc4eD4mvw6bX89QO+syKDuktj1KN2KKdwNyWzHF+yGZ/ZDMQSy7yBzUziTpz2jxDaZFR3SHqp9Xv3OX3pwl86d5hmUt7igN+e3wyP599p2YVE0lW7L2FSKlhFk7Ng5TPLaMvyzr3/boHlPow6iopOfW0wPilT5BQUwpb6x4DIpqfKIk5HYjOgsuvUnR+tL72s3KN4fk1nQclm8FuCPL1OcDDi96FG+0KX16SGslyfbXPKY/tOy8kseO43f6kw7pPYfUgerXzoricZJ1S+SL9ua3DHd+I0OQk9Zr7i1LlNLROywbTv2jNo2LoXafhejIRmfuDcYNkEIh+vMRE1f17/n7luL7STIHEfm7YygrgN6sJb3lkbtvqJ03DKYtUcIy/VHMYMoejUknP5Fh2f4zMmN70L23V0SGfiBcTNYhtxliXdh/DoYlYaOHFzx2X/CUU3UQ0z87wA3GNvEXQ5Z+GNGbU+2ZaBpawxSjqYjyrYHUqG2H7KalftKju6D7PCw6dJZVz09c6z/UGnkkFqrXMfRnYmVrNgRr7L6ipI78Qgv/a1XcuT5Pv3oDWj7xQYpnXr1OMBHS/lYXJ4D8HYfsTsxoIqZ5zCN9GDMqaUjQOq7a8/CSOmMngs1vuGrQzqeY/njE1utFbv1vpqheskx+akkfBBoUnIko3JXSM7NrKd0NaH+7Q3YDUjVLsqn5uzs0LP2wTaLuULoTU1gPOXjCYe5NcQr8juX+d4tMv+0yrMRMH6/i9SzlD/ZpnFKSX7oa0zwFJ//JgOZxl8qVLnFCR2TtjMfUxz1yW/IxdUKYuBFRuis3k+U/G5JfE0ehcAeyGyJ9O5Gw3uJNNXCtUzD7To9kQxOy2IM4KQ3a3nM+UVrWm925sTtgVehInIDGCZfpHwiLDfKG3G2P5orP5GeWcDqg8LU91rcrLH0P1n4hRfOkyDrdRcOobFn4WUi6FpHbDkgdGpb/qMatv/Nwob2PRDOVmVmyT77y79A87o6bFUPj6SETk23q62UmPnZI19WwWKM6LPZ0BE5/e5N7W5Mk1pIka9od2ici5n8CQcahu6Dp0PybAxonkwwqqn+z2xZvaDl4ypBdF0k5yIoquPEtw9wbcrF+IP/oLsVkt8aM9/EoNkqaoyzP5ik57jXOSww4mNBEK7cZ0VoZJw2mobMc43UNc+9EdGdc+tNaBLk1S/H+kHvfTVD5zJDZC4nSDt1ph/oTEbm7HqOCSNrDsoIn0vuCzbZeEwnbjIWGzSdGlD/waVyI8VtjRr2rjIIo6eB3IjZ+3uXEb/cJ8z6jvEtnYZwvOxr7mbZj2osevXlLUIyP5C9OoIUfZmHunT7bL6VZeKPH3jMZWk8MMS2f/H1ZKllH0vPsfY8wo4eifB06C2JpDSYUevz+P/4/fDWaKYDDS+4YAoH2i30YujRvTDD1rhZbb8rBCS3VS0bJdGfUobYGKSYn29gzXfozgrLyd10GE/q6iWshCz/p01lI0J8y9GdjJm6ofnNHMUEponVaefSjgsghhVvSA7VOh4RpsaqKN3WperOG6XcbSkpuKCA4zBiO/06b7gLMvWmODCd6M5bcRp8oBY1LIeGLLVL7DvM/C+lNudQuxYwKlsIdNXnr30qS3nPoTxkOL/l0ZxzKN0eUP5MRcJAfG+Na4cuF+yHNYyJ99OYM6ao0YfmrCcKMPAysq7jzMKWw49p5h+6Mh7fQY/trWdqLvmTY2xFBTpowWQOJJWZOdcgutAkzlpkPRsSedvhEw7L1tTRhznL/l9O0nxrg1nzyd0VocYby3jJjL1eva5j5UJaW/fmIzrI9Elo+zOuRWKgmUo0UpkXcnZ9s8EtPf0ZmW2QKdyQoo3rRVTLI7QivbZj6ZETv7UmqNyusTlWZenKPxlkUEHtamqrmMY/bf8+lXxFhefZtqJ+SD/3e0y4rf2BJ7TnEvjDYzqKCvVrHfJzh+Bi/DZUvBngd7Sh3/naJ3pxY8dZRvXjvV/M4gWFQMhy8EjLKOYTFiK3XcnIMPHTpV9NSj35dE6/Jjx3ihKV+YQzbDA39aZE4nEBiuP1nkkdyjeJNeeZntzX5CXKq9dyBpmvy/o9xR8Jx0/uWoBSR2VM+Vm5TCErmIGLpP9Msv7WqcqE76zL5+YCgAJOXQ7rzDu2zAcHI47XFO8QJy+7zCU32zjwg58gm6fSL93H9GH+5CzxgwqnZrXwizdpwQiZ4TgDzP5EnV7JhGZW/Ys1Ub95SuKudYmOzwh9/9DgwlpdE8rRP1gQh9SccivdiGicTYvfP9dhslOiPfEZTIdktZSPNvRnRXbQktmX2lV+PSTZCTYUmDamqIfZ1DPWnLZ3lcURQzlJ7UnBYmFKY2cGlFI2LIZOfDTn5Dw+IvfHv9UyP5R8OjlhAvTlD6TOfxjnI3ldMZezB3DshqR2f/JpA8wdA+onf7hOl5Poy/VHI5KeG1glLbjtmWJIS1gmRRVBZDiO9WUP5el9UxKEYXKMCFK416c5p8fanZTBx7Pcs/SloL/v0ZuQOffD3etz9e4LgCvdEs8xvhtRPJyndjGmc9MjsWMofeRybrR7di4nrkqpnNxw6S4j0nYIvriyTSo8Y7mRonVNyYH/OktkW8N99sg92PAjxBZmZWCnUM+9/hSImTaA6sXlKQLqfCUhP9sbBt+NgglAgcZyQUULtnBmbNEC4l6HXStG4VwbP4n3nUP76mx2xduqSiaRqEa1lJSCnD8eZpac98pshzkg1ZaIlHDN3x6N+Kaa9IuKHiSF/22P7lSRrvzZNel8cgsTlDGFK6gAnkK17f1pSFr8tN5CpTwe0Fz1MIBFj5XPhu4fPRmx9PUvpmpSr/YqL149J7zvyxV/Te7ZGPILcptUgoQ3rv5Aem2toYFG6HdFfzmtE3DD4bY1Ae1Me2S2xvEaViInLhsRPC7iJiDBrjsbSUUIYcJgaW8j3Y/JbETs/XSQz1WXmXZmA9KdUl3t9Jfr1zwy59Nh94thQvuJQuuzhDi1RUvzWKAWFd9Nkdo1SVcb0vkRT0Ftv8it09LsjjSTL13Rsu9ezDAc+nZd7pF85lEFZVXVRZkejyfS+wfow8/4Qm4ypTLaxiZiF+RrZhOhsw8k0xVuahAwnIxqnfBrnLPmNEUHWqJYKoDflMnEtonxVjVd/PsIZwdxPBGxPXB/SOiW4aOqzkMyuZZSXPj+7bdl9IUF+XWSN4v0hlcuSzKg+dI44tulDHXMHT2mnyd/2tBAdcANonkAkm6olKKipDHJCFrqLakKmPnAYlZRB1TgtbqdwVZfGCY1PhxMxhbt6uKKkSogobSld9TCx+KXH/l9iVe29aBlMSq0bJccjYAudOZdEfcTobJ/Xlu/QXpao0u8oqC29b6l8bqn8NMHt6iTuW0VBgVWZABdvytY9NSZ+p6qWzhIMC1qw/Wl5MxQ2wodaI4/EQlWdp1DaZMNQuhlTeDtNFDrUWxmib9VprcrCpzen3M5E2zJxNaJ5PEH+lkchNWDp+AG7V6bZOCgz9fo2+8+IZZXfDjGRobOk8ITd55NgxvmhJcjuRDRPSOrSn4nJ3RPbPsgo0XpQ8Ukeunh9S7I64vC5iGQdijfbqmnLItLUz8H6N5OM8rIrn/mgI1v2kh4Ar68TIsxrUZvxqTeYNDRPGNIH0vg3zltMoPdoXTGm0nuG/uqIxumxpPuZmChlKd2Eqc9HjEoy7k00lUjSmzXUz8vDVPWupTdrKd3q0582VB9Lk9005O+4JOuS/vgdR5h13hClYfvVDGyn+GBvmeGTXdoXRnRWYoq3VM7ULhqiX61hrULk6qddWisO/UXxT4clQ7IR0100NM5ylLPgDcTwch/OFR14hBaqO9Q0yjqw/7x8S5O3U2Q/yDC8WmI0FRF/rYE516H2XEDtAtTPumQO5HC38eECp4oHJJoOxXyP9Z0JUoeW/EZI7YzH9PsyjRiVLL3lEL9jWfpD2dE0Tnj4bcVIFm8Z+rMWdyj/qihtGBRVU/XmDY1TaUqXPVqnY7ZeL5KqWrLrDq1ll7m3I7yuUdJ03+HOr2fpzViKdyOaJ6F5QiZnfsulN2MEgEfivg6nw6P48cpnhkRbcuhEy0AsPXzlHZ/UoZq+0/9Nl+JNLZj1b/oa125InTv5maCg6Q/t0SjUJnRcN06lKdyTF5QzsoxKgpu6cw6Tn0XiQQwlzUnvW6JSyCDwSCRDji/vc+aJdaLv1gl+oUniTIvgZxX6WzmmLu4zmI7pnR/glIc6qebEBQgzep+DCU2i+hOC3ArrMWH6K3T0P2C4F2+N3U86Crud+TCgfD0gyFnmf2TwflQi+Uae0qcJStcht2HZe94hfQBBIeJn94/Tnw8ppQd4CbF29p/ycQew94osHBNNQ+myp5ot54jE21bt1FoeB5yV1HDJ00okX78rJpfft1SuDSAWPDYsG/Kb6tSbqx4T1yPSe5pIlW5CdhuS9fAo/Cv2OTLU9dsaGzfPQO6ON44Mkl6rdDuivewy9Wkonq6jbNjOqnbSzrEsvVlz5HEAHFlA9qf0YO380kik8AIs/4kaxSgBzZMSTBbW5XNVuB+S3YkVDBFyxADDgewdn/j9Et6fF6n30syk2zhOjO9GdGtpvvUvvIvNRuzdnYTJIdnLKfJvZ/D7sRQRFY2JreHIwaY/ZZj7iUP9jMMw/3BL8NEA/KeX7Oq/+ve/7ApnRVHTTRWRIcipTu3PGBINjU9NpHlxYT2kccLj2K/c5fKdRU6v7OK7EbfeOkZuDdK1mM68gmSbZwVEZzdFJ+zO62fOvhdTPa8JEUY7ld8dE6LP90h9miG7HdMf50I9+DcnhO6KIs+XvifV66gEo3JMelcDgtiH4p0RncUErdUxD3NNVMXK5R6dlTTNVaWTWFcphqkD8Tz9tna/5vmIykdagAcvRqR2PLKbCuIAMcRKt7S4M7sx1UuyBirejkkfBOw9myTMyY0+6ijMAAEAAElEQVQ7Tshx229bEp2Yw8edozIkSoA3MCTq8odNNSK60y5hVsTr1klLou4QXujC/QxBJcRtuUx+It1Z44SGJHFStMPK5wqxyK+Ncee8oXVS729U1DW4++9/RdhTJhJ0lNmLSdUUw5iq2rEuXLmZ3uCBQew46KCkmhagX1F+0dWPj/Hs6XvcvDvHeqM0XmzjpiIhJWX5ssHrORTuRXTnZSvj9UTj668E8g1YiMjsKTHEelD8cZpEQwK8oCDd0rAsr6hE05K/7ZHe9GgvqeP1+pC751C4H48tIi3bX0sSpmHm/YDchkR79QsW6znUzzrk12OaZ6B5WqwtgPap8ChJZPpdiRwBMmveeOdTTWtCy8z7Q1rHdCJYB0rXIbNraZxy6M36BIWxG3QHFn4akGiKON2dGy+sBEx8YcnsGdy+GsGDF0NMqLo6WbN0lpH6NmlJv5fFO9Xm8TPr5NYdOosOo6xDdzXE62n3zN+Vl1Xxlu5xvyLUYPHHgRS8nnbYh3k9EgsVI6e8zpLeSHtZYHKUlHIytxnTPB2R/8VdoqTsHzN7Y1OxKTH/C2sxyZrDpxuL5Kc69G+UCIsRgwmHxkmX4YTkGdYTFzVMOSTrgpDya5b2skPupoLQpt6Vb3/mIGbmg+CooSncVUKy1zH4HU3H+lOCz5INjqQfhXsRcRI68w7tFYf6WY08Y9cwmHDpLBtap2NSBw6br6dJHcLhU9JCTX4iTPcBVbA3r1Cy2gVzpOY0sbDeB5ZGnWNjB5lxA+1EOr4HFWnQOgua2/enNP5tHvNpnBkrVwfCr7Obqok7y/H4pAK/6rH/lIfflveXO9BGkawboq81GW7kuLI5f5TiFyfAr7uEGfBbmvFPfRpTv/Bl41i6E3P/O67MlZuKUn+Y16OxUK2iIMOMpXBfDUT9oo53YYAOiz+2bN+cAiDIG2qPaeyYaCqbdJSTQC9oJzhdOcA51uXM6S16c5bpD4dHCSuyKo8Ylgz5DY0cm6e0o3u9MZRVNnRnXLpzDluvebgD7cyJbow7hJkPRjgj4aGDOdHXTCQIJsgZDp6U6jNKwtSnAfNvBoRpAfbdeWmWEnVd+tyGxpte15DdcmitOiRrhtxuSP6WS2bbMP1xdGSJ017RAitfg+1XPRIt4azd+QTFtZDBhBZm7CmOM8hC9/QIEykRpnwjIr8VHu1yyeY4o3VgOXjcI3ffkRVl0bLyvQGxbxmOv+fMB4Ga3mnLRLaHO9cDIyZVdi+m+tKI/D2oXAuxruQwTmRJ78twYliGxgkHZ2TIrTnySpj/CvFRrQvVx1XLNU849KcthVuSJruBxqrNFQ8KIakDQ2Y3xgnEbUw0LbHvEKUMiYbBbXrs9/KcmD6kPUoy/XFMd87n4GmRWYYTltSey3ACWqsuxXsRTijX5/6sGFJBVoOEUQFm3o/pLElQ15t0RAJ53SfRtOx+LWb+x+rCyzfH3jRGdouZ3Vg+qY/5bHzLY+KKbkiUVD2pWExL85S+Jr2nmi27JTHgwSWf1hmhE61lD3coG6HKZUv5ZjRmhBmyOxFTHyuztTPrEiclo3ZHAvkL92Nm/tyjcE8+WkHWYe2XRQjP7qnBymwZGmeEkeZ2YiY/kXPg/tNpJi/LYim9bzl4wqc7Z/Cbmh6GI4+omaD6hKW14jD1kwTpWsz+v9jHCS1eB/IfbdOficnsBZRua1uNk5ZRWQ9KduMrNEI1ERRuGUrXxTstXxvPvvvC3EYlucwd+0cSpfWnHIKSYhIHk4bDx32Kd0Py6zHzb0UEscP1tTkOPp7h4HFH6tC+wmTLV3VD8velhqydE34aj2smN7BEaUt/NqZyNaJ5zCV9IBpcb86w8U2X7LqhvapacfclLcCNbyTEXp8X6H34hMoWdwSF2zLISNVkMVm8rXpxVJKEZvh0h+LdgJU/7tE6qbRmJ4T0jjgJxXuq+6oXDbUL6qRHBZlxtFZcDp8wSqlegOmPYrqzes8YPfj1c3J16cy7dBYN5c9dBtPQXNXPidI6Sby+pTs7VhUYxasPyobGGYfWCcakcx3jkzMtisUe6aneOBJUzdT212H6H6dpr6hJ3fvWEiY0HD6WZO8FTR7TWy65dUtuK6Sz8nBr5JFYqLEnsnN/xjBxRTfVWEkXDp5wjrydqheSDMtqLk79d0Pya2IS5ddjamdF9Di84DH4oxnOH9tm4okD/Astac7rhtymdrnS3ZDGGUhvumS2LdaV+0eckDX47Hshiz9U2Kw7RDHsHTUnE59L3ly4q79PfqzftXBXnzf9gWTQklDL/a7+VIh1xXwq3NWuOiqqycpuW9xrOe1Wi2kql2UtFOQtYcYy+EabxkmP7E5MUFZd15sTx7VyWbVset/QPT/ExIbGcemlpj/pH7mp5O9JahP7WpBRyhCm9f3dwDKoCDfuzWqUm90NKd6N2X5FJYTbh4WfjJh5PzqSZNduTzCTb2M+KuAEevirFwyJqkv1vIc7Un0dpWWjFBSkxkjvK0ykN2NY/9sRJv4KHf3GioJWvhHSm1GET+u4unwnMGNrSOgci5Vw7ELjdEa6pQB6M7JIj12Ye2dIlIRrn66Q8kK6tTQHzyvjPkw98FFSt1/5IlTkT9/gjiRiixNQPeez95xPsqZdWBfa0lpFvlGPDUjXIkYFLZogq1l846S8l8KsJXVgWPhpQGcZErselSuWQdnQPqY6s3wdWsfV9OTX5Gq3+xLUzjk0LgVkdlS3em8X8MZZAsUvXHJbMW5f36O97NCbE5Zaei9B6hC6KxGFexCmPZwRTH8UUH0xUD7WgpWZRkqeUuZMh8PH3DHeawny8dhZ0aNxymH6A20i2R1LmJVHVeYgIrsfUbpmuPv+Ms/9ymVGJUuQ18MaZkXqjhLqMZrnIgq3IVmF9EHM/vOW+tkxFfKzFCt/1HqoNfJILFTriEix9bpSjZ1AKs3umFWfbGpEWf7CULo9IkopZmfyakhhXQl+YdpQ2AjZfiVJUNDiaw8TmETM7M/k42ndcSBtxaV4Jyaz1aN1Qjtja9kjfWBJHWjOPioLKks0NEmZ/ijixG+3sJ6Fls+g5FK+GZA61ENkYuithHgDHesYuP+rDl7XHPn/BzmhBekDQ+O0HoD2yhgjPrTExQC/C6XP/aPmyQk5yhbtLgnhmPp0hDtQbVy5EtE9OcLvyoGkeMMlShr2nxQktflzLpNv+bQujMhuKsHQ66lOdj7Pk9uwFO4PaZyFmfflUtObGZu0ZQzzP22QqkcMSi6jnMPhRY/OGNLCwjvfv0iiPuYJJJVAY2I9ZGYsUT98ISTRVgrL5IkaYU6NZ+lOyP3v5h9qjTwSgH9qfsk+8dq/y6AsrHDySkjtnEf5Zkh7wRPs0ZaeyETqUMOMMERvMA5vWHV01PYNiYZ8TTvzLu5QILMaCU28+lOG+FKb1Jt52se1k1oHSrdi0ocR7SXviJbWnxH2aEIpJh8MJaxRHT37bpe7v5rBifRwJOtiSKVqQgAmvghonPTxBkId0ocxe88bklUNA6KkEI/6Oc3o0/vQn1GJMCzJnii3KdVrsqmyoXJlxO7zCbyBplj5jZjOokN3JSR/yztKdgmzQk0OXozwGy5eV+JGryOc2sRfogi5DXMkme5Pydp84nPtfLXHY4o3XMo3hjROJo5m9GHGHEnLoxTEnqV8QxqyREfWR4lOzKAsLnC6FlO94OKORA8s31Re1fv/w1eE4e+EOv6HZbF8Di96ZLdjdl9Qukn6QErK2AMTW4r3R/QWLFOfhwzLdqyutMx8KAqc9SBIOzTPhRTuB0rgS0hSHeQMhXsxzud5WidiJj7T/H3imnyT2gse1WdD6heVUJfdUhrzqPhABCf3FSfQgtp6LUvxFqR3BAd1liyJlhqvznJMkHMYVgSBdZbH7KiaFml3XuTrKCmicf6+ZTClmjL29XCmahZjdYzK0RoOnkwwKiki08Q6TayB+T93GExaMjuGwkbExPWI9jHDqf9+xOSnliBrCTMx0x8H0pQVhBxEuZjOsgYAJrZMfzKg8qlYTqlmTPmqQ+PpEe3lhKC7BXNUcqQP9L5NKGVwb9phMGkYlKWwSB/K8qj6dER6byC4cTYm0WLcFH6FalTrQuSrTkrvSSeeaMcExYjDx/Xm608HZPZi6s8G3P+OjsZBSdLoYUVpcAePK4JS6ceG1I7H/lOJMStKQV5uXypIkUEe2OQY2ksO/ZmYZNuSu+2T2RYWmjlQmK07Gke1HxNs5XeVTpJfj4kTclTuLIkhFSXVSJWuGYKsw9ybQ7yefE+xKhMyBxG5TUFNiZaV5PuMwhqapxUEl9sJya+H7L2qDKmpz0LKN2L6szHFm4YgbVj8fpPaYzHlWxG1cw6LPwlonompn3bpl6Wa3XkxQ3/CIbNjSB24DMouw4JDfzYmv2bJ3dF1TNUsbiCv/uoTluZJh/1nNITx9326C4agaJn6TM2WE0B+PaK7oDIsvf/lQMLvaThigpj6eTCBw9ovZRiWDXE5kHEz4r8+zOuRWKgmlNlsnFC9uvULMY0THpWPXKY/kn147kaCMGXwd30mP1bSc+yLQpesG9qLHtaxJDqW7E5Esm6JfZU15VsDMCKodJctg0ldzInrGhHmNnUsu315rEZjU+F0NZan0lA7aWfJsPo7TZxQIcPZbSuqW5KxTEUiusG0pXgb6pciDp+OqZ1LMvl5NDYEjnFGsP2KJjjtJZdRTk1jUIoo3RAv9/CiR3PVx4ks8z90yN9zqJ7zqJ9xKNxyiNLypKpdKlC45RKOuav1kwn8jkqG1knZoee2FR8ZZlTT1s8bevMGZ2TIHITCqwdQuwCHj2mcnFuTzHn197rEl9qUv4D+sRGFO3DwuEd3Thq29pI0/dVznhZ0R1OtYcEZ229myd+DzLbSZdonQypvJhRGN/bAepjXI7FQnQg68x5hyjB4tU3hqk92N6azDBh1/Z2zI6KkIbNjqJ/T13XnDakDUcgwcu44fNISZhwaZ9RRT14OGZZ8/LZgksIdiNJamGHaobtoSNdEKkk0DaO8GqEgp1020bL0Zy2Nb3eVjPKdEul9wWLtY4bK5xpUdI5ZSrdiUgdy/xsVDJWPXCqfOAwnYPc5EY8PnlWmgOxsRGJ2Az2sJ34zxBtq1Fu8Kzy2dibB4eMO7kA3NLOnB603p523N6emKsgJhzaxXLpn3w1IVmUovP+sSgbrCkstX5P0Jb2vWM/cuojc2U1DUIrl6h2KrbXzUo7KP81SOw+Fz8fu3HWY/rAvQ5B7IYMpS38xOjKt88dZXtbVn0H+yylV6YqnRMQJ+dYW732FiNNhRvqZ1ukIczWPE8Lh49ohDsf5pv6esqAwMP1hTLIRE3sylpj/s316s6oPpz6Cgye1QzmLPXb+5QHL//5NRhd7Go8WVZclm7D3/JgHW3aZ/mhE50RIbjvC78CwpKOwteqQvwvH/68Rua34qBsflOVHevikGqypj9SsDScUtZ5oWQZTyqNPtMBZ7TL5eUTuvo5ghZ6pm3eHlkQbdp9P0Tjp0llwiBI62vNbEek9+RJ4PYnxJi9HFMfWQA/cCh/4RnW+3sXrW9a+ayjdiXD7ISf+6YDukhZOqhYrgyuE8vWAyueWKA2FNfF6k1WXzJ6leCegdcwZx/Y4lK9rWtc8bemsWNorSWIfwpSDdSypHZfWBVkSdY7p/rgDnURhRkEZg7JDf0axSit/ElD5IjpSJfxlr0diobpDmLhiSO25pKowmIKJK4JLZt+TdWF+DRJtQTB7z+lGpg8Ul771N6YFXg8M3bFzdPrpKrnsgN96/r9kMtkh7OtJdiJheqMCpHfkJtI86dCd9Tn5PwSM8g7tYzGLP+0Tjil0jXOW5rk8jZMyLks1IrpLMhBb/b0e5ZshJrbSYu0bVn9foQtOoMbIhLD0n3u0l1RDt4/H+F2RlRd+OqB+ztEuuRTRW4hINNTY9GYNtTOuTC5yhvaJCOtZRllHOz/q+geTcPhSQPO0JfFpDm8Qs/JHYtzHSZf9pzJMf6RAjd6MTgm3bzm85NM6po480YqY+Shg8Yc9hiVDd1bp3ekDOX4Pi0o5WfmTgMId5XW5Qzl5ez3DxPWIxJ6H17fMvg3VS+Lajopm7D+rE27iC4W49af8MWH+q3T0jwQZmQi68yrCO0s60mrnPXoLYup7PckXpj5RF9xZjql8bohdseJTVe0uqarl6ZlNcskR/9Ha3+TjwyWcpscoL7xyVBR26kQwvNhjMB1SPwf7T6dlknDVsPdMGjMG+pNVhzApHufuS1IEpPcMc2/B1tez1E97YAyDknag+7/kk2ioc3dCPVyN40nJk0/GpPYdRV4moH46ydybQ1my77m4XYfOijruySshvdVA/vsWCrcVwV69JJQkWZXuKczFlD5OsPy9YHwKeew94+MOYfulNJmDmM1vyVtr8vJQSENKu3+yrgnZwRM+Ydph67UM7uiBDaih/qI4DFFKMNXd33BpH4NhJT5irVWuCCPN7ChDIcgYCne1+xfWQ6IkLPywTm9WCI11obXq0J1xib5KzdQDo9hUdfx0WRmPtY45DCeE7VUv+FS/NmLq04DDx+UX77cdqpe0kz0wRkjWLfXzlpOZfTbuTvHF5WW2rs4Ql0IqV0f0ZtTwHD4vwkfqswz525I1OyN1q1OfdIThrkryGyc0LBCR2xk711lq58WE8vowKMlvNLsdM3FFPyOzIzcWkK9WlITZd/V+d18QFW5QMRw+niRKGop3pEc69rv1cdS6C7GhNxeTbMV4Xbn2uUNdq/aJGK9vyN9z6C1Ydp9PMpoKKd+MSVXFX3BHUD/j4HYcGqccqueT5NdihmWY+SCm/ngkoV2kiPjpTwP605bw2Ta9eQtdj9arfc7+8k2sb8nNdlh9boMoG9N4YcgoL2O29oJL+jCmecaqyVpRP3BwySO7bemcKDD/lsinhfXoyHoyt/1whNRHYqEGOTGHrKMCfzCpDKJhWQvF7Qt4nvhZgv2nfJJ1Q/F+JGXlDfE0Ex35mPpdizvb5z9/83V+5bmPsbmQufP7ANTOJeguyTqx/Kk3Hi4EhBkYzQXiG1Qcdl7RtKR0E7K7Me5A9WSyPs6gSllGUyGLP+yNE+skDZ57d0D9nBkvJtXPYcpIyDeGpVorGi4s/jhWcspYAdC8GLD3tYjukuHGv1GgeiFF+xhMv+3Kddo3cnq2usGDCSk9Szdi2iuW7KZOifxNn8KNJrEPy//dHbyelKCpQ4fyjUhd9pgs0zjh4nY0DSzdiUg2LPv/ap/Tr92jlOvxzKvXWTm9SyHXZ6dbkPy6m+Tm7TkeO79Orthn8IstEheatFdh51tqjIKMIZiICArCqK2RqrUz55JqxHRnXQZTYom1VtyHWiOPxGQqM71kl/+3f5+pz0K6s65qmOkvpydTHzbpHM+z+csRi3/o0p1xSDUsYVKLIshBULTMvhvj9mM2vuWyfHGHte0KqdspiGHi1V2a/RThpyXihGX+jYD2sk9/Urht9YKroNpQ49zefEzpho5c57tV+J0K3sBy+IQ8QRNNSSseyFFMpAjGiStjFUJWpUuU0ntUVoAlVY8Z5RxaJ+SB2l2OmP9zwVTuwNI5BrPvRtTOegxLOk0mrlmSjYiDx5VkXbkW0Fz1aa/GEGssO1gImHpLAWjWHZNL9hTwkKrH5Db6tI9l2H3ZkjxwwYHiLcULNU9AlImhEGJqPt5Cj3/zwpu821gljB3KiT7vbB5j2PcxriUaaHJH4GAyIS+cuMdHPzmroLrb0DwJflcBGsnGWPadVVNZWAtprnp4AwtjtOLK/+Pv/6WTqUciFQVg5sOAxnHlb4ZZS/kLewRDOUFRdc6fQOOkZCdRWjejfCtg5yVvzBWAic88UgeG9m/Ok5mUhKL2ZMTzU/f5/vpZkmML8OrFhOTFdYHO7lBy3mHFUL4V4gQegwnJUuxWkey0xpsTl2XuZV1INSNGocPhU5C77zD9oeXgSUXo+F3V3V5XzCGvK4dq6ziEGUNKmzz52y6tFfnnxx5Urgxpnkwrc/XqWLrsw+Elf1xTjnmtCc3V/Y6Y+8kdn3QtpHreOzIm9vsusWcIMg7VC1mlD8Yxw+mI/G2X3qxDZyXipWdu8O69Vb51+hrf++gSQd/nH956AWMs7Z08JjBMXHbIhoKarCNn7+S/vkOjl+Zus0LqQoPpfIc7+XnSm2KPJZqS5ZhIzWCYlte/3xFrq3XMIdF8uPXxSBz9WBiWXJzQkr+vHT7IiAidvyfv99jXcYoVDppoaLfZ+VeGeF2NRTNbkiGnqqpb+9Mx7WPw2Pl1Pm8sEMdG6Xh1sYhK1wVBRQmVHsOyFkVnzqV8K6BwP2biU4fcLZ/iHdVyYUa4b5AX8F0/a8huOAR52H3JHoWSuWNP++5yTLKmncTvqlb1ehYnEuPdujqGFXVjOLyUGevqxVFwR+IDTH4WyExjCDsveUxeGTH5uWKC8uuQPpCnVnZH9j6JpnK4gjzULlkKGwGZ/Zhk3cG6lvxmRGc1YvX8Du/cWSVq+vxs8ziZqS6p20m8H5Vwv18mve2R2RKoXz/PeMcWZ7b9m/O07pY4qOUJQpe1gzJOT0vKRGqk6uc1Gk8dWirXAgaTmtzltkakDmWu8TCvR2KhuiN7lE3UXTDk1sU4SrShecaSaEc4oaV21iO3LZilPyMXveT7ORJtSB8EVK6ER+B5a9Uhs+Nw/HfatIYp2sMkvf0shTuAUfpedlcykgdjPyfQIgyzYh9hBNzntmJ2XzTMvdljUDF05y25dU2+Cve0uzgjmH5PVu75u3K7DvLKgIpSsPu86jJnJD5ofjM8qk/7UxyRZ4y1NM+F5O/rYRgVRKTpzHsU70sXnTowdOZ8urMS8wU5WRZFKS2kyrUI4i95CcmqQ3fGp7PokN2wJKouB4+rwbp3dR5bT5C77+H8pET2Dwty9O5aOsc0sbKOBgnuSE2cNV9it5lth9TVNINegqCdxG+r9Em0LY2TPrl1yG3JPaW56jP7XkB3AcK0ksGjh4NRH42Fal0FenXnYf5nfXkftcWbTO07tBc96hdl+lU/4+AOx8fIcwMyuzHdBYvXC6mf9o407s5IHIDbf9/DGIsxUotWXwyE9b0bsP2KN04TkRNJ7H+paXdCYYXJhmAsExl2X8jgt8VlHVQM2U0RQtrLztgZT3bo1RcDUocj8vehdUy7cLIuT1djtaMcXPLJHEb0Z2PK15VYPSwb6o9HTHzmjnVYouOBLH+2XvPHRBCRa0Cc1vLNQLGVDbG6qudcupcG1J8KNarM2iMB3mBKJ1X6wOANDKf/+w6pPeG74ZhXW7gnYZ7fNGMjX0vjtK6R3E80oIi9cUTQy4fYrkduUqbKyTo6JctiWA1KDn5X07TGSZ/Jy5ZR3j16zw/zeiQWauwpzcPrGe7976QUHVQ0WUm0VRctfd+S2Re5N3UA5RsxhQ9ShBlDet9QvZihP20p/+I2o5KQg2TNkEwGrG1XqLWyJOuWyrui3FUv+GS3xjvFwNJdsGR3YtK7+nuYtUe7R3fWoXxN3b111F0X70U0LsRaSEYW57lNpbWk7yVoryglcObD4Ejol2jqphTWIkZly/7TLrNvWRqnHZqnLaVbEX5diX5+11K+HRJmLYdPKrEws2UIM5baY4bJK33ZBHUNG98YL+zYUr8U4Xdh9o8TZO759KdE9AFRHyeuhVS+CAiyQi9u//N5hpWYKDUOLq7FpOoRpVvSN9UeU4Pn9qXrTx9IDDgqqCTzWw6DwMPpO1yY3hUXtW+pX1DgWpiGwTRHAsb0YUzjhEN/Sma/hbWvEDxlHdHy0vuW+CBFZl9z4/2nHYZF7RI7L7kMi4bSDUOqFtObUbzPoGLGCRtqYjq/NYffFhw0/a1NevtZ/pMXfpf8D7OMinqC+xVHi9BVbE6YNhTuwd7XIxnlmvHxuiRLn/RhTG9WkpP8pmrVRDPE7Tv4bXXug7JD/ayofDgqZ6yj8ewD95HcprwKujMu6V0D1lA7JxJy5TNDen9EdsOQaOhYqJ3xMBaWvj/C62mhuUND8SZ0FpLktiNWfr/G4o9jRiXLcMKQ3vIUkOtC5WpInLT05lSH9yYdrGOon/ZJVS1eW01k/r5D+abelze09Cdd+hOKBPI6huO/3SJV07Wun9cDN/NhwGDCYf7VTaJIySvv3zmmE8oDZ2ZAbez76gylEhhMx7RWhI3HLqSqhvqZh4OnHomFGiegddyQrsfk1hxayx5OoHGkG4wzoEpyKR7lDX5fhFvGLHNlGMlivPa4/PNHMwEbhyUunNvg//Ter9I4re44SujiNc+IxGwdudmZEHI3fIKsEV2vJ8OxKCkRnMjTsP069OYj9p9OklsXyH/yn0j+cuofHjD3dsRwIsYdWCa+iKh/t0d3NWI0EdE4pRo8cxDTPhMw9WlEZteSPhBZeff5NNaD3RfTmqLVLMWb0DiVYFhWgxn72q3aSw4bf8Oy/fMTVC94OKFh4lpElLHkNjUs2fy1kPSOQ3ZT0ZaF9ZCtnxPxvLsob4DsJjSfGDHKaaOonVEK99RnPbrzKhHu/nqBMCWKX2ZHwW/1Uz6Jbx+w8eEC3zlxhcGLHbKfp+hPq8ld/G89EaoTalg7C4aJz9R7pA+ke8vs6vR8mNcjAU+ZEMrXY3qTqgmHJUN+S3jj/osRxS88Jj51j2wU66c90vv2aKKS35IcpTvrUrrqsP2yg9O2TC3W+GJtDqwhuyULmyitJz5/Vz87vW8woT1qALyBkjxGJegcj5h6T47JqVpMf9KhdMXBuobi/YC9Z32IYeelDE4Im788LZhrAza/oeDg6f8xTfO4O052UdPWXHXIrDnsvmhJ7+qmVa5EOJFI404g/LGzPCZrVyHZgOrjYms1TmsRpw69I2eVzN7YjzQx9tnvw/QPEpgopjft0Fk0WMcjf1vKXl0TSXOm/9xXxM9uhHW9MT7sEqUt7oGyCbJ7EYm2Q+u4pXUp4NzqNverE0RJy9XmHO6VHGEWireFgQ/LCbw+QmmSwqrzdzu0VwukDiA9UP2drD3cGnkkdlRj5ULs97RQBpMiXngDuYWMClB7LjgKmyjci/R5FS28rW9Y9p5zaD4/YPDzbS68dpvK6So5f0Sl0sFPB/Sf75L65/Zonon4pX/7DcKMoX7aoXkmon1cdXDsG/ZeUNmQX48oXHfpzo1zTfOGzF5MsjU2ZJjzmPxMRmxREmbe7ZLdicfZWJb0ri5tlDQU70Z4PcWn92dVD6b3LeUv1Nl7Xdh+TT6tD9hZzVMicuTva3xbvjkita+Ul8z2OKOpLYgt9jT6bS3r64OMYTBhGRYcOgsO7ZPREWG7uyiWV3p/fFqckEAwyI1N40r6nTe/kcAZ6eOVawPSewPN9p+oMjXT5HiuiuvGmOkhG7+3SqKpZMX+9DgKvan600Qw94YyD3ZeLX6pOJ4W36Cw9hWi+WFV0/ldS+2coXwN8Ul9cAYOiRac+7/WSdViyjdH1E+71M4bFp/a5uVf/gwnH2CX+iTTAf1Giqs7c+STQ86VdoktjHo+rx2/jefE2GTMP7r8HK3zAcOTA+ZOHeC31LjFLnhdHYEw5oqORA0cFcVVjV2NBVMNwWSpQ1nxbP58lu6sQ/1MgvxGRPeEAsWymwNGeenim8cdiMUBiBJyeyndjMnuxGQ3lTSdrMt1MLehBVu7AKU7AVtf83FeqdObs3jj3ah1zBE55/yQUc7QPh7JJ2DCkGjJxU/aJ+WsducUqzmoyDg59hQXmWiIVJJsxPBki+53WhIPXmrRORFR/L9ssP7tDAe/PMQYy2tzt/nB3TN0WymigUt3SdxSdyDpjnUN9fOWzrxLf9ay+c0SrRUXv2txAsPWa5LipPdj6mce7lB/NI7+ceN38IQrH6gVQ+oQEt2I7JaH37Xc+tenlf4xk6B0J8b8vX2q3QzbtRO8ePwe79xdJVjPQjHk2aV1jmWq/HD7DP1hAm8/wRvblwDItg3d0xb8GOoJDpM5MjXYfkU+VNkNQ2vVYn2H8heAUbjCAwHg4ZOW3LqDNY52j7qw1N6MgPsgq+lL7jb4vYi1X05jgZU/G7L3dIrcukPrhMXriWbYWVLtnFsXXc9Eutl+T3Vdfg12X/Ao3IXuoEy2ammcVeDa5Gcj7v+qw8pvOxw8YcjddwmzkGhY0rWY9pIrVlk9HqfOKJc1VZOLiomheSxBexWC2RF115I2ltVKjb/1t/+Mf7jxMl879Ql/cP8xguMDnM0UTS8itoZRK8nx1T02DktEyYhO1id736M7D9OfBPgdj+K9PrUghTOS/dKwrNPDxA6NU5a5tyNy2yO+eIg18kgsVJl1yf++ftEy8bmO4v6EzBSilME6Si0OCha+3mQ13WW1UONwkOXy/hzx0GXlsV22Dkt88kfnudyD1sURpU8TTN8J2H/ax+trIU391Kd+Dia+gCiZpXE2pnzVYF1L7emQqXdECaw+ZsmtG9I76vhbpzTLD9MiKUvrJJwUdJTGvmpRrw9B2jD/U3FVDx9L0T0WMfszQ5xwmP5wSHs5Qe2ivmeiHWEil1Rdqs3GWdVvg0lD+VosbqyVV1S051BYi9h9PsGx3wvYf9I/ohQmmhoARCkXr6tFnVtTSZBbNwymLMW1CBPLvK1zfsTMD32Kz+0xle7w/voyX3y2wg+TXTYOynQmknS6KV4+cYebE9PUWhnudStgLLE1hCOP1I2Ukl9awmu7sx7Vxy3NUykSDQ085n82JPYd9p7ziX2pEA6e9HECH378l6+RR+Lod0eSKUx+3MIZShZRvBvSWTRUrgjLzK3pVw1yltEnZX7v1J/xevk6f3P2UxxjWViosf3xHFHgiNtZt0y96WMiS3du7Eg30lEdJzSBaq0KuJ//mVL+RkVD7paPO7LECe307khH5KhoSe9I6IaF3rQh9gz9J/qk9wQ7JRqWwVwkTXxOdff+Mz71Mwl6s5bCDZfetMPin9bYeTkpH//7Dt05wTSlOyMSnRhjhQE7ASz9oEPzpEPlsxbL32/LQMNIk5RowijvHEV09mc0FJn6fEDxXkB/Wtqyzoo9UvoGqwNqZzxStYjcJmRuJhmWHHb/YJkbtWlmy21Kq3V+pfIpMxMt7nSmeHZljTdvnWR/v0jy4xy1f3AM03Op/ukC03+WYDQRk6or7j3MGA5eCUjtO6QOJP+ZvBzSn/IZVFwmL0fk1gXXpQ80qXuY1yOxUMPxNGjntSLWk+lEb9qlfFNmX95ACyZZh9l3oHgn5vjv/pu83TzJ/+0H3yGIXOo/nSW9Z3AOEnTnDYdPx3TnzVFUzqgkGMZYsdWzmzCYC8ltyA6osxLJrOzQ0pmXl31mRxhqoi1GVelOJMD7hGq5IA/5t9LEviQjo5Jh4hOHymXBXaU7Ie5Q4Hj+/pcg+ea3JyjeiSndimg+O5BL9YElyLl0ZxTP86CLv/crWfrHh2x9o8jmN/IMyg5+R0B6b8HSOCXHv8FkTOUzwWWHF1P0pjxyW5bW6ZDCHVkEzbzXJvdRmvSBpfqYGPaJNiSbMb0FS2/oU//RHMPA4z/86FdxjeXy+jxfK98k+3kK2h5eF7Ze80hWFbbWnXdwB6IhdlZDknVL+UPNRfuzlvL1gN0X3CP5S5DW6RRkVfP35r5Cs34nUPcJsPK9gNZxQ7IZc/CkoX7KJUwJJ21eCNl9SRhi+XOHP79xmsy2g3m7yOixno7Gmgx0Ew2HUUHOIqOiTCmO/w9qYjpnR9Qfj1j5fcvBs8BzTVJ7wg9jTyrSVC2mdWkkWfGMJb8R05uS+djkJ5D//TzWGfuQTskVWrzOIe1lhyAvpn2iYY9ynAZzEZkDNW1OYOksuOQ+T7H5TWGP9dNapCaG4r0hE9dlD7n6j2H+DUURRWnoLuqapfcMCz/tiYC87lA/JxtLjISHo5zBGTnk10P8tmHn5TyjEv8zY+P8psLP/KYh/acFAHqHGTJvZwn+mxn8eyn+n//9r4wD0IQBT31i5TSzKzXGxBVLuhYx8akrk5AYwic6zP80pLXikd4RXyHRjGivyH7d71q6izH5+w+3Rh4JPmquvGjn/72/T+pAoH2iYQmzRlr8A7Hl/bbiX4KJGKfnkKpK/JdoaEYfJ2E4ESmHdFJZSXHCULo1Yut1n2N/0OP2v5TCaztMfyBMNL8R0jzuk92Va5+JxQ/I3xtnCtzps/90hmDsP2VicPuMlZWQX5NXU2fBOTIgC/IqL1L1mMZpsfqjTDyeYn0Z6la+GWEsdKcdfb+s3qeJJXbMblrS9Ri/E7H3dIL0gYjYCz/tUz+TUlTO7oDahQyjgurkiWsRW9+K8evekVYrSn3pDpPZjenNaqGMSlC8HdNZUvJLsmnpzqlhTB+omctuDdh5MUOcUBPkdy2tk2Lum1AY7LDk0F5WkN2wIjNlrCZRw5LojvVz8kgt3B/nBNixMd5WjDeMeet3/49fDaeUKCkT3vnv7WAdcSnlWixdjztgfBwaVv4wxuvD5OVAxmOTcp82ASz8uSY8yZpodrEHifqQ7IZh/dsZElWH+TdCdl4xdBfE8fR6ltpZl+6xSK4koWrkxinYey6D19WEyjpfEkQexHm3jjlEvkwoinfiL72gFuXblL8Py382Yvl7MaVr4ieAbmL1gktnVsI96xoqXwQU1vQ9Mrtjh2YLOy8lCHOW3E5I5WrE9itp+lOqae/+WubIaWXu7SFBxqHwhc/Ux5bMfkz12QgTW1rHxH2o/tKA9IFwWaWUaOI2mDAMJuSyMv/TNrEvaU3zRBonkC1QbjtSfFFNk7vuomH/aYfqk5GM7PaFHfsd0TB7s4bMfsxgwjD/pjbDxkmRdx6kVkcJQ2vpKwRPxb4M0GovzDCsiESRPpCs1+0L8E/WtXj3nvZJ1qB63pcv07qIzDMfBXRnPAprEdWLLsm6LsjmN/JMXI/wOw61S5bWisf0B3LWSzZjhkWJzUwoHZax4wzRrHapMPWl/inZihlMiL3l9UTNG1QkzQ4zqgO7S4bCbYN1LIcvB3h9n9pFg98yJG8L7G9ciMC1jMou8X2H4lpIkHXYfcVSuCkwv7QVUj/jkdoX7bAz78n+cjCOhFyLcQdyfNG0LkFQkM9B/bQ+PvWOyoDYtyz9YMTm62mihHZt64hIk2xENE76+v1LEY2zOUwIjYuW4g3V6E6k5i3ImSMvWRMDDsy87TAo63cyVqdF6kD8i2FBXlf1U658Xm/H9Kekgq0+bildl4fCw7weiR3VHSksN3bldBymIHZV66SqInf43bG1+aQ628GkJcoIeA8KltS2wslq5yUpyezF5LYj5t4dcPiYK5+lwZim1oyUdpIxZPYjCvfsuMMXb6C7IIue4l3tgE7A2MVl7Ll0tjuW/Gp2nd2Lmfo8IPbHNkGuFAj5LxKSE3fMeN6t8sJvuVTe94TNeoprjxKGuTcUGKzUlZDK1eBoLDosK1i3vRoRJcd0wZQaKr8N9ScivK4WzLASs/SDFrEnk46ZjyL2nk2Sqo1DOir2aJS7/YqHMxKl0oQijlsXpj6SuYXfV22pCCOxxlb+eED+vmXurQhrhNO2jykcI7tlRQNs6XeOE/IMeMAR7s5bEm3lFzzg9j7M65FYqM5o/MZKhuKdSLSzpOoiJ4DpTwM6S4be8RH5ezo+chvKT5UE2rDz2gT5dQHp829GuIFl71mHjW8k8XpaTIU72kl2XvDGpYGh+pi0/c5Iu2J3SUZrw7Kcrac/0QIclgyJjryrvKtZhpWxIW9JD1B32mPyM1m1D0tCMtqnQvqzsW5UpAUwysv1uj8j3meyrkYsVYvoTTqMyoqtjH1HsNxOBI4sg4I85O67Si0xqokfcFZNoK0pXY3J3Xdor2aVzxrCwRMe5ZsRhbWIqU+HWFcPoTtUboJ1VZeaSFOt3pwkz9ltjaajtEPzhEP7m1260y6bP5fm8Osj+hMujTOw93JMqmrIrHsMKiKiFNZDvIGCOQZlpdr4bXmo9isOgxnlBKT3v0J8VOuIVDysWPafcZj6WDdoMCm77faCcNC5H4gw0XgsPLIxb5y1tE5oIdQe05HUOO5RPeeS2ZW+J0rrGDKxjNjihG7uYFoXrn7BMvtuRPlGhNfR18y8H+O3Lbsv+My+P6KwFrPzokeYlTu134bibcvkp5buvMOoZGitjkNtm7D0gxbZ+6Lp1c84hGn5D1hXIL6JxIVNtmJqFxSN3p+1LP9pwChvuPe3LbsvegRpw2Amonx7pFp9PqawruGH7DgN5VsRxDpum6supTshtQuqu3MbMcmqpC5b3w1JNEdMfKEa2utb/I68/IOsofSFobAWM3k5ZOJan3Q1JnahtSRSTeqdHE5kqVyNKL2bOLKTn31DD3uiATjQPA1+JyT2oXAvprMkxpbXk6SoeQoyWw7pXQV+PMzrkVio8QPGT9eQrApI333OpXDPUn3MUL45xB1Ce8UhTMHUey5ez5LbDJn82JDdlFN0oq7E6eyeGq7sbkx+TRcTI4A/MXbhC7OSEYcp5QfsP+3SXnSpXNXxVr3ocvBawPSHIftPJRTvsz4Oxrjo0l1Q/TcsGLzO2CayxdgHCtxaBxPDyp+EJOtjx8K0hg5BTg1V85Qj71AeOKoY6meS6vpvJBkV9TMqHzsQyykvu+HQr+gkyW7HDKZiamddyl+I/BIULNWLHl4Hpt4XKWVUhPRhRPmdBGYYULtoqD4nMojfkiY/KGhnHxYMzRWPzlKKw8fccTyQTrhk3TL9Tp3qBZd0Vce9iZU6DRoB96djZt+NWf+FJJndsaetFTOtdUqNaHpPm0ZmX+jLw7wejYXqwXBiTFNLSSeEIwaRCWHvmRTNM5GwP1c2kXHCcPCEzyivI3tUAowWQjw2i2itOIQpM3bfi4kTAvMr10KilHbi/owhyKvZ8buWxkmHZCPEGqi85XP4mHeUkZTfCEnUFSocZ2JMrPiZ9gntvtYTjGV9uPt35sluxXTmfHqzktZMfxQoO3VLxrz5+9pljOUIfxxOCGbyW8KEHwDjh5eS9ObiMWvMjHFYh+IN1cnyzIoZlVQ6hFnJYLK7sbBVpFDYfXWC0jVY/a2Y7uK4HClBZme821fEXd1/2pBfU0BanFEIb2fZcPh0GYAoYSh/oRghbyCX7uy2ZfHPI7ZeVzx79TFofH1Aat9gHUN6Vw4que2YwaSldt7QXXy4NfJILFSvxzh0NibZUOd48r/cIrMD7khTmsJtF6+nGypLSQge79BZloDN62i+XrqjzjKza5l9b8DU5wOpQaddJq5olBhk5E8//4YWTX4jZvH7NQn8Qoh9h8yeHYd/Webf6DCcsLSXPVJVS+OMQ+LQlYXkEObf0NGZrCmgl/HC7i7IJjy7JSXBweM+6WrMKG8YTooS11lyiH1LfjMmyFuGlZiDJ8WHTR1C5UofHB35Sz+I6U8Ki41S6phLd0aYALxuxMa/GLL8ZzGNM6IHPgjLDTPQnXNpnJYqtTdn2Pj5BOk9CReLd2JKd1SLT14OMLFl+QeBLOdTkNiX63f+vqWzYjj+X6/TnXOUJztWTSSahs4yjPIuJ39zSJiG/Jph8R97GlcfE96d2wnZfVWyo+wmpB6yRn0kAP9Cdt6e/fV/j/0XLeQCZv4sgdcXv7JybcT2ywnF7cwbooxY797A4vdi2ovukVJyWBZslGzGNE86/3ORWVb8z9hTjRj7gpzax6F8zUqunIDBCx3cKznxRW8MaZxMEqZFrEjWZeQbZiyZHT0MxspPqT9jmf44pjMn3wEn/JKkYj2IXZm4pQ8kGfHbcgyMUhYnkgf+/JshUdIh0Qzpzfi0Vhxym5baxbEj9XpM9TFFNoIWoHUNw6Ls1Z2xVDvRFKRWuBdTvSirzv7MWGZeVz7XYFJNT/lmRPO4q9r3jkN/ymJ9OP7cOoPQp9rNMLhZPBqwxL7I5n5HHN2JawF7z/vk1iyNM5KplG9GtI65dBfVO+TvOnSXRRIfToyx2GWZEXcX4PZ/9BXJQrW+Q+FOn9IVh/T1FIOyw/5TMhw7vJiAc22ap0UkKdxWeJpc5CI6X+vx7P/+E1rf7KpO7Km5Wf2nVenmHS2a/H0tZFmwa0jgDaUJ6k07khsnwPs0R+xpd95/KsWg8iXWVz8vWCd/36GzGirMzOq49tuqrTH6WieA0h1JTcK0NoPsjvJEw4x8B8KsZfqTWLS+lqFx3Kc749BZSFC81T0KBk405DuQX+uRaBiCrBqp+mORFLtdyZsHU2LYx56MiA+e1NChfSJm5gOVLdOfBEeTviijh2w4YUk05N86/8I2/qkWNzdnqHYzTOc7PPbibZwRLP1oQPmaRIBe39K+OKJ5wh9HARmy27pQtfMuU58OKdx2SO/KarJwRw/u0g8GtM4FWN/Sm9eJ+TCvR2Khxp4hKCbGs2xlncbj4h0g/6c5vJ5S6JJtxWtbF+79mkc4cnl3Z4UoHI9Ai6r1bv5rZeU1FaB8I6KzJHwyyMncoX5eaXMzH43I7MfMvT2ksKbomelPQtor6v69nn6PytWA6Q/V9HTnLQs/EiwWZpQ3CnD4hAYVo6Lqwe6MS/2CZf5nEaVxpKMzElbcm9ZxeXjJZfKTNu5ID1VnScysm383o7HscDwQuW658+tZKVrTqsPnfqohRqpqx+nPWqTpfUvtiZiFN0JaJyAqySrJiWAw4ZJsxvRnDQt/Ls7DzPvSjpmX63hOzKCfoPheis5ujs33Frh5OE37yQH7T6RJdGI6y4bizS7HfhNKt0ak9wzdRcFc7hAWf9ileiFJoq0NYfbdgO6CkI7DS2kqH3iktyVwfCBv/8tej8ZC9XUBY0/692RLkEV/2tA+FY6NDvT0D4p68oNLHWwqwk+F1Ks54pZPsm5pPTbSHLrmHMFBXl8ueX5bVpXzP7OUrxhGEzF+Y4h1DLsvJmked+gtxHKg2xqz+8vaCfae9ceMf0t2W3mnyYZY804AE9cjpj+I2fuFQBKP02J8ZTcctl9VOdCfkjMgY6OzIC9LoTu/kad8QxE3D0zQZt4x5NclufG7UD8n1n5+MyLIQWcF+lMOcTaidKOjPKhtEaKHE4bCTZfetEeqanBrPmFGPUB31qEz75JoQJh1mXlTfx9NRrT3cjT6aWys0qj8mRSiw4FPOjekdS4UNyGGW/9yhvaiT+NUQinY+Zjsth7S7deyJFoq1YKceKdRSoa+D0q0VFXqWPNw6NSjsVCdUMavo6KMteqnZNc9KsdHAWZugOCgCcPTr9wg6PtMzTbJpoeYnsfsapXmKZj7oUdmP6Z4L6Y7TkruznrjTlnwTnpvJA+l6SHbX89TPyeScnbbUvpCn9cdp36kavr9UgfiJDRPOXSWhMXuvxgpea9vGeUc2osu2atJopTUrpXLXbyeJZxSCrTfFcmGsYeriTmq+TZ/zjkyaBtMxwQZw/6zY/x1nIidOlCkZOlmTOWybnTpc5/DJ3IK2CiZLx1fRpZUQ6kp5S/GYsElxbjHCVEQY0+lUG/B4hVHLB07JP6DCunrKTrLUvPm1qFc7OK5MV5hRJgxjEoWr+PQOiW/1/LnLqldl92vWaKkFt/hzw3pLUZ4PZh7e6CA4Lrud/mWfrZ8FR5yjfy1rb5/hpc1UoVOXAvx+lbpdAPL/BuWpT/TuLB6ydI+FpP42iFX92cpVzqkvJCLUzvk7rpUP52mdFNTj+ZxR4nUP4pINCC7E9I5O1JkTRnu/lqSIA/+3fTY+kaWOJ1FQ+Vqn+JdmUnMvt8XqtCT+a87tOTWLHE6JnVomH7HPRqjHjwf6/uXVG/m1wzVx7Jk92NKHybIHERYR8K3UYGxw56S9qIkFO46mNgyfKpL+YoMJ1KHIsikaooqSldjhmVL9ZLqVBMKN/YGgpWyuxFLP2yTqspyvXbWOxq/hmkYlixuX7X4/pMe2Y0eu98ICTMxYdcn448k5vsiZOKKYfN1KR2an04SRg7GEZ66/IOAZN3gdQ2JukPjxaGs2ZORJoY7Me5OUrv6rKF1LEnqUCyuKAm9SSkZTKwH5mFejwwpZVg2tE66nPjNFmu/XMTrgjt0aB/TjjL3dszWrwTU704wf2af6Uyba3uzuE5M59IA2/PIr8ccPp7AWOGpmftN6mcm2X/ap/KupTdt8FuAVceeaAvLK94Cb6B0wDu/nsIJDKmqYVRIk6hDc1XTpvayK536nsuoIJFc4a58WUtXHWoX1MWb0NB+qYd7N02YlgFF9bzHqKiuuXTTUjvv4ncgve0SpaA9IWK4uZOheH9I5sDDCWKqFxK0j0HzlALNnAgSVfC7MQtvjDh4InmkMG0vufRm8oRpmP4oxO+EhBmXw0s+TgOSDV3LzK6yug6fyDH/pzGtVZcg67JeLpM/sBxc8ghzFr+rpm00FzCX77LfytFbCtko+kd25yaC0q0EQcbijhJCPAJY/HHA/V9xmH7XxR1ZDk6Lr9pdinECKY6DnKGRe7gl+EjsqCbSrDmzY9h7vkjplhg7/Sm5wM2+H7D5Tcv8TINnnr7F1v1Jru3NMmgnub85yeJsHRMbkod9Yhfm3+qT3wrZ/MVJKpeHDKZiqi+PSFfVLHk9kTW6c+MkZVdBZTsvSRoy+25M6kBPf38xIr8R0zwXScZdskeha37LkN8Ij1JUrAcTlw3pPch+kCG7pSO4syxZd34NstuG1qpKDWtEsknW5GfVnXdI7xk6CwnqZzwOnkyQ3Y3Jbmjq5reBGFqn5eJ39285VK4GtE6oc5ZTYUz5Vsio4LDzYoq17xoyO5bqE/FYbQrt43DweJIoZTh8wmGUtwynQnwvYv/VkJmPAiqfWUo3VHqVJ9ucKByS9ENMZPC7hvmf9RgVdfT3KyKYOCM9MDtfj9l5KYHfdBnl5c2V3Bf7a/lPZSbcWnHGgRxfJVJKKL8it2/pLVgGZdn1pA/FWTy8KGe9QnLAB7eOMbHQ4O+ceR8vFUJkOGjlWD61R+c/7tM7OaJ5LCW3lREcXhLVKH8lSeOUQ/lmQJC3kqFsRmN7HJlTTFwxlO6oDW2d0O9mMyG59QGVjyUxKd7myPJnVI5Z/7ZD79ke5Vshy382ItWImX2zRvlGQOZAOqLSTZlIeD3oLojA0Z+xcrPuGcq3A2LfytIoofSQYcmSX9eI0cQyKh6WLZOXI0yonTV/w6dxwid1CFidOu1lh93nXRKtmPS+JbOu/KfUvkuiY+kc19enqpqWJeqGRNtAIiaTHOHnR+w/6dOddxgWDeETHY6Vavz07kl6gwSJuqI8N38+Q3rP4owMpbvB+GGFubfapLc90vuaWmGEmgynQwZlh86CR34zUtjynAzbHmqN/LWsvH/Gl4ksh98ZALDw04DGBaXlxZ5Aba8PJCNy/hDHj6mvlfnB3lmsNbx47g7Dvk+tl+bgoxmmf+LLHuhQsYnRy03KV4UUzL4ns2CvZzjxmyGtVdVQqaqlddzQPA2N4z4mtgRF1VD5K0nufyctS8ekIUqIXFK6HZLecyhfcUh+lqF2xmP3uSStFZe7f6tM7ZyvLNaBauTegoKG45Rq8ERDMYyxD+0Fj2RdKSi9WXuE2x48qe47vxGKKTYbktnpM/WBQ+2spDN+V5p868jkOChaxblPuQQFlSy9x/viz44subsuxTsaOlh37A7Tg8LVBEv5BtnMcBzhDt53Dokjh9u1SQq5Ps7VHKNSzInfauJ1xJCa/sgesdGCnGXn5TzHfqeqoOVIkUqFeyNW/sAy85N9gqy4BU4oGC2//nDd1COxUMO0gY00M2/WWftFj/LnwkSrT8hZJNGyuFWfDz86hZ8ISdQctmtF8rk+zxTXAGhv5ynd0JEeZmOSzQiMZfn/HJGqxWNysQjCXg8OHk+RrFqCvFyRRxMRflOMosYJj9x9l8nPh0QJpbCALB4fcGEPnpStUGEtIFkXvc8djtnrh2pegqxKh51XPJKHhnQ1ZvJDh+6sQ34jGh/VhjBrGJYs2R3LxBXGcToGvy1stLnqEycAYxlMpZTvlBZqEKbE0A8KQhqCiZDSNUNnRdxWDEz8OAUGDi9pgrb/9YAowZFTSfGu/n79d88wDDwyrx5QenGPxXyDZ1fWiCKHxkaJytUIt+dw71dLCrFb8DCRJdGW9XlmT4jJvV+vAILPyl9A7XySw8d9as9OkaprOOD1FUv/lYqYdEeQWzNsfWuC+TfGrHQPitddpj6SrUyUjUkdOjif5RlVYqZ+O43zJ2X+0x9/k//g2T9laqVOb1ocykTTYVh0OflP2tz7W2WGJUmLyzcDxelMiBjROiloqDcr3HH2vSFBTmPORMuy92ySwUxMbq1PqmYZzYZKn7sBQV5s/cPHEuQ35SadbFia50NyW5EmX30oXXZwRhodNk84JFsxybpUtqP5gO5x2VKWbkLtovDX9mpMmIbCfYsTWExsqVwZ4h1ql36AN0dpuWRjVC+7ARz/zZgoZZj8PMIJVUMevhyQPhBrqfKZxTvw6c/GpKox+0/6NI/J3c+E0K+mObxdod7O8MnVVd65eZzBep5E1WHvWWVKeT30MLU1xk40BPZn9mPdu6Qe7PRBTOvE2N0vEPtt/xV5ttaeiMYoxldooTojOXtYR5KHzH58dCy1l7UzpHY8+gshmT1LetMlszOkedryN178lJ/UzlCt58CR+tIaGJQcdl4pMpwPxKAKoH5K8pWZ9wLqpyQpGVQsqUN5Xu0/ncTrSwhnHUmnC7ccdl7J0p90mPhASSrDsqF4Q54Ao5LcoPvTsq+c/ZnD9jeEBFSuDMVGqmqXTR0IPotSWozpuwkyaz6NiyGHT2s3Te8ZEk3niCVmXd3krdcTJJqG1gk92GEahtPRWBLDmJUV01xNEGShO+OQ3rMkDx0SOz69aYHvxookU/5CGKoTih+Q3wiUmBIZXn/+Cscma+BZ5ufqJGsOyarBANZT6dJdiqmfdYiSwn9HxZh+RTt06Yaof0FOvgyJpqWzHON1YeYNF68LxWsetQsKQ36oNfLXugIf8hUlzTgEVoB+46RL8Y5ql+GEuKXpA4szUHZpogW3/q5P8tDhh/dOM4g80pfTDCYsg9mIOKG6rbtoSW4lKN8c0Tor8HswLRjKHcLUp6EsEHMaaUbPt5h6vyXFaShlaG/OHqWZ9OZETnGHaraKd9QVRwk1Y71Zy94LltJleZRWLyaPdmd3JLJM9emI9qr8UJWaYnGyAeXLqn2DrJQN2S3VjvWnwiN7zeFkjHU18SndjvEbLp1jMeUbwVGIL0Y2j9Y1CjVLqd4elbXDVy8qQ2BYMoTJcXDEJhw+npANe93lzR88xs1rizx99h7b9yfJ7MoPa/rDmFE5pnbBkL/rKPWkFjCYNBRvGQr3QwYll0FZG0Pj1YGiOLsWr2sYTkDtotCRwaTq6vghAdJHYqFalyNd1IOk4+rjY9D7QNKTRNtSvG5I71ryWyEL35ec4fTMAZ/eXWbweI+pTyyTHzjj2bMht2GY+TBk+2s+6U2X7qIu2P5TPpk94YWZPYXjZnYtxd/JsfadIs3zIamGQsOcQHzW3Z8LGU5HFG/FZPci0vuG/e8OOfbH3SMiysqf9Miuu5jY4oyNJ4KcYVRQYzioOCQOXNW5E0pMmXsroPJDZUpl90LVmimpbk1kmXrToz8pcD2951C+ashtWhqnHHXMBvaf9AkykK6K05DbDmmeVsJ2fg2ymy5eRyJJE+sB7K5EFO8OCPKWoCDSixNC4nyT6EQf61rqwwwXzm7QeL2P17P0phzmfiaOQpjR5vGA6D0sGRqn1APMvtNmOAGTf5YaS9J1eqX3oHAL5v+8gd/S7L83+xWCp4yFwjd3cQLBPmFG/FQcxed0Fyx7r8QUNkISHUuQcaidd1k6v8up3D75Uo/UZxn8bkz9rLRFs+/J/rtxQnE+3lhfZD2RVLrz41rLyHqnvarfZeajgMoHLt1Zl9QhTFyLSXQsbsNj8QcK8fXbkVhZGyn2ns3RPC2O6drfkE/qsKQjO05Y8hsifLePR2T2YipX7dGNxoHqxQT1c2pEtl7ziJKW/LoMKRrnNB/vrchxxW/bsU5eDaHfgsqnhvm3BrKNzzpUrljqJ33O/H9aWKMJ2ORnI3KbqiOdUMTombcNW19PK2T3eEj+Lvr9sj2ijs+5M5scfG+Re99f5bmVNWpPRnQXGC86qFwNKKwFeH1JUKTQsGMdWo7inZjOoiHIucQJnT7F+wF+37LzaonBtCWcH5H8KsFT1oHam7O0j8nUNrsjL6fsdiw45bZq1PopH2sUS9OfD1nfrvC7V5+g10vSm49pnBIbP9GEfmXMpiqrqB+WLTMfxGQ3jBxPCjJdGJTUoc98GJFsRmy+7pLbVn5nsmHpTTvUTznMvWUZlCTNaJxIEHsQTCm4wgmhvWpJjHcJ7/m6/PV7ht2XVcfOvmVINkL2nhPQ/0C3H+Sk23KHlqmPLakDh/bK2HfqhwosS+169OasjOOmHLIb4iGYWJzQ+pkkxLD7SkxzVbf01t8tEmYN1fMurWM++y+r9Jl9ZzTe4aXFyq3DxCcinwzKDrPZFhdOb3L7vRW5egOf/Nk5yvNNMtuKSWqdD6if9jl8LIHfjY9UCr05Q+PJEe4IRjmNiwHSezoBmqs+9TMOTmDJbhjKbycIM1+hHTX2xw4hKcvM+wFhWrtf9ZK65d6MjskwDe3l8U7oWCanWjy1uk7YFTbaujiieAvaxyyDCYcoLafmvVdjhovB0SChswLJmqG7ICJMfyGiftrj4HGf4k1D46TPsCJycXvFUrkW0Zt2GJa0gIprAaU7aoZSjZjMthKtg/yYDfSTMkHeUrwbM/GZoXxN3XHtfILUgYM3gO6pEeUbAdltuRU6I9kale7EhGlL8X5EkHHoLMfEvmXiqqh9UULUxcL9mOxejBNKmpJqxGQ29KAGBbHHZAAsqXJy3x3nQ/mMClA/B5MfqKwKsobOikx+P/jgNFevL/HCa1cVLTQfMVgMOF05oPNKj8MnDV7dU7nTs2x9Q/GeqUNLsgH+gU/1CXFNWydULgR5XbdRAfJrQnXCrMq52Xe+Qka+JtT0wsTQWfQYFQ35rZDcfUOyPg4rq0hSPfvBiMG05euPX6d+ZZKPPz9Bes0nKEVkbifozRot2pMxg8WRiMeHLrnrCUZFQ7oe4Q7UUMy8F1C6HZHadYl8mPw8JFWPSbT1lAc5wUZ7zzmkqzH9Gctg2jLKuYQpw2A+ojWe/wcFS/m6JdmMKd4PiY73cUJL87Q0YLVz3lGSndu3FD9PYF2Zi4HA+PLNiO6Mg98xeF2hIHNvKpihcYYjEkduc2wKUXTI7UTM/6xP9TGdIKOSiNqlOzGjojRgYcbgjAyVy9Lhy4ISDl4Kj5qZhZ+MKNwFv6klsZqp0lsOKS03wLV8vXyDqJZk+oMYv616d1AxLPzY0DgtTsbkpz1mPoiZeU/fv3jT0LgoI7rSrbHUpiydlNuH3Vdjqo89XNDUo7FQI4HPqUN135ldy/bLOu5m3u+TaBrm3opIH8bUTyWIPctPb57COdbFazqMihYTGPqzMTMfDCnesSTrDrmbCYVExIo7jBKw+XOGyctKPxlUXGpnlUpXuhOz/5TH4SWHRDse714S34VZ5Ql4XcPS90cMS4bmSZh5S3bpkp1YahcMnUWFZcz90yT9Cfkw5TdCOqcCgqxKm+Zp2VIOx7HpzccCdr8W43eEQ1pHadqdBQc30Ng2vWfG/gMS9bkjZU11p13JV+ow9VlA5bJMgWUYJ2cTd6gGtV+R4YWJVeOmtn06K8I/d16U/0HpZozXdPknf/I1nFxAs5nBzwT8p//drzD9ruHwcTkdxglLbtOSrIU4Q5GADp7KcHjJpTcp/6/sXsTUey79KYf8vS5hVjX33FshqZql8pFL4d5fkVzaGHPGGPPpX/ivZYz5d40xE8aYHxhjbo3/LP+Fr/kPjTG3jTE3jDG/8Jf9DOup+xtOKKO+dkHRLk5k2Pj5NMMJS2q/T+O0Q3fJUrhjSN5JkUoGrD6/QerQkNlySe07HDyVpHnCkKjDzPsS9i39aEDmIMYdQHrHpV928Poxfs+SqmnmXT/jECcs6QPVkuVrwj5rFwxTHzi0Vg2z7w6pXpCGKrcOB09BdifAOujBWNfItz9t2X/SoX0cirdg/Zcc5n6oHa98PSJ1IHOJ9q+26a5EZO/4uB2XzZ8TrOWEkNnT8b/3tFCC9GFM5YuIrdc9Yg+2XlNO06hgGFR8Em3Lxjc8auccukuyIkofqH4eFQxhFtrHpK+fe7OLE8qRr3AH0gchw6mY5klhwcmaVLmFd9Ms/pYP9zKyIjrlSLS3Ho/D4ODwsaQyGOYjevNCO4KCKJvtBY8gZyjfGLH/TJ7Cben/95/2GUwIj65d+GtIlzbGuMAW8DzwbwE1a+0/MMb8B0DZWvvvG2POA/8EeA6YB34InLbW/i8+OpnpJfv46/8O3Vllvj/IkJr58S73/uU5klXZNKbqSnd+EG+YW9ef1ScsNhEz8YnLqGSY+nhIdy5B+5h2lPS+PZJ6BDnZyIQFjQPzdx3aJ2KcoRbfA/+mw+cjpt5xGU4YzDg9eurTLgdPZjGR0vyipIYRQUHCPROpJqyfcsjsq7Z+kDwt3oKcQWJfiSuZDVes/DnDsCzS8Ynf7rPzSpbOqYDCVY1OK1cDurMetUvyg3qQBpPdlktLZwVm3o84eFyJKpk9iRCTddWGbl8Jz9aF0t0AZxjTWpGBxLAkXDpMa8ARu3JxSTRVg9YvaGMYTBgy+/r3B9Gb/RnZe4ZZeSU44YN/szihxr9RSnTA4YTWWepAZUPplkw0hiXDlf/7X54u/c969H8DuGOtXQN+Bfhvxx//b4G/Of7/XwH+R2vt0Fp7D7g9XrT/i68HE6lUTcz20p0Q68Duz8+SrGmXMnZ8UZuaqEx/HJPfCunNiZo39a5L87SSUkYlDyeykq8sB5gxbhn7qofT+4aJz1wSTU2JrAMTVyW8s0YxjLk7HlNvbEsEl1RXv/tCluK9gNgXOcUd6OJ7vfFNSVuaqw5RxpLdiQTmj3SzCmsx5etqJHoLMW5PNozNk2Mpy1VLdtOhfi6DdWHiI4+goJzVQVkE7fSuQ+uk3AWxcPjqiDgx1m6lHAr3ZXo8mFRdn92OGRXkLcvYjbB2xqdxMjGWsuhaFu8FlG+pqenNaeF2Vux4xOzQH/v9h2nDqGRUZw8ht+Fw/L9eY+IKY8sey2DCUr4md0W/LQmR7IeUPJjZU726+2rMqGD+2qQo/wLaLQFmrLU7AOM/p8cfXwA2/sLXbI4/9j97GWP+DWPMh8aYD+NOl/60obvgSKTWjxhMqSMPsl82Wom2ZfFHbUrXFQ5xeNFnOBGTrI0ZRwYy26rbBmVNsaZ/5lF9NhT7KTPOWDLSY6X3FJmYv+fQWpXn6rCiVGqvD7f+zXmipKTBiz8JGVYsG9906S6pxnRCyG+Mx70ty/RHMZOXA/L3oTMvndSoKGtwJ7AMKg6VK0OiTKxk7J+PmPgixm9ZrDEU70VH1uvtY5Jzt1bGfqJjZMQEX2aqTv4sgTuwzLzfk73kqppE68Dkx4aDpwRjDScshfsh2R3L5BWx+L3uGErryk5o7zmXzoKS/JTgHZPZMVQuBySrKs+mP+xqZLwkSC/RtBz+3DKDSUP32R7Wk67LG0rUl6rJuwCgcjWieT6SVWfCkL/tjRURf8UL1RiTAL4L/PZf9qn/fz72/1NfWGv/C2vtM9baZ0whS/lmRGZH8M76t31GpVh+pbGIvlHCcPiEYf1beXozhoU3YrwerP7+kMYZHa0z7wn2yO5FYxxTC7Jw1T/SJzVPyo+/eEfkjf2X9Ug/GHVmNw2la0INZt6PSTZiBlOGnRc8vI5uMEB3NZS5Q9owKsiAornq0l4UH6CzDJXLsnxsHvPYeU3CwurFJH7Dxe0Zcjd92kuCkcKMcMxR0dJbjuRB4CuWsrOg0NvOsfCIFFK+qbzUVCNm94UMg7J4orWntTM2T8o/K7epY7lx0qM7Z+jO+kcnVFCw5Leio13NxOMyJSFxYezC7otKpfG6sPGtrHxVpyKKd2NKtwYcPBuT3o+JD1JgBZs1V1UyjQpG/v0zlsOLLhOfyjuhPyUHl960Q+HuXz3N7xeBj621e+O/7xlj5gDGf+6PP74JLP2Fr1sEtv/XvrHCsgxOpPFo+Qt5jPZmDfmNmPw91Uv5ezCcigmzlkHRJbMfUz+TIn9fR9bBk4pZrJ8W3zGza5n+aEBhPZLR7L7F66vGbZx2cIeWzJrHxBfytmqegtxWROO81KM7LzoMKg6Tl0PcoRxbytcsyaohveHh92PK1weUr5mx459qQb9lKdwV5lu8pfzU2Z8ZJq4YnJFl6mM56Y1KssLpzalWi5Lg9Q0LP9R1GRWVx+p3LEEW0jvekYfVzkuG+jkpSEu3Q4KcYfEnIcf+JzmkjKYiurPiw3ZXIoK8RtUgBxavB0s/CuhOO+TvCSGYvBIymLSkD2LqZ4VQ+G3DYNIwmLYif3cNNh1TP+ew/q0Uy9970FNoNw+ywrz9tsqexkmPiSuQqso0JEpb5t7uipyThFTtr94k7W/z5bEP8AfAvzL+/38F+P2/8PF/wRiTNMasAqeA9//XvrE1sPCTmOYJhyjhjKluenJjT/XR4XMic+TvOmAN6WpEd9ahcUZ4XrIGpRvQOqaRXaoWE2Zg7ReTjHKi1/k9S+VKxGguYFQQEB5mLPvPJOksyqCtdtZl4nMzzro3YyqfR6IpCl53Xv78y99v0150OXgyTXdOD9SoEhFm9ftkDiKipCXRtbQXZKfj9S3WMzRPuJTuRkxclYbKCWDmwyH1J0Lyaxa/G9FaNbh9NUbOSL9nsgZTn8p0YuGNmNy6xe9Y6me0gO//Lcv9XzMMZ0MqH8pyqLNomPjUObKo3H8lxBvID3XvmQROxNFxv/eMR/naOAr9jhoemRlD6bpKr8FkTOGKjzOEzK5h6+tyLEwdGJKtGOIvo42GZUOiMXa4SY5j5WuG+tkM7lCN4F8pjmqMyQDfBH7nL3z4HwDfNMbcGv/bPwCw1l4Ffgv4AvhT4N/6X+v4QbS13qRLZteSOgxY+d6A1IHBbzmSiXQssz+TIe6wpG63dtajN2/JbegoDjNw+Ky8oBLNccxiReC/348JMg69STHjV37HkFt3GBVld2P+QmeeaDOujQ1BIaZzekTsW/qzwlMf+Ije+pey5DcjSncCBosB+Y0hhRue0lBux1TPe2T2DN1ZR8z6ddXGD3bvIC3ScHpPtuLrv5Age8+nP2WoXkhQuh2T21EtW302Ej8gDZvfiegsQ7/sMpgQpXFYtrz+z3+AW/OYWaqTuefjhDo1UlURlcMs4qBu+zLzndbwIR7vssm6xboixDD22G8vOQzmA8KcGsphyZBbc0QXjDVpyq3r+pfuhtRPuyTbVuTs5y2dEyrBMrvSmsXJL8uxKGUo3VF9/le2UK21PWttxVrb/Asfq1prv2GtPTX+s/YX/u0/ttaesNaesdZ+72F+RpjRRT94IsnWq2n8jowawpyakPay/OGH0xG1JxS1k91UqESQV4c7+4ah8dSI3tyYXzmARAtq51xMJLC5cF/UvGTd0luIcSJd2MoXsofszltaTw9oXgr41dff45cev8xwPsCc6uBN9Tl4JWDmw5DiDcOw4BB7hsw9n2HZxwmgc36kVJScNGDuQO+j9picmw+eSDIsQaIbj0MZYpqnoPK5ZVSwMuvditn9WozXj+ksaNavqB1LYstn+QdDOiva9XvzYlD94A+fJX+qwcH1SYKipXlSQ4jUr+3RfHxE6hAyO4ptL96PSDQMvXEDFKVEW/S6hu68IdGxTH7aongvonjVZ/6NiERLu3dmPybK2HE2mKwmc+OA5URbsqLl7/Xxmw7lzxzcEWQOxTYr3o7FlrKCGw+eMrReHfzVLdS/7leY1VHRXZA894GHfuHu+BOs+JlhGqbeczCR1KnuUE9j5YpiePafg6U/0ty9edwn0RTJt3AvZlBx6C5HDIuG3rRH9UnBQbUnIiauwOFFj+xeROlilVK5y/Pn7/A/XXmSH90/jZsJWanUOT17wNNn7rP1dWcc2jaktSynuyDjEGYhfTdBbismWBoxmDLUng9kduHCsOyOZ/Ex+09pK6ufk+rVmrFsuwfZrQGZdY+Nb4/VBg0RZEZ5nRDtxSRhWjN2J4BwfshgPsD9wzJxPmLmPdX1T796g5V8Ha/m059Sc9abG/up3oyY+oMUTmCJXcF+EzcEqdXOGTa+VWSUG7txhyLn9Kf1PkaViNKtkDAlY44gY9h/ztK4EHLwhENrNS3BoBlLg9IO/oOx9JI8ANwhTH1sWfgnX6ERqt8Wl7JyRdbjctVQ/Va8JUudMP1lwl/hpm5yb07epl7f4rUNs+9YgrRDsmapPxuQOYw4eFJyXXdgmfrQIXMY0zoOM+9qGpba8xQ3bmDrF2LqLeVFTiR60PKxFuLAIcZw7ZMVOkGSqBSSeqpG40TyKK6neVyY7LASUztvKL2XoHAvpvRRQlkEBw71M6pTnUDmEXEC7bbnXA6esRTWRHe/9XcSDCZjkvvukQjP631pxd46IRLLKG949m9exhhLamJA4/UB+Ws+e8/Jk6oxTPPunVWZBxdjMjsC7/uzEWFKpr1eT9FH+Y2Y/acdZt+N8HpqDrsLAvHbCx7FtZDBtMI2/Lquf6qmXb5xGjLbLtk1j8yu6tHcZkzsGop3Yhon1Dg1Thvyn6aIUvrew6KhM/9wk6lHYqGGKWgvuNTPuBTvh8LlFgVGN09qJNk4K8VikDWkavIArVxRCbD/lNLj7JhbigGn6VE754mwe6DOuj8pQ9mJq1axO/eVvxqe7zKYiTAjh7iWpL5Z5HufX8QUR7y4fJ/luRpr7y+SXW1y4+4cXjoktsILsTJEG8xF9JcCvI45SvxLNiNGRbnfBTk5XCeblsJtlSTdeS247LalcEfJ1MmapXTFo/yFIbspeM4dyiI+2dRY1wSQ3VK5dL0+TanUxfdDcu+lCfJi4hdvOuz9zgqZ3JDRTEicivH6dmyB7jDKyeAtTEN6X/aUxbGRW34jPvJW7SzHBAWD14uZ+NzQfHJI8aZ4Cp1FRQNNfCHsdvkPq0f5sI1TDv1ZS3tZdXKQ1/XuLMckGjIrdkYyIH6Y1yOxUJ0Akq2xOcKiJ3peHRpnpNJM7xuWvx/h92KihCG3OWL5+4pDb10aEubFFip9tEd+K1IaRyZiWNZUJLsTkaxJ0Fc771C9aOhPGfyOOlzPj/j1l98nM9/BOpbcXIdzx7epTHR48+4J6r00v/Htt+isF5hdqPPsyhrtrQKtFeGCrRNgjWXynTF8lIJEO+bwMZ/yzUgTGCt8MtkQepHbigiKscapi2rorKPF053XgxUnGO+c2j1NpKPeHWkU3D4eMwpd6vfL2LfLuGP4p73sjoVz0GulOHZsn/J8k8OnpKLIb0XUnwxJNDV9ap4Q4bq1alj/Fdl2TlzVoCBZdchtxew9nWBQMZQ+SDKcECdg5oOY4ZSsihJNqD1Zpjuv5nCwGBw5C/5/qfvvWFuzM70T+60v7pxPTjfnUDkXyWJVN0O3xO5WS2ppRmrJ1mg8DvJobBgaGDYwGMOeP2QYMGDD0liwAqSW1VIHstlsZlaxyAqscHO+99yTw875219a/uP97ikNLHffMSTMrQ0QJKvOPfecvde31rve93l+T6ojxEGnH5PZEvll95CJPRaT4+O8ngiQb3puSc/+vf8ca6Qo3oHeEUX5lvjq914UHqk11gdYRSOSXTjIiaepelX4/BKDKN/T6ctRmd+I6R0yyG3G7D8vKR3W6FEaIAyeG3NorokXWjR7WV5fecCel2ftD44QW9IPbD8bcuzILv2JS9bxWduv4Dgh2T/OE9tSX0eO9H0fZa+OZ2KibCy0vCXjgPFvj6TWFJqdnAKZPVHfF+/77D/n4rY1mf2I2FaMagaRq+gfjUjtmzJOiYUEePKlh9zankEB9o1MohGV37m/KAssTCsyf3WH9iiNoTSdnQKpHQu3DZVbPoN5m+GcKM8WfqIxvZjuEZtMXcbZziBOerUB+8/YpPe1IOsTkraQYBCvVkN+L78g6Pb8ZkTjnEXxgWDk+8vS1003YnZf10y/p8jUQ97+7t/7fIB8QXSX2gR7FJN/KB/mpKRINcSV2nxOoBRmIBbi7F7EcDE+EBSn9zWFBzGjlRCloXQvILMXM6oJZ2qwaDD3rsBrCw9lGtI7G5DNe7xcW6U7SpN2A3780/Os/f6RxJ+vye1GmB2Lew9nyDo+o8Amm5nwpZV7eFVJsRssGQyWFfPvjuif9mXhFiNyqxZhSvqh5lga3mFabCiTsgweig9iwiQG0qtaxKb4pRrnLbpHTNEVpGH6Q0X5ToxWMpUCuLk5S600QBkx4+WA6Y/Feu1npebuHjXoHoX1OzOMxw79YYqnz6ySqifonVck4Hjl9/YxPUV/yTyIz7RGMcM5RXZ9RORC47yNNqF9IWb2A5E2FtZCUm059rUhD8ikJO3EzpmYzTcMCmvyXo+rBpED0x8PiBxFetukfUbRPu481vp4IhaqVhJY67QVoykJfm1fiKlekTdpsGAw81MTe6BxBpJnb040hQcGg0Vprs+818Ueawq3rIRWZ0s/0JVa0O5pOsdMcg8N2qflya9+YDHopvkXP32FhWIX7+MKlauK3LbYKyJX5HSPdJ1eaOGHJsHHZX78R89IsseOono9IHI0O69kqL4nfcrcbZtJWajWsQvDw1KSjKeFd/pIsOyVFYNDGrcTkVsbyY74UDN1KSRV10wqEGR10p4Th61XNvirX3uHYmFEb5Qi+26OzKpN44LYwb0pxWgulqzYviK7ZnJmfpdifsSn91YwQmmdLX13iDcd8+C/TuN0ZPbfW7Yo3o/Yf86iejNk4yt5IdUAsaVJ7ZhYw5hD3xoyqpn0lwR1VL2i6R4xmJSlu1C6bpDZNWidlsyqVEcz/WlA40KSf+XLkKB76nNUo6JgdNxPIGSQ2Y8oXzG491dSpBpynA8WDJrPxQynRRTSX7QIU0mwlx9z/y8V2XlZ2FJOT3Yd2ZUljnGwIi2eICdjwSCrRI96N0X+nsnuN5cpPhCi9c6roi2YlBXTHwr3yUxH7NyepnO3wng5ILOTsKJs2H/Gxp8Xo9ukKpaU3JYo9PtLBrXLPsZY4BiRC0Em0Zs+iOmeFKZ986xN4+kcozlN6zy0TlmEablQWSPF6KjPxi+L0GX41oB/dulFqtkR7k8KSQtJSpnxjAwJ7IGMNZ2ONPZvvX2EUtpjdq5N+xUfr6bYeTXL9IdQ+b0spXsyZrU8+d21oZkUTCo3Ikn2ywj12ulD7Ci8qRTFh+KPGs0YjGvSow2zCdPWlHC7dF3s0eOaQfeQReQqyncCTF9KM3P0OYrvMQJIPXSEodmB/efFFeo2TWEjeZrCWkzpqsFoTtT+bjcms6cJXUXzjEOqqZj9IMarmAxWpL4trIcEhTiJY0y8V6MESbmvmRSlI9C74It3KCNTlqmPJGIy1dJ0ThhcePUu0chCVSfEpZD0Qxu/IKVJ71QIMTi7NpOivPl2X471sBJiD2CwYJPek+RmcyJ/93hKYsqrl0TmFzlysqSaivyqfE1+O5Qb9xiyd+WIjFMx1id5nHTAXKaLOdZUbgW4HZ0YBEXVFJ4Y4bZEheV0RSexvl9hEljkSyMye5rRYsykoGiflIXmdAQFaXny8ES2uAMsT+zgoxlJpm6cs2idEpyPX5DJmFfTTMow/9OY9jm5yLntUCIkExjzoySX3Zdspi6PCXKfTRb/zDXyH2jt/fd6xZaMPbM7Asx1OorSbbkYFO+NcbuaxkX5JUt3YzbfMrA8TfuM3LCjlPRZeysm7VMCl50UDfaet3HaBt3DBrVP5OY9nhPV+2hWoS05goy+RaojQpFJwcCIBCYxKSm8xYBB4KLGBo4bkq8MyW5rpi5NpJVzx6LwUITXhTXZRSMXOicVxas2vWMxpp+01jJCWS7dkz87XJCau3xHJHjqUaurKh/extdEjOL0NKOliNJNxcx7itF8RBhY7I/zEgk5ZdF8TvCXw1mDSVmx8g/F5nLk9/qM5mScHLVcTEPTa2foHoPKFTk1/FJM94TM5mNLyrDsToxfVHgVk94h6UqYHnSeCqjejIhSMFiU5Ov8Q0Fq1q6GTAomRtXHL8fsvOpSvgn5zVAUWzFUrye/+6yLigVa8TivJ2Khmr4IGR4lH6tIFOSTimb79YyMD7vypu79coBW0DxrUbmmk7l8Yqt4GLH0wwm5TXmKC6tyQSs8jA/cq4X70qLJbkmImV+UMN/OCfEnjWaUyOr6Cv+pIYWpAbfvz2P4imJ2zPBBkex+SPO8CyT+pvPyEKRaEZUbEU4P0IpJWToWYUp2rdGc0E8e/A05RfyyMPVHNZPmBZ2kPScROfWQ0mVb6CZf8LG6cvv3Kgba0UQDi3ufLh0kEtbet+gdhUlNgjHqT6eIHWg8lSdyZLHrTETOnXBseZ8wqyndkeKzfE2x8p1QugVHowPWgUS9S+1auh8R5iD7wGb7NVm4nROKzLY60A53D1l0jylSV9MYE1FRFdYn7D1rMylpUs2Y3RddicqcET5A69TnCOQbJUx5tNSRlVshYVrS31ItjZ8TkbNf0FTfdiQSZlY8735BU7oLwzlDYiEvpvCLitZ5CYbIPxTrdOwmgbrJv+ucMGg8G5PbjjHHCmJJZ3lkn4jOD8hmJvS7aYgUh5/eovvhNLk1g+4hm/GUxgzElWr6jzKdbNL1AL/AgQHO7gk0Irujya+D24+Z/Y6D6WvsrjTDx9MikukeNslteBg+bL5lYk6EZ7ryL8WPVVwNMT1N7oFFfmZAnNIMjgWsfKePX1Kk6sIb3X3BJrZg6lKAV1VMfxJQWA8xmzYb9TJdL8XKd0J2X8owXogwA2gfc6g/BygZRkwq8kD7JQmpqz9lkmpIXzpVl6CN9K70fIO86ACssWb605D5t4dS5vhw/y9bZPblFIstRe1KSOlexNxPWpie5N8+zuuJWKgYsPOKSeuCfKj7zwhv04iksM/uRYRZdYCV9Mua2idSI6b3pKYyfREpPwJOuE2DnTejhA6iD3SVQUF229qViNxDk94hkcDZA6g/qxm9OCL1fJO/dfZnHK/WMW2Rrp0t7TD18g698z5BgnMcTUvf1/Cl3py65NM55qJNMcBZQ3WQFWoPNG5HLhr7zwquKL0H/cOJbsGTC8v+sxmZQO0blO5PWPlmxGDeIswqDF/4V4PDclxqN0KNTVa/kcNP9KZhRlO+LVkCe8+L56p5xmbvb42Jaj4vrKzh/8kUgzmb7I78PL0/N2BwSPxYdsfAHAvZb1JSLP5Y7ONuS8ql7I6cUiqS0GO3I5M4P6/INGL8rMHGL2cB6D7vMfW+UKf9qTCxnmh2XjFY+0aF/rHw3y2z/3cvkf/hXyqC4p0kaGzBIMjHcuHZlaK8e8RChZrm03KBqlyVhr/dFzVPkJbGs3wziS8HsLIB+Y1A8OXAaD6iek2+bvdllaSQaIYnfaJXu9SONfntc+/TWS2z4VXoTtIs1DpYJZ8/+tHz7LYKYIDxUpswFxMUREJnehBmFa1TDqM5ReWmXOZyG/rAiDialaGFnzfI7ChyWyKpW/mjHs5QUpcn5fjA22T60D3ssv26jdMXj379aYfY1sz/SDHYLPDqmXtceGqVMKfhfB9iCf6tPy25o2gYLwWMZ2MOVVucXNnlw4eHMCfSfUh1Ipa/E2N+lMcaSgzm1KWY2BGPl3ABLILMZ+kl3WPyPYO8INfdtoZYVGDpXY/hvEGqKQ9N7UcuzTclEWX2bZNJFUYzonkt3ouZ+sDEbX6OLlNoGCwn0eQXRxQeGMRJ/zSzJ4a9KK0o3ZQ6bTSrDtiak6KIqLvHSfzsUlNpA1KXMgxnbdonbawhlK8bBGkxyWkL+odijD/f5EtnbuONHbqDNP/0+ouUDrf57v3TAGx/PMf5xS3iKZ+FWofzRzfpt7Kcu7BGbGsGL46xv9AUTE8Ek3KMn1WMZuQyM1hW+CX5IBsXLEbTKqnRpKe6/rUCrdMi2qhelt011dSU7oVoE5Z+OGE4Z2IEitFSxGQ6onPMZO4dxSffPsPa7x7FmhthWRG82uHol1fRy2N6J+RrXz57j7/79W9z55NlIm3AdorRrHij1v6convIpvggorCqqV2OGE6bdM7FlD9uYPhCn4nS0sVQsaifsg9sgYKUlGB8LoSMZgx2Xs0SppOJ4ENxyzr30pTvBMSWYDfR0DkluVmV6338wufo6DdCqNyQaJnsLzJi+srD4p+0pDWzJVOi6Y8G9I7HidvR+IyhumRSuS4XML8WHYB0VShv7nhaM1gRu3XkSq/Q6ivyqwaN3QKrvSqnF3fx2ynCVopi2uP4TJ3GKIPpKXaGBZ45ukbemeAYIS+fvM9Gp4SuBPz66Ut02lnSDdGSlm8qcrsCqy2uBuQ2NaMjPvYgQa1vyDgxtxkz9774lVa+3adySxZm5Mhcv3FOsrHWvubSveBjjWD+x2B3TFItzfaXhJbSX9EEE4ti2uP01B63tmd45fADtBOTmR/wweoh/v67XyV2NGsfLFK7JDVl6xykti3SrZjhnIlfUAzmTLQFmXWT/demsMZQvCRCmd7JELsHzppLel/eR9OXbo3dMgkK0nqr3BJx+WhWSpkwLfzYyBbXRX9FnBTZTegdy6Hiz9OOCowrBtUbAZOypn1KAnc3fqUiSXRlqYEmFRe3aSQ4Gzm691+RJ3s4L43l4nWTSUFSOsbT0uAv3pUpiNOT+XnkysRqPKNZ+pbBdrPI9dV5/hev/YDsQ5OddoHmOEO3n8E/PiaKDT759Ci3dqb5dG2Jy7vzDEcu2jf45t3z0LcxfY03JSVL66RNkBWkORpKlxxUnCB75kRqOFgyGFVNCqsxrbM5rFFE86KojLxpUU2JPFCRXnUorkbsvWBQvSzClMplk6lLAdlthWo4+L8zw6c/P0E0tPnptZNcPLXOsJ7hy8fvoMYmVt8gLEiM+sLbEdZYgow7x0VzOikKvzVypF0YpWQXzeyLuHz+x4r+YU1ujYO5vj3QuC1Js3Ha0Dsa01uRkIrMjoh+pj+Ok9NLBEDF+3ESdiwn5tSlz9FkKrZl4dSfsineFazM1JUxsx96aFNocOMZaJ90mJRjRtOKMCeN7PSmyWhZtJK5DfH6hxmF09UsvB0S5OXikt2N6S9ZBwA2uyuM/N2XTNSDDKpv8U//4VcFmNtJ0fvpDMZamtI7Kcr/dZriLRP7Sg7HDQlDE5Tm4sl1/FaK8hWDyJaSYlKWuX1uXW7zzWcigizsvmiz/4xBuhEznDEp345ovhLgVQxaX/YYztlkN4wEGCF6ALct06/MnmYwaxIWItxeRPcEZOoxW1+0SNdjMrsGo1npXGTv22Bobv/wKLM/MXnv9y5ijpWYGk1N85xJ44KViLQV1esRgyMR9ggmZYPy3RC3I31pbUitWX8OYltRvCfMABXLAMDpaTqnElFOWhwC+Q1ZmEqLsXL7l2JSrZilHwp2flI2kuBklRBqPkd61NiGqU89Vv6gQX9FeoWN82map1OU70QCud2SuXdmW1o6pTuxQBnqmun3TLSpmZQNuodFatc/BLErDX1tQOn2kCAjAg+vJsA1ewCVaxpzrKhcMRI/O6Q2bTK74jaNbcXWl3IUHwZCFtnJkH43h3kzx50fHCW1Jzfy3tEkHOyeBOV2zmhyG5r8fYvynYjifU1uQ5JVqtc9TC/Gato4Pc3KPzGSS4hmPC1TtMyOJJeMZ0Qc7tUgu2qx97xNekeR2p/gdOW9AllQpTvyu1bftw/+2fTHEwk5VnDqHwzQptTAnVOgTU3nqEnxholX1Xg18EompqepXfXlMnXcwG0YDOaNg/e7fQamPhkSpqUdlt0LcduSEbv7qqZ/JGI8rdh/NkVm1ZaElpzFcEHwSdY4JsiSJFI/Xo36RCT3qRB2X0qhwpS0PSoypZqUxAAXZjXd44riPZHTTSrCii+sKuovxBRvmxALrELFcilzO4qdV0yMiaZ6TbP+lTyZXc1wORkGrEtju37BYv5nE3rLDulGjNOT5nXjOfn3qR2L7I6mkbglK5cN3H7MaEEEzEYgeoLKdY3Smu0vmGS2FdOfeuw9l2b6E49xzSa36bP9ukvsaCaVlODed2GwCF7VxfAFpWN6Yl1WsXQMTO+ztlN6HzxHUbs6Yf+5DHZfanQjkPdKGzLv10rUZM0Liv6SS+lezGDBYPOtEnM/8+kvSp813fosjSW2Zbd3hjGtUxZB3iF2Y9Jr0v8UOaV8T60gyNtk9kIazxi0RzaTstT+1U/FNNk7HZD52CLliWqsddqifEOTbkW0TksGGEoGG4/zeiJ21EdTqVRLM16ISNc145pQ/YazBul9g8yu1KnVGxMqtyImJenPVT81UJEmvStHSrqu0ZaQ+9ymwpwICMEIZZct3RTEeOeYRX9RUqQ3v+SQ3RMZ4Xhaai+nZZJ7ILuz25HejN2T79E6ZVC9opl7T9JQslua/pLUem5TUb4b8PDrKTK7MduvpQjTBrsvufhlGS5MyhoVSXsqdsCbEoOhigQ44RflaM2vxeQ3ImJL/P6DZTkFQLoIo1lN86K04/IbEeYEFr61Sft8jF9SZDflohI5SqLlbdh/RtCctcvSvM9tCvgCoHfIYPNN2eViB8rXFZm6RB/NfCQj4Ec5r1tv2Gx/0aJwT3hhKGnDxbbQYYrXREWGgsZFxdRlH7+oWP816d0GRaFxP+4KfCIWqjkGbzZkuKDQTozb1cy/O5Y5d1du78UHUnS3Trm0TpkMFg1mftEnTCs6pzSTWsx4OmawpMitCdQitxUnIQ9y040tuammmprhgnQPgoISD1MkKqPBaZ+pd3Yp35Rxp/DyBWY2ntXUropMz6sYhGmD4KU+o1lZCI/wO90VG9NT7L0eYw+gfUouKNMfCsg2uymWGHMCy98dJCRn2T17RxIe1L2RkPU6oVx80pqFd0LMsaa/5LD/DQ+n9+hnh+5hi8x+wL3/ZEFSBq+F5DcjwpzszEFGABqxC7MfTGidEalj6xmJXy/fDahdC8WmUlDM/WxEkFW0j5sUH0Y0T0vAmtuW93Hmw5j0jvjVUu3ooO+dbsbk1mQRmr5sQMW7sP2ajTYgd9thuCzdDq1gMPd4h/qTo/D/L/9zMlsGo7kYFSkyu4owlQAYQqi/FIGG3KrFpKSZey9iXJUxo9z6wfTVgQugcjvCK8m8un1askbdphT5AGjx72d2RckU5BPm53qScjLS5NcDCSOzFX5RFu0jYET3mOgPnK4m04jxSuLfnxRFJeUXRWOQX5vQO5QizEi7qnHeYXAkIr1pMp6LsLsGTl8xXIxwOuKwtQfJCNaW6M3RnEB49581KN+U6VyqJa5dEmVYMOuz8EcW+btd7vyNEkagWPhJiFcxxZqNaE3TjZj956ByTbofxdWQrS9ZHP+/bTB4aj7hEch71DkFqX35uqkrAf1FCb5QmoP4y+oNn90XhUNLjPACBvI7pJsCFXFbj5gJMtYOskkXp6ZhZcT93/rffT4U/lpB5bJKbrkS41O+E5LfkFZM/aWIpe9A4Y4lU6BiTG/JIt2I6B41aF+IsMby5oRp+Z67LxkMFxSRA8f+eRu7B7ldgTd4lUQ5FYirFUN2Oret0EpsH6W7E/afdWg/F+D0JQCidV7TX/7s7whyUr9pQ3xTk5L0dlOtGKcrHPvm2RT152NUBPWLDt6UpnjTJLelmXlPkduU6ZjdlTSXMJ2gcM6FDJ4bS7z5lha4QztxPTQ16UaMNjTFe7FoIlZdvIrB7usVnK5BWA5pnbHZeyNkNGXSPWyJ0bAT4XTFGuNVFO3jNrVPNd7xGVQs3KvxlKJ7HNK74vlP1zXdwza9o1J6ZLclldvtaJqnHfIPNb0zAZn9GNMTGmD/qBgGY1tSuCcVIcjYQ01/BYZLMcXbkHsn81hr5IlYqEYgE5wgLwona6CoX7CovzVhUlIUblsM5kwGyzGF9ZDyFcm2337NJLulKd6wyG4nIbRdOfYrV2XUZ05g86sVwqxI+NyWIspopj4Ziog5iXdsn9Wk6uKVz26aOPsD3JbmyO/Ijj2uKUq3FLMfyGI3PUl7HqxoeisSQtE9HVG9PqF1TsLHRnMKewhTvzBovhjiTQmwbLgoDfO9rwQHqPBH5cfR3+2BguVvi007s6vJ7soRPilrpi6NCfKK7hGTmQ8jsjsB47mY3EYS2ZiWmjm1aVO5FVC47iRze9nVUjsj3LZQsC1PyqLYguGcTf2CzeL3pDdqDeV7pRoaeyTvbaohtEPp0sTUX5SYzOyOT/lji8GSwaQiu3duVfgKRiDEmdgV4UrjvDxw0x9C+5ymcnPyeGvkP+D6e+zXo4AyEMcohkyrUndTkOg7+0dg/qea5mmLdDMm3ZKmcfMZcXLWX4wI04rpj0fEtsB9jYGJGegDbI3lSSO9ck3jlx3KdyXlOL8miiAQR2iYhp0v14gdRX9JMujHs3L52HnVPEiWnlQ10x+JjynMyk7ZPeyQ3lPJjVbKC6+qmHnbxBrJqZHZkgEFXRt7IPGQ9lDAwPf/YoEgB04/oPyBgzYQFmIMUUaz80oar6YprEV4ZZOdV4UzMFhSDJZ1Erir8FZ89p+ysQea8Yz0UDM7msaz8v3bJ2zG06Im6x416JwQ0kn9okWQTyzmK2Jdj01FqhNRvRbIAKIZ07hgMPtTg9oVTfO8K1C6Hc3MBzJIiV3ZidFQuC8UxCArGVl2X9N4Wi62/WX3sdbIE7FQQZr6md2YvRflqFNaosP9grSvFn4cEjly7E3yBr1DBsU7ULph4HSll5rfiuicSJPZkaM0uyl1Y2zD/DsjBvNGUl8arP9FMdUNFmXRqBhSHfEZzXwUSCLIlD5wv1avyG7rthT5TdkhVAz2UMQpUx9D56wohDL7MZNKjFeVh8zpasZVg6lPQ7xf7uFN6cRvZeBVpRadVMSGU7wv/dj1X0pJwG1aMa6ZaBtqH4tyLLcu75k2RF5YuOYwdSVk7udiQ5mUYfn3RSfROyypz4KaF+meuBCEOKgtaX+pGDpP+WKj2RZ+Qv6OJaSWYczuyyaN87YY/S6YVK8LDtMrGdg9maiNZiUaKbOT6G0L0nUwEqivV5P7gDclGuDBkvp3AEn/3a8nYqGGWeETjWsGxTtSxwyXIuyR1KiWB17VpL9sUHwoGsuDyYmhaD0T0zwP9aekRgxyityGJrsbs/e8QXpf8/BX01IfHYlx21B63xGfz0AiflJ1ka85XUV6o4/b0mS3oPzBtgwIBjFGIAzVndc00+8LBmfjLZPMttSj6U2Lys2A0bRB6aZkoFpD2eUeUfei6wUKzzbIbGvsgexC2kxcDXdi2l/0cNtSL3tllfQnNaVbiW73ltTA6brPcEEdkPl2XxDH6uxPWxTvx2y/biYQYkVQjvGqsnsCFO9pWl+cYH+jztnfvEnhuTpBIaZ42aF2WdNfNLCHsmsP5xTdQ3KKSGasmA3HNeMgWr35Unhg+5Y2mJLYy5TwGYKson8sonZFOineMY+5n4fMfCSA5cd5PRELVYUwmjJxO7KDTIqK5e8kc+dE3CGJeZr2CYcgL2lxud0Qa6wp3DZZeDsiVf/sCR4sK9x2xMJPQqY/HuD0pEda+1ihQk1uJyK7Iz1CayR/L0jrqn2+lAAjFNtfX6R6XdOftxhXDcp3JthdscLULguCJ7YVraciCmsx+8/ZZHdjJlWxdfQPiR62+XzEaD5m+sVdFvMdln/7Ht3TEV5NHeQLtM4oKj9MSclxLKRzRgQek4KivyJZUNndkPrTBt1DKRGEDIQXO/dehBFomk+Xya9PsPuKwYrI8FK7pkyxpiR5cDytSN9MMZvtc/cfnaK+VUKFilO/dQv/L7XRFqDlwY8dYX4ZvuQbANQuyykVO9KBKFyXdOrRjKipgixMiiZBTtM7HtM/EbH4ffk9UKAnJvvP2Gx9wTyw3fxZrydjocZCFslvTETcUVVMiiZuVzOpRPQXLbxafIBZTO/LMdJbsmi94SWZpXEyOFB4U7DwjkfznEj87v4dGWMWHsS0z8jCapwXl2dQlP7paF7TXzSpXeozmjXI7AUSfDEnoV7WWNM7rtl5KcWxf7TFJG/QOeaINK8dJ3lLiqUvrbP3KxOMiRzh2S1hX8392MBpiQDm/u8f5+rmAsbIwBqKN2z2Q4+598MEmhFy5HcjytdEXBNmFKW7MZXrmp2XbRbeCRnNKio3J7RO2bjtZIKXkjqwfSpF4WFMWAoZzstDPTgaHqQaomF0YsLdRo3GawGH/1XM9Mea3WGBanbE2W/ckjDfLck7mHtvwqSaoImyMFg0SNelD127JuL0SUlRuhcRm6JRCLIKfWzIwo9jitdNNn9ZkduVy1dqyybManLrivz64xGnn4gR6iM1f/uEi19QlO9GDGdMwgxktuTIm38nZufVmNyWon9IWPHeFEz/sUv54z1235xmsBQz+76MImNbyoDe8YDqOym234hJbxos/WBCmDbprVg0L2oOfStg+wsuK98Zce9vG0xdkguM3QvwqhZBQaNiuewU70gKyvbXFwnywmSNUor+qZD8LVvif6yApd+x2PqSZv6nEe3jFt5UTJBXlG/G9HSG4ld3sHyb1Pke4dUZRrOKB38D8p/YhFkIsxbjGZPlP/HRprTIBn0LFShql2Jap22KqzGjWVuwOy3RJDyaMDk96QBkHtrkNjWVSy0sr3zQL7ZGYDZt/D0HJ4TGBVGaLf6fa9RPOqwux7zxP77Ce793keGxABW7GBNBmj8i8cWWYJEsT1qAwZQGTMZLIWZgsfDDLiouMJwBDKh9aNA9LMDlR2Q/yxOS4OO8nogdNbJFrBHk5Lh02yG9l8ZEKTmKVQx+3uDI73liayjJyG72vQmjWYOdt2akjXVfHKf2QNE451C7GrD4HbGauHsm6X1N56jD+q8kvc62waRi4Tbh/m+mSN9OUX86RWZX0zqTQcXgtgzyD6WFk98Qzajpi/UlvS/tm/SaTfatPeaf2eF+q8rar4rmcphwlR61tUYzBpUbEbvtPK2NEs9PrTOaVmS3Yqo/cZlUxBVgelK+7L3gMp5WVD60Se1Js380I7aWSUHROqMIM7LLtU8ajOZlLNs9KgKS4n0Jdrv72yVRceWk9WR6migXUz7fkPe7oynfkoxXayx5rFujIuqlDkf/RcTUpyMyu5p0Q+NNaWZ+3qX2QQMQm43TTYYOBShfMvEqmvWvFQlyijCrKN/xiW0gBpScaLGjaJ/msY/+J2JHfXR0mB70Twes5x10J8abjiA2MQOYVBSRkxbBriemvvGUjeHLwvFmNGFOEeRjKIRkbrkEWRO0AB1ql8S2PJw1OfN/3Obuf7ZEYVWz/4xBdlMWk9OTY2g0LR+6NTQ48s+2aT8/S+usYlK2yW3GDOdlt7Z7OkHYQH+cYmjE2GaE3TGxRlInGwGgFH5BEOLjmkEcG5CK+ZP7p5mcDMhtmQwX5TLl5+WksHsy9dKGSlIG5Vj3i+II9Uom5VvSTZBGfhJvvhST2TRw+prMvs/Gmy6Fuwqnb5DbntA55jKaVdhtk2DORCHxlbl1uYFbY8lmvXVjifmjdVoniwyWwekpchsxtcuw9WZRgG8tyTytXYmxR9LBULGUC+MZTfGBZu9FiE0Ht6MPat/hknAGKtckHfFxXk/EjmpOYDgP4xnI3bEJZ3yWviu/QKopfbfslqZ3VBTy6T3xx2tTWjtuNwnObSvcpknuikt2S96kvRdFpxmmFEFGUED7by1x+A8GGAEsf2+CPRTvVWwlqctTmsy2iDNu/Z1ZIkfQim5HYBil+xHl2xHZ/Yj+UfG4H6k2eX52g3Y9LySWHIk7QWIyxzOa+R+1iVxF1HOwUiEnZur8R8+/z3jKoHg/pv4spDqayBGBijkRqNqkINZjtyvoyo2vye/eXzIorEd485JpOv1JQG5NFqkKoXE+xfxPAwnQPWTQOeqS2wyJHE12S9Hrpale3Cf/EPqHRSAdphTGhS6Gp2j0snTOxUx/Ekt7qSrRO35ZCwQurTAmivYJARSnGsIGKKxFzPwipv6rHtMfSomU25afURuQe6jQpjAN0u3PUY2KwUGiXW4rxghcmuegcFeivi1PHJC5DfEa2QNxq7bOir88yJtkNyX/SIIZ5D+pfUNquZpisCgfbroOoxUNKodXVQznUrhtSZobzss4tHotxs+KgMUaCNGuv2gSpaH1UojRtsmvCste2xJKe21tnvnpDmYqIr8ui36wLOXM9mtp8quaO3+jSHpXfuVwYHOvXuPu/hTeRR/zQ5uZDzSNp6VsKK5GON0Qd3fI3mtlYkd6tmBQ/YUox5yuHKNT7xlYXkzzjE26IfjI7LYc8dYoYu7niq0vWARZReOiTeWG2GGGmynmVjbhr/T49M4K/ldHjMcOeuww+75mf87EbZhsfc1n7vuSR+DnxPvk10LMcQKA86F0a8D2F/KoCFqnZOHm3ssAYljc+pJFqi4PX/+wTsh/0Dn6eRJOGzL5cTsi6wszkNuQiZNAF5LocyQKp3NSsktVqMivid9f8khlQvQIfhtb0DpjHAhK0nUh//kLEgpWXI1INXWi8hHdqzcTS6Z8Ej5h94VyZyU93bk/sTn0RwFhFoqrMcf+uc/wN3tYTkgpNUZtpGheFIV+Zld+rnRDM55VzL4nLR2rZ3Li6A7LlTanZ3Zxcj79w+D0Igr35BQJ04rtLzg8/M0KnXMxxQcSS+RVVDJo0IRZIaN4VZWAeaHxXEzhgeyMgyVY+3qK7mEL008i3O8LrzXIyPe5sTvLp7cOUZnpkfm9IsZaGt10mZQM8j9NY17oiuuhoJIAYxkoWG1L8J8ZkVE++As5GUnnxcFqBCJC7y8bLP5wjBFIp8b0NIs/DhktR8ln9jk6+rUN+QfQOyLpGrEj9VZ6/zPMTZiWefVoyqB4VxZodkszWJYOAMgbU7oDpbvSqE61NEf+yZbYIwzpx3rTmsp7Dpm9mN6yyaSsyG+Ket/0FNZACfayIF6tIK/pnYywJqJxza8OaZ51mXt3RPO8Yvv1DJYRE04sgsgkmp+Q2lcHJOXe6ZD+MuTWNPWn5NYbVgO6kxSWEXOmsEsh6xHM+Kz/qoHbkxzT0FWYY0WU0uTvmYRp8YQZPjhDTe+toYiqk7uIMBGgcMdEafDL8l6Ub8qFpXQ7ZriomZTEtjKaVZgn+yxVOzjFCa2tEqMZxcyHMdlNg94R6B7XzP5fXWZ+bNE+L/LL4UpEqgH5h/J9jVD4VEaYnEAjeXi0+Qi1CbsvpSnf1OTW5BLVPWTjNAQ7+Wh69We9noij35hAphlhTQzsQcikonDapjSaw4TYUZR8z8yegB+y25rcdkSUsugfklzNqBBQvSbU6lRL5HH3/kfzANSuavaetSncf8RYkg9eG9BbEmWQMZFWj1eF4XJM+boiv6lpXDToHhH4wsYv57FGsPXFDJltGCxpdGCRzk1oe2lmprrsnlUMjhvYTVDpEH8+plEzUL5B9s/tEYzSmEbMvf0aOXuCF1g8e2yNzX6JbaPC1AcmRqjJ7II/UgfTmyCnMXzF9nMhuY9yFNZDGhcschuaoCDj4dgSkIbU/bJrejMRQd5g+bs+jQRFtPjWOkppNtolasUBo5/mqNyYsPmGQzrhp+YfGgznHfrLBk4L9p/T5O9JoHJQUFRuhQxnTCYlRfGuxu1FBGlplQ3n5GHde97AGithCaREBphfFRhc/7D+fPn6tQmTvEn3iMHWF1MUb5lJAR6JHXpZdKXlm9JET9f1QcamBFIoTvzjFtM/s9j6ooVfFM5/kBVkoxHJ0agtTeekyPOkhwczv5gwmlPUPpXZtyAbNeVrYtKb5AXmUL4jAWreCY8wAxgSKV66A8ZHBbyRw2yuzySwsNMB2JqF57cpVwZky2NOnNimdqyJa4UMO2lsI2al2uZ+u8Z8oUfKDNnbK2IODQZLcjQ3n4npnQxJ14WJWrusyexqau/auF1N84yV2FHkd7UHcoxOfygP//SnPul9TfG22GO6Rx16ZwKspzpMIou7l5cYD12i2CDIwvbrLkFBU70+wZjIIt99K5TE6l15iPvHhfPq1TSjqUehE5p0I6JzRNi23UQO2D5hsviTMHFqmJLRlfRhJ2U48m88gsLjrZEnYqGqGLJ7kgJtDWXxaUPSQiL3Mz7RaFbRPwTtczGlexH1py0mZaGVrP9qlfqbE6pXZNI0nBXDX3pfH2SVOl1BAM18FJDbEs/9YNHBCKSWGqwkLM8ZqefGVcFfalPs2eMZRfZKisyOJr+mMQMYzilKdyOUAQ/bZdr1PJWCFNR5Z0K7mWdYz3D31gKtbpbdbp7Th7dZ26zhxyajic3au8t8+q0zOJsOlasSw16+M2Hup4pj/8KndUFGvkFGUX8tpHdYHqjxrARITIoG+TW5DE7KGm0q5n8WsvqXwe3FDJY0flkyEeyCT8oO6f7hPEagIFYMPJfMrpZ5/12D0bQt7a4c5K87gkuy5PK28EP5u4NCzHBBJYneiu0vmPhlzWj6szIst6nZf1qU/dZIY49jZj+IaJ2Xz2XtV9Jo43NUo0au1Frjw6Leye6JIimzpynf8aR+PRVQfBCT3QKnbTKcE2iX21JUbgY4PY37IMVoxqB4P0gYnwapjmTKj+Y0mT1hpa79imL3RUdgDwVxprodzcwvArnIZDWNZzRGKDvyYEUcr5GTxLFXFa2zsP96iNuFzjGTwrspnG+XUCOT/tjl9dN3uHZ1hdxVl/SGTfG6yfI/tnB+WCRj+VSmerSGGfioiN2XAN3MTqLmWoLOURe05sFvpEjtGWy9JT3b8scWTl+a5dVL6kA1rxO4mT8VJeAKk/nvWuy+ZFC6DdrQHH16k7CRov9JVTCQIZSrfYa7WeqvBwfto8hVB5+LCmWSZE40k6og04sPQ9LbJpGjcToGw0Mhcz+LKN2CxR8PyW5Jj7l9RmpWp69Jt2IiW9FfNCndUAJ8myjc9ufo6DcnsP+sQ2rNwRwrhnMGU59IXbr3XFqyTndtvAREUbobJ+opaXDXLwroYfaDAG9K0zptM5pWiUnQwBonsTcRDBelb6ct6YnOfDggVZeUkvoFm1RbUu0qV6UN5hc1M+/LRWH+ZxMWvtsi1ZBFVfmFxWhOuhV+UY7r/D0T/36Bm//tWeZ/DP0TAWiRELZPOAwOaRYzHQDCyMB4ocPgSMjKn3gYiW8rzMolMrYUcTrGHkLunpWE+kL5tqRMhxlFZCu6RwyGC9I7Xf62EK3nf6oZzEkgr19QzL4XcndjhtINg9wGVK/JCda7WSUzMyS95pDdNOieD+iclAXqdEWJH6YVuZ0IR35sGhcslv7+R0mWq8bIBvQXLJTWtE9mGM3L4qxeTQLb0iJG6ZwwsDy5j7hNL0EgfY52VG3Kf6rXIyq3JD8zTMmt3+lqpq6ERK4IHfKbMeMpuTioWKMdiYe0xrD1RUv4pwNNbuuzS1OQS9ypZoJMbJjElsB2d18WHHnjvIURwuYbBrmNmPaZ5O9bFc6+imDveZfms2XsoU4ogeI6dXqa3EZMmBWvVO6hPFCDBZOj/1JctW47IrsTEWVjNkcl/sLKZUbNDIOdHPl7Fu0Toj9N10Vd7xcVbjcitWthevL3oThA75gTccdGKakfjRD8Euy8bOL0ZZe3PGEDuB3NYMHC2nYIv9KhdwQ6x2ysoWLu4i5RZBzEbqqxQeWa9KlB9A2xA70lCyPUrH1dZJPNv/Ys3WMG5liRupE+SK4eziuqV2Ny2z7jqkGQh/4KdI6YlO5IgEX9okX/UIbI0QdZBn/W68lYqEpmxbsvK7beFJtw74j4g5rPREwKBqmWDARUpKncDBjPaNoXI9JbJrsvS4M5vacYLihm/tUNvIoAfrsnYqKUpn8ouVApiaTMr4n9Qysx/oH4tGqfKhFoNwxazwtMQlvScdCG+KW6R4zEviHGt+G8Qe+wQZgVveVoQeP0tQS3HZNbtj0I2X4D8gs9Lm0s8t++9wXswoTCXYsgB+OakhAIX5L9jEAePLclvcpJRaR1hY2Q3jFhV7VPiXvALwjLqfhASpviakh6T8uodCDTLcH0aIb9FJVrmt7zHt50zH43h1KSmzqeUhRvm9Sfg+bTmpkPhpi+HP3eNGT3YuyuaCe8qmh+5SKl6Z4OiWyRLLZPmuw9K4bGdF1u+YXEDOh2BMyx85pYguZ/9jny9asYZt+LKNxNYl1OG7gdKD4MyT20GC7IU2z6UiNufckis60oXZHjsHAX8hsTnJ4cnff/12eYlDWLP/aZ+UCOfWlAy03WCDVeReG2oX8ixJuKKazFjKZM/KL0GK0RzH/PILehpAVlJEfhIomwAtyuiIUzu4JozK2LDQONcAIKBv0VSLVjdl5NU7hjYny/jOsGmD0LdT9L75xP6V58gO3Z+bK4bSdF8fLXrnqM5jXpfVFw5S/tMvdzTeMpKN5DyoqW/N7dIwbGRLJHO6eEAZuuxwzmDQYrcpqUSkP2fykAYPbUPk8tbKGu5OWEiEQ3kd0wKN1QPPyGGO/GM+K62HlF0ECDJZWgNIVd2zsClUum8P9H8j7NvzvA9GVBh9mEBqil/h1PK6qXFcXbirU//3hr5IlYqEYAneMWfkka3NZImvv9eQutpDcYpkXuN5w2iS1R6mtLLM4YsP1qCqcf4zY19kBhDxV7z7uoSP4cGsyxZvrjCcNpA286ZrCoyazJ3zFYeIQel4lTuh5jj2JG81oE2Urkc8V7osq3xpKol1/32XtVJjFomP5IoMHdwzbOQFO5LrFBo1MTxjNyKfHW8uLJnyiUZ+L0Y0GyH9Kkdmxy65JEmNqHzTdSGBPwnhsyOOWz89UFWidNspsGKPCmxeHaOSmxjWFGs/O6Jr0nF5f8wzH5zQgVisVZ/WEVo2FTq/bZuzbNL35xgsyOWFWCrOzs9kATudKWG83KSRbZisyOQe1qiF+Kk3G0JnQlnj5TjwlygJLWU+tMljD1CCwS07iQ8FqnNcX7kehVTT6bWPxZa+Q/3PJ7/FecqGpiC0q3ZTGZExKuqGbmIx+vqkg3BJtTuaYoXxc3p+FLy8rtQOusyXBJUXwghjsjkGNZxdLXm5RlLBk7iqUfRGhLFkT1stzoH0F3B/MCYWietQUW8Uj1s6Cwh9KWsfua7Lam/pRL4bZFZicZbY5ipj/1MQJN42lBsBsBzP2xTfGOaBVyawbZLSG6GJ6ieVawQal6UoqM5RY/ntWU7sakWgr7apbCVYcgLynQqZam+klbPPSjmNw6NC6azH4QM/uuHMHpZsz2F7LEljy4YUYRpRS1TyH49hRu0yC9Y9C6GDOe0UzmArI7EjuZ3ZMRdpDTzL7vk2nETCqCVNIWTH06oXjfJ7bAm4vY/qIANSYlmHvfZzgnP2fpvgT9SqklD4Q9jA+YAsp/vIX6RAAoMtNLevk//S/I7siRbA803pQ8camGtEm8mmbmQ8HK1J+D9K4hRrQNeSi9KSEgD5YlxKH2sSilGk+LcCV25Eizh3I0l2+PWf9amlRdZthBXt5ow9dk96ME7CVC6TCrQUlQhZ9XyWVKtLN+EQoPpAZsn5Xs0MJDYT2lmvpgzPgISpFqyu+T2RUd6aQso16tYDQvXnoVy225+UpA6WOH8p0Jm286qEg8XaO5mKlPYP8lTfmqQaqtaZ02hOZSU4wXIiqX5II0KQv50JjI72ENFVFa6uDg9Ij0RxmKqxG9QyZGID/nI/xPkFOJikvLpTQvF64wneyyTkKeaUhpJBtG0l1ZkM6FORb98CO4Xee0xvQUcz8PaZ2yKd0P+dnv/28+HwAKFNSuh1iemMO0pYgcTeQKk98vwOz7EfvPGQwWDVL7Bum6tD+mPu4LcGxfLg+1KwJNi21kkUZSy2U3xOfkVRXlOx6r30hLIh9C3nt0LJkTaJ+wBIOTg/y6PCBzP48O1OhOTy5Ksf3InaCp3BySW5f2i1eRY9ntxQeMqMGKtJlUlGQ2zSraFyL8sqjc/aJi5Y8GdM9GFDZCBsuKmR9ZFDZC9l5wMSdi0gtygCG8qOy6eI6GMwa1qxGRK3Zkp2UI98kQd2uqkThWH8hizexImZD7eYYwA3svGPgFKWWsoWgahvNi5oscmfkHOfCL0oqb/niMPXhExo5oPi2sqzhJi9SGTBEP/+EYw4eZD+R3nlQUdk/0v80zNoPlmP784137n4hZf5TS+FmD/rLB1OUAFWli2yE25RcMctBfsrAGMo2Zf0fTPGMS5mH39SyVSzD48hC/69L1bPykN5dfhfbFiMgx8WpSy879PGBSsUk1RW3kTUc4LZPS/ZC9F0xGc4rCQ4nVyexKIvLeiwbEkNmR1pdfUAdMe2sI6UbI3gs5kRY2pDGf24oJMkJrqdwQiMPOqzGFu0m+wBhWvqUZLMjlLswqVn89R+UT6B420YbUiZOi/BxeVQjZYUZjrUo9aUw4IBjuPWdQuiMSv1QdSg8CItcgckxJYBnJw5Delz9jrccEWRP/9JjcB2mG85ripkAzUk1N9VpE96iJ09EoLQHAu/MiNdx8M03lesSkaLD7soHha0bzMcRSQk0qmtEcjGYyZHYFIBelhNu//7JGGyapJqImsz5HelRrqNj7QsTidyXJI0xrYicmvWccOET9gtg0/JJiXJXwsdhUjKdkVJr7Ufbg4uR2xN8znoHiDen/2QOp21QE3f+kj/HTMsUHMaOmfJCTgknhvijO8w+GxGYWbSgG8yaVa0lsTSfEGseMqxbjKeEJdE9qtCXH8vBwQO0qDJYtwpaY/vYPaaKUxeRwgLuaIsxKuyfIKbyqSe+oqLbyDzVuR1wBsx8EaGXTPaZxepKW57Y0bj8GZWCPBC+kQklliVKKwkNpbdWXI0zfIcia1J9RgnqshwwWrIP0a3MC9Qs2ldsRw14Kw5d2nelDeG5A5l+nqT9tkNmGTCNm8xmN2zY58gcT6hdSpPeSpOgJlG5JWSBjVtEBG5HELY2noHuCxDYu4m13H+whspsegaXvfo6I09qA1LZNZMvtungfcmsieh5PK8p3JaqxeH8kdZSZXHqWpNnuVWWUGLmK4ZI0q2MXFn8wIr8p2MZHEembX7ZI/ZsS9kDqKnukaT0d0zonNdl4RrH3Up5UW/ioo3lN96iidUHq0nHVYvfNkMGS5NAX7ilKd0PGMxpjZBJkBJI2mlHsPW+QmRriNuHYP4gwJpB9Yx8jklZN64xi/l2JsOkdkcnW4k88ac6Ppd1lBJJ75dUUrZMmjac1aA2xonpVpmu9o9Kj3PxKTCkxJ3aOyW18XFM0z9l0TotO1whg5oMhRgCtk4KMnFQUlVsTlNYUv5fFGsdkN6Wk2XvBQE0MCceYcijfC2TEqsGrwXha0XouZDyV1K2uLFa/KBC0sBBJFtgDCYqzRkgkZssgs2Ww/5z9WGvkidhRY0d0lruv6oOMI78InpckgVRNinc0jYsZ3JZCKyEhBwVNmAMVyuTHz0mSnDZlbn//L7sYnoHTUQcCaSOA3JbP3vPiHXK6sPh9Tft4Ame4H9O4qAhyIqYAEaD0jioa5wy8+RCrIaVDbGp6RzUqMnE6Irw2fVkQtashgwWTeKuIPdR4Uy5TlwIa8TRmDoYLMdktg3HVIruhcfua9gmD/adTOF1N62JMal9iyaOUwDDCjGLuPZ+dl1MU7ks50bhgMvNhzKSQ2JDTsvuOpxWpVlIaRJDdNCjdDWmdkY/cm9YYE1E4+SXNaNrBz6uDh1EbivxGSGxZOH1oXozZq0HppkVsw3A5IrVvUrkZ4nQtohRoJWyq3JpoJ9AGQdcivx4znJYxrz1U7LxsJWFxovd9nNcTsaM+og8X7plkdhSNl8MDmO/0+4LfjhzRN1ojWQgoITOn9kU4XX/apneEJLpHsf2aRWZLwBSZPRmnlu56uGc7tE65AscdSv3bPi4qrOG8pnvUIKpI7r3Tk5t+djfE7snFbO7HBpm9hDVqqQMyi92XG7+fkyieIGfIjFvLcKB5xmLnNYvMbgIZbouTob+iaD4XHdTjbkfTPyyoInG6ygc5npHLSue4S2zD4IsjGVFmNcVrHaKUIr8u3YLBivQ4DT/pVqzH9I5LDR5bUH8qy8KPQ1SyRop3ZTQ7XJRIo8GilE2dYxa5nYjBgiJVN7B7Bs5AxEK5VRM0bL8u06bxtMabks+k/YJP/Vkh+hmRZGwFOUVYDSneF89b6RYEZ0bC6XqcNfIfZun993wpuVE+Ct+tfij+Hm8+Ylw1yG1I/zS2FFFaEaZFCmd68sfTe5r0nnxIflFEGdZAMf3RhHHNJN2KGR0J2Hk1TfRBWdRSbajcCinfiZPQYMXUp3KJqL5nEznJkCGlGE2Lvz+zL31RayjtqMGiQoWyWEsPAtl5Dwkdu3le2ldBXhYySEbTo8ua0JkllKJ40yJOMJiPwA+xLabE2FYs/tCXSM2UiD3cDjiXs/SOQuk2YMrlrvGsiHVy6zD33R2MABZ/2Gc4K0CzyVTE7AcTLE/TOWZz6Jt9cluazhlN8+kIfWTEsWc2iJ/uc+QLD+ldmLD7mxOCC0OGKyGTWkRky9g2vyEw3qUfRHhVUWgd+sMOqbrGToUs/ijCGkrLUE6xmNJlm9GMOHxjG8rfTVO5GTzWEnkyFqqWMackKUvvL7OvqX4sHnYjEmboaNogsxvTPxITubIQlJZWSuQqglJMqiXbRGEtpnnWpXMK6hdNaj+3UKFYI3KbMYNDmsYFi96K3KrT+yLQDrKGXMpSmsUf9vCqsuu5LdlR3WYSbWOK3jKqBmS3Y7Zfsxksyj8/+RduYwRKXKMt+dmdLlR+6koQ7mEDuyd2jnQrkln/tHCmtAGjOdnp7aHGbQvDILea0KyPGIyntZwGBam/7/+VEgDlawbFBzFRSnHvb86R6sRsfyGP09MM5wysvsnWF1z6K0LkGxzK0l9RHL+wwdPnVnlueZ3bdxbwJxZ3d6YpfeQSjiyCoY3dNXFaog8O06LVDYqazlEba6TpLyu23iwxXFSkPsoSpo0DbUS6HkvcUiD3guymCGW0Cd1Dj1ejPhELVUVJXIwp+e+5zRhnIBrQVEe87hjQPaFpnhcS3KQiScqRA0s/6DNYFh9+6+mI4RtD0DK7BrnlDlZEWBLk5PaZ2ZIJluHLsMAeabI74s7M7odUr2vWvl7AL8ZUr3+G8ga5lKkQULDwxyaNpxT5hzCeicm/WOcXtw/jLfu0zsc0X/GZlKH4MMDPC2xi7l0RVk9KiuYZi9iC2feGmBe78pBuQ5iTRT6eNlj7up1Ag6XzUbmu6Z4NSW9ZdE8Kat1tS2apPYzxapqlH03Y+pJ0S9pnkinXLQhzMYVVTZARsor1VIfbdxbYG+W5+/88RWrbovKjFLP/2iVy4eQ/mJCtjEmf6hDm4iSNW4Q6Ux+LYuyRAsptyQMf5CSSXhhVghqalJOL75RmuCix9rEt08fHeT0RCzVMSy2W3YTS/ZjeIYOdV+QXa1yQVLlUO8aciEK/ck0izp2OonYtoH06R6qlOPJvBuQeWBT+JCuhaWUpAQqrMZkdjdNTlG+LNC63E9NfMuidDslux5iBXLaGcwZBxqB12sDpwcLbEkTrF0XUYo2FCDJclPyq5hlpa6lIZH/13SJ23cbNTfjtL/6UwmWX8dEJ9Yu2hD18xad1No3pC/5xtBAdkFny/zrPyr9YJ3YUTlvh9CMmJUjvGdhD6dFKyWFQ+9DEHsnRa/iazkkZpXaOi2Bn402X0i1FUIxFMxokJr+bSvhblrgWBjs5vvH8J/TGKbonPvtMGhfEDrT2tTzGz4qUM2OKJ1rktiSBMExB77D0p2NLrM+DJRnTGr6cBp2jIlj3pmQHHU8ZpOpSn1pDAap9rmR+IO2X/mHoLUtOaem2YrggAhC/qMk9HJLZUoxnpeFujQXbOK5YhClhmTbP5ySjsyLIRrRwOTP7AcN56SE2LihSDdlV/ZLGzAeYE42KBVhRuh+hTelllu8G1J+2GM6LXK58xxff//WQbAKklYBauehE5RC7blO8DVP/MsM//84XmVQ0mYJH6pUG/4e/9U/5+unrHP1rdxguCK9UW5rW8yHDRUWQVuz86pJICM97BFnBYhZWRaM7XAS/JOVFYc2nfyxiNGXRPSGR5K2TFk5HJ9pSSdOufSLKpfxDKK5G+EVF7qHQvd22Bifm27fPMRq6mMcGvPYrl+n+0ojCA6mFTR8GFz12PpyjtVWivyzc/iAP1gBql8Q203naT0QmULkl2WCZPelWGL68R6W7AZndmNxOwOwHApJ73HTpJ6I9ZQ81xQcxnWPScko1ZAcNU/KbxzbsvVg4UDKNZwRc5lXkhq9C0QQYoWbxhz4PftMkt2pRvR7h5w12X3BJ70vaXZiW2zZA+YamP0jTOSbKospVRehKz7V90iY2Ib2riR15MB7+qsn82zF+LrEDTwA0gxUht6z8rqJ9StE7ohl6JvPvBHQP2XhxEfOh5n8799eJXUmAKXRg+pMxD+ZcrLbUv5OKYHz6x0Pm/8imcdHA8CHVDOkdNjF8RWwKy7V+0aX6UdJDbitiF4ZHA+yBZEjVrvrsP+PQPQY6HTGatumcVOQfJBfOgqawpil96tA/YqGnfGw75ER2l525AjdeWMbuGiy+uEXW9rkaLlL81KGwHvLwz5moIMbpGvRWDIK0Q+G66CKKqyH9RYv0vpRyXlUe8iAHuy9b5NZh52VHNqCS/nxZUSJHMVgQNLeKFeMp4eJHrtSPblvAvTO/iMnsx1gjEQtrU2bTpi+Kq/6SYucVl5Vvy7E6rhkHwgltwN6LssMGObFimL7M9rUhFOrBkuw+fk4x8/6Q8WzMaE52gs4JmHlPsEBBVm6+rfOa7gkZZfZXROgsbS3F1OWQ3RdtUh3B4Tzqr/rHxhgjgzALvUMp3IZB6ZZctoT2HFH51GTndZj7WQjn++y86hLk9IHUsHcM/JK03IqrEZNazHBek1mVPmqYAWLhHrgtRWZVkgjzD0WUM6loglKEijWFtZD8A4PKT10yv1vkg/Zh7uxOsXxij1/7+nu4Zsi1h/OYbYtJGfafNTn8ByGVa4owK+9duiWgYGsMG78m39caicBovCjCaG3A7HshmYZYWryqaFuD7Oeoj6qQ+ssaioDCPN0XSl5doiRTTZk2hSlF95hB7bKgy90WNJ8XbePmWw6TqRinB9uvWji9ZIw6LRcKFYE5SqIat6U9018y6Z8KkuxPjdsWWeCkqth6I0v5uiK9B+tfN1j+fkBsC/JyNCvHZuGBwdF/3ma0HGINpZ9buzrGnMD6b4izYDQlVuT604qgoHHupoVzvxBhefGBYmywJDX67ssmnVMatyGw20P/VYDpCbGldDcmvyl+qZU/HtI/GdA4Z1L7RJHdkssiSOdk60syoRKIHPSOwXBeU385YukHAYU7Fu3jJht/KcT0Na0LMbGtuPLucYKhw34vx7++/rSkaHsmUx9LmRJbsPuCK2x/WywwrVOmZKsOofSxy2BFNAMqRiSQ+zGmB/vP2YQp6buOVmQBO93P0Y5q+NKq8KpynNf+WUbaNRVp9PeOSCN8XDNY+aMem2+ZaBOmLnuoUG7sblP6jpOKWEyCLJ817R9IqKwRKbnxJpaUVEtTuGEzmhVPf5z4klJ1Te2qvJH5rRC3aWIEscQ4KsmA6p4Qesv6n6tQvGmRbmhK92L2n0qLZcaMWXg7PLBj5DYUr715Fb8c0ztsULpqsPeiwunH+EWYfT8mUxeBRn5VVFgoSXQB2ZG8isFg3iKzrZlUXTJrNrGr8WoiFYzmJkQpOdrdloyj/ZKA5fKrkmRy8h+N2XnFkYuZA9krKVpPxRz5NwGDRUn7e/nUfaYLA0wzppwZc+LYDuFfbWF6ivIt8M6MyewmFvalSHDnVdGpDhY1tU81Mx8EVK+HVG74dI+ILmI8HzKuGFLzd0yMiTxEj7VG/j2vuf+/XmFGUbo/kUlMXTOaNnFbwqZPN0LSu4r8urDt179aYOEnggMfLDjk75rSo/NBRRJ6lmrKAlWxQCwmZdGaxrZO7BJyDLfOSiaqNUq8R0U5xsfTEqVjRNA4Z1G8F7P1hRTV6xO0CbO/GGONZCdI1zXVaxNGM4rOUSNJWdbUfuzSW5adPUwr/vrf/hN+dOU0cSrGbQnaEeRnnPlFwN4LhsQLNUWPm1+XSB63m4Ra9EWGF2ZkQTz63uUbmtHzI/Zfjlj8fQunC5NK/N9xdzbPCVa9ccFg57U8TldOm9rVCKVh7ifQuJCiel16uh0/Tfeb8xS/n8H/ZzPc25mi2coRFKRfmvs4LZkGvqJ408QvSbkxWI7Rtuy6k5LJ+q/H9Jds/FJM/5DkylpjabtVrokew88/3hp5IhaqOYG951PiRlXScxvPiEdoNG0J13QhJa0MBW7Lp3gHWmcV05fGDE75ZHfleInSULviMZqVXKbeiknt6gRtyI4SW8nUB9GpTgoGk4q4ViVGSNpU+8/YdE5C6Z740VMN6B5ysIfw8GspVCypf15NsfrXRZjtJIG5g0PQ+/qAMCu9wvGM5jt7Z8nds8muWiKmDoTwUn/aoHHBpnZZsJqZfcET9Zfk0hemhOc/+04LpysIIb8oySthRkafpR+kKd6wGM6YjOY0Mx9Cuq7ofHWINQK/JNrZ2Q8j/LzsYrntmHFFchGCrEGqGbP9umJ4wufuzjSRKyyw2FTUvpui9kMXYii/tisTO1sxdSkktpEOyBAqV8UL1T1msPvLAWbHks1izaB0FxZ/PGY0KyWB6UOqLkC4x3k9EQv1UeCr0xXow3BB+p7DefnwrDEM5kTTaI1g5+UMQU4x9WnM2ldSTL1r0zqrSDWkoa609EztgUoSQ2Qi1F8ySTXlie8fFbvKcF5RuiPijep12X0lbUVgukYkixElt/LYFNdq4b4sGK2g9hOH3HrSkN+UVtFcucd4SsTFy89usbZfYXA4xAilZ2wEguB5NF1Dy+h0UpBpXGZXJ/YZmPuJQeuZMt3TkUTuGNLuyW7HeFWx4gQ5OX5jR9M9ImPe8rezklG6ZWINFduvmmAkO7OrDi6Pk5JisGRg9w3OH9tE64T83ZKvyW9M6B9WpOuK7fUq8TN9dt6I2XpDPpN/mybdX1ZUbsbU3nGISqEEULwypHMS0Bq/nDgETCjd8+W9fYzXE7FQrUEob146SVDuKYoPJ0x/HON0JCUl1dZ0XpqgQsjuCq2jeV7m6kJ9lt3Lmmj2nkvjF+Tm783EzL3nCQUkcY8Ol2JKNxR+SS44WskM3Ssa+GVN6X5IZi8BS2SNJDxBHqTcppCTR3NyMYtd6B5NiNiuzP/tgZYPO6Wx/6M9qqkhWiucisdgJaL1uk/r+ZDBkrhcs1sC8W1djBguS56VX5DfK0pJV2M8JQ7d0ZzYondehf0X5aZt+ALb0Armfq6Z+cWE8bRExJsTnainNFOXdCLa4WDCNCkrpi57pJoaf2XCnb0p3GtptAHTn3jYfeituNg9mP5ozNTPLObLXTJrFsU7KulzJzt8kvTXOi0ilNq7Nk4P0h9mMXzFvb9pUb0stX39GUWUfvzl98R4pp760v8SbShCVyWxiyIDQ8kuagRSB2W3NUFGarvYUVhDTePFCKtjYg/lg81sK0xfPuxH6BprJH9em5J72jkl0xGQsaTb/8xq0l82DxRd3dMRi9/TeGWTwZIIgoO87ESl+xHDacmnH8+ILyi2JWQs1ZBF5PzWHls7ZZSCV07c5+efnkSbmoXvK3qHTGnrDCUIN8xoqtdEEV+9HtA+bhOlJX59PIMAHBYMJiXN7AcRkWswyct7ZIRSBgRZkRiOpk3G00ICjE1pbVWv6gMuqjwEwvVK7yVdjL+6xSS0GPzxLF5NepyphgCS598ZsfXFDPZIxOkv/8ZlfnD1NJn7Dk4POhcDnH0rGTRo5n4WMVgwsYea9inhfj1K9zZ8qNz2aZxzSDU1H//j/9XnxDMFtI+btE4Z9A6LMmf2/ZjI1UQpmR97NVFIaSWjzCglsj+vqlj4gaRR232RrAU5GQ86XX1QP3mvDBifnOAXZQELr14uXUEOvJJJ46JB87zJpCy2ZruvsdsG7RMW/RW5kIVZRW4zFkuxoyhsiPA5uyXmtdFSSP6hxIC3z8ChQgur7mBvOFz7nTNkV02wY3qHTAoPI7KbCr8SJ/BhgWRkdgVLNFyKcToy8LCHULozJL8WY/oqIfmJmMEIRIoYWzIN2vglcdGm6ppJLSL/ECrXNfVnZRoWZMWgWLkdYg3EPpLbDgkiwZ2Mp8URkdmT0TIaNn45zcI7Y/lnHvzg07M4eR+UGPkKN21Wvi0aBnOs8AuGULH78UH+lzWSPrA2JMA4uyPagcd5PRkLVYmApJDYMbpHTAZzJuXbmtJtaJ0XY5qfBGoFWSjdDyQsNxb846NoxdGsYupycMAHnVQU6UZMtJFh8fdN5t4Ts13kJmERZbkBO4OYIK9J1UVRP1yUozy9p1j4URdrLEemCqFxUdFfgeZ5xeaXTcHq7MWEGUVqz6Kw7hHkDKJszM/vHCWzI3blTD3GGkPpY4fKzYDBvJgQ534mP7/bkkU0npKdsXQzKU9yQuDe/kKe4ZxBbGnCnGbnNYsorXCGMXvP2uS2xW+f3RRtQqYZkdoz6ZyS985pC71b6cTd0BedQWZP0zxjs7lbptHLkjrXoXIjpHlOiVPAk13Uqzk0LyjymxErR/aJNjIJWkgxWIpZ/fUMpfshYUZKjNKDiO4hizALS39Uxy/LzzGe0xQeRpKXUPscXaa0Ic3iyBVJmF9IDG9pRe+QvMGdkyTtDNkN/ZxJmJLxa/OsKMyXfjQmzGh2X7CxxuKJ96rCkDICRfuEhRFojEimXfZIU74dM/uBz6RgUPtUpGiVa2JjUVr87VtvFhkuR2R2NP3DMZkdOcqKd2UxWSO5nYdZKU3u/4aLV1YsH9/DWXeoXZmAhu5hSQ6xxhIrPvtuF3OiaR+XvrDTk34viMywdUFj94TI3bwghkFtQGZbMfteRP6B9KC3fkkEyu1T4j4NctLyap20qF0NcXqK1rMhQUEzmtPJ0eux+6JL+7zAL1CgfRN/5JB2AtZ/LcYI5BRJNaQM2H3RQCvY/4tjNhslUeknlp7yDWHMNk9bTF0SdL0RSD2f2dFsfWWK8i25wM6/HTKYN2k+pQ+GFH/W64lYqKYnx0v/kNSQc+8FDBfkcuV25Jac3lcMF6ODxQACpxguJEISA3ZfSJN/KP+8+VLApKioXZa6LLspT+/ecxZOW9780bT8+nvPO3gVUUMVHwaMa4r+Sop0M2b/GZn4TH1gkOpK1v3UlTFOF+ovSba9ih6ZCQVOkdkxGH5pyG4nT1DQNC4If8rtfJbuYgSa5lMFuqeSo72vhbdVkQ9eAGwGQUHGruZE0VsxDujT3SNiCbHGoDzJ0ErVE3NjG7yyHOPt4xZhVlP+1CJVV+TWROG092wat6VZ/KHU0rEFubs2emJQdD0MN8KvRsQvdak/pxmd8XBP9Mic7BB0Ujg3MqR3xeAX5DTppgRsGCG4nYgorWmcswjTJB0TGdgU1iI6xyT4rXRLkX/4eEvwiVioYUYYpaXbomraedmmeBdSzZjcdkTvKNK+WTeZ/miCM4hpPKXIbQfkNsQqXL4THrS5ah/1OPS7Yt9on5ImvBHKdKh6XdT1nTNamtQJKCK/IT3W+kVHWlkrBp1jYo0R+rJmVDPxizG9lRS5nYj0ppXspjKnL91UZDc1k5Im9E3iBzmhkexLDRo5wmoq3fks9r1yRSVaV3Gi2j0hFjpdJYr89z2KP0uhkrZWZlfTecZnNC8c2f6KSoIc5KQJijJu1oZAfa2xCJUnJZHadZ7xmftph9iG2pURmc0Rppe0w3qa7AOb5Vybr526wamzG1RzI+zZEeaey7FqAw08fWZVbDNJlGR6T9G4aNI6J5rcrS9aGL6EtaXrYh9X8SMKuAh6yneEbtg78TmDpIEspsJDSfUwQo1XNegclR6gNqR2bJ12aZ01Ofz7A/aeFXWQNRKxiFfT2KOYrbeK9A7Z5Nc0Tk92mdbFCK8KflZRuu8z9Qu5gRbWIgYrj/I99QF0DA2Vm+IurV2LmJQFDVR4YNA6C4MFUxbBrkzJVAStizH2SAYGh+aa6OUxWgnNOnKhfyxi88tGEkmksSaye8YuhDmovyw1Y5BWDJZjsusm7ZMuRiCDh95RCZUwuxaZLdHCqgimP5Lxb+d8yMp3JvQPiQgmlwiqVaSZuhLQPQGLf2zSPypQtMFymge/kRcwxCimsCH4zo//8QW+++NneFCv8p8eege/5xKWQm7/6Cj9Vpbuf7WMXxTSd+dpH3ugKd6LcVuKmQ8l1FebwukaT4mxUhua1jdGjKcVua2I5lmTxgWL2Xcebwk+ETI/05MPejAvsTmTkixKbUm/c7CsiCMo345onTHJrWu2v5DHiCT9bTQDqXZE5JrEljTCtaFoPK2pXhaCnIqUoBWnFfZIJibjGU2QM3EbchmyR5pJRbiiYQY2v2yQ3pUFYXoypcEQw5+Kpc+qYk31qsQ6xkOD3iGFX4rYeXuR6oMYaxLRWxERsjk2KNwTK3V6X9M9LrrRKKUp3dbElkX3uKZ8Q2rfSUXKhb2XwGkbmGOpT70p8WIVV8X90Lio0Iam9gsTuz0gtZ+id0hq5/ZphTVUdJVN6ZamdVIuf5PpECOwmP4kYjhrMikYdI9I4mDpro81chjv5/k/ffyXUYsRCz+C3Zc0+WsO9ack8cSb0ji7NuNpcV3YQ+ityIZRvAONN3yO/z8C/JLLaNYm/d0sfhF2Xof0DnhTMbmNz9GtP0zLUxal5Ri1+zD1iSa7blBY89GWXCr8vJFYmIVYUngY0zus8EuarS9YVC9JDONoxqB9FmqfijzQHkD+vkmYi3HbsosNFmTRZLfkqOweMZjkJe3Zz8otPbdukGpopi5LWVG96YsouSvTKj+n6B41aZ5XTH8SYA8U1esh0x+KRjbMKAkHi8AvyCUG5NhXWlpATkeO+9iWWji9L7qDzumkk1EUqFrsaKo3I3EieGLr7h6RzNNUXTH7QUyYUmz+conK7QC/JHWj2xLBSmEjpH1Kat/JdMj8jw0mVVF3mZ6m+UKI29aU7wY0z0lNHduiZbB74qitXpFuSWZPgGmFu7DwE5/ig4jS/RCnp5NWnfRqp37osPlGjoe/pqRropKLWdMkKGhyGwaDxcdbqE/Ejqp0QkgeyU5hhGLk86Y0ftFi9v2I1ikLbzqJo6kpjAl0jiqmPw3FouwoOqdFrqciCZ4NsrKg5n4+wWkMafTKDBYVvaNQvCe1U3E1ords4i0EZPYsKjcnbH7ZkQRmT4Qpe8+bLP1wTP1iGruvk/AxjduS3qBWCq9iYoRCfH5EZOmcgLn3IpqnTapXZGrVPi8As+Gsgd1LFnxBrOH5h+IBM6JkOrcaCbJyISL30KS3bDJ+aUDQTuHVhOCS2Q/Z+qJFqmXQPR2R3jGZFE2yWwprHJPfiBjNmIQpRWFVQh/SdfMgxjJMK8jA1HvCmu0vWsJUqJK4eoWj0D0szFptSHvOGimsSczmmw7hnI+16xwknjzqUUcGzH4wodNycHqy+4/mpCbPrUH/kMbtfI52VIHryhOs/q1ZeOW6xhrHwkS1oXBfLgf5tZjxnFyANr9syKKuCc/THihqlxROL1H/R9A96rD3ahkjlIfCbSVj2YZm9yVR7y99R1G55dFbcXB6Aq71KjKWtXuK0YzLaE5odJOywNgmFYXhy0BiPGVQWI2xB9IK06Zm6YchrVMmU5dDeocFYOY2zcSCAkFe4VUl2TlKCYzMLwnHtXJDxrexJaeB2xYxsnkjhzkyKN6D9mnhB0RpmbStfDtm6lLAuCqsrMYFkyBrMCklrtFE8F1/JcT0YfrjmNFCzPQnHlpJmZFqx1RuxoKJDKX2LazK51S651O5HWFEUL0WSXixgvLPHdI7ogXwSzGjuZjcVsh4WrH9uotXVYymDbL7Eam6Zvb9CYNleYgfyQX/rNcTsVB10r4IM6IDDfKSDJLbnjCYtxguKMK0xvJigcBmFLk1yOyHpHeNA4pc5VaAX4kIssK3N32N25YjUNidCqcjD4b3zAgzkOSU4bzsmkHWYjytGC7LxGtSk+M0diQg1+4p3KaAIfpLAmAwfdnhwyR+PL0vR7zbVgxnLNJ70uet3pAW2vRHIYMFUbaHWVj+nk+qHSVHooE3GzJYgd0vSqurciuifySifSZhPE3AackJMv2JZud12fky24rBvMX+szZhTkQ3C+94eBXZsUbzopjK7MXMvG0SOVB/ysDwFSqIDyZWYVrQQlFKugaV2yGRo3C7sP+MgxFoZj6IGVfFZp7ZlTS+VFtTXA2pXlZktwzaJ20m1Zj0nghzemcC2icswqyiv+SQasrpOJr5HO2osS1K78w2tE5ZWCPN3Lshe8+mqV0ekNnTlO7CYN5ksCCxkMMF2H1eVqhfElxl+7jNzM8MtKlEx9qUdkLzrMl4SnZOr5ZgEb+bpndIgLpBQZM516Z9wpbEkw2TwVMeanpCblvYSRJJLrtM7YrHcF5/dgpoDtj0mYZMn/widE5/9vs1zpsEWcXOaxIMll8TRsDGLzsM5gQfOXUlJHffQoWK3AOLSSVm92VFesckvWtQXA1EfGLLRWb7SzH5VZOZD6WzEeQU8+96DJci3IZB95CLNdYH4uTRtMHu6zGt84rxlIFfFHzS3osZYhOaz0iQR/sMkrUbQm/FYjSrqV0aEWY1kWsQ24reMc1wTuxA42mNn1fsPSeDl3RdBgHu4T7akkvdzE/NxGYummBzIsEejzLE/qzXE7FQ3W4iFN6JWP79PYKcYuMrcglpXMjRX5GpVflOgBForLFcOoK8SOHKNxMSsikpdqanGczbbH1J3pBHav3IlZLgUVBs6a5oOt22wdR/44KWEazpweF/rKh9x2Xrq+LRsjwJQKvcDIlSJtWr0qTP1CXRr39IdofOEZNJBZw2rHzHI3akW2GNIbsXUftUiIRGCN3TIbk1DqDFQVY+OHMC2e2Y4h2F2xAc+dI39wjyIvIwxxywXb2aBB0/ghCvfdUls2mS3ZFBQqolixEtF8vqx6bkdp0Mya0LCdocQ/Vqn8Id8anFlrQHyzdE/1q+pemcSON0FPWnFNtfFm6W29aM5kRA3T0lk7vmy5JXFeQUxntFUfZPCUPA6epkp4bekRinZ1C9/DlaqKGraJ80qF+0WP2tGcI0LPwkZjyrDy4mop5KflwNhfWQ2NF0nvVx/toebicmvxnRPSIxPM1nY1ACEmsft2R60pBF1TsKqU7EcM4Q5fk+1J/JHpQIlqdpnHdpPAW19ywKqxqvIiTp2FGsf9VMaNPSZAcRw1RuhRTWIrEHW7D5xTTppmbvRdGt1p+ROtFMJkGpPQt7kKAu10QI4zalgR+mZMQb26Lmf/gXZwjSir2XOYg1UoHwUk1PQMHZ/ZjSbXm/pt7ZEblgsntldjS5rZjl377H4M0hmTVLRDt7EcNFzeqv5YVZkNU4XYPapQHts1C8A+0z4hNDiRbX6pgEWRgsia6gdi2kdMOgsB4y+30r+fnlYpWphxRXI4xQ03xe3ptUS/rRYVpChB/n9UQsVJBfqnxb0kEKazH9BQtrpJKeY+L6PGNiD0U7Opg1cdsGaNht59l/RbP/nPx/r6rIrpss/kgnraMJ2ob+okGmLpCu3opF73TA0vd9VCzilOGcQfVGQJgW2Z3TMWif1aTaEtYQuor2cZP8fSMJEtYM5iRceDiv2HveYu8laXA7Hc3y9waMq4o4Lbz/2mW51asImi+GotzKCmbT9DWDBRl7FtYiuifEwRDmxNuUrgtRW4UCekh1Y8Gw3zMYzyicrkH7hMloVlG+F/LwL8/j56TN1T0RM55V7L+g+fTWIX712DVAmvF7z8mD/SjR2+1IcvfgUJb0rqJz5rOHJ9WQlpk9VMlureidDahfENdC/aIQENunhfQiqeAGfl74BPk70mQq3QvonAuZ+7k4Vh/n9WS0p2LZJXIbY4wwhdOPBINYjEkFBmokx1N2R2wh2pYC3hoBgUG0k8GeH+GrFNWLTUylKaXG3FhYQdsRG286ZDfEK2WNpW/qVRT5Ozbdo9JJSNfliNx52RL5XwcWfzikfSrDuGrSuRBi9UxS+yQmNoPspgalsTyRyhljSDUMiqs+D79homKBC1vVMcP5HLEjHnfTA7sub73SULo7pnUqLTTrvGL/WXngVCzAsyALrZcCrLrNzIdgTkI6xyyx6sxpjFAcB5OSYjQnwhClIdXRpN/XdP7qAPdIgB67vHX4DmkzwB7KYGXmo5DBnOAtR3PIGHckOghrLOHFYQZyOxG9JVNm9Hek1aRi+T3sAYmvS9M7YuB0ZRgRuqK5HS7G2P1HpG3JRy3chu4hSLU/R0d/bIMz0EwqLoMFk52XHMbTmtol+VDdbsxwSWqxVCeisDahey6gfyrg0O8LOMK6lMNpm+yuVdl6WGN/mOPUhXXy96yDES0KeodMIkdhebD4rT38vCLTCEUwshdhjUUt7ww061/NitK+rDA8g+lfSJ7UaE52Rq8qAo9JSSKHIgcGL4/Yf9rB6hu4PfE/pd/LETuiEAtT0HnGZ+HtgPKdALej2X0xQ+uCpF3HNpi+THrSe4rOKWFi1d61hWeQEy1q5EoyiT1QOG3hS8180EdFMuAYz8bUf2PE7osGrh3Q6WZ56/AdbnZnuNRZpHciAgOsUZSkBGqC42MGC4oo4ZapWBwATl8zrkgKX5AVmJyfF35qZlcE2OOZZOffl4ttet+nfVo2l/yqgDQi95FyS1BF2uIg5PfPej0RO6o9lFYSiLnO6cuIcVwzyOzHDOcM0ntyMegesQgyFqltqbtMf0L3GETpmLgYYrYt0MKAchYivBcG2FdyhFnhTY2nTFIDTWQrbv4XVbKrsPuCLUd92yC/HuOVDcZTUL0uLP8gA4an2HtRHpyVP57QOeYmYGBBCE1d8egtu+jtNCgRhUSuTIXcXkyrYFJYj0nv+2xZKVqnSWo+qcMzOwbDRfkw82siYBnNaZa+H7PzH3tUv5kh1ZSHxm0L/bpxwSU2IdORXbX+TI50XRZW7qGBddTDOTekfmMKZib8yd3TuNcyQoVJK1KtiOGsjd3XNM8p4qZ0UWJbUXgY0jhvsfzHbYaH8uSv7rP2F+eZfT9m/1mD0h0xXrrtJHOrJJTDICtl3P2/YnL4d31GszaFhyGRq2ifNPGL4Axk5IuWOvxxXk/EjuoXDcZVg9YZi8oNj+xOknea1IXWUKYo3WOixne7HEQoPvh1m3RdERdDSh87FO8oFt6OSW04HC/VKX43KwKMIyH7TxvkNmOJ5SlCesOScWYsKXfDJVmkQS4xFC4IacXpySWidFPQPVtfTEkI72n5GqejePDbArsoPpyIb2ooySX2UPSmUUpCdHdeTrH0vSFuRx9ES3o1EtOhPgCudZ/2sfuKxgUL43aOcc1gUoTSvRC/pKhdGSf6BLlhd1/wDviqgwXFyd+8zchz6Xwww1/48vuYVkTUdcQ2bgl2snnWFCllT+w7pZsGfjmmdU54tMXVmL1XSnQPWey9OYeKxVRYuC8ozt4RReGhR+eUGBp7x0OskeyiuXs2e8+7dI8ZjGum3C/60r0ZzUhw8CMLzeO8noiFanoyyis8jGmcTxO6khodZqWGq9yeYIQSO+N0YLgozW/DB+2KLWTpm6JqTzdjdl8w4WyfD751ntGsZHbOvm0w/25A5CTWi01N7XpIuh7jLQRoJTfY3HaEESYxNi7UrgUEeSg8jBnNilwuvSeRO5ltuWQMDoc4ay7F1YgwI62hIAvDJSFYm75YQVBJN+CtLFrB1Kch2R2pjb2Kwp8PaL4qlzkCCYDLbksdHGYkP6u3YjE85rP2KylJGqxKlJCemHTeHMOX2vjlmIfdCrn0BG/R5w+/9TLOpzmKNyyy28koc0p4q9qUkehgRXa26mVJX4kcg0khgX90Na3zMbUrPoM5Icmk9yVeafPLaUxP6li7J+2t0bS0y8p3pWUlwRiQ3RWdqjbl8zYC6eY8zuuJWKjagMJaSOiKXrO/IjHkMu2B5tkUkSNe/yAnBDm5FIG7J7S9/WfEkOcVDYJijNdIE7uaMKtpPCdR3uMpi0wjwh6I2mpcNukdMah+YEnrKM8B2jzMKkr3IgbzsuvaI02UEdngaE4ILkYgcIjMhkXxvgg8BvMWkS1/NrcGo+UQb0rKGBXD/Lse6T2NGWj8vLg1J5WYIKcpXnLI3XDI7kYsfM9gOKfonIDcBtQuBxi+/J1G32LxBz6tc9K+0n+tgTEyyWYmLBS7LF3coXG/ghdYEBiUb8p7NanAYEkMiH5eHrjBImR2RXCSbsR0TsolyytJfPzhPxwzWJTdtnHeIbsXM/MLD63UQbR8al/qTLeR9Kk78r70VkzhKXwsJ1TvsEmY0ZRvByJi+Uqd4fznaDIFsP+MxWhGsfOqwpsLCVOJyr0tvcr5tzuS3HxYlPvDBaHHFe4nDoDdJOn5qwNqR1vYZY9UQ3H4D/o4LYPeYal/B/NSJzXPWdJ6akiNaXrS+nnkjR/Oi306toXfOpgzZR7ejdGGlB1BTlG6FzDziwmTxDJseiKs2XvBINXSpHYs8quPfPew83LqAM9oj2PcdkztkmL5ex7mRMaqRohkPHWlsV++4+FVTNL7MozI7Bjs/k8mXHjhPgtP77C/XyS1Z+AHFjPpPmu3ZuVNfa/E9M9NhnMGhYch2e1E/bQfU3ogY+JUQtLef1EzrhoU7sN4ISTMKAaLiv6hFPbgke4WgoyieTZF55xwZd2OnHDWCLmsHo3pHRapYG5LUm0Gi9LFMD2wewaDOYvmWQvjd6rY/cdbH0/EQlWxuCzTDZF+5R5YB2kgTlczKcHey0WxNZhyc4wdgZo1n9b0joku1DvhEUWKyfen0GtZekdj1r9SQMWKQ98aMK4ZxJa4PIfHfGqXxAXg1TRzP5+w8gd1ohQYE3WQDxDbMgGzRpLj6fRjUg0Jm0g3NI2LNnsvuGTqMf1Fk+5xOU5rlwRHntmVn7d2RUxvRiAtmdGMwc7LJpOS1MGN82mUFivxuGYwXIokj9SB1qkUvcNGEt0oHABv6ND107RGaaxthzf/wi/wOimu/qNzZDdM0NKNeJTx6ucNxjU5skfTBpEtCjCnoxnNamZ/LvkI/cNw9HdDui96WCOJGHK6mt4h2Q2DnGJShOonJntf9RlPCcBNG2JHL98UP9ruSw6TggB+S/ciJlXZ/b3FgOGiAJg7J+SEeJzXYy1UpdTfVUpdV0pdU0r9jlIqpZSqKKW+r5S6m/x3+d/6+v9SKXVPKXVbKfWVP+v7R44IUx7x3u2+JL/VLsdEaek9hhmxZribDpELcz8PKd33ya0ZzLyvGb4x5H//4h8RdFIybr0ptt1UU9Tn7TM50ZJ2knCJPVsK+Vi+tnvEYe3XpvCLcomY/iSmfM/DbYuhLtWJGT41JkwbTCoinB4sKeweoAVxo2IB5qIFY9m74BNbiuG8onvEYvZ9MRu2zghNu3hXpG7ZPWkVhWnFwk+G5DdC8vdM+ouyOL2K8KaCnIhg9l6PeePkHR7cmSXr+pie4vvfep7CDZvxlKJyI6R8Q2R9o3kRe0euIihK6yhKSY6A0xPBTaohMZVeReOXYuoXUlTedhnPyKjaL8pCTDXkFCvdFzuP+yDFyncGTF0JMSKY/iRkuJBAlvsc3Oi9okF+FSo3Awo3bJnmaZnOtZ/69xSIppRaAP4O8JzW+hxgAr8F/D3gh1rr48APk/+PUupM8u/PAl8F/u9KKfNP/TsSUUd/RWb1mUZMdktumJMSTF3xqF2ekN42WfnOgMhV7D1v0znmJH3DmGBi8X+59SZ2yWO0HFF/JaR8S75Hf8mQMOAhjKcNYkcRrng0z4osze0J/96ciC27eV4sxY3zIlzJrSl2XzYov5PCbYfk1mW3dFtyOSqsxRihXBL8vLSkalciClccjFCgYOlGzGjaILcpSSqZukyfjBCGsybVax5Tn4y4+zcd2idsjEg0CX5JnAQyepQPfuo9kx/dOIXKREwCi6krIZnkAqciaJy3sIeawaI6GGeiweko9l7TOJ1EShlC56TUk7Et06mpj6RkCXJSDlljTboeMzg/IV3XxK6IzPuHFaU7Mc1zWdonLLyaZv1rctnN7Qh8zauIKLt7EpxBzGjaYjydcGILJuVbYHX+1KXx+As1eVlAWillARlgG/gG8E+Sf/9PgF9L/vc3gH+ptZ5orVeBe8ALf+oP4UvLxBoptr9g0Dxjkm5Kkoa2oH3CZTxlU7wfs/lGjsFKTKouu1p2V+bvuUspzO+WMAwNuQA779N8y0PFIh10O7KTpPdjzLEmdT2NihWl2yIk6R4WuWDvMEx/FLP7opkgxkUJld1UNJ+OePjrBsNFhTclO2GYEfuFTpr1YUZ2r/rTBsPFODH0xWS3faEEhnL7nxRMKlehcE9gZL1DLvf/kovVtshtR4K39GW375yL8Woxua2I7HYsSdgtC2XFdNpZtr5gEKUUxfvxgcKrd8gQ/JCS2fp4RpwQ2YfmAVxNGwiWZy9m/MyIuZ9L3do7ImZLpy9ft/+Sxt5wSDejg+C3lW/3RUiTV4ynY8K0pnjLwvChc9TCTKLcYwuWvzuRTaesyK9JmfcIsnz4m6N/PwtVa70F/H1gHdgBulrr7wEzWuud5Gt2gOnkjywAG//Wt9hM/tl/56WU+ttKqY+UUh8FwRDT12R3NNbAwJsT1b42pKlfXJUexnBeftz5nwpcLEopRjWDzS/LjN4ag7qRx2g4ZN7LYmymmLzVY/3XxLtu+LLj5XZDSveE66kNwdrk12NiE+Z/FuFVDKY+ldotdpIPrAjFWxbuvkl2SxDtO68q5t8ektv8LBq8ckuIeZOFAHMsE5/QVax91SW7KcELj3BBvaNyIUvvi6Yhtya35NbpxNdUls5H9qHJwjsS4+PnFINFcTAoBX/+3BWefeGu7OYF2UELDyNym7ILGwEixomS4Dktn3qchOsagSxq52ZGyoyaxpsV6orpSUyQNRTGV29FdupUQ1F/OkfoCh3cHhig5LIYJrglbQiNBaD+VIruEQNjAsX7Pl5iCY9tuPs3nH8/CzWpPb8BHAbmgaxS6j/+0/7Iv+Of/X9VzFrrf6i1fk5r/ZyZzYIWq8ZkeUL+nkW6ER0IedsnHCJXUXgYkW5o+osm2gKvKozN3JrB1hct+suK6Y9CsusGk3KSXXW3AKHkqk7KCm0p6hdsWqdFNR/kFZXrml5itWicsyRuJq+IMqJoGs2IKzVdFzzNow/dmChG8yk6J2WGPZ6P2H49adFsOlhjQbhPiorculxUZj/wCTOa3KaPNYDiw1DCwu5FB5cweyAWF3ugD6wd268ZtE5ZBAVF6baUBdHQ4rsPTnO7MU3vRJjAz0SR1DklKjEM2HtJYL6LP56gLYlHMsdi24nthEC4J1E7U5/GpLdE72BOZEGnd2QogYb2GRGj+wWxlZRvyfDF6RgU74lQJ9XUBHkOFGZeRXxetSsjtr7kCJtAy0mS2v73lzP1FrCqta5rrQPg94BXgD2l1BxA8t/7yddvAkv/1p9fREqF/58vFctlKbOrKVxxSe9rtr9gMp7RDBdjnJ44NgfzUs84PXkj0HIBs8bylGe3NY3zFvqNtniefHmTjbHB/rOK3ilpexXWYkE6VhTl2yGtcyqh90mgmjkRTr6KFFOfDimsfiZEKWzIxSHMQm5D4ecNmcHnFZXLBpVrUhYUVsXT3jsiQurxDKSaip2XHCo3NHvPCwpzUjQZVxTN8yZ2X8h9hTVRFdlDWUhBXnY1v6jJ7GjGM4rsfkT2gY0/sfADC2NikNuJKKzK5ctIhgzWEDLbBqMpg/YJVxDxE2lT5dYNuqc0+28EdE9qdl802P6S7HTl2yHDOQn2GM0ldWURKtcU7VMSLFd4KCeVikXHILh06ByX4z3VkQfNGknnIMiLusptPSqH1EF217+PhboOvKSUyiilFPAmcBP4JvDbydf8NvCHyf/+JvBbSilXKXUYOA58+Kf9BZENfkFkZ9XrvuDEd6VmLdxTDBYUdk/RPSNkkiCvCFOazJ5idMyXmm2SgGddiN8r41U1kwr0zgbExZD8Q0X5sklhLSK77Yul937E7ssm2Q15w8KMStA8EjcZZWIe/koWtxdTuhMzeXbAYNak/oJYLNxujNuNya/Hye8hH1qUkhKjvwKZXYPGczG1qxHVqxOmroR0jxkHkeGDRVHL59ZF/L399ZDeiqQ+9w5JPZzeh8miT+XZfdRfqQvfdcESZ8O9NJX8kDgTMZwx8WoiwtaG1IFGKLtZ7CTYocwj129MkIP5dzRGzyK1bzDzUcz0+4rZ9wMBJreTHb6vWPlmG2so37d2WS5uj3JeVQhTlyYYPuTWFZWbQgsP0orS/VBSFJ+P2HnFonxLyqPM3oTynYDKzfhPWxqPv1C11h8A/xr4BLia/Jl/CPw3wC8ppe4Cv5T8f7TW14F/BdwA/gT4n2mt/9QehBHJE+h2YPtVgUqU74SYnvQcx3MR1RshtQ8NvJrUkuWb0H3Bw6rb+AVNqimUO7vHARnPmw2pzXexGjZ+QYQejYsG4ykbbUHrjElsSsPaL0iZ4ZdhNCOZpqk9IaX0F0wiW1H4QRYUTH0gWVhoGQSAZEBFKfFluS2JYy/f1HhVTe1jg8GcydaXHJqnLdymLIIoJbf63JoIqv0SVN918IvSv7XG0iSPUqDGJsdLdfY2yhx68yG9l8eYRweUXtxj7/o0qW2byg2P8UzM6FDAzEcRYRo6p2RH7J0JpB2n5Da++0shQV7TOWoSp2KUht1f9+keNRgsWpgTYasaoWZSi2k+XUJp+bnS9YBUS0qF8XxEphFRf0rMj9ZY01sxGBwN6R0VtFBsKWbfNshtSo2994KBOfDxyiajqce7zz8RfNRsdUmf+vW/y2BREOTpZsz+c4KSdJLMzN4x2WWzuzG7b4Wk1hyitMYaKOa+tMlOp8C4lWZ2qUW9nefwTJNYKzY+XKDwgAOOv1YcmAEfsT/dDmR3IvrLppQVGTm+7YGifEcsy6W7ApLI7MXsfSWg8ImLVxWx92DeEBBwVXz9vWW5uY9rIm+bVMT6HKQVZgDDWYP+IXmwUnVR/FtjTe0XbR7+ZgViYaXWLos+dVJWjM56qIbD3Jl9tnbKECmOHNpno1EiGDpk7zp4NdF9xrbGHMtCDzPSBbD7UL7rs/u84N0Xvr1L9+lpvLLQA1MNQQTZQ1FUBRm5rJbvBGy+YaEiqF0RdqvpSemTX5Ouhl9KchEiqWHzD0XL6ucVQUF23Oy2kFVK9wJ6y5bkiKUNwhRc+gefEz6q0nLsZ/Y0flE+yMWfhGR2YwbLctRXr2gK62IfqXxgU7oT4zYF2rD900XCO3kWlpvs7ZXIZCbcuz/L+icLuG1ZoF5VYsHHs5ql/9dtwgy0z2km0xHjaU3kyg27vyI/U2ZXFt1wRhj/QUZusbtfjtAj8UWV7kgLzUxu10vf6zGaNmWiZMsOHaWkUb//rMGkZNC4oDAnmvyqgd0VKJo11rTPwNZXKviFOAkRFp9W/7DU2vmPU1SvKDo/mYVIkV51WL09R+VbGVLrDtOf+FhDgyglNvPstmZwREQgo/lI8qWKJrXroYx435ihcUEueEYI1ZserdMmhh+Troe0n4rI7sYMZyxy6wKbG84Kp6t3Mkx6sIrqtQFacRArmdmVeJ7eYTjyjftoQ36WMCV1av0pO8kIU7TOyMj6cV5PxEIVUUrMYFnqF4AwbTCekslMkE12oSUx/GX3IvyC7FKFB5ICYnqK3vdmyV9yCT8qY/bMJA1P+pZRLpbifqx4+J+dZOpyiNMyKF0XmZ49kvidzI6ifyLCq8ixldmPKd1KWi4K0us2bt0iyMc0npEPOV2PabwQced/7pLZj7DGMV5ZuKy1K2OWvj8k1RBTm7/gS6ss0Fge5FcToMbiiElJk2oYeFXpbXaOmwcahPGMxhlIokpqwyF2NYvf0wwXDNw27D0nx0T1imb3lyWH69AfhOQfQOmmwWhWEVvC6KpeE+t24QEUb8vvNZpxhVB4MUX3sE3xuoVXVgwXRd86nJMQt9xWSGrPono1wOko0IkA5Ri4vQi7r2mdlb7ztV8cBi2tt+x+RO+wKNeilGiQCw8gt/U50qNq87OmeX9RbvutUxIIm9mVXp9KbvjagP1nTAZLYkHO7UQU7hlYIxFb2APRAKhQEp8fxezk75p0jhlJ/QrD6UfBE8I1bZ0y2XkrYrCkMUYGU5fl1mtOxBZshHK0zb/roS1N9ZKieklu1/0Vg/SWRe6Km3ioZBLWOqfYfzbD6q9lZDQ8gMxdV3ajszHjKRG3ZLdjZv/fLumk9VW5HbH3ekyqLijJVDsmuyW1XfX6mMqNiIW3fUZTJrnNmMiViVSqKdrUwmWXwRLsPe+Kin5WZvz2WKiCjYsmMz9vo025HxiBhHmYnmZS1RQfBtgDWYBBThaSgI+hc8xOiIUmk4rm9v80hRFwcMEL8orMjqJ2fUw85bPw9oTa1ZDtLwmoOV0XAcukKBPCz5WvHy3YxpXveGT2Y9yOhEAEZZkpR2l5s/yCCEFKdyTlrnYlYlQzGS7KhMrPJZDfAoSVEK8qUjbgIHShcjMiva8PmuMAlZuyiA7/ribKxmImLIuudFyVnu1g3qT1TEznqEvhvnz4fl5YV+YEKjeigxHnYM6gdD/GTrj1xbty07b70hbKbutEsicj3Pqz0D5hUbvqSSurapDetAhyshNV3t8TcfW+onkuTe+QSX/JYbAkutJUUz78IAflWx5BTgyDAK2nZLCR3QtpH7PwSorI0bSeKtNfkZ0tSovmdDyjWPq+J8dzSdDz2tIMFgRZiSE8geGS7M6F+1C44kiIXEPo3rElD037WIraj12G8zaN8xbVjw2qHzXRhiK7E9M8rw74ro/zejIWKoCG9bdSxDZMSpqpTzSZdZPRnJIj6kGE5cni6K2I83JcldHh0vd9Ikecqu2zYo1Y/BOD3IYsnHQjZDyrqNyOaJ02Gc6Jekeb4v4MXZU01W2WvqcP9JqT0iPRhmY0r1n4gdSL7dOQ2ZVhw3AxcVumxDefaovIOXIU3tkx/bM+QUYEyP3D4mcaziny9w0qN2KKdxXmRAmHqmrjtjW9I+DNSDjZaMqg/vos9kDeJnsgEyfL0/jliNGc/I7/n/b+K1jS88zvBH/vZ9L74/2pOuVRBaDgQYBkk0Sz2VKrnUYtM5qOmdFubOxOxMbOTGxs7M3ezs3erYldzWp8SC2pW6bVJLvJpgVBeBSAQnl/vEnvMz/z7sU/63AmYlbERkiaQgS/CAZAoJAnT+Zrnuf//E2yKcy0eilNmNPiso50/MmGpfaUT6KtxZjZMzRPq2GLfWUfNM9A6U7E3pfSutk2IpobHjMfQn7L0luG/FbEaClgNC3OcP2Zx6cux87Ug3nV/GFGp+WoKDtPd2w5/NIUXl/2Rxt/3CVVtYr9+RzPE7FQlc+knKUoYY6NuDL7IlBUn49pryqSsPncGCcStJNsW7qrlqPLSYXiTpSb/QXL9jfVdeZ3Qh79DhONzi9+Zm/eo70hUsZgxkzKBmUxeQO9ht+x9BYN1jFMfyzysyLYRZ4O07/wsUpXQxFk+jGJjuq1uT9LMvWOLz+pkRaI3xUZ5PH7HRcMxdtgPSXCuGNL/iH4TdkD9Rd0nRfva3LV2nAYTpxJTCiGlDuyOJE8V9P1eEIsV0nVW9YYt3g/AqORdOtSQJi3OIFhOIlVn/7UkmxFE/6tpXJFeVf7b4TErhJkgrSD2/SwaTVpfltE9mTdTg4YQ/maw3g6ovgwpHcyIHMU0zwvuK9zYvKeliy1izmZfYy/QDWqiXX1hBldByZWbVl7xtLZiJl72xxfq/lrCUwoDuRg2uH0398nzOi6NBEs/URWi8VrHqV7Y/Ze8cjf8jGxobvgUrobs/TTvhZcz7DwlrrjUdFQeBDhdwzuQGz+zroamWQzpnVK5N9RWQEUw4pKCRND93RA/UKCRGcy3jUC/92R0IJEC4Kc0vNEWRS6sPtGdGx1nj6S/1PrpCPrG6sR6MzHMa1LAa2TKkUSLf35/ryRrWZKn4Pfj2l8aURnxaH2lAYP2T3R9jorDvVzLrEr1Wzxqo/X0ShWebOQ3Rtx+FyC3LZl8c3wOACu/L6PNymbqs9bZt8Hr+oTpYQTWwe6azAqWxIdS+FhgN9y2XvNZeW7Yq7N/9yS3xLMiPkFfzW7Fx+jLL/seSIWapzQqZCqTU63PcPgxJjZ9yG76dBddjh6Kab4IMYJNUL1epBsxDRenKPwQIt49sMRR097ZA4s3VcGeL0QdyzVpjPSGLQ37ygvqWsJ8pbOqosJxbkclh3GlZj+IhCrOVn/40PaJx0GSyGFh2OxpDyNT1NVIQrT73p4PSuoaqQOvXHOZf9lh+nPRmQOYla+H1K5qW68ckOL1m15JJpQuxzTWXNor4mTYCxkt2H+3YjuokvlQ3nhd1bVgU99apn+NKR9cgJhbUQcvODibyeZuhaQORAsVrw/Zupj/b11xakt3wkYl3QQpJrxsd16a11N0bBi6C571F4b4/XFOdj/isbAlU8M9QuGufdj/JajsilnSTQM4ymhNc1TCRIN6cmsUXPYWXapPifdlHWU6zWqCL77vCvwiVio7nCSSVo2x8YLC3/h6RcBBjOWqSsOyUaIO5xEz3QlGcZA45xe5/CFJEHekt0LmPuXSTprKQaLIeOias1xUcB68Z6c8pxII8DizUlIQsviDg3OWPaVXt/y8A9mCdOWxR8YWhuJ49Ou+dyY+rMxUYJjO/SZj7pYByrX9aX4PcPBi0na6w67r/u4Q+GZ1kB2V7/buADzPzd0NwK8Phy+MCGJlw2Dintcw5kIZj4JGZWNYKN5l8y+pXJzeBwhmT4w9Ge0aYK8Qo2dUHqpVFWq2P6Mx/y7Y6IU5O626C0LsK8/bemuR2QORCyfeitBkFWA8al/FAhZiNUYVi+5kuvMuwSzet9r/8oeh9b1zo5pXQqOTYKDHHgdfa6xrzHucEaBxM74i9T1M4E/unKqC/Ki9D2WN1hXJ+3210Rq6KxD+e6Q4t2+PoyxofgoxHmlwdwHMY0zCcY5Q2/JoXJF06a1b7dJH6h2a52Ul2nhrnDW4ZRqx3FBgjUMZHb1c7I7lrAQU7sojqwINJDaTJCsuSTagtZiH7a/kSdMG0YFkbSTNRhcHJBoA454tc1TDlFSNXHptkbHvTkHr+GRqses/9kYr28p3dMJVbk1wgkgvxlTPy++Z29Jc/veouHRt1IYq5OvuxbjjZRkOP2JqJDjvKG3qGt5MCux5NbXfUZzIdvfqhD7lsGsZenHMZWPHVL1SLbxs3Kiyexb2mspEm1L8yw0z+pENJPs08InCfqLcj+pvh7gjCF9P0F60ydd0w2YOZTvQOVmjDNW3V24B9Wnzefu+p8IAwprpPp8zEQflSyzH6kATzYs7kis8zhhGMypw9z5cprCw5ggD+UbMVvfgsy7ZYZlmdGCPPeH0xPD3i8VMCEcvRiRrMmzvvgwoHNClLYooeCEqbf9iTGC2FPWVWeaqkKiHVG6qwYEo/c9/WmfwxeyxydtlJIPgB1rA+TfTTMuQP6+RpJR2lJ9TlY5UdIQ5GH2w4Ao7dNZNXRXUiTrlnFOyEZ/PjlpzmK6pwNO/9dj6hcypOsR/YFD9kChY37Xkr/vgI05+U8itr/mEhRjklX5dfUWAGNJH4m17wQusWdJH0pVsf+yS+pIyonsrp0kQFuql8V99buQ2WNSuvSJvTSjipozbwDpekQjTpA5igk7htYZ2Pl1S+7u5ARdDBhveWQPI8Ylj+6ywu6M/TdESvl38fhdmZCZWNr1/EOOR27puqVzYUyQNWR21Uy4Q+3q9rp8jhrnHV09ocDr2kUzUQxo0QzmFFvZfDbABMJW+wuWgxd8Kp9Zpq7pKkxtJwjy6mT7v9di+qqarPJnDoNZS+18gnFWJrdRQmbDd/9mWjaKpya+q7eF34oWJ+jKGwiaGk6p+080HfoL+jlYZHpbkhJBJBRDb0GoR6pu6azJUqf8kcfWG1kyRxH1cy6FzZC9V116KzH9RUPrbMTetwIe/CEEqyN++7UPGa4EFF86ZHBxgDnfJfdX9sn/wS7tk7JRGla0YYp31StYV5OyUUl1f/GOtFbhxKcgzMDWGxn6C6r9m2eVSdtZcYkTCn0zkVhXsz9XmvXiz0bM/NxTlteMS28tIrelW/To+c+3Rp6IhTouusfs8uymQ39RDiKPHfTSDxKMKpN0k4litbsiFWWqHuO3xeqJPVnsFCeAfKKjD3uwGHLwokPhho871BdTuiMla3/eobPiM3NlhDMSJli5GZH6V0WCjE5y60q27QTCL0v3AryhXO5KN3S6jmdCTCzZd28lJl2L8Lv6d+lqTKKtay59IBw0s2uPcwAy+zEG2PqmumGxkPRFJluWyrXJxOlhSKIDrXXZFllHk6XsjsqkylWH1IMk7kGC4rsp3vx/v0jlA4/6pzMk7qaJ7uY4vDLH5tUFyjdEMLGeSqHOuiZ0J/5pnTBjmX83JnYNxQeyMinfioj9yWdRhzAX03gmIpof03phRJQAjEaiw2khEc0zUHgQ0TirAyDMCEOe+lDhFbEHhXtfINvJx1dm4Z7GpJk9CdmitKG9YSjdk68p5rEmScB7UIk4eCOgtxLTOO2R6EDjnCHZVJSh3xdvsnDTw2z0SB9pwYxLYCKL1zd0nh0S5Ay7rycZTcVEKcvRZfFFg6x+VpTSNCXMaEM0zijCMrsXYz3Zrc/83CO3JcnH3DtSCiQb8p0KU2KBzX6kePLi/ZjeimIqhzPQn5eh7eJPYDir5Oj8lmrFcd7QOiVOQeOsT+X6+Hii0171RBUs6GodlQ2rf94h9i29VcF9TgDZTTUsuUdioE1/LITDGasGzW9a5t8J8LrQeqokrHNOhhxOEFO8pRHz1CeG4WJA69kxxIa//fI7/I1nPsQ0fbzBL0zgYk/j7Ow27P/2mPxWRLoa4/cgKIg+mNuLNRxJfb418mQs1FALs7csckp/Qd5G4zykjuRbOv1ZiBNIh/545p955JG5nSS9rzGiiSyJtqF45RDrQnPDwQkgKMDsH6UFjI/V5AQ54ZvZq6njsWZmV572Xk/Ut8dRPSaSrmjlO1WCrBbwYNqhfcLB79hJSEPEwVciyrdEeB4sRWK/T5JHMtWY6iURPwaVSWbUfszUZxGjsqW7pFMxu2nIf3rIsOLQnxNOmn+gjZyqWw5eSlB8EJE5jAm/1YR5NVtT76vd2P9SHpCyIVVTrhZMFvZpZZM2zkOUhvxORPg3arQ2DN0F+aSOiobSTfUBfhfq55JkD8Vd7awZMg99ZuZbLD69zx9dfYFb7Tmc2SHtUxH9xZjRr7cVsWRVo7tbKaynAU5uOyb3yKG14dA461C436Pw6N+QXPrfxRMnoHVaNWpv0SEoSDeUbKomSjQth88JrmqdUZ2a34xJNi3DGZn/hqmJPWQIt/6TOZEwetIW5R9ZWidc6k+5zL3Xx+8pGjHIw9S1gOyuJdXUaZjdtcfRjIVHIaOCHOqSDcvtv1dhOG1I1uUF6ndET2x+q6dhwic++Ud9Ei1L/o4r0H7dZVyA1gkJ91qXAqK06uzOqkPtKZelH4+p3Arpzyi89+HfXKR9gsmgALKH0XGGwGB9TPOUy/7rMYN+kuS1NAs/HxJkFXvePhOSf+BQfBQSppnET0J2R9inM1G2Fu9HhClD6r+vULkRE2blY5VsWTJHIemGXKJHZdj6df0uQU5j4+47Mxy2cviPktz64QZRIwnFgMsv3GXQSZFoy/Ai2RIMVz/nYV3dnImOJVWXdqx5Nktr49+sXPrf/hNrTJfZs6T3Hpu+CnZJ1yMSbQ0F4nJAnJQb87hgKNx36KyK3ZRsaJTnDlUbBnnJeZ3QktuNiRKW7TcyYuE3dDo3T/lECRhnRfYdVQzZHRlE7PyaR+crA831E4bT/11LmfXVmGASye4E4H+am7xfy6PfzGE9uQU6Y52+1oP5d7RB5n4iuGz2o4HKm4zl0W8m2P6Gw2AORaX3oXRbkZtzH45pr2jcO6wYlr7rMpqyVD52CYce/VNjahd1ankjS/JIpOTerAIskg1LlLK0TjmsfbuFO0EjHjv5uWOReaxRTE/tkmr4w+dcjURvx+QeOIynIikuDlWjFr6do3QHFt4ek952cQ6TnMjWePbkpkyBH1MZe5bBXExx8vt0lw2DGejN+Rz8WngcEvfLnidioTqhUlGCvD2WL3efVu04nIKjy2KYh1mDafvEnmbsCkaAlR/0dW2V1IC5g0nwVgj9RWmRjp6VYiB1qFOqeR66q1pETqhatLumMiTRttReDKlcA+9Wht6SoX0qonO6wOavp+jPOrTPhhMPp5Cp6xo5BjnD8o+H8lkaycV6XLTHadhhxnD4G2OipKF6Kc1gWhHvxTtKks5tW3pzDt31mPZJhAicSjCYt+QnAsPOksvMh8JFy+/5nFnbp/jbu/SWYvozgpHqF4R6zHwSSGNWjogTlrt/qyCjth/UCXLSUu296tJbMsx+1Kd5TqjE9u9FVK5pYtU47eD3LaldjzBnye1HzHzUo7ssLVt/1mPxZwPm3rf85dZZiokhg1mVGOO8hgnu0NC4YDl8zmH2o4D0oW7Q4icJRuUv0Kz/MbDv9UX0iBOWk/8NtM5HRClR2HI7Ed2TEWvfVWBBmNK0JkpB7WJaxGZPAP5oNqJyPSa3bck9VKfpBLquR1MC8fP3wTvTIfbkvx/kJCB0xxMy9JasZzJ7GpOa2PziZxTBb7g0zjgcvOiy9yXFA8UJ2HslRZCHREtfQOWa6r6dr2QY52Huuwnp5QeiLGqaI2vM/GZwrPeKE0I3+ouWE3/SJHMocwljZVTWXXEYzBoevL9Cxh9jKwHGQn8xxhtoNDkquTgBJOoulWuW/EOpFKrPl5n6TBhpoimu6sFLGXKPVNac/G8lH88chhQeyvjNRJCsO3TnXe7/fkZ5V21Nux79lRR7X7Z0eil+dn+DMC1D4kTHkjmIidKWZMMhVdPtFmZl1zT/sxazV75AOKoTiA1VvCfXk2TNoXYhxfyb5tjo9eAlh9x9l6NnffyuJdGBoGCPzcyOXo50Ip/qkd5xaa86x3qh4oOYoGBJ9Oxx/Pi4aBg00sqbL1iKDyMKDzUS9QYxU9cjGk+pdPDbMPse1M+6ZHaksUofGKauSWGQaBjW/0WdzEHM4pt9iQ/vjOkuG+oXLaNp2V/iyO4y1YhpnRaCYKzoc60NzckX//KI3CMwAdSf0k2z/+Uy1WceDz8kHQ9yqtWDQsSDH6+T2EzQORFTvCWvLhNKZPfYxj1zEDAqSRlqPZFmrGcZzsrYWIMWoSHjohCMo2d9Yl+QWVBUCeH3LXPva2ScbMWMKoKqnMqYV9YfErYTuEO9byUhOiz9SCre2Iedr/qEGdj5VszD3y0yzn6B4ClrJIvwe4KhsjvajWHSTLzyLeUbk3z43IQDmVUNOJwW5W3+p/KI8j/Nkd21DObjY2WpNbpancDSW4sYzGqYUPjMp3w7oHAf9l4T9ucEcippr7msfG9MommPLXlK9wRRjYsw/emI+gVxW/0udM4UlfT8bIbKrYjOSoLsnrgDiaaZcD9VE49zzrH1ZJTQVAxHr7v1V2eENMSG9T8bEHvQOhuR3dGGnvmwgztUeFyYhcrH7rGGHyPlwLiopD5jpf3K7BlaJxNkDpVxxWREWrrhaLHOiHgeFCy9FY4Xz+LPBqRrkcgnUwHEk6FMVohM84xOYHdqhOPEbHbKOEOHKKNeoX42xXBGMaD7vxnQ3QgIs5apV/ZZ/J5EgqPyF2jWL9cTQ/OUR+t8hDuy+H0RR/yuoXxD9U7sC3g3kUi/mQNZHCY6ltL1FqV7AamqpfpSzPSVib9qU+EI7lhXZuUjRa17Q33xu695REl5ooKuuERHCcqtEwlapwVNPfidBJ1lR6z/hYjucoL8I53s2YOY9pqi0qMkhP9xbXJCCeqKkrD005hEQ6eTiWHppyGpuq7V3oJDkBN7LLsfk38wwOtD61SaRAcWfwKdr/Vob8DDv1bACaB1TpOs7qpOu968w/y5Q9ypEbErFUTlRkh/UT/DHYlaaB2kIUtrg8y87yjOaN3BHYqb2jzjSHVwLkWyPqZ51iFzN6FgiST4A52u86/scfnfv0omM2K23GHr0TRvfOkT/uDrP+f5v/YZjUsxcTngSy/exEuGXD7/kPPPPmJnp0J/5nESy+erUZ+IWf9jHDVdjXFHAttbiwogqz4t5n6UgOLVaEKzMwQ1wT7FezHdJYf7f1Ai9iVu81oO44KYSbFnjv2Qak9LLZCsKepm+Y8f8fAP13CHltXvxOy/4hLkY0q3DAevWGwiJlkZkP2zHCZymP50RPtEgsWfADaWK3NPGOjjCHB3BIPvz9K9HJM6VG08nIlJ1gJ6v66N0p83dJf9/wkhY/FNNS6dEwBpxiWLO5JKNdF2mC11Gd3LkjmSHNvvOvjdMcU7Lu2ThsxRzFEjj3c7Q3YyXs5tiW4XpaAzJQpimAZnbJj6LKJ2wcXvywa+vyBcOX0E9UtaPNbA/ssZybdzFmMNI9dw5j+8Sedwgd16gX7gk0mO6Y99EsUR33//aaY+dKhftOBZ/IMEn125gFuCm5+dJnq6y9TbPo2vDii8m6ZyM+be51gjT8SJ+rhB6M07VG6MSDUVYjsqGObfi+kvxsxeCdh/xSVMPYau5Jc0rMiIF6tacTDlUL6h0zJ1BDOfjJn5JGL6akRYjKi/HNA+BfntiIPfXCV9ZOmuQmvDJ9mUu13rjKXyqcPUBy7mep7s7hgnhK03EoxKgnac0NL8+hATq3EKU0Y1W1nX+cr3IxJNRUU6Y8OD301SuOkymNXGGU2LiVV8EDKasgoBBiqfyXAtmA4pPgxxRobsfsDBx3M0Lli23tAmdELL1teTABQeKCsgnR7/gjNRigVP1dW0pQ/krr3wzpjeakxv3iV9aKlc65Pbi0jVhLZESU2vclsS381+NGS4GOK3Db3TY0YzMd0wyVKxRfF7WY52SzTen6V1r0zp21n8pqJ94nTM7Lvq+LtrsRhkBtxPcwynDLPfSWJCS2v98+GoT4QBRWppxZ7/a/+pjAx6Ogmzu4r2Tu8bOqci8ncfO5JYCveFKRZfPyDx/5ii+rSH39E1Mv2JpXHWoXgvpnVadZDfQa4m62q+/K6u4FFJZrqZaqSw4AGTUZ860e6CS5xQybH8nSPqz03hDxRPHuTkjhIUY9I7EgD6bY5Z7EEeRjMRqT1JW0SwiTl40ZHR2v8o+yrZsBPcUTdLfmtMb8FnnBcBZjgluCf/VI1nZ3Z5f3+F3B8Vaa87x95NwdM9gqFH+n6SpZ8O2PxmispnOu1hYkQRPHbaUwpL8WFAZ8knSgpaG03LYn5UUenQn3OO7S9bl0ekcmNOTNdwjOXRt09M8mmFRnRWHbrnxkz93Gc4PeERz8W4I026BtNmkgQjGXlv3iG7F9M843Dn//KffUEMKELVk4Nn+ziBZTgX01s0eF1Df8mSeyh9vbEi7hYehZTuxkT/eJZx3qF8K5K/fEUZ8rNXQlobDss/GuAOtfDdwOK3hFl2Twe4A5j9SCSL/VddJeqNLNVnRe51Ql2HUUqGYQdfmaY/7zDOys9pOC2i8sY/Hh97TyWb0kF5PTVOiZrSq5N1LcQgLb7rcFZj2dLdkETT0jgPpXsR7giKdwf0Z336cw7dVchtC1mYey9m9LNpfvaXl+j3Uhz81ohxycILLUaX+qQ+yDL7owQzV0J2Xk+TPpgkSJcsYU6QX5izjMuK8GxcjHn4W+6E/S/0oXRD34d1dGO5I8gcWIZT8O898xGnZ4+4cW+Ra/eXyBxYsjuqja3RZ+RV/eNhSuluRKLlMPNhTP2SeBHNs1C6Fx5vmMGMc4zC/LLniahRMbou3ftpdbyhtO3jnINz3zIqKqezfGtM7Ds0N3xMBMUHAfXzPuU7+sKX/8GA7W/kOXraI7tnaZxJMS7ow449+T01z8HaP4Odr0qRmT4yFG+JZwCGZE1qg9ZJ9zg2cf9ll0Rbk6jhtKF9JqL0mUOYhsMX0iSalsr1iP0vGebetWAt/QWXoBCTbsRs/RXL+f9rnXt/OEvxrpQJvSXBQIPFkMyWx6igTKrqMxnaGzGpKiy+GVJ7yifIQXfVpXhHPIKXT97jo70V0hd7VPeKEAl/bp2CYc0jTlpo61TLbgtG8gcx298QZbH+lCG9L4rkqAyDc0Pmv5OYyL9h4e2Qo6dlB/nYL+Cf//BlTGzI1iehw64l3YiovgyjfY/hfIzX1Ombqqu08Hri8Bbu+rRPGAr3LYOKS+OCxJZefxKM9jmeJ+JEdccQpQ3Tn2qHTl+B5kmXzqqhteHQen1IqhnjDSPa6x6ti9IetTZ8htOW5oaH34e7f5BT8t5sTOMp1a/uWNdad1XpcommobXhk91V/TTz8ZAgbyjdicFqbNtbMpTuRMfKAnckgockJDEz77i4YzmdJJqyqNx73RBPj0lVAxpnXFJVQWKDskv2vs/OX5kjuw3Vy5bsTkzxLv8TObM/0VwlOhY7iR169NuG8u2Q0p2YxZ+OiJKG6c9C3v7JU9j3i6T/PyVyt3zSW5qlz34Y4Q2kYsXA0j97OLEzckjvDkhWNRY1kXyj0tWY0t0Y/1GS1gnR8nCgdsEnXX3s+aVNnr/vULmqxtDEUrc2T3pUrrgU7lnm3nTI7hoWv7NHkHFINmKWftiifi6BdZQhNioraaVwD0o3YTQTHhuE/LLniViosatJkzUTe/EJKJ2aBILlPlRIWmclid+1pPZ8opSy5Ff+csxgztI6BbMfiGNZumlwJtMdE8m4Nn1gjk0aOuvSBY1LMbXzqeMaa+p6RJDVz21uuGR31Kh4A+F946Jl79cUSaOJlp3UpoZk1aH4forWiQRuoFMl9qG7ZojdSdrJnPirrVMOqWbM8o8GtM5Aum45eFURmIluzNSHDsm6Yfo9l63fjQgyhqPLSWIPtn4Dsjv6XfZedemeDHFHqj8PXpRzXjzxxu1fXCTZiQmyMJpOkTrSn8s9goNvqDwaFYSzzr83In0o2/NxSZsyt2VZ+Z5k4MWHAfVLkDpyJmQVlTuDGcPRyzEHXwvJ7kXs/uYCR18KSdciems5nEgW9o2zGpHHkwzZ2ksh5U/dY7+CX/Y8EQvVuoJ1hmVzbN2Y3bH0liC/LR1OZ9VhVHQmJ5KA/+lPLHtfSpJsyF68dtEwmrLkdiPm3o+Z/VAmE2FKDUL6KKbwQP5OXt+S3XEYzkir5XWhccYlKKjr9rs6TUdlYYegBTL3M+mjul/tMfuhTIZjX83J1LUhnRNQuR4qlietWjS7J/8BE8qpMFmH9qrL/qtpMrt636UbCuTtLrgMK4ZRWQyv2R8laH11SG5HA4zMlicnlEchxsL8mw7dMwEr3++QaBr6i3rfJobqpQSN0y6xB/uveJrZ1y3+wErz1VCD13w65MHvSumaOlTt3VuS88nmb3pEKXj0WxoUJOuW5jnLzEdiQmV3LUs/gOydBI1zYlkVP/PZf8WnveqSqmlBL/w8JNnQ9e9EMP8TV0bL1S/QCBWUUd+8EFO5GeENZXCb2xYpI9mUSGzmox6pQ4fMUUSiIzjLCXUd61rW1R4lHVrrLu01j+F0zGDOYCKJ/bJ7wfFEJLcdk2jJha6/aMltxyJ/uBL2BQUxshJtRc90TsqKZuaTgGg/Q/O0y8ErWqQA7dUkybqhP+sex/OkjibpJJFedzAtzDXMqIksPgzxuyLQYHXiT10LSNYlVSlfbZO+kqaz6tBfjBks6POpXvRI7xn2vx6ReeCz+c08U9cCsFB4EJOqxeS3Y2VJtRWn/tg9pXHWgVhNVP2cy9ybDvNvSXz32L/L70D7TEjuoaZoufsec+/HFB8GZHfk55psxgynDd0ll1TV4reF0yplRQYeB69aopTh6LLP1NVgwlXVkKO3YGif+AKNUN1AQrDKVYdRXmTk0lf3Kd8a0V+yNM4Jttr9Sha/D4cTNlWiZUkdKbfJHWlggNXJ7A11Lc18KBgq0YKlH7Son0/oy3ykQcGoIsgks2uo/uaIMGOo3BTBoHRDTd5wGvZe8yjdMKRqhs6yR+6hQ38ponTT0DwvPmeQVU08nNbN0Dqtmjf/yOL1JsSYnngG7lBlzuFzHt3zqj+H04b8ZkyUVmM1WI44eLVInND72PiTIalDl/m3FHeersUki0OMFVNr81sulevy8G9tyDCjdEenVvO8SpjYF1yXqqvk8vqa+jXOOtSecmicNTTPiUGW2vPI7sn/K8jDzm/ENE4nKN6PaJ5VRGaybsnuReR3ZEXZX4qVmxXrc/BbDuXbgaaAX/HwRtKRWSNr9c/7PBELNXZh6n25OoPA7PE/maN5Okl630g+UpSRWqoqUrU7gM5JaDxlmboqD6Vh2TlueqavDoh9OSYX7ivho7ORnxjrynqn+DA69pmKfSj8LMW4KKe5+Xdj6k9b0o2Y1KFm6f0F1YadNW2Swm2X/GaIO3RItMRPSFdjwgkxeuYjS+MpaDwFg/kIJ5CFT+dkTLpqj7HNxK6MxrBCNzrLuv6XfiCVa3ZHU6/D5zKkqpbapRzuCPKPRnAzhzUiz1Q+E1+hc0Ib9vAFlRNBzlD5VJsonnzGYUZwWuxLd+X1VUenDyfmEa6w7N6iQ24nJrcpLq2JLLULrsqmKfmrHr7gsPtlj1FZUfLJRkjsK8vK6xu23nCZnthUdhd1o2QOLc2zauw+z/NELFQngtrrYwbzluZ5LRQTQX/WMC5A5WZI+U6EOzDUL+k0VBoKnPznI7orzkQCPEmoyxtq51M4oRZvf06+/FFCoQ+lOzH9KZfWCVdGv2PVqe2TkN2WN//RZYfUkUNvVnzMxllHMpNXB3gDRZ4P5izbX/dINCRBrl9UsnLqSDzZMGVY/GlE7pEhs+viDeW7VL42mZn3YO3PmsQu5HaFo4YZwV/ZXUvjjEv7ZJrWKQ0TjEUDgnZMbidm+2tpnED1aG9J6del+wFr3wkYV+Q/mpj4Zx19RVaSsa+ElMdWntNXA9onBM4XHsW0ngqPfaTqXx4phfCEQ/HheNIEinZpXU2ynEB5B6vfHZDbssx+FFI/l2T2I22uVM2S3XY4vOzLcK5mqT0tA7Vk/fN1/PCELNTYgcXv+mR2ZV/YXddp4w2lCK2f88jf7ZDoQO6RobukOMTuKuy9lqZ8KyTIGdobIltYVyZmcUJMK2+o2mkw7bD00xH9OYcwa+gvyhi3fVJiu6WfRoymDM44pnxTJJHW60Oqz8eUb8YcPWeY+fMkmV0rxewOnPjTIZkDy2DWIZwd01uyLPzoiPZJXdd7r2ncmmhaXbVdw8x7TWJfkuSt3yhRuQ61p9zjPKiZd5VwPS5Zjl6I8fpajMmGavWj5x36cw6VmzGD1YBUTdr7woOYvVc8qk8nmfpYHIlRyVB4YMncS9A8J4ZWa8NhOGUp3exSfVpQXVTS1V28Lu2U9WD+OwkGs1bBahNXmtolQ/dUCBY2/mFjcqA4HL6YwQktjTMe+Z2Q5oaSs/vzUktEad2GvWXxflNVxV16ny8P7clYqFFywuoBuidCMjtyEqncCKg95dLdCKk/XaS7GuOOpQKY+kwLxe9Cf9o9JluYkEmYF2S3JnDMtlKQFTaWZDArdao7MJRvDph7V/xPE1umPxnTWZHKNEoYcu+nKV1z6M86VK5a6hcNqVZM/oFO1ebpFN1VdcS5G0lyW4bdN2Y48afdY6c+E0qdUL4bUHgUc/9vlMjuy5DtsZvJ7EcBw7JDlBJ8444hWTNMfeyQ31Kyc3/OIdmMyW6b4zBer+FNJkki3aSP1Pm3TxqWf9ATsfwZbTq/LTQlu2Mp3oX6xRyDuZgwBfkbCZqnHAqPFIb2OAHFHRqcyLLzVZ/0viV9YFj9M32mvZMFwqzkJsX7IaOyNkDrhAYuflfcgjBtyOxp7J3btAxOjuUvW/iFpP2XPU/EQnVH6gLHJUPyUK4f7TWX6tM+pTsxmU2P3oKMGxrnNdob5w2lO2M665r+9JcjOYo01Tip89SJYl0RiWMfBrP6wryhXOV2v5zBG8WSMM95bH3TI7sfcfCiQ347or8kvVSU0kIJ58eMszIIc8b6MvMPLf0lS/e85oHjAjz4bWGI45JOFXcI1Ys+YVKA97DsMFiMCHOW3oqlP+sxLnLsvZWuRvhdXdsHr8Vkd0UBjJKGwayltxrRPK0w3Mej3sI9bfriHdWH/cWUxJENh96SXjtdjWmdguqLMc0zkN2W536qpv/Vz3kU70caQvRjsnvasIV7TDRilv6sy7DiULugiHeZIougXbwDQUaz/cKjseyVitBdeYymGOZ+5B1zDrrLXyDAP0pB+6mxYKPZUOEKWxFBzjIsO6QP5O+Z27LExVDYnJX1d+Ge4nXWvh3TXRFHQB7xHGfTtzZk+JvfijGRGPqdkxF+V13wwQsu3Umw78yH6sRTNUP1kovXN5PANZ3k5bcTinMsGvrr0iRVL8vRznQUSpvfFJvfHQreSrZ+IbfprahEcYdaQGEhJkpa6pcEI3VPRHTXYkwsAk360LL0A0N/1kjQmDe4A0N20yX/6Bcpf9ZA/YWQ8p2Qzqpevzfvkt22zF4J5A79MKZ1wiW3Dfm7LqVbIp53VxRyFuRkcNxdcomSgsBGZSP7oMcZAHnJWcKMmq10Naa3ElN7SrdQ+yQs/aSPdeH+31H2a7IOsx+qrPF68lsoPBIGHWa/QCcqjqVwLcHcez3S296kE3eYfzcizMkO5/BFeTGt/Ynh8GWRd7N7lvy2cu13X/NI1tX1RwlNn4KC6HeghWIizfuLDwPKnymkt3hPVugm4jgcM32k0mDmk5Dh6lik43VtqNjVCRnkofSJT+luxPQV4aSrfx7LtnJOtufWlRSleXnM1LUR46mIwn3LzJWY6nNWco0rDjMfKF0v9qFwxyVZc2ic8Ul0Y6LUJOcqrfffW3zsFC2pyrAixGL6akhq26c77+L35P2UOYxINWM6Sx7dFRFPnJBJ7cokgdCSbCrkzDrQnfeIkhPLckfoRm/WJf8oJrcXUbqr5Gh3JFw29gxRNj6ONk/VDUHBl7jxQUKf3Wt9hhWFYnh9uQGKLG6Yf+cLpOt3xoKfDp/PYqzseqY+adNZcsnuxiSaRiZjPgymPRINMXZ6i5o3m0hzaa+vL8LEUL5mWPrxWErW+Yjy7ZD+rIMztgzLrqQuKahfUk2X3VMe6Tg/qZ1aSoAufZBgXJCSwO8o4ts6OsGcUNei39OYcu9Vj/KnjmQwRf2zmQ9h7oce+68kqVxxqD0t7unCW5b8Vqx5+rpD+XbIuASFTUWaD2eEhyZaii3KP7JMfyqhXGJCOEk0VeONKjHtdY/srmC64r2I/EMY5xx6cy6jivxm974qvuzimwPSuAEq9AAAOrJJREFUR5OkxI+ll2qvO2CkxC3djRgXJbFxxvKoqj0jru/BSw7tVZHbiw+ERad3PYazE++vETRO+8fZVnECoqYILzjQeioi/1C811Td0l79fLyoJ2KhgmCh3rLqvsqNmOpzBcKMmgpvKChl5lOZKgznQlLN6NiPs3HGJX1gKDyKZPj1QPlUW28InyzccukuumT3I0YVuSAHWTnrpY4mYP+zgrLaJzjO/RyVJmkooT1OSs7frFO+FRElLd0VjXY7K7qGRRKG9KHD3LvaOO11/axUTXPu7JZYV8OyQ26zT5gxJFuKCirdjqldkNFGdktUu3FRgwQntOy9ZqhcFQKQqv/i2nRCNXP9eUOqHhOmHEYlw2DOECe12Ba+55G/65HdsVQvpaWw3Y/pLWi8HBQ0PEk2NK3yuxZ3KOw4yBq8E12m/2CLp1+7w3DaEr3Wov47fcI0TF2LqHymAQdG5J8oMaH0jWH6A5fFn7YZFaH8qRCNxzzax+F0v+x5IhaqorvHMCGltNcc2hsTw9wNh/xWSH9Bp2d3RYyj5oaPEwhGGRct0VdbHPz1Ec0LIcmm6tn0kWb2ZqKRClMiUitm0TBeG9FbiinfGrP6F2NyuyHlm/pi85uW0n1JkIcz4qhmjmJu/6+m6M+JAhgnJlbnaV2nqZpOdhPB0bOGwbxl6kZIpiqMs/ggpLseM5iPSf71A2qXssqNqsV0VlWDZvYs49LECG5ZNj+1SxoXJ6sOnbWJb35L067htIH1Hn7fMpyLsK4WdX4zltTbQm43prfo0DkVUn99PAnhFXcWhC4U7kLjksUdSAdWfTGi+cqI2a/t8KX/zQesVhrcvrPIvfo0Zq3Ppfk9wrFL9tUqrT9syw27JQhLLoqWgxc9+vOyJWqezTE4ORaHYlZTuvTf3fufyR3/n3+eCD5qlIDqM0lSVU2pkk07uXYs5dsh9bM+Xl+nSOd0TKoF6dtjOmtJOhsW61vsgzx2bkRiasj217LESUucDsk88umvhpjIkL8t10C/I5J0sJ0kyFl2vubjbnQZtpIk9iDMxvTHhnTN0D4BM1cinEjpe9bVCVu6DY0Lv3C+DjO6IvP3tZAKDwRLDUsu3WU5+T38XcPyX1hS9TFbX5/HnRZhpXHesPTjEdtfTxyHWQwr4iIEGTm3OGNpqNLVmMZZMZhy25ZR2eB/kiP2LGt/FrPzFan3wlxM4Y5D5jDm8EUFr5U/cUk1HYZl6Hx5gPuXaaZuhAxLrjynrqtJtC7MvONS/2aAYyx/9vZzUAhwsgHF9JCz04fca0wThw61eo7nTmxy/VkXPiqII3zWZzD7mIBiaZ10CAoOpuuRakQ0Lih6svMnC3SeDT/XGnkiFiqOFuv01ZDenEvrDBRvq0PurHssvqVAqJ2veMz93NBedahdSBM/1cUJXZZnGmwdlMnmhpyZOiKxHLHbK7L12Tyji33OzFXZaRUZVYsKazCSC7t9mLpqaa8ZCn+S5eAVGM9EpHY9RlMxvTlNrpqnJvHrTen58zuhvPU/E7ziBJNwXV+wV+eEZTCnn5PZMZRvRaTqIV43SaI94vC5NGE2Jn2oBTf/bsTmbyTwu5JrtE5PPpZQ3gReTydo87RhXHBIdOSut/izIe2NFNmJJ36yLX+rRNuw8LalehEWftSk+kyZcX7iIxBrEyz+owTjSdhZ7WnL/DuWyFcNHmQdahcN7v0U1feWSRZhmI7IX02wP5Wif3+B4YyhOJI06CNW8ZMhhS9VObLTeAO5BLpjMdGye/prsubQXhMxJj+RE6UOvkA1qhNA+XZEf8al9vqYVFXhD8s/GmBiQ2/OY/8V/1gqnT6S1OOvn/mYE/NVdqol3jh7kzNTRwB88NNzbN6e0zW8nebu1WV8NyJZlz6pc0oOLCf+q/skOpGCetOG3AOH9JYnS/A9h+L9MelDnVx+V02G17McXvaUaXVB739cVCJdlFQJEPuqfVf/YszS96v0Z1z2XkkymLccXk4qNvyjCf9gReSR/AMxrcYFjRYX3g4ZF2UMHBQ0ji3diUnVLL1FNS7d5QRBXrLy9JFlMGVI1Q1RytKfcXFCqD1fJlU1muptWQ0ebkf0Zl0S3ZjWmsfcu9BdcuiuOEQJM7EEgvFsiDcUmSV3O0GYgelPlf+a2xZj7fR/W8cOXYKBT3W/IPx1YjN/+OKEYllUvGZ+UwSg1JGh+VRM6V7MeOPzeaM/EeK+5PKKPfXv/2dEKUU4Hj2TYjg9wS1vqjtN1RSFY2Ir1nnKUl5rMBz7jIYJMJbKX6TV5c9A5bpkG7GHrv7AYH1L/o6HO5hYjS+JU2A9LcL/MQw0c2VEd1lMq6OXI/L3PLwJn/PgZSEV5ZtyPqk95TE4OyR7NTUheKgOG6wEmIFL4a5ORr+jGjyeNBOpmp2wpqSYHcxrGDGqWFJVna6ZQyWjKI5RXvhhWoTx1skEnRNymfF60oPF3iTN+n5E87Q7gdQsqbqmdpmjmMPnlZvVPuFMTj95/w+LDkHBULoT0l736C1bLDqhw7QarVRNDV52P8ZEloOXHOLFIXHgkrueoLceUb7qMPNBmzjp8fC3MySrYqSNipLBLL4VMs67dJZlJHzjv/iCiPucYCJvrloGM5pkOKFqs2Q7IkoKisoeRlRujZj61JC/49L9ZIrgToH822lyb2eIfWGduUeWYXly+h5YZlYa2FyIX1cGVJyUy/LjeToWGUiMJcQbTlmitNzwql8bk6i7avJOCZeNczLZHZUNnSUPJ9BcXIwhyZQzu5bMAzlcD+ZlgeMGls7lESbUz/UGyhuYujYx3egovCyzJ1AdK7Pe3F5E63SM3xN/If9Qw476cxG5Tcg9cgizIlqHGQ082msu5dsRqUNtxGTdEqWhft4h/9AcM5fcwWT2X3aovRKSaClW0x1Zpq9YchMTYJl/SNTYWZf1Ue8PW2TONlmZa+CnA/qLMcWb8uGqPVOgtZEhnORrjQsO7ZMO6SPdkI2zExbV3hcI8PcmpmGddeguOvgdGFWiyehTJ1GiDTvfsOx8JUXzrLrKqasWbxKuBaL8JZo6iTOHMaOypfZaQL2VxbR9ooz8Pys3Ak780YG4og8heqFDkFWnPZjRSXnwgka5K38ijmeUAq8rEnH5isfq90bSwNcm+GvBMJwRF6F4Pz6ORvf6Cg+2BrJ7Ecv/3CVdFfWweVYdcW/WJdmOmb4q2xyMyNy9RUOyHdNe8Tjz33eZ/WjM3HvRsSxm8QeGYcUcUwSnbgyxnurDzGFM7bxLkBfRJ9WKKDyIKN6TBipKiEVlrEbSUdIw9a4sK8M01C5HdNYc8juyfK9cE7MLKznKwesxw7FPJdtnu1qi/J0M3sCQPlIm1WOPrMJtl0RLSI7Xk1gy9gVpjUtfMPbUuCQLyPwDjTYTLUv5mkOQhfoFzYWJYeZdF6xMGuKE6qrHkZKDWWUWJTuW3pJKhETT4KZC5ittknVZ3jTOWxpnfI5en6W/IO2UcyWP34X0UYjfVaM1dU01WPWiNwm+gNz2hEqYMBxdTqkrz2qjRAlDZtehN6+pS/U5S29dWO9gWjDZsOyy9ZtKIclvhZJ8LBiiNOz/zpj9l32ipPKtxjlD5tCSrkaMynD/P3cZFV06K/IfcELhrJkD4Z0zH0JrPUn5pmh73SVJkU0kZOLwskfzlEv114cUNiO8gXJKO2u6hbyeJCyPVROLP3Ym8hxPCt623kfmULXla0/fJryf49HONKUfpTl6MSb3SKT1qWsB7XWH7LYUC+1TskV3hxJO5nYi4aixavrP8zwRC9XrG8aVCL8HicbEqfmEKH6PiR+9FWnjKzdjBjN621FS+Gh227DywxGZPUOirZP46LInYnDgsLM1RZix9P90nvwDh/6i5CHpAwfrGRZ+Lk/P9po2hYl0AnROCBIbzEhGnK7JhS+3K9LGqOjQOgWNsy7F+8Ex6SW3HSmn9NZEYdrl+EqufOTSWXbpzXu4Y4s/ERwmb6SPPaH6C0bs/WbM/ks+yz/sYR9l2PuyTkKVNIY4ITA+yMkMLtmJaZzzKdybOG1vxXhDjX1zW3Lum/uzJL05h3Exxh1qjJvZ1ylXuRlhHXOcWh1Ock6nPwvpLIv+1153OPr9Pu9trrLx4qa+QAul6w7NC2ryHv2mS5Sy5LfCSYqNYeHHdbyBZeqziNZJj+6yNFPe4At09VsQa2ogkNo6kDkwctibV/KJ1zP0NgJqT+mk9QZCCqKE6qf6uaQyRlfUtaf3LbXXxuSKA/7ey29SvDMJn8jJ2RpHbJ7MYUxnJcnC25pquSNL57QmKlE6ZpxXDqvfhe68S/6ByB69RWnpvb6oc0fP+mT2rXReZ0X4KD4MCfLiY3oDS/Os3J5HFaluB9OKRk/VAAeCrKX4QOhElDTs/ZoM2/ZfzZLZk01k+sgSyKafRFPIwNR1bZLugquwhwkFsD/rMKzotXO7MhsO03KUTrTFnXgspgzTGpE+lqokmzHDitz/qk95tM5G9BdihnMRvh/Bgyy77QIYS24vxO9BnNYhkaw5lG/JuXBUAScwNJ8qSd2w7NJbmkwgr4/k2fU5nidioRogt2kZ5UX1iz256ZVuGvzWJIHvRkThmi/GUE9SkqNnJXjzO/a4+81th8cSj0snd+g9LPIP3voKjQsKS5h/d8TozIClHw+xRplUJrYiV6QhXY/BWDqnI4q3XMIc1F4ORLyeV1kR5KB4PyYxqbd6ixN8csZM4CX9Ulvfmow2F3TqlW5Jwp1oQuusPSbKPJYmL/8opH7WxQnkbZrademsx/g9S+XmmMon7rGcO8hBlHbI7loe/Z42UvtUPPF4VaCbOxbLPtG2dBc8Zj7QP7OuRtYznwTMvyeMurMREaUmXqh5lRYbf9Ilt6Nru3jbpXjL4bde/YjFUhsTQ+afFim9k6Rx2scbxqS3lazojpU5ax2RgEbzAa1TDuOibqX8A+QRu5GQ5ebneJ6IhWoN5HeU89lbV3apNxBIHGZFd+suqfMOM5qctE66ZA4MSz8WXBKlY5zAUH3ap3wzpvZKyNXbK8T5kJl3RAJJtODgpSSFd9LsvZrSqWp0ejXOuPSXI2LPUPnIY/ZtLeAwZUnuJCaZUBNwfwwHLwmwDgqq2VITDZTXn+RE9S3pbcFdUUonXLoq7ZETieKXaEP1pYjekjbR0TO+DDSmDHFSGzK7JZzVemqcuuscR0rufNmjsw6Fqwm6K4byNUPzjEPv5T7Jtgg2qRocvhYyLonL0F3SGLlzAo6e8enPeJrzH7okG/HELFgb+O7fzNLccMjtxgqJ+2qX9VSNUeip619yKGxqslS/oKFIkDfMfjASAXxaLLQTf2Kp3FC93p9zKN8eayM2FN3+eZ4nYqG6Y0tn2SNVs1SuSFvfm3NwR4ohHE7BzJUB8+8OSFel9yndFbH48HmH/FbE8g/sca3Vn3VIlYYYP8bPBBLYnRUAjp0oAI4bDUP1RVHN5n9mOHpOzH13krQsEjLkN/XfJdr22CY8c2CJUzH9aYfWWcFi/Xk1ekFOiXf5R5b8A52ArRPyBDWRld1PLab0mcfUVUvqSPzOMGMoPIpxB2ZCSrFkDyL6Mx5+V43kOG/w+5byTRn1puqqDYfTQgBSn2ZItMLjJOxE1WP66pjMYayJUWApX5/AXyc5tsd8HJKMnQTAhRoOVC8Zkq9VMcbyf//o1+j9s3npnSbk9O56zPw7Y0wkK/n2eoL0kRZg+lB/bW4oZDhVtfQWfEbTMe0157jf+GXPE7FQx3lxPMdFfWD5+wqmjZKGzqph6ScjRlM+O19Js/8qECt0wRvK+Kt5yqW74FJ4FJDbFqnj0sIuTiIi6Pmkqoap6yHjvE4jJ1C+vIlFJs7f9Zj5NKC95hBlYjK7lvp5R96hIzshWliGs9Koh2lp54cVw/yb4rWWbqhW9vrSBEVJudhZR6ym7K6ltxxrlu6okesuy0IyyIrl1F+YTHGKci5xAnEGqk97+L2YzsmYxjnhn6OyQnvdoT0OikvWtSn6KxFHz6Qo344YzlgWfh7SWvcZF/Q5T3/QJtGVQLJ0S/hqfismTOvfJxuKMMrsGlonXQqXazSbWaIHOfytpBy/C1LdNk575O857H7ZF15alKpCWaqGqU/b7L6WINnU+8xvj+ktOHg9ISFe/wt0ojpjnXDuSDVgmJHrXX47InNg2f5GgsYppdSlqs6EYQ/DkqF4U/Y/o5Lh8LkE7siSeq7O+59t8PTyDsVPEwpTeNFlsBSJ9NKOSbTEtxxMO/SWYqqXZL9YvO7SOg1z7wf0F+SFFWag9owhSlr89iQt5Lw+6FFBOQOJnozWjFWe62MrSRNPdF0LhtSRw6hsaZ2NSB9ZEi1tlnQtxutJDVDYjHFHsPnrCQazIkDPfhCw/6ohWXdItAxzH/QVDVTQZGswbVj98y7eUIs+tecSvN5m98tiNHUXhGY8RlDu/Ad5goyD159ke61pVDqY17g4yBumr6qJ7S/GGKNFZlcGJBoGZwylW/pZ6SM1iibWjTCc0jCjt2io3Ii48x/kKd1RemH7BBw9m1QkfTmmeSE+Dmz7pWvk3/Ia/FyPieW4l2oocMLvCq88+tt9+guG8ZRsJeffixgshhQ25YfkDSSaq1yPmH9vJFfpZwwLhTZvXL5GbB38rqX6QkzlM8vKn8vUtrssq/Spa0PCrCY7j0MrRhWY+diy9YbLxj/p4IRidJVuIM3WRckt0oeG3hI0no1wAsvBi4blH43IbcfMfiC8ML8VC9rK6UqM0pb5t0P8lsNgRjVn+lBym96ybpbmhjrv4j0NO07+4zbdJY+ZD+VrkGwowWQ4bVn4WUSUgsGcpXEuK+piEaWQ/LBA+sAh0dRizlSjY4PkzJ5DkDH4bemi8nc8+tMOQT6mv2CYvhrQm3eoPmNYOHtItZaHahLnoZywa5dVWw6n5fkV5A3pfZUCWMg/gvl3AwYV1dfjSQ5qqmboz2v6l3vgsvI9e+yT9cueJ2KhPmZPtVcdOqvOxNoxxv04T5ixlD9xqVzXLD770JM/04UBtWfUwGT2R+y9ksQdQvFijVP5I/7ys/Pc/v4GzbOajhw9D62TSjj2u8LvmhtJ0geW9vlQWU6blsW3lBOVf+hw92/ldOIdqVNOtCH/wDl2oFv93pDcfY/BtHIGTKCcz9ZJdbyJTsxwWiewmVj6HLzsM/OxplnZPUvmKNJp5Sg0bbAU0V8wZA61CKvPFaRRShlmPpxYB50LSDaM2F11i/UsnVVD8UE0UYUK1ku0LMm2TDha6x5bv6XTMr8VMy4Z2qd1dSdalkw1ZupjcR82f8MlyMHMxUP29sukMmMKdx2yO4bsDsy+C3tfifE7htymyqgoDbVX5YjSOG9pnPaVdfUBYKF/ejxx5zak65bSvZCjZz3FzH++JfK//GNCqD2vhibMWLoXRiSrDpWbYtKnq7HCcl+Qb1SqaVn6Y5/lHypkt7WRxg2gvREz/sE0/+qjZ8ndUhiE3zH0FyyFu5OwtLEA++GUoCRjlanqtwzpWsTB8yka58XCn/pUuKqJRPSIPXXucUJRN0HekyHalObZtYtp2usupbuyZtz9sisz4nllqzojQ3CmT2fZZecNjWt3/47szINCzPYbGbIPXebeC4iSk8mSVcBZf9HQXXZkkfmJT6Jlj7mjJjSk6kpzcUe6mufej0hMrHPGeb33qfc8MALtu+sRJ/75kNhX2XJ4WUYdieYkE/bFJs9Pb2NjQ+KnBUZFycnTNUtnzaH8qTtJwtZp6nctS9+V3frq9yIWv3eIibUIhzMG48X0lnQYNM46HF32KNyPWfxZ8LnWyBOxUJ3IktmcwCRHhvT9JF4fjp6VW4mJIf9gQOmjBJWbYzrLDv1pl+z9pmqwJcXxpA90kvk1j9kPR8T+xHbykcai45Ku8eYpjWJnPh6JgZ6VKrJ2waN8NyR1KGZTd1mL2QlF5csciPgR+2LB773mEeQc1r8dELuyMc/uKrKxdCdi8c2I8u1o0liA9Szrf1819tzPxESa+k6KMG2UGXA1YjRlaW74dFZUtvh9lQzjgqW7Jpphd02bJ8gLXahcQzLrSFTBhbf7BFmHwoMhiY54B+OiVbR8pEjz6Q8d9r+UASSnAXEEWmel2V8stPnum5dZ+6ePhxJCFtqrziTK02B9iD01r7kdScx7S4bmKZ+7/9EMxsJgVrVw5c0kfuexIUYsa6Wyw+Z/+PmI00/EQrVG12qyIZucZG2SMdpWsX7wssPBS1lMbKk+nZDXfCPm/t+cJrsj95AooSyncVGZVLWnkgRZw2gqJr8TKYxsIhVBmD61p5KTL9wyrMgwrX7OAwfaG5MEv0MrS+9bmiQ9bkg6J3TaRwnD/b/lkD2M6M8rc7R+0SqratWjdlGwTJSU+/P215PSFeU1DVIwhJJIhmWH8vVJBFBGcFNn1VG5MYCFnxnyW+GxYvaxI3b1sqV1LqJ0LyLZtjRPp3ECy9HlDO1Vj8GMMldTdbmTdJfcST0rzNTrwcwnMtot3hZcePAvVinfMAynXDrrkuGk6jHjkvRj4xIU7onJVbwXs/sVl8p1DV2I5QgzmJb8Jr8lhOQxWaV2yUzM2aDyvfTnWiNPxEKNE/KYesx8whGRwokE7UxfmciiD+NjD8/GaZepa/GExCEGVm5L8FB/XlzOcQlm3xexIkrpFCjdjSYyai0IWU06dNYmKsqxPsDkJBOqdUZ8zMevqRGjoXzNUYOTgeSeR2/OJdEyxJ4l/8Ah2RTmWrxn8Qcx6UN7zKYazFg6J4WnHr6gzbD4pqX2ckjjgvyi0gfSww/m4mNcczBlaK955LYUm5ndsbTOaOMkqy6tdVc+siVDe809VsriQOm6EUehFVN8EBAUAAtz7w8YTsP+K4Z0dcL4vyTKY/OM3Femr0a01xTy63cMB98I8PpSN1gXjp4zLP0oxB1ZCg8E02HB70Nzw6N+wWHlzxukqpb5P98msy+M2OtD8+znWyNPxELVB6ar2gkmsE/FkN+MGKzKU7N+wWVQcVT/TU7BcU5XSZBn4iAyAa5TkGxFJGuCn0ZlS+Uz6f3r511Kt3WSxL4c77y+mFYLPwuJkopYHzwj5nmiZeitRkSTUenxgnUNw4pDd10nZXsjZvpqSOG+5BzdZWVZtU7BYMphVNHCF2VRdLnhlGH+bbHe26su+Rs+uU15CPSWDK0zMYU7htx+hDOSn2m6brGORrFRUgs/feAQFGKmr0qRgIHKrYBxEeovBYRpbYz2qsf2b0U0TvlMfxIxcyXm3r8nxUF2y1GKYNKQaMqxMLsjkzd3qOZnnJMq+Px/0cQJYOpGiDeQa2H1aX/iaSWPg4W3R7reS1r83Y0CxYdDHv6dFXpLYp2la0pN+TzPE7FQraMc0M66qGD1S5Mw25Rh8fuinZVvRgQ5Q25nhDuyuAM1OoM5wR3jPBPfU7GwBlMupXtjcnsiXtcvqj4z0STRr2Enc3eZiRkL44JL5sDSW3Ao/Th1/CGnd1yKD7RBhvORGFXzklxUPrNUrg9wx4b9V13aJ0VacULpiXJbuhUGswL7+/OaCCWahuHpEeO8JnCpuoSMlZsiVk9di6l8qo3Xn5GjYJiGw+cNg1kFQLhDGE0pIn7uHUvjTAJvoFupfs7HHUP5Q5/CfQe/Y+itWBa+59FfkmUSwPQVlVfDGSECIIw60dbn1V1yGUy5RInJe5m37P36HGEG+jMunTVRA4O86ub+SoTfsbTXEqTqMeVbuh26iy6Hz6UpPIyVpL0TMyo6xxbrv+x5IhZqnFRwQ/pI48PKVU1cCg8GYAz1Cw5B1mE4I026dWXL09pwmHtfJg6Zg0nczO5QgWMlw9HlBNWnRScLShG9lZigqBop9g0zn4THQHVuS/hq9iBkNG2pPR+RaMnMAaB0vY03gOShS/OMQ/EOdNZ0MjbPpFn53pjpj2NKd2T8W7gn3LCzptd3Qp2C7ljX5eyVMblPk4zzclQZTBuOnvWoXkyR24LOikP9GeGa9Wcias9Y5t/uM/uhZTQr4k3jAmS3XJGRPSWtjIqT7Ne6Zfl7TU33Xh1INv0QDl9EMpSLY9rrUp+GGY1e00cxg1kRZPwudE4KBx5OGw6f1wQus2+YudIju6uboPBAPYI7kGNLas9lVNbNMyoZinf7TF0fHrPcqs/IT8G6ULoXMPfu51sjT4YK1Qq+6S8qurz6jCHKxAynsjJbqEk7H6XUdQ9nYubftoxKLo3TYswPZuVwd+cPk5Suqv6zIwUc+G1DblMfYJjRKRemoTfrUngUM6w4DGYnpJUXZH8Z1TzcIZrgDKB9psC4JGa+3xdWmmirNj54I6DxlI/XM2S3ZQsOssMZlRWDnn9gsUZeAaOiy9YbPnEixusYopNDcu+kdYtkxbHFQnbbxRlB5YeGxmmH0VQCrJygU1WVH4mWcNLqZY1unUC19mDWcPBqkanPRmT3kgRZyV6Mlat08VNJZ4bT0kIV74g47oQenWWHwZyldMNMtF6W9CEYa3HqFuuKhwuWwiNLZ02IgTe0jMqOxroYGs+GJDoZcjtjrMOx6w1GHNnWukfp/hcInjIx5HZiln6s5ih9aHCGDsaqLBjMTIggVYXmJusOu1+H3I6MxdobiNTqwOxbE2lzV4vE7RsG86LkWQOl2wp9iFIc1429JYVX9Oct0bkeT//tz0hNGovmWZi+OsIbxALTh5b6hYmsomvpzzgUPkmSaOqUjROGYGWEO7Y0LlgKdzlmaaUalt3XErTXXCpXwbrSR8WBQ+YoZum7B1hHENrjEWv7fMjO15W+EiUdGudcivdURphY9ezhN8fMPn3AcCYm9TcOSFdj8lsxsWc4ejZJf05uLYOKVKaJ1i88E0q3panqzxsOXkxQfcYweK5P4T40noknGLaass6KM9GaaVYP0maZCJrnLZmDgOEUFDYDkk3L0vcdhhWH5qkk3TMB6WqkqWIsB8DeMmx/7fONpp6MhRpqhzmhIJrhtCX/SN381PUQJ5KW3gmgva7w2en35b5nYsviT0Omrwb4bS1AdyyYKMjJqib/wJE0Yk8ncqqpxOXyLVHhCvdg9vwRqY02v3P2U9785ByNCyIgB8tjRmVPpm0/V/3nt6F0NyBKGGY/6JDbjanckG7KHVmS91JMvb3PzIdKVZFfv5mYSVhG0xZjLckjF79nSd1NEiUMt/53M3h9wVKP7RjdjsvMey6jqZhgwm6yrkoOJ4Li0zVWF+rE1vDs5XvsVRU92Z9zyO1Gk/gcBZ8luqqtO2dCRhXD3pdcDl+LjmvxMGPJPQL3Xppx0VC84cpavWDY/1rE/LsjBssRtUtJ/K4azSgpdCOz67D3epLUEex82af2rKV1wlUtO2fwqx67r3syf+tIg7bwVkjq6PM1U0/G1Q+0ThnMbX1x/qQrdsYy0Uo0ob3i6ZTwYZwTg2lUsfRyFmt8jRmzYKwhsy+FXG5bufKDOUvhniZByfpEojwFo5LL/HsBD38fvEaeKHT49v2nyN31KN2XTHn9HxrqZ136r3XxruawrkahjXO+jIWDPJ1VmPswxutbqi9HnPjjiN1vLco9ZWJUOy7IY+px3mvrlDDZ4v0ho0paRhRvWUwc0Z9xVJu+FDD1c5/YhYU3LfuvChqrf2vAXKXNzsNpukd52rsJvK7Bftnwnzz7E/5fm79BbotjIsrj1MLaRcPSj8YcvJhkOBOT3XZwRx65TUu6FrL3qosTKV3GCSyNi5b8fRm4pQ892muw9JcxYTqm9rTys3Jb5jj4w++Ke+FEBuvHE7KRYEF3aOheGBMlfRItqR4aZ3yy+1+gZgoj0kbjgqGzLksfd2SPmTXtcyFxwtBbcEh0YpJtxUN6PUNmx8EN5MrxWMYSJQ2zH4Yk2gr3Glci6hcl8zAxJDqSe3hD2PmqB47lr579DIDMn+cn1pUqC/ZfSjAqQ9BPMFgPyG9aVn44IPZg4Wea9oxnQ4ZFl9xuwPR7LkfPJjFWGaTtk9D66oDynZD6xYkEZX3MzMfKc91/VYB3b94hyDjULrp0V/WxeDWfTE3OeoMph/S+Q3C2T3yQovruPGv/0nL6vwwVtpGElBfyX/7xt1j93gi/H1M/Jzw0yE+yTYeGvdeTTN0ISR1pTOz1JvGeI0uqJgTEOqJczr0Dc+91CdLOMa2wfs6lP6Or3x3B0r/cIkwbuquG1lmB/bMfhsy/KclNf1EHRexDcttn+hN9d4MZDRGap75AfFSsIiaz29IzBZNU5fJtzYcLtzzaGwK+O0vShFsDK99vHzPnq8+aSYGvk7N+1qO7pFS64g2PZE3a9fZJRSiGWRgXLJzoUZrp8uf3zpO6laK7Yqi+FB2/rhNC6dUDzMDFSYV0lw33fj9Jdi8md7+LO4aZtz3aJ7Wpwoxh9HxPFMU9y+wHMemPMuy/4h5r5P3smCDt0FlxxI0tqcxwA7HHEi1DsqFY961vQm85prv2uMky5DZFHmmve9z/vQyjgmHj1x7w6O4s7hCOnk3Rn3WJ0ip1cltaKGHWkmhCohUSZnQC5rZjrGc4vOxPQt1ChRwP5AK+/3KOzqoilfJb4v8aqxO6cs2y/bsrjMtKOIyyMaV7Ie0Vj72vR8QJ3SbZbTHdohRUL8uacnB2yLDiHHvS/rLniVioxkJ/TjS/3kaAM0kcbpxxyByIVLL0E0t+J6K/ICgpt2V58Ht50ofijFauiV42LqrxwhFWaicivmRLDPZER/DQ4lsDgpURi5U2rYclgrFHbkdfQuWKcMPWSVHujuoFBYjVkySbULzlcPQ8bP6VItm9mNiFwn2LO5YxQ/H7GaKkTG6rTzv6lCc3XJSC7Js5Et2Yhbe6JJqWVFUZo0fPae4epSad9pHFbykqJ7utVO3Mh2lmPxqQaEHz1RHxzJjGayOu3V4mf1cSkVFZv/PMxzHZPQXCjUq6fhNty/7LKcq3NBhonnYghqUf9+jPGXa+6hAUhST4bcvsRwPFumfldh1mhef256Q0iJMwdTUSmfymR2fZJU7C7JseuU3F2A+nBDcu/SRk7TvBsWHHcIrP7eb3ZCzUCLK7hva6Q+G60pQfk4q7KzFLPwlJ1cZ0lmQGIZI1k1BZEZNjD6xjGM1ElO6H0lmtQn47prcoQm+QFUTSPzfi/v8a0rkRnVECUxkRj+TFVLhnaVyUi16yDoPlgEx2yNq5fUCvUXwUkGhIWDfOayIWJWHv1ZRUrkPL0TMOnZNKVxkXLWFe41/rWtqnYpqnPPZey9GfF/F74x/3KN4RqD6aktAuShkSzcc+T4bd1yaW5ZfSapQOE6RyI37v4sfM/FxRkOkjS/6hhhHNUy71Cw7ZHTG+gmL8i1tnSlxRdwyZo4iDl7PMfBoQpxUXn98JSddjNr+VYlSxMm87H+G3HPwe4EDjYkx2AtwPpyZj0aFQi/z2iPaXB/SWLaW7Svjrz3jsvZIk2Y5JtiLGpwZk9r9ANL/HeF6ipVjCMCeGeJCzrH87IPvhJttfS9M5M3Hv2JZLCRZO/pMmQQ5ql9XMzK/XjnVR059aak/JKjL2tMAbZw3F95PEfY/hVp7G/QrT30tR/sCns+JSvD+gcNcVX/WimD3dR0UWMm2Z8GbAHagkeKw+LTxUTulwOia/KRzTHSpqvLfkMHMlZulHMHh2gN81LLxlWfr2gRxDroe4Q8v+a3lap3RFzr5rJg7ainocFaWR8rsaT4YpiRvjlMXzIr7zp69w9KUQvzMRHOaV42QiJTg3LhjGpZj5n1vyj34RA9lfEFQX+2pe9171cIaSzWy94TLOOSRrhtwjbcaFn8hNJnMQkTo0LP9APFlvqEzUzgn9/OG0ZfdLKaK+hzs29BYdxmXdPPktEYB6cx65D9Of2y3lien6RwshyZrYDInmxG+0ZWitJ9j6xklym5bKdUN/Go6eThEUIFWFzqkCo5Kh8olDb9FQ+B9mqD2l06Kz4lC4b+msO0xdE7FiPB8S+x6Xzm5xfXuBjf+nZVRJ0F/wKDyIaJ9I0VmPYXYEgcPvP32FT5tLfPDmOXJd1XW1p5Kyr2QS33janQjbHIZTUHvWMnVFI8n+gqV1UrRC/3aa/lKEE7gMi7PaOKc9RhWNjEu3oHnWQuzgBJredDY0SrWOpCfpe0ymWUAuZDTymb4e0en6DKcFvM982GX/S3msI3cZb6hatbMqTwSMyqzhtEP3ZIj1vIlpm8jXW78TUbySxAmF1+YfSnQ4Kjki8yREzPaGLt1lCMoWv2HY+EdN9r9cJrclFlz6UYLMvjT8sesQJyavU4bsXoyJHTKHXyDAH6vgBr+v+tIJYOqziOL9mKCgtGNvCGHS0F+aFOg7Una2110S7ccWN9A4P6mjcjqdo6R8jgYVjWBxLedffcDV2ytUSl02v5mh/h93cUaw9n+4zcn/7S2ef+kOtpHg7z3/Fu8crXPYyZFo6vQcVXQCeAMhE0HGEPmPQ9jiY6eVxgXhi9MfK3TNCaB8M2b2HTHse8uq22avjEjVhLV21iH/SLzZzGFMflNfonWgeV6ix2RL/lmDuZiNlUOCwzQHr4jwXHigRO6dr+Up3wnIb8mn6uBVkVwGM5bRVExuU4v95P/tFqXr0lOFOctgxtI86ZG/mmRUUnOXaEHjoqC/UcmQ3xSwn6zJyM0dG2bf0ZDm4e+WVbJF4mJYMwm2qExsMb/elYo3lCr48LWI5sbn80Z/IhaqiWH2g+4xoz3RtnRWXdki9gQODyuGo1dEDHEizdnDjBatiURy9nt24m4nPmRQMKTrMb15V1aJW4ZTawd89miRldUqGT8g/Wyd7lGWzkbEld1l2kGKhBPxjZc+4x98+BrF5JDevaIc+IoiUWeOYmqXYzLViMNXZXkznLYU7jt0TkRkdh0K94Wddhcno9u0HAQPvhpRfUY8g9xeRPVS8jjfNX1oKD4IKN6fnNIW0vuChnKPRJw5fFH0ui+9eJPWMIXNheTvO7RP6TPJbktK3tzwGUw5TH02In/fIbttCaZC4nxI/XJMqmG5/X8+TXZPJJKgFBPMhCKozGrRHz7nkN2PyT10CDMT0sm8oT/v4w1hVJChXeOcsgPcoRhdQU6befajEBNC+XYglfF2ls66pOjWg+I1Dzf4AtWoTgj3/vcqyB83J/nNCL+jf28iNU/52y7Tnwqysq6ulCjJMd7qjpTqMZwyUmgmoLPiiuvakotdZ5QkVxiwc1QCILIGNxuyeu4Arua5cWWNzU6Zn9w/xctnHnDQzVG5aijfDRVC25XjYP6uS+2CR/LI5eiyR2ZHpBDriQ2fPYgIstA+r4maGEox5Y88ln84pjvvUr3okWwIL041NK5srfk0TzmUb0XUzicoPIrpL1icQCdp/r4grbdubeC7EZnCkO5r/QkRRZ9F46x+52TLUj+fZFQWbDb/E5eVf+VAJEbWiT8d0V10scYw95Zh+buOpN3bhlQjwh0a9l8X8TxKyUQ4s6dRavNSoOlaQ66KvSVHPN+0kloSLcjeaygJZd2ntx7ijmTYkdmzjNZH9JZFovlca+TfxsL7//eJXSj+OM30pyNSdX3RvXlZJrZPyYIxexARJ7XwCndFk8s/1EL1u3Izzu0GVK4JYpp/J2Lug4DFH9TJP4ppPB0LeP4nM3QfFbH1BJ0/WcD+vEz+rTS7Hy2QOVDA7NaDGUqFPmk3oPvJFMMpQ2vNI8hqZGsi2ftkd9RhJxoQFGQO4Q4EaR08L/2QCXTSD2YNsx8IL+6sJGidlcSkcR7F5ixpzJuuxQR5O2F9WRqnHfIPxN18bIDhDmHmJwl635mn9Ec5/sb5j4hycjPxezCaiqncjBiWnYnXFvQXLcOSw9GzHqkjl1EFdr4s9CDZjsltjag95VK6K1Vt87RH5sCS2fI4fN4hva8YzzghV8Ppdzxib8LPbUUka6IvOoG0XMl2zMPfnxGicxCx/H2VcFFWEJ4NHMJS+AWbTKHRW2s9wWBaFDy/pxPRhKqNGmdclr7fIshBqhkzLlqKD8aMZtSZ+/2Y3dd9goyhd2pM7YLH4WWf3a9V6M872GTM2neHxK7Cx1IHwkorN0I6G2Ix9ZbkrHf+7Dbl1IAff3oOd2hwBxqF5ncVDuz3pPTMHIZ0Vydy6H0F90YZOaAExVhS7k+l43LG0DzlULob0tqQ8mD+7Qh3cis8pik2zjsU7yq20hnD0k8HZKoxXj8m8kW5G85a6hdVU9bPu/yjd14htSc6Y6JtyT102P2qrlfriPBjDbTPxJRv6e8feyn0FlzCpKG3mCT/SNhq46IWU3sD1v60dmxl1F0T0tE+qUlUsqkTtXHap3MSind1cgdZQYVx0jKqWGmp5jSkyDzy8EYWp+OS3vQZlr9IkykHhpUJv3EzZjg9saeZvLtUXaPQh79XwB2IseP1xQxyxo6IzJVJ8t1mQOFqAq8Po4sDwqy4maltn4MX06Ra8onyBuKNHj7vUblqWPl+h9Jty3Ax4PbuHNV+hsJsl8LDmFQzpnFebnnKvbKEGUtn1YMY3EDdvzWQu+dRP+vitx3cgRhUcz+tUnwQkduWt/7UZ5b+rENzQymFK9/v0jqtDFVnJHa93xevob2e4vA5I1L3UYQ7sKx9Z0T6wCH/EMWiTw0Yzv5C7jIuQOrQZVyU0cOwrEUy9bGhN+dQvi2DjKnrAb0lS39ROVsglld61yVVs3h9w72/U9ECO5A+yjrK+fIGUL80YUEtqTeIEpqiBUXlV5VuW0o3pRTw+uK7ekMo3eoxfcUwfU22Q59zifwv/3h9jQ3jpLLhnbG+sDClOMb+nPiS2W0oPojIHMh/qnwnxBnJ9qd9UnlJsTuRQB/GzP+zBOOirqL5d8ayoDnjKup8wqKPPUt+a0x3PSsQ3IF8bkC7k6F9qAlSf84htzXJRjrS/08fqfHJ7orBPizJj36woHS93KYcWaY+Vtp0ZndI9ZWQZDs+jnqce79H4T5sfivH8g/GpI+gdC8mWZeVTqJj6awbCg8E0NfPe3TXYf/lFIm2PXYcGdXTJBuObB3XJaJLVXUiT31icAN5+INm+tWnDeNvtqmf8ynfgOLdWKSRJd1cw5kYb2jlcfoA5n/eIfagdUqnsN90J2ktQmMK97VRx0VDouWQ2RGHN0wbWqdkbXn0JdH7whQcvJin9oyletEjc/gFuvqDnOQmiZbCeb2eTtHKNahdciapeJCpxlgHDl8RXLL/skvukRI/5j6IKT6I2H/Fp3J9RGfN4eBFh5UfBMQe7L7uM5jWl6iYb0tn2SG3BVgxz8t/dZe/+uyn9AYJnl/bJHHo4Q4VEpauKpcUBEUNK5ZRySPVijl6XZY41oHlH0R4Pcms91/VVe33LHuvZ/HrHqO8w+HzWuQPfidDZ01qzs5KgjAlzX3h0WOPA8PSj/ryhFqSd1WyZn5hQFazWMeydvJQMpkpkbY75wMGc/pzR5MUaSwEGb1/Y8F+UJwoUg2Nc8I400da/Kt/ERJ70us7IXTXs7ROQf6B4trzj8QFsA70FuWhuvd7CrQr3RYnYzgtR5nSbRmzTb/j0Vt0yH/5cGJ76VB4GJPofL6F+kSkouTKK3bh//ifkjqSnEGeSvLXD9OadCy8HVG7IO1/5ZrGq6AvNrcdU78oL9XSvZjaUw65bTsx/dI0ZpwX2yfIw8pftBjOZ9h/Sea7Qd6QfxSz/7o99ul317uMjzJkH7hUbobsve6SrBnGJcvadwYcvJQh0RQVbuEt1WEm/AVHs3RXaXmZA41jY1/1Y29xknua1+Z8bO8+KkH+kWrP5R/FBJOo9eYpFxPqBO4viAI5nI2ZPXfE/oMp3J5L5UKVzrsz8oHqCzNNVeXjlX8ob4DeokN3RdlWw+mY8nVtovKtkNYJuXOnqxHtVcWxFx5K+eAORWhJdCyJbsxgyqX6ekD6QYLRmQHJm2lKd5VdEKbkAzvOG7J7k7TAMwGzb03mShaaZ8DEhvwjZTbECbj3f/rPvxipKLGHxp5J7XZ/Iiwr3RnTPRGx8Q8b1M8LmE7WzMQ7SnWpO4CDr0Yka7qGenMOqaqmJ35HAbfuCHAmoRY96K7n2HrDxRlPoK+hqIEXLm0SpSxmtYf/fp70tos7kpveYzeQ1JHh6JnMJNbcKPZxyWXxZyGJtmCxzKGleUowUvUZQ7KtTdPakPtJ7YIn87WkoXQvxO9IGj0uihklF0OH9qoLsQYgmQPFpb/yO58Sp2MOawUWT1bx1rqYfzzN/LtjGs9E1J7VYu2uKMIo9mHvG1KxZnccUlW5xvTn1PT0ZzVVS3RjGmckDfG7Yk4Np9S4xr7iL1snPPyeZe2filNb/nGKZAMa5xza6+5k+GHIHCg6afEnbcxQuqzukpEJcjE+jhTNP9Co+fM8T8RCdSLwe/rlg7yu/cI92P5Ggvxdl7t/t0zswvzbfaKkkIBx1iHRtbSfGWOGSmE2oXaoNxD7Zzitic9oEuUTZAyd54dUn3bJbkt63bgo0kqQhWu3ljGhIWgnsS6MpmXYlqzLdKH5yoggJ5VmdjemfVouJH7XsvmboqwV7nGcVle8J4+o2kXD1PUAv6dmxR1DYTMmzMDBCy7DabGaYk/Dj2HJoXIjZFSG8t2QMKtxcGKlx88enWRxvYpxLNUP58j8KMfRCzFHzyZwBg65h84k1NeQ6Eh1cOq/CxjOipGFFVln+mpIbkvNV6oeU31azejOVz3sBAbLbVvGeb3e/LsRhc2IYdmw/XVFpTfPqSEFfR6DWYMTSSRZuRnx4Hfz5DYdumsxo+mY3FZMes+V9GXOl6p48wsE+MceTH0W0j4dM/d+IGPfdUQBMzBzRb/M3b+b0JftKKv+8LWQ6Z/5TF1x6K7IcaTwUBn1ibbc5pyxFnZ+S9fTyf9aDCZr5H688j2LXR1gjaF0VemAqR0fvw2Vq+ZYpz4qGSpvJgny0ml1Vh3W/iwQi8uD0me6+hvnOSbMDCvqnufej9h71Wc4o/cUJTU08PpQvmEp3ovorCs0zBotpP1XFHvTWfbwO1rA+W/nyH8vS/gP51j9r9QuR0mJ5byBXAmz+zGle8EkONhnXLRsfTPN+r9oStv0DRFj9r7kyp7I1WukjsSviBPatK3TTPKh9Pp+RyZ1rXOW2Q+EdSeahjBnmXsvYDDjkK5q4QcFubHMfhjTn7fMvQf5e44kNKhRbm0IM3Y+n6PPk7FQQXje7LswKrpEvqF80xLMBvg9kTPCrGXmbZf+vK62RMuQvyVhWOwpQM0dQvO0KzuddYf0oSAukYwdGs9EtE4kKd2aCNoWLUeXPXJvZ47NeqOUndDfRORItiz1Z+RZ2p/XKNM6WvgHLyXpz+taD3LCN72+6thMNSLRFfZZvegxLsVUrlqijQHukGP9feu0RIyZfUtzwyPMaJRqXd0u47xq0yAP7RMySxvnDPXzSSrXhFMWb7oUNkNhmuccxvlfpF8n2gLaxxUZySWvZggKFm9gGJfFXXACjv328/cdsjuWymcqifoTV8Deos/MpwNSB86x+2CcVCnUOCNv2dbGJFurKx5D9ZLLzEeW5oZD54Q+1+mrCr0o3YkZTMl+8/M8T8xCVQqHQ3/Gobtq6M87LP25Fm1/ziEo62pxR+qi00dCAjprIj14PZi6Fmj+H+uEweqDjn116ZlNj9pz8kHKbcUkGlrc+Z1I0ZUbIjbnH4mqNyoJkiFSPTcux+T2I4bTcPCyoJpUVc1emIPKVUuYFTTVm3VpnfBEpbNaANVnDeW/TDOclo5oMKuSJExJ+jEqI+frjZjCHahcj+ivhlKq+pZkQ0mEo7I2X/OUNmPvy12CjMPRqxHZHY04/Y4lSugzKt0dUz8vDwFviIzoDizlGwrqaJ8wlG8Hx/qq2NemFIFbtWWYNrTWU2T35MbdOmtxRnIJDLP67FN1Q3ZvjBPpNfyeyjqAxbciBrMy6cg/immfcMhvR4Spz7c+noiu3xhzBPSA6r/Bl53+1es9Ea/1eV5vzVo78697gSdioQIYYz74ZRDFr17v383rPYnv7Ym5+n/1/Or51z2/Wqi/er4Qz5O0UP/+r17viXm9J+69PTE16q+eXz3/uudJOlF/9fzq+f/5/Gqh/ur5Qjy/Wqi/er4Qz68W6q+eL8Tzq4X6q+cL8fx/AZD/pXdCfv6FAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "for batch in train_loader:\n", + " data, targets = get_batch(batch, seq_len, digits_per_batch)\n", + " \n", + " plt.matshow((data.view(-1, data.shape[-1]) * mnist_std + mnist_mean).numpy())\n", + " plt.show()\n", + " break" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.884123, + "end_time": "2021-01-22T05:47:55.461320", + "exception": false, + "start_time": "2021-01-22T05:47:54.577197", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "### Average pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-22T05:47:57.244082Z", + "iopub.status.busy": "2021-01-22T05:47:57.243328Z", + "iopub.status.idle": "2021-01-22T10:39:49.076238Z", + "shell.execute_reply": "2021-01-22T10:39:49.076661Z" + }, + "papermill": { + "duration": 17512.732799, + "end_time": "2021-01-22T10:39:49.076819", + "exception": false, + "start_time": "2021-01-22T05:47:56.344020", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 71.71 | loss 76.41 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 71.30 | loss 30.88 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 71.32 | loss 21.27 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 85.47 | loss 18.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 77.66s | valid loss 25.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 83.71 | loss 15.61 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 83.86 | loss 13.37 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 86.86 | loss 12.41 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 81.42 | loss 12.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 84.67s | valid loss 18.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 86.64 | loss 10.24 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 75.73 | loss 10.60 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 81.76 | loss 9.95 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 81.87 | loss 9.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 83.37s | valid loss 14.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 94.72 | loss 8.42 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 77.94 | loss 8.66 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 80.76 | loss 9.25 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 81.51 | loss 8.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 84.36s | valid loss 10.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 84.04 | loss 8.32 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 81.32 | loss 7.46 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 81.14 | loss 7.77 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 81.43 | loss 6.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 82.77s | valid loss 12.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 83.32 | loss 6.57 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 82.53 | loss 7.28 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 82.51 | loss 6.70 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 83.59 | loss 7.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 81.88s | valid loss 11.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 72.45 | loss 6.17 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 71.97 | loss 6.38 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 87.26 | loss 7.19 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 82.36 | loss 6.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 81.41s | valid loss 9.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 77.81 | loss 6.93 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 79.64 | loss 6.20 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 87.53 | loss 5.67 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 86.36 | loss 6.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 83.96s | valid loss 13.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 82.75 | loss 6.00 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 82.32 | loss 6.48 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 84.71 | loss 5.84 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 83.97 | loss 6.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 84.83s | valid loss 7.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 74.50 | loss 5.66 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 81.72 | loss 5.39 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 81.64 | loss 5.84 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 81.68 | loss 5.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 81.53s | valid loss 7.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 95.57 | loss 4.99 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 92.74 | loss 5.79 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 83.36 | loss 5.59 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 81.30 | loss 4.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 88.28s | valid loss 9.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 83.18 | loss 5.24 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 82.40 | loss 4.98 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 72.28 | loss 6.00 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 72.45 | loss 4.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 79.58s | valid loss 7.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 81.41 | loss 4.62 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 81.21 | loss 4.83 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 81.45 | loss 5.07 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 82.70 | loss 4.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 83.06s | valid loss 7.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 81.82 | loss 5.03 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 81.20 | loss 4.39 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 81.65 | loss 5.52 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 81.84 | loss 4.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 82.96s | valid loss 7.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 91.10 | loss 4.48 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 94.42 | loss 4.31 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 94.26 | loss 4.71 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 84.28 | loss 5.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 89.86s | valid loss 9.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 89.22 | loss 3.90 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 94.02 | loss 4.33 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 83.09 | loss 4.15 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 89.17 | loss 4.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 88.59s | valid loss 7.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 81.50 | loss 3.63 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 81.28 | loss 4.56 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 83.62 | loss 4.43 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 89.50 | loss 4.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 84.57s | valid loss 8.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 81.48 | loss 4.24 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 91.07 | loss 3.85 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 82.23 | loss 4.34 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 81.08 | loss 3.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 84.77s | valid loss 7.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 81.41 | loss 3.29 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 81.01 | loss 3.39 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 80.93 | loss 3.74 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 84.11 | loss 3.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 82.98s | valid loss 6.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 81.27 | loss 3.06 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 80.79 | loss 3.20 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 81.06 | loss 3.93 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 80.46 | loss 3.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 82.14s | valid loss 8.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 86.91 | loss 3.42 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 80.69 | loss 2.77 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 80.62 | loss 3.55 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 92.44 | loss 3.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 85.94s | valid loss 5.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 70.95 | loss 2.31 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 71.03 | loss 3.73 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 71.15 | loss 2.81 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 71.33 | loss 2.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 75.15s | valid loss 8.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 94.21 | loss 2.89 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 87.97 | loss 2.83 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 80.32 | loss 2.84 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 80.64 | loss 3.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 85.15s | valid loss 8.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 88.83 | loss 2.83 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 84.89 | loss 3.16 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 89.33 | loss 2.83 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 74.11 | loss 2.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 82.59s | valid loss 8.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 71.34 | loss 3.35 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 72.75 | loss 2.25 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 81.36 | loss 3.00 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 78.41 | loss 2.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 76.52s | valid loss 8.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 84.67 | loss 2.55 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 92.13 | loss 2.33 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 88.64 | loss 3.18 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 71.05 | loss 2.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 82.51s | valid loss 7.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 72.11 | loss 2.31 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 71.14 | loss 2.94 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 71.04 | loss 1.85 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 71.10 | loss 2.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 74.23s | valid loss 7.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 83.42 | loss 2.17 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 87.03 | loss 2.48 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 93.23 | loss 2.63 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 93.52 | loss 2.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 90.21s | valid loss 7.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 78.50 | loss 2.35 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 80.81 | loss 2.49 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 81.04 | loss 2.18 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 81.16 | loss 1.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 81.64s | valid loss 6.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 71.27 | loss 2.43 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 71.09 | loss 1.66 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 70.89 | loss 2.37 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 71.03 | loss 1.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 74.18s | valid loss 8.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 31 | 200/ 938 batches | lr 0.00021 | ms/batch 93.75 | loss 2.24 |\n", + "| epoch 31 | 400/ 938 batches | lr 0.00021 | ms/batch 84.64 | loss 2.25 |\n", + "| epoch 31 | 600/ 938 batches | lr 0.00021 | ms/batch 85.37 | loss 2.61 |\n", + "| epoch 31 | 800/ 938 batches | lr 0.00021 | ms/batch 92.95 | loss 1.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 31 | time: 89.57s | valid loss 8.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 32 | 200/ 938 batches | lr 0.00020 | ms/batch 71.22 | loss 1.85 |\n", + "| epoch 32 | 400/ 938 batches | lr 0.00020 | ms/batch 80.19 | loss 1.87 |\n", + "| epoch 32 | 600/ 938 batches | lr 0.00020 | ms/batch 80.29 | loss 2.06 |\n", + "| epoch 32 | 800/ 938 batches | lr 0.00020 | ms/batch 80.81 | loss 1.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 32 | time: 80.30s | valid loss 8.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.99 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 72.51 | loss 88.23 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 72.16 | loss 41.16 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 72.15 | loss 29.01 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 80.13 | loss 23.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 77.37s | valid loss 25.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 82.34 | loss 16.61 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 81.89 | loss 13.85 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 81.95 | loss 14.34 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 92.34 | loss 12.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 86.21s | valid loss 16.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 84.88 | loss 11.57 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 82.39 | loss 10.76 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 81.91 | loss 10.35 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 81.91 | loss 10.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 82.94s | valid loss 13.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 95.08 | loss 8.85 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 84.45 | loss 9.29 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 84.18 | loss 8.57 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 85.76 | loss 8.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 88.44s | valid loss 16.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 90.90 | loss 7.53 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 94.42 | loss 8.29 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 85.70 | loss 7.69 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 81.91 | loss 7.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 88.04s | valid loss 10.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 75.08 | loss 6.99 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 81.84 | loss 6.41 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 81.99 | loss 6.65 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 83.25 | loss 7.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 82.14s | valid loss 10.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 82.93 | loss 6.26 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 80.35 | loss 6.59 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 72.05 | loss 7.38 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 72.07 | loss 6.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 79.00s | valid loss 9.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 75.83 | loss 5.88 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 72.04 | loss 6.49 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 72.20 | loss 6.22 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 72.24 | loss 6.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 73.77s | valid loss 10.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 80.19 | loss 7.06 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 83.14 | loss 6.49 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 84.90 | loss 5.39 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 86.44 | loss 5.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 86.27s | valid loss 10.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 76.84 | loss 5.85 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 71.91 | loss 5.29 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 71.84 | loss 5.79 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 81.39 | loss 5.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 79.53s | valid loss 9.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 87.92 | loss 5.35 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 88.29 | loss 5.27 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 81.77 | loss 5.61 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 84.33 | loss 4.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 84.96s | valid loss 8.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 71.75 | loss 4.84 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 71.38 | loss 4.78 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 73.99 | loss 5.74 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 75.73 | loss 6.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 75.25s | valid loss 7.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 81.67 | loss 4.71 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 88.74 | loss 5.11 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 91.09 | loss 5.00 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 90.25 | loss 5.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 89.02s | valid loss 8.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 72.03 | loss 4.63 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 79.80 | loss 5.17 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 81.48 | loss 4.95 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 81.43 | loss 4.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 80.67s | valid loss 9.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 75.78 | loss 3.75 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 81.27 | loss 4.08 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 86.63 | loss 4.45 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 71.66 | loss 3.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 78.73s | valid loss 7.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 79.32 | loss 3.70 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 77.30 | loss 4.76 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 83.67 | loss 4.15 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 90.85 | loss 3.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 85.58s | valid loss 8.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 80.48 | loss 4.15 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 81.32 | loss 4.82 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 81.17 | loss 3.71 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 82.79 | loss 3.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 81.23s | valid loss 8.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 71.23 | loss 3.30 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 70.82 | loss 4.18 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 73.04 | loss 3.65 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 72.10 | loss 3.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 74.92s | valid loss 10.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 86.68 | loss 3.60 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 93.82 | loss 3.48 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 93.76 | loss 3.37 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 79.69 | loss 3.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 86.56s | valid loss 10.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 81.62 | loss 3.44 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 84.95 | loss 3.68 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 91.28 | loss 3.52 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 81.38 | loss 3.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 85.43s | valid loss 9.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 83.98 | loss 3.31 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 93.80 | loss 3.08 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 93.74 | loss 3.53 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 94.00 | loss 3.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 93.29s | valid loss 9.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 93.53 | loss 3.58 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 87.35 | loss 3.02 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 89.43 | loss 3.74 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 77.91 | loss 3.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 84.76s | valid loss 7.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 72.56 | loss 3.04 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 82.10 | loss 2.46 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 93.86 | loss 3.55 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 88.35 | loss 2.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 86.55s | valid loss 7.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 80.91 | loss 3.04 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 78.84 | loss 2.31 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 80.16 | loss 3.41 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 74.90 | loss 2.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 78.02s | valid loss 8.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 82.22 | loss 3.09 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 82.27 | loss 3.04 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 80.60 | loss 2.53 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 80.51 | loss 2.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 82.55s | valid loss 8.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 81.30 | loss 2.08 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 82.49 | loss 2.57 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 72.06 | loss 2.22 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 71.10 | loss 2.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 78.81s | valid loss 9.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 80.93 | loss 2.50 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 80.83 | loss 2.66 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 81.11 | loss 2.77 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 81.25 | loss 2.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 81.81s | valid loss 9.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 80.56 | loss 2.28 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 80.50 | loss 2.56 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 80.45 | loss 2.21 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 80.34 | loss 2.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 81.52s | valid loss 8.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 78.06 | loss 2.36 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 72.04 | loss 1.96 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 79.92 | loss 1.87 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 80.99 | loss 2.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 79.67s | valid loss 8.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 81.60 | loss 2.03 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 87.25 | loss 2.37 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 80.33 | loss 2.19 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 85.47 | loss 2.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 84.75s | valid loss 8.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 7.37 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 89.52 | loss 69.04 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 96.20 | loss 30.76 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 78.58 | loss 21.44 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 82.45 | loss 18.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 86.84s | valid loss 31.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 83.35 | loss 15.80 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 84.27 | loss 13.25 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 72.12 | loss 12.84 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 87.54 | loss 12.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 83.27s | valid loss 15.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 86.11 | loss 11.30 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 93.28 | loss 10.76 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 77.77 | loss 10.14 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 82.13 | loss 9.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 85.44s | valid loss 11.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 89.80 | loss 8.84 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 81.90 | loss 7.83 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 82.44 | loss 9.22 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 82.05 | loss 8.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 83.76s | valid loss 9.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 75.79 | loss 8.12 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 79.95 | loss 8.56 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 81.78 | loss 7.81 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 84.50 | loss 7.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 81.59s | valid loss 13.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 93.18 | loss 7.48 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 78.71 | loss 7.53 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 76.73 | loss 6.76 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 85.86 | loss 6.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 86.67s | valid loss 10.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 83.06 | loss 6.47 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 84.60 | loss 6.41 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 81.87 | loss 5.95 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 81.85 | loss 6.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 84.67s | valid loss 9.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 95.51 | loss 5.69 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 80.49 | loss 6.34 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 80.68 | loss 6.23 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 81.66 | loss 6.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 85.35s | valid loss 10.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 96.05 | loss 5.76 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 90.01 | loss 4.88 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 81.74 | loss 6.70 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 93.33 | loss 5.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 90.03s | valid loss 8.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 82.30 | loss 5.43 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 82.68 | loss 5.64 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 82.08 | loss 5.72 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 83.55 | loss 5.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 85.46s | valid loss 10.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 75.34 | loss 4.88 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 81.44 | loss 5.34 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 81.68 | loss 5.40 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 84.11 | loss 4.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 80.73s | valid loss 9.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 72.07 | loss 5.11 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 72.66 | loss 6.05 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 84.01 | loss 5.19 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 95.25 | loss 5.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 82.34s | valid loss 10.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 89.96 | loss 5.62 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 86.05 | loss 4.99 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 81.99 | loss 4.98 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 81.38 | loss 4.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 85.40s | valid loss 7.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 82.57 | loss 5.00 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 82.90 | loss 5.50 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 81.33 | loss 4.09 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 91.23 | loss 4.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 85.76s | valid loss 9.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 93.46 | loss 4.38 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 84.40 | loss 4.07 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 82.35 | loss 5.13 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 90.18 | loss 5.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 87.99s | valid loss 10.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 94.73 | loss 4.11 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 90.30 | loss 4.31 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 94.23 | loss 4.08 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 94.69 | loss 4.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 91.53s | valid loss 6.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 91.08 | loss 3.32 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 81.15 | loss 3.45 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 82.94 | loss 4.04 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 71.57 | loss 4.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 82.36s | valid loss 7.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 81.98 | loss 3.39 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 81.32 | loss 4.55 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 79.05 | loss 3.23 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 71.29 | loss 4.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 80.54s | valid loss 7.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 81.41 | loss 3.20 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 70.89 | loss 3.77 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 76.80 | loss 3.92 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 78.64 | loss 4.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 79.15s | valid loss 6.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 82.71 | loss 3.12 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 80.66 | loss 3.23 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 80.85 | loss 3.76 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 88.56 | loss 3.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 85.06s | valid loss 7.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 94.62 | loss 3.08 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 80.05 | loss 2.73 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 81.27 | loss 2.67 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 81.46 | loss 3.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 85.92s | valid loss 8.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 88.84 | loss 3.47 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 93.46 | loss 2.90 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 90.10 | loss 3.41 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 82.41 | loss 3.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 87.90s | valid loss 6.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 84.22 | loss 3.23 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 83.60 | loss 3.14 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 94.17 | loss 3.68 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 93.87 | loss 2.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 87.24s | valid loss 8.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 72.60 | loss 2.58 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 77.96 | loss 2.60 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 73.11 | loss 3.15 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 75.28 | loss 3.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 76.67s | valid loss 7.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 81.13 | loss 2.53 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 91.28 | loss 2.70 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 93.54 | loss 2.71 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 87.19 | loss 2.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 88.40s | valid loss 7.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 86.27 | loss 2.50 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 81.15 | loss 2.43 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 81.05 | loss 2.54 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 84.10 | loss 2.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 84.87s | valid loss 7.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 77.42 | loss 2.43 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 78.27 | loss 2.84 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 80.54 | loss 2.36 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 80.57 | loss 2.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 80.81s | valid loss 8.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 80.22 | loss 2.43 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 80.69 | loss 2.43 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 88.48 | loss 2.54 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 90.42 | loss 2.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 86.51s | valid loss 6.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 83.58 | loss 2.16 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 80.53 | loss 2.55 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 80.95 | loss 2.39 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 85.18 | loss 2.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 83.57s | valid loss 6.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 80.65 | loss 2.15 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 74.79 | loss 2.28 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 70.96 | loss 2.11 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 70.93 | loss 2.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 75.28s | valid loss 7.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 31 | 200/ 938 batches | lr 0.00021 | ms/batch 80.67 | loss 2.46 |\n", + "| epoch 31 | 400/ 938 batches | lr 0.00021 | ms/batch 92.33 | loss 2.03 |\n", + "| epoch 31 | 600/ 938 batches | lr 0.00021 | ms/batch 86.50 | loss 1.88 |\n", + "| epoch 31 | 800/ 938 batches | lr 0.00021 | ms/batch 88.90 | loss 2.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 31 | time: 85.61s | valid loss 6.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 32 | 200/ 938 batches | lr 0.00020 | ms/batch 79.18 | loss 1.87 |\n", + "| epoch 32 | 400/ 938 batches | lr 0.00020 | ms/batch 80.10 | loss 2.17 |\n", + "| epoch 32 | 600/ 938 batches | lr 0.00020 | ms/batch 80.15 | loss 2.11 |\n", + "| epoch 32 | 800/ 938 batches | lr 0.00020 | ms/batch 80.07 | loss 2.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 32 | time: 80.75s | valid loss 7.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 33 | 200/ 938 batches | lr 0.00019 | ms/batch 85.45 | loss 1.99 |\n", + "| epoch 33 | 400/ 938 batches | lr 0.00019 | ms/batch 80.39 | loss 1.58 |\n", + "| epoch 33 | 600/ 938 batches | lr 0.00019 | ms/batch 83.70 | loss 2.32 |\n", + "| epoch 33 | 800/ 938 batches | lr 0.00019 | ms/batch 78.25 | loss 2.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 33 | time: 80.66s | valid loss 6.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 34 | 200/ 938 batches | lr 0.00018 | ms/batch 71.04 | loss 2.14 |\n", + "| epoch 34 | 400/ 938 batches | lr 0.00018 | ms/batch 78.74 | loss 1.48 |\n", + "| epoch 34 | 600/ 938 batches | lr 0.00018 | ms/batch 72.37 | loss 1.65 |\n", + "| epoch 34 | 800/ 938 batches | lr 0.00018 | ms/batch 75.16 | loss 1.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 34 | time: 76.37s | valid loss 8.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 35 | 200/ 938 batches | lr 0.00017 | ms/batch 82.73 | loss 1.48 |\n", + "| epoch 35 | 400/ 938 batches | lr 0.00017 | ms/batch 85.00 | loss 1.86 |\n", + "| epoch 35 | 600/ 938 batches | lr 0.00017 | ms/batch 84.27 | loss 1.38 |\n", + "| epoch 35 | 800/ 938 batches | lr 0.00017 | ms/batch 91.21 | loss 1.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 35 | time: 86.22s | valid loss 7.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 36 | 200/ 938 batches | lr 0.00017 | ms/batch 79.95 | loss 1.81 |\n", + "| epoch 36 | 400/ 938 batches | lr 0.00017 | ms/batch 84.06 | loss 1.63 |\n", + "| epoch 36 | 600/ 938 batches | lr 0.00017 | ms/batch 79.33 | loss 1.71 |\n", + "| epoch 36 | 800/ 938 batches | lr 0.00017 | ms/batch 78.01 | loss 1.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 36 | time: 81.65s | valid loss 6.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 37 | 200/ 938 batches | lr 0.00016 | ms/batch 80.59 | loss 1.53 |\n", + "| epoch 37 | 400/ 938 batches | lr 0.00016 | ms/batch 79.83 | loss 1.50 |\n", + "| epoch 37 | 600/ 938 batches | lr 0.00016 | ms/batch 85.08 | loss 1.47 |\n", + "| epoch 37 | 800/ 938 batches | lr 0.00016 | ms/batch 92.25 | loss 1.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 37 | time: 86.12s | valid loss 6.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 38 | 200/ 938 batches | lr 0.00015 | ms/batch 72.19 | loss 1.56 |\n", + "| epoch 38 | 400/ 938 batches | lr 0.00015 | ms/batch 82.62 | loss 1.41 |\n", + "| epoch 38 | 600/ 938 batches | lr 0.00015 | ms/batch 87.43 | loss 1.31 |\n", + "| epoch 38 | 800/ 938 batches | lr 0.00015 | ms/batch 88.32 | loss 1.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 38 | time: 83.55s | valid loss 8.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 39 | 200/ 938 batches | lr 0.00014 | ms/batch 81.54 | loss 1.69 |\n", + "| epoch 39 | 400/ 938 batches | lr 0.00014 | ms/batch 75.47 | loss 1.18 |\n", + "| epoch 39 | 600/ 938 batches | lr 0.00014 | ms/batch 80.64 | loss 1.09 |\n", + "| epoch 39 | 800/ 938 batches | lr 0.00014 | ms/batch 81.86 | loss 1.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 39 | time: 79.70s | valid loss 7.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 40 | 200/ 938 batches | lr 0.00014 | ms/batch 78.92 | loss 1.57 |\n", + "| epoch 40 | 400/ 938 batches | lr 0.00014 | ms/batch 80.34 | loss 1.87 |\n", + "| epoch 40 | 600/ 938 batches | lr 0.00014 | ms/batch 86.51 | loss 1.75 |\n", + "| epoch 40 | 800/ 938 batches | lr 0.00014 | ms/batch 81.09 | loss 0.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 40 | time: 82.77s | valid loss 7.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 41 | 200/ 938 batches | lr 0.00013 | ms/batch 81.03 | loss 1.35 |\n", + "| epoch 41 | 400/ 938 batches | lr 0.00013 | ms/batch 80.63 | loss 1.10 |\n", + "| epoch 41 | 600/ 938 batches | lr 0.00013 | ms/batch 80.70 | loss 1.72 |\n", + "| epoch 41 | 800/ 938 batches | lr 0.00013 | ms/batch 80.78 | loss 1.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 41 | time: 82.11s | valid loss 7.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 42 | 200/ 938 batches | lr 0.00012 | ms/batch 93.86 | loss 1.57 |\n", + "| epoch 42 | 400/ 938 batches | lr 0.00012 | ms/batch 82.25 | loss 0.88 |\n", + "| epoch 42 | 600/ 938 batches | lr 0.00012 | ms/batch 80.25 | loss 1.31 |\n", + "| epoch 42 | 800/ 938 batches | lr 0.00012 | ms/batch 80.90 | loss 1.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 42 | time: 86.23s | valid loss 7.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 43 | 200/ 938 batches | lr 0.00012 | ms/batch 85.86 | loss 1.51 |\n", + "| epoch 43 | 400/ 938 batches | lr 0.00012 | ms/batch 84.51 | loss 1.31 |\n", + "| epoch 43 | 600/ 938 batches | lr 0.00012 | ms/batch 93.74 | loss 1.15 |\n", + "| epoch 43 | 800/ 938 batches | lr 0.00012 | ms/batch 93.75 | loss 1.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 43 | time: 90.30s | valid loss 7.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 44 | 200/ 938 batches | lr 0.00011 | ms/batch 70.99 | loss 1.34 |\n", + "| epoch 44 | 400/ 938 batches | lr 0.00011 | ms/batch 70.50 | loss 1.25 |\n", + "| epoch 44 | 600/ 938 batches | lr 0.00011 | ms/batch 80.99 | loss 1.25 |\n", + "| epoch 44 | 800/ 938 batches | lr 0.00011 | ms/batch 81.18 | loss 1.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 44 | time: 75.79s | valid loss 7.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 45 | 200/ 938 batches | lr 0.00010 | ms/batch 70.48 | loss 1.50 |\n", + "| epoch 45 | 400/ 938 batches | lr 0.00010 | ms/batch 84.77 | loss 1.21 |\n", + "| epoch 45 | 600/ 938 batches | lr 0.00010 | ms/batch 92.58 | loss 1.03 |\n", + "| epoch 45 | 800/ 938 batches | lr 0.00010 | ms/batch 92.48 | loss 1.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 45 | time: 86.19s | valid loss 8.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 46 | 200/ 938 batches | lr 0.00010 | ms/batch 81.19 | loss 1.50 |\n", + "| epoch 46 | 400/ 938 batches | lr 0.00010 | ms/batch 81.04 | loss 0.67 |\n", + "| epoch 46 | 600/ 938 batches | lr 0.00010 | ms/batch 82.75 | loss 1.20 |\n", + "| epoch 46 | 800/ 938 batches | lr 0.00010 | ms/batch 80.35 | loss 1.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 46 | time: 82.53s | valid loss 8.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 47 | 200/ 938 batches | lr 0.00009 | ms/batch 72.33 | loss 1.66 |\n", + "| epoch 47 | 400/ 938 batches | lr 0.00009 | ms/batch 83.93 | loss 0.87 |\n", + "| epoch 47 | 600/ 938 batches | lr 0.00009 | ms/batch 80.56 | loss 1.22 |\n", + "| epoch 47 | 800/ 938 batches | lr 0.00009 | ms/batch 82.32 | loss 1.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 47 | time: 80.63s | valid loss 6.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 48 | 200/ 938 batches | lr 0.00009 | ms/batch 75.06 | loss 1.36 |\n", + "| epoch 48 | 400/ 938 batches | lr 0.00009 | ms/batch 80.28 | loss 0.71 |\n", + "| epoch 48 | 600/ 938 batches | lr 0.00009 | ms/batch 80.38 | loss 1.23 |\n", + "| epoch 48 | 800/ 938 batches | lr 0.00009 | ms/batch 80.35 | loss 1.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 48 | time: 82.53s | valid loss 7.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 49 | 200/ 938 batches | lr 0.00009 | ms/batch 70.55 | loss 1.10 |\n", + "| epoch 49 | 400/ 938 batches | lr 0.00009 | ms/batch 69.93 | loss 0.79 |\n", + "| epoch 49 | 600/ 938 batches | lr 0.00009 | ms/batch 70.04 | loss 0.92 |\n", + "| epoch 49 | 800/ 938 batches | lr 0.00009 | ms/batch 69.85 | loss 1.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 49 | time: 71.05s | valid loss 7.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 50 | 200/ 938 batches | lr 0.00008 | ms/batch 70.30 | loss 0.89 |\n", + "| epoch 50 | 400/ 938 batches | lr 0.00008 | ms/batch 81.24 | loss 1.13 |\n", + "| epoch 50 | 600/ 938 batches | lr 0.00008 | ms/batch 84.96 | loss 0.83 |\n", + "| epoch 50 | 800/ 938 batches | lr 0.00008 | ms/batch 93.48 | loss 1.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 50 | time: 84.81s | valid loss 7.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 51 | 200/ 938 batches | lr 0.00008 | ms/batch 82.49 | loss 1.06 |\n", + "| epoch 51 | 400/ 938 batches | lr 0.00008 | ms/batch 79.34 | loss 1.09 |\n", + "| epoch 51 | 600/ 938 batches | lr 0.00008 | ms/batch 80.02 | loss 1.19 |\n", + "| epoch 51 | 800/ 938 batches | lr 0.00008 | ms/batch 80.40 | loss 0.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 51 | time: 81.79s | valid loss 7.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 52 | 200/ 938 batches | lr 0.00007 | ms/batch 90.45 | loss 0.66 |\n", + "| epoch 52 | 400/ 938 batches | lr 0.00007 | ms/batch 93.29 | loss 0.93 |\n", + "| epoch 52 | 600/ 938 batches | lr 0.00007 | ms/batch 85.25 | loss 1.10 |\n", + "| epoch 52 | 800/ 938 batches | lr 0.00007 | ms/batch 82.05 | loss 0.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 52 | time: 87.72s | valid loss 7.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 53 | 200/ 938 batches | lr 0.00007 | ms/batch 72.66 | loss 0.89 |\n", + "| epoch 53 | 400/ 938 batches | lr 0.00007 | ms/batch 76.05 | loss 1.03 |\n", + "| epoch 53 | 600/ 938 batches | lr 0.00007 | ms/batch 79.63 | loss 0.72 |\n", + "| epoch 53 | 800/ 938 batches | lr 0.00007 | ms/batch 78.71 | loss 0.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 53 | time: 77.60s | valid loss 7.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 54 | 200/ 938 batches | lr 0.00007 | ms/batch 80.09 | loss 0.80 |\n", + "| epoch 54 | 400/ 938 batches | lr 0.00007 | ms/batch 79.64 | loss 0.80 |\n", + "| epoch 54 | 600/ 938 batches | lr 0.00007 | ms/batch 79.75 | loss 1.02 |\n", + "| epoch 54 | 800/ 938 batches | lr 0.00007 | ms/batch 79.98 | loss 1.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 54 | time: 79.71s | valid loss 7.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 55 | 200/ 938 batches | lr 0.00006 | ms/batch 70.60 | loss 0.85 |\n", + "| epoch 55 | 400/ 938 batches | lr 0.00006 | ms/batch 70.03 | loss 0.53 |\n", + "| epoch 55 | 600/ 938 batches | lr 0.00006 | ms/batch 70.00 | loss 0.62 |\n", + "| epoch 55 | 800/ 938 batches | lr 0.00006 | ms/batch 70.03 | loss 0.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 55 | time: 73.34s | valid loss 8.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 56 | 200/ 938 batches | lr 0.00006 | ms/batch 80.97 | loss 1.22 |\n", + "| epoch 56 | 400/ 938 batches | lr 0.00006 | ms/batch 80.25 | loss 0.77 |\n", + "| epoch 56 | 600/ 938 batches | lr 0.00006 | ms/batch 89.04 | loss 0.87 |\n", + "| epoch 56 | 800/ 938 batches | lr 0.00006 | ms/batch 87.47 | loss 1.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 56 | time: 84.95s | valid loss 7.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 57 | 200/ 938 batches | lr 0.00006 | ms/batch 80.24 | loss 0.71 |\n", + "| epoch 57 | 400/ 938 batches | lr 0.00006 | ms/batch 79.67 | loss 1.09 |\n", + "| epoch 57 | 600/ 938 batches | lr 0.00006 | ms/batch 79.69 | loss 0.86 |\n", + "| epoch 57 | 800/ 938 batches | lr 0.00006 | ms/batch 79.49 | loss 0.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 57 | time: 81.13s | valid loss 8.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 58 | 200/ 938 batches | lr 0.00005 | ms/batch 77.99 | loss 0.91 |\n", + "| epoch 58 | 400/ 938 batches | lr 0.00005 | ms/batch 70.34 | loss 0.81 |\n", + "| epoch 58 | 600/ 938 batches | lr 0.00005 | ms/batch 70.36 | loss 0.79 |\n", + "| epoch 58 | 800/ 938 batches | lr 0.00005 | ms/batch 70.68 | loss 0.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 58 | time: 73.02s | valid loss 7.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 59 | 200/ 938 batches | lr 0.00005 | ms/batch 77.33 | loss 0.71 |\n", + "| epoch 59 | 400/ 938 batches | lr 0.00005 | ms/batch 87.45 | loss 1.08 |\n", + "| epoch 59 | 600/ 938 batches | lr 0.00005 | ms/batch 83.62 | loss 0.65 |\n", + "| epoch 59 | 800/ 938 batches | lr 0.00005 | ms/batch 79.68 | loss 1.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 59 | time: 83.40s | valid loss 7.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 60 | 200/ 938 batches | lr 0.00005 | ms/batch 80.53 | loss 0.67 |\n", + "| epoch 60 | 400/ 938 batches | lr 0.00005 | ms/batch 69.92 | loss 0.75 |\n", + "| epoch 60 | 600/ 938 batches | lr 0.00005 | ms/batch 69.94 | loss 0.52 |\n", + "| epoch 60 | 800/ 938 batches | lr 0.00005 | ms/batch 76.40 | loss 1.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 60 | time: 75.95s | valid loss 7.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 61 | 200/ 938 batches | lr 0.00005 | ms/batch 79.42 | loss 0.76 |\n", + "| epoch 61 | 400/ 938 batches | lr 0.00005 | ms/batch 79.73 | loss 1.35 |\n", + "| epoch 61 | 600/ 938 batches | lr 0.00005 | ms/batch 82.49 | loss 0.61 |\n", + "| epoch 61 | 800/ 938 batches | lr 0.00005 | ms/batch 80.29 | loss 1.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 61 | time: 81.84s | valid loss 7.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 62 | 200/ 938 batches | lr 0.00004 | ms/batch 78.24 | loss 0.66 |\n", + "| epoch 62 | 400/ 938 batches | lr 0.00004 | ms/batch 80.49 | loss 1.19 |\n", + "| epoch 62 | 600/ 938 batches | lr 0.00004 | ms/batch 80.45 | loss 0.80 |\n", + "| epoch 62 | 800/ 938 batches | lr 0.00004 | ms/batch 80.34 | loss 0.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 62 | time: 81.82s | valid loss 7.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 63 | 200/ 938 batches | lr 0.00004 | ms/batch 92.67 | loss 0.79 |\n", + "| epoch 63 | 400/ 938 batches | lr 0.00004 | ms/batch 85.17 | loss 0.99 |\n", + "| epoch 63 | 600/ 938 batches | lr 0.00004 | ms/batch 81.07 | loss 0.94 |\n", + "| epoch 63 | 800/ 938 batches | lr 0.00004 | ms/batch 81.26 | loss 0.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 63 | time: 86.50s | valid loss 8.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 64 | 200/ 938 batches | lr 0.00004 | ms/batch 83.66 | loss 0.52 |\n", + "| epoch 64 | 400/ 938 batches | lr 0.00004 | ms/batch 92.42 | loss 0.63 |\n", + "| epoch 64 | 600/ 938 batches | lr 0.00004 | ms/batch 92.55 | loss 1.31 |\n", + "| epoch 64 | 800/ 938 batches | lr 0.00004 | ms/batch 82.01 | loss 1.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 64 | time: 87.31s | valid loss 7.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 65 | 200/ 938 batches | lr 0.00004 | ms/batch 80.51 | loss 0.37 |\n", + "| epoch 65 | 400/ 938 batches | lr 0.00004 | ms/batch 87.62 | loss 1.04 |\n", + "| epoch 65 | 600/ 938 batches | lr 0.00004 | ms/batch 76.82 | loss 0.56 |\n", + "| epoch 65 | 800/ 938 batches | lr 0.00004 | ms/batch 85.35 | loss 0.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 65 | time: 83.14s | valid loss 7.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 66 | 200/ 938 batches | lr 0.00004 | ms/batch 90.65 | loss 0.63 |\n", + "| epoch 66 | 400/ 938 batches | lr 0.00004 | ms/batch 79.20 | loss 0.76 |\n", + "| epoch 66 | 600/ 938 batches | lr 0.00004 | ms/batch 79.50 | loss 0.79 |\n", + "| epoch 66 | 800/ 938 batches | lr 0.00004 | ms/batch 70.41 | loss 1.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 66 | time: 82.23s | valid loss 7.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 67 | 200/ 938 batches | lr 0.00003 | ms/batch 70.70 | loss 0.69 |\n", + "| epoch 67 | 400/ 938 batches | lr 0.00003 | ms/batch 79.58 | loss 0.81 |\n", + "| epoch 67 | 600/ 938 batches | lr 0.00003 | ms/batch 83.05 | loss 0.91 |\n", + "| epoch 67 | 800/ 938 batches | lr 0.00003 | ms/batch 81.08 | loss 0.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 67 | time: 80.24s | valid loss 7.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 68 | 200/ 938 batches | lr 0.00003 | ms/batch 80.51 | loss 0.47 |\n", + "| epoch 68 | 400/ 938 batches | lr 0.00003 | ms/batch 80.07 | loss 0.61 |\n", + "| epoch 68 | 600/ 938 batches | lr 0.00003 | ms/batch 80.26 | loss 0.78 |\n", + "| epoch 68 | 800/ 938 batches | lr 0.00003 | ms/batch 80.70 | loss 0.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 68 | time: 81.51s | valid loss 8.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 69 | 200/ 938 batches | lr 0.00003 | ms/batch 70.69 | loss 1.01 |\n", + "| epoch 69 | 400/ 938 batches | lr 0.00003 | ms/batch 70.23 | loss 0.82 |\n", + "| epoch 69 | 600/ 938 batches | lr 0.00003 | ms/batch 78.64 | loss 0.70 |\n", + "| epoch 69 | 800/ 938 batches | lr 0.00003 | ms/batch 80.21 | loss 1.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 69 | time: 78.08s | valid loss 7.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 70 | 200/ 938 batches | lr 0.00003 | ms/batch 79.96 | loss 0.65 |\n", + "| epoch 70 | 400/ 938 batches | lr 0.00003 | ms/batch 79.55 | loss 0.56 |\n", + "| epoch 70 | 600/ 938 batches | lr 0.00003 | ms/batch 79.69 | loss 1.22 |\n", + "| epoch 70 | 800/ 938 batches | lr 0.00003 | ms/batch 79.80 | loss 0.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 70 | time: 81.20s | valid loss 7.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 71 | 200/ 938 batches | lr 0.00003 | ms/batch 90.88 | loss 0.90 |\n", + "| epoch 71 | 400/ 938 batches | lr 0.00003 | ms/batch 87.77 | loss 0.79 |\n", + "| epoch 71 | 600/ 938 batches | lr 0.00003 | ms/batch 79.70 | loss 0.58 |\n", + "| epoch 71 | 800/ 938 batches | lr 0.00003 | ms/batch 79.52 | loss 0.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 71 | time: 85.16s | valid loss 7.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.32 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 82.20 | loss 65.45 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 94.90 | loss 29.18 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 81.63 | loss 19.88 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 91.75 | loss 17.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 88.10s | valid loss 18.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 92.06 | loss 13.51 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 87.81 | loss 12.68 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 82.31 | loss 12.33 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 82.32 | loss 11.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 86.47s | valid loss 20.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 79.44 | loss 9.71 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 71.98 | loss 10.58 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 71.95 | loss 9.71 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 71.98 | loss 9.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 74.87s | valid loss 17.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 82.86 | loss 9.03 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 76.22 | loss 8.92 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 81.57 | loss 8.39 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 83.72 | loss 8.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 83.78s | valid loss 14.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 83.84 | loss 7.45 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 81.72 | loss 7.82 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 81.84 | loss 7.23 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 85.06 | loss 7.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 83.19s | valid loss 8.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 83.31 | loss 6.52 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 81.63 | loss 7.36 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 81.84 | loss 6.62 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 74.94 | loss 7.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 82.03s | valid loss 13.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 82.25 | loss 6.77 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 82.26 | loss 6.67 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 82.21 | loss 6.47 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 88.93 | loss 6.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 83.83s | valid loss 15.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 73.78 | loss 6.31 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 86.00 | loss 6.25 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 94.87 | loss 6.17 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 95.27 | loss 5.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 88.38s | valid loss 8.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 83.01 | loss 5.74 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 81.66 | loss 5.34 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 81.73 | loss 6.03 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 89.26 | loss 5.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 85.02s | valid loss 7.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 81.06 | loss 5.78 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 81.44 | loss 5.90 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 81.50 | loss 6.09 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 81.64 | loss 5.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 81.49s | valid loss 7.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 74.47 | loss 5.48 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 71.27 | loss 5.79 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 82.46 | loss 5.15 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 91.01 | loss 5.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 81.67s | valid loss 8.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 81.68 | loss 4.39 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 87.93 | loss 5.39 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 81.36 | loss 5.36 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 82.15 | loss 4.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 85.25s | valid loss 9.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 95.82 | loss 4.96 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 83.22 | loss 4.82 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 82.25 | loss 5.44 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 81.58 | loss 4.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 84.22s | valid loss 6.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 92.56 | loss 3.77 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 81.17 | loss 4.94 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 83.81 | loss 4.59 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 77.19 | loss 5.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 84.46s | valid loss 9.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 81.61 | loss 4.34 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 81.12 | loss 4.37 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 81.16 | loss 4.82 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 81.23 | loss 3.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 82.54s | valid loss 9.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 92.28 | loss 4.66 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 87.30 | loss 4.56 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 81.39 | loss 3.44 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 81.65 | loss 3.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 84.45s | valid loss 8.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 81.20 | loss 4.21 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 81.55 | loss 3.57 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 74.36 | loss 4.46 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 79.30 | loss 3.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 80.91s | valid loss 6.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 81.42 | loss 3.67 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 80.78 | loss 3.17 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 81.48 | loss 4.22 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 81.41 | loss 3.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 82.59s | valid loss 6.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 81.91 | loss 3.56 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 80.58 | loss 3.33 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 80.72 | loss 3.34 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 81.42 | loss 3.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 80.90s | valid loss 8.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 82.81 | loss 3.56 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 86.02 | loss 3.30 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 88.54 | loss 3.90 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 84.70 | loss 2.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 86.23s | valid loss 7.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 81.48 | loss 2.74 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 81.66 | loss 2.62 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 76.00 | loss 3.20 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 70.90 | loss 4.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 78.08s | valid loss 7.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 83.43 | loss 3.16 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 80.37 | loss 2.72 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 80.59 | loss 2.95 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 79.80 | loss 3.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 82.79s | valid loss 7.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 74.46 | loss 3.11 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 80.56 | loss 2.80 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 80.49 | loss 3.02 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 80.46 | loss 2.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 80.05s | valid loss 6.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 89.37 | loss 2.53 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 90.80 | loss 3.07 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 94.06 | loss 2.83 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 82.34 | loss 3.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 88.72s | valid loss 7.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 80.75 | loss 2.58 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 80.43 | loss 2.30 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 80.33 | loss 2.64 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 80.38 | loss 3.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 81.78s | valid loss 8.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 91.70 | loss 1.78 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 93.44 | loss 3.07 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 84.62 | loss 2.45 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 80.18 | loss 2.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 86.83s | valid loss 8.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 78.01 | loss 2.45 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 80.35 | loss 2.47 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 80.19 | loss 2.10 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 77.80 | loss 2.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 78.37s | valid loss 8.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 71.06 | loss 2.11 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 86.72 | loss 1.92 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 94.24 | loss 2.74 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 89.79 | loss 2.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 86.22s | valid loss 6.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 83.34 | loss 2.50 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 74.37 | loss 1.94 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 80.42 | loss 2.21 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 87.89 | loss 1.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 80.41s | valid loss 7.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 70.66 | loss 1.71 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 70.25 | loss 1.56 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 70.34 | loss 2.20 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 72.65 | loss 1.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 74.37s | valid loss 7.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 31 | 200/ 938 batches | lr 0.00021 | ms/batch 89.56 | loss 1.92 |\n", + "| epoch 31 | 400/ 938 batches | lr 0.00021 | ms/batch 81.09 | loss 1.57 |\n", + "| epoch 31 | 600/ 938 batches | lr 0.00021 | ms/batch 87.54 | loss 2.06 |\n", + "| epoch 31 | 800/ 938 batches | lr 0.00021 | ms/batch 80.92 | loss 1.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 31 | time: 85.28s | valid loss 8.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 32 | 200/ 938 batches | lr 0.00020 | ms/batch 80.94 | loss 1.96 |\n", + "| epoch 32 | 400/ 938 batches | lr 0.00020 | ms/batch 80.78 | loss 1.96 |\n", + "| epoch 32 | 600/ 938 batches | lr 0.00020 | ms/batch 88.02 | loss 2.50 |\n", + "| epoch 32 | 800/ 938 batches | lr 0.00020 | ms/batch 77.41 | loss 1.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 32 | time: 81.37s | valid loss 9.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 33 | 200/ 938 batches | lr 0.00019 | ms/batch 84.93 | loss 1.74 |\n", + "| epoch 33 | 400/ 938 batches | lr 0.00019 | ms/batch 80.11 | loss 1.54 |\n", + "| epoch 33 | 600/ 938 batches | lr 0.00019 | ms/batch 80.06 | loss 1.80 |\n", + "| epoch 33 | 800/ 938 batches | lr 0.00019 | ms/batch 80.53 | loss 2.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 33 | time: 82.53s | valid loss 8.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 34 | 200/ 938 batches | lr 0.00018 | ms/batch 79.88 | loss 1.77 |\n", + "| epoch 34 | 400/ 938 batches | lr 0.00018 | ms/batch 70.72 | loss 1.48 |\n", + "| epoch 34 | 600/ 938 batches | lr 0.00018 | ms/batch 70.82 | loss 1.89 |\n", + "| epoch 34 | 800/ 938 batches | lr 0.00018 | ms/batch 73.22 | loss 2.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 34 | time: 78.16s | valid loss 7.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 35 | 200/ 938 batches | lr 0.00017 | ms/batch 81.35 | loss 2.16 |\n", + "| epoch 35 | 400/ 938 batches | lr 0.00017 | ms/batch 79.76 | loss 1.58 |\n", + "| epoch 35 | 600/ 938 batches | lr 0.00017 | ms/batch 80.07 | loss 1.10 |\n", + "| epoch 35 | 800/ 938 batches | lr 0.00017 | ms/batch 80.26 | loss 1.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 35 | time: 82.28s | valid loss 8.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.53 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 84.75 | loss 77.96 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 92.74 | loss 34.66 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 87.85 | loss 24.17 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 85.42 | loss 19.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 88.43s | valid loss 31.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 82.78 | loss 15.55 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 82.78 | loss 13.87 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 82.26 | loss 13.89 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 82.99 | loss 11.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 84.28s | valid loss 19.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 85.61 | loss 11.35 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 77.21 | loss 10.76 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 74.75 | loss 9.83 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 80.99 | loss 10.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 81.84s | valid loss 16.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 85.97 | loss 8.55 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 82.90 | loss 9.05 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 81.78 | loss 8.71 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 81.77 | loss 9.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 84.04s | valid loss 13.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 82.16 | loss 8.52 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 84.40 | loss 7.69 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 72.21 | loss 7.79 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 77.57 | loss 7.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 78.45s | valid loss 11.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 88.20 | loss 7.19 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 79.03 | loss 7.19 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 81.82 | loss 6.37 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 82.07 | loss 7.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 83.93s | valid loss 10.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 82.46 | loss 6.03 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 81.70 | loss 7.26 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 81.65 | loss 6.02 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 83.77 | loss 6.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 81.09s | valid loss 8.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 88.80 | loss 6.03 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 71.96 | loss 5.99 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 74.51 | loss 6.94 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 82.12 | loss 5.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 79.28s | valid loss 10.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 89.02 | loss 6.17 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 90.21 | loss 5.98 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 84.91 | loss 6.34 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 81.58 | loss 5.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 86.68s | valid loss 10.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 81.99 | loss 5.88 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 82.27 | loss 6.31 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 85.34 | loss 5.49 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 95.76 | loss 6.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 87.70s | valid loss 8.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 87.95 | loss 5.33 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 81.94 | loss 5.71 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 81.76 | loss 5.24 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 81.94 | loss 4.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 84.53s | valid loss 7.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 83.47 | loss 5.35 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 81.99 | loss 5.42 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 82.17 | loss 5.23 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 81.23 | loss 5.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 83.30s | valid loss 9.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 81.71 | loss 5.60 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 81.69 | loss 4.33 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 81.45 | loss 4.66 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 80.44 | loss 5.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 82.57s | valid loss 8.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 78.42 | loss 5.61 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 73.83 | loss 4.50 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 71.15 | loss 5.03 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 71.16 | loss 4.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 74.55s | valid loss 9.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 79.53 | loss 3.94 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 78.79 | loss 4.41 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 86.92 | loss 5.12 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 84.88 | loss 4.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 85.04s | valid loss 8.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 82.23 | loss 5.04 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 81.61 | loss 4.21 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 81.86 | loss 3.84 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 79.27 | loss 3.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 80.25s | valid loss 7.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 71.70 | loss 3.82 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 83.57 | loss 4.82 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 94.12 | loss 4.13 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 94.01 | loss 4.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 87.20s | valid loss 8.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 81.52 | loss 3.99 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 81.72 | loss 4.16 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 76.98 | loss 3.37 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 76.95 | loss 3.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 82.02s | valid loss 7.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 89.75 | loss 3.33 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 81.13 | loss 3.93 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 83.94 | loss 3.34 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 81.06 | loss 3.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 84.66s | valid loss 7.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 81.50 | loss 3.09 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 80.81 | loss 2.61 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 81.02 | loss 3.66 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 89.48 | loss 3.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 86.19s | valid loss 8.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 79.05 | loss 3.44 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 81.76 | loss 2.79 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 81.09 | loss 2.48 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 75.53 | loss 3.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 79.23s | valid loss 8.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 73.87 | loss 3.07 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 73.32 | loss 3.54 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 73.56 | loss 3.67 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 78.22 | loss 4.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 75.54s | valid loss 7.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 71.65 | loss 3.07 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 80.89 | loss 3.08 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 76.28 | loss 3.06 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 82.00 | loss 2.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 81.56s | valid loss 6.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 90.52 | loss 2.67 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 90.91 | loss 3.18 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 84.27 | loss 2.73 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 80.46 | loss 2.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 86.55s | valid loss 10.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 84.99 | loss 2.27 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 80.22 | loss 3.48 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 86.99 | loss 2.97 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 86.72 | loss 3.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 85.16s | valid loss 7.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 81.06 | loss 2.23 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 81.50 | loss 2.36 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 82.53 | loss 2.64 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 80.73 | loss 3.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 82.62s | valid loss 7.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 87.94 | loss 2.33 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 81.98 | loss 1.98 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 80.99 | loss 2.34 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 81.10 | loss 2.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 83.97s | valid loss 7.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 94.92 | loss 2.24 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 94.26 | loss 2.67 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 80.03 | loss 2.35 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 78.52 | loss 2.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 87.02s | valid loss 6.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 71.32 | loss 2.01 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 71.06 | loss 2.74 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 70.97 | loss 2.22 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 82.54 | loss 1.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 76.61s | valid loss 7.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 91.84 | loss 1.86 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 81.74 | loss 2.46 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 80.82 | loss 1.89 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 80.36 | loss 2.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 84.22s | valid loss 6.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 31 | 200/ 938 batches | lr 0.00021 | ms/batch 86.83 | loss 2.44 |\n", + "| epoch 31 | 400/ 938 batches | lr 0.00021 | ms/batch 80.44 | loss 2.15 |\n", + "| epoch 31 | 600/ 938 batches | lr 0.00021 | ms/batch 80.55 | loss 2.64 |\n", + "| epoch 31 | 800/ 938 batches | lr 0.00021 | ms/batch 80.50 | loss 2.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 31 | time: 83.12s | valid loss 7.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 32 | 200/ 938 batches | lr 0.00020 | ms/batch 81.15 | loss 2.27 |\n", + "| epoch 32 | 400/ 938 batches | lr 0.00020 | ms/batch 81.24 | loss 1.74 |\n", + "| epoch 32 | 600/ 938 batches | lr 0.00020 | ms/batch 84.55 | loss 1.85 |\n", + "| epoch 32 | 800/ 938 batches | lr 0.00020 | ms/batch 78.70 | loss 2.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 32 | time: 81.18s | valid loss 7.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 33 | 200/ 938 batches | lr 0.00019 | ms/batch 86.49 | loss 1.60 |\n", + "| epoch 33 | 400/ 938 batches | lr 0.00019 | ms/batch 80.90 | loss 2.54 |\n", + "| epoch 33 | 600/ 938 batches | lr 0.00019 | ms/batch 80.75 | loss 1.78 |\n", + "| epoch 33 | 800/ 938 batches | lr 0.00019 | ms/batch 91.72 | loss 1.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 33 | time: 86.86s | valid loss 7.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 34 | 200/ 938 batches | lr 0.00018 | ms/batch 86.83 | loss 1.65 |\n", + "| epoch 34 | 400/ 938 batches | lr 0.00018 | ms/batch 85.80 | loss 1.78 |\n", + "| epoch 34 | 600/ 938 batches | lr 0.00018 | ms/batch 81.63 | loss 1.39 |\n", + "| epoch 34 | 800/ 938 batches | lr 0.00018 | ms/batch 85.99 | loss 1.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 34 | time: 85.27s | valid loss 8.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 35 | 200/ 938 batches | lr 0.00017 | ms/batch 78.37 | loss 1.51 |\n", + "| epoch 35 | 400/ 938 batches | lr 0.00017 | ms/batch 91.80 | loss 1.74 |\n", + "| epoch 35 | 600/ 938 batches | lr 0.00017 | ms/batch 81.36 | loss 1.44 |\n", + "| epoch 35 | 800/ 938 batches | lr 0.00017 | ms/batch 80.65 | loss 1.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 35 | time: 83.70s | valid loss 7.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 36 | 200/ 938 batches | lr 0.00017 | ms/batch 81.72 | loss 1.84 |\n", + "| epoch 36 | 400/ 938 batches | lr 0.00017 | ms/batch 84.79 | loss 1.72 |\n", + "| epoch 36 | 600/ 938 batches | lr 0.00017 | ms/batch 88.90 | loss 1.62 |\n", + "| epoch 36 | 800/ 938 batches | lr 0.00017 | ms/batch 80.43 | loss 2.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 36 | time: 84.55s | valid loss 7.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 37 | 200/ 938 batches | lr 0.00016 | ms/batch 80.69 | loss 1.04 |\n", + "| epoch 37 | 400/ 938 batches | lr 0.00016 | ms/batch 80.97 | loss 1.70 |\n", + "| epoch 37 | 600/ 938 batches | lr 0.00016 | ms/batch 81.79 | loss 1.91 |\n", + "| epoch 37 | 800/ 938 batches | lr 0.00016 | ms/batch 76.50 | loss 1.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 37 | time: 81.48s | valid loss 7.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 38 | 200/ 938 batches | lr 0.00015 | ms/batch 80.94 | loss 1.70 |\n", + "| epoch 38 | 400/ 938 batches | lr 0.00015 | ms/batch 80.28 | loss 1.41 |\n", + "| epoch 38 | 600/ 938 batches | lr 0.00015 | ms/batch 80.95 | loss 1.62 |\n", + "| epoch 38 | 800/ 938 batches | lr 0.00015 | ms/batch 80.75 | loss 2.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 38 | time: 82.04s | valid loss 7.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 39 | 200/ 938 batches | lr 0.00014 | ms/batch 90.24 | loss 1.44 |\n", + "| epoch 39 | 400/ 938 batches | lr 0.00014 | ms/batch 93.22 | loss 1.72 |\n", + "| epoch 39 | 600/ 938 batches | lr 0.00014 | ms/batch 82.29 | loss 1.27 |\n", + "| epoch 39 | 800/ 938 batches | lr 0.00014 | ms/batch 81.03 | loss 1.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 39 | time: 87.46s | valid loss 8.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 40 | 200/ 938 batches | lr 0.00014 | ms/batch 87.06 | loss 1.21 |\n", + "| epoch 40 | 400/ 938 batches | lr 0.00014 | ms/batch 72.82 | loss 1.19 |\n", + "| epoch 40 | 600/ 938 batches | lr 0.00014 | ms/batch 81.42 | loss 1.03 |\n", + "| epoch 40 | 800/ 938 batches | lr 0.00014 | ms/batch 83.48 | loss 1.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 40 | time: 82.34s | valid loss 7.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 41 | 200/ 938 batches | lr 0.00013 | ms/batch 70.51 | loss 1.57 |\n", + "| epoch 41 | 400/ 938 batches | lr 0.00013 | ms/batch 70.18 | loss 1.14 |\n", + "| epoch 41 | 600/ 938 batches | lr 0.00013 | ms/batch 72.93 | loss 1.19 |\n", + "| epoch 41 | 800/ 938 batches | lr 0.00013 | ms/batch 75.83 | loss 1.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 41 | time: 75.21s | valid loss 7.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 42 | 200/ 938 batches | lr 0.00012 | ms/batch 90.66 | loss 1.38 |\n", + "| epoch 42 | 400/ 938 batches | lr 0.00012 | ms/batch 80.54 | loss 1.20 |\n", + "| epoch 42 | 600/ 938 batches | lr 0.00012 | ms/batch 82.41 | loss 1.50 |\n", + "| epoch 42 | 800/ 938 batches | lr 0.00012 | ms/batch 81.45 | loss 1.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 42 | time: 84.46s | valid loss 7.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 43 | 200/ 938 batches | lr 0.00012 | ms/batch 80.89 | loss 1.18 |\n", + "| epoch 43 | 400/ 938 batches | lr 0.00012 | ms/batch 80.66 | loss 1.14 |\n", + "| epoch 43 | 600/ 938 batches | lr 0.00012 | ms/batch 80.56 | loss 1.15 |\n", + "| epoch 43 | 800/ 938 batches | lr 0.00012 | ms/batch 84.43 | loss 1.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 43 | time: 83.49s | valid loss 8.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.84 |\n", + "=========================================================================================\n", + "Result over 5 runs: 5.994574546813965\n" + ] + } + ], + "source": [ + "num_runs = 5\n", + "best_model = None\n", + "best_result = 1000000\n", + "results = []\n", + "\n", + "for i in range(num_runs):\n", + " features = nn.Sequential(nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(64),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + "\n", + " nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(128),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + "\n", + " nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(256),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + "\n", + " nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=(0,1)),\n", + " nn.BatchNorm2d(512),\n", + " nn.ReLU()\n", + " )\n", + "\n", + " classifier = nn.Sequential(nn.Linear(512, 1028),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=1028, bias=True),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=10, bias=True))\n", + "\n", + " conv = nn.Sequential(nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.AvgPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.AvgPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.AvgPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.AvgPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.AvgPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.AvgPool1d(kernel_size=2, stride=2))\n", + "\n", + " model = ConvMNIST(features, conv, classifier)\n", + " model = model.to(device)\n", + "\n", + " optimizer = torch.optim.Adam(model.parameters())\n", + " scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "\n", + " epochs = 30\n", + " model, result = train_model(model, epochs, train_loader, test_loader, seq_len, digits_per_batch, optimizer, scheduler)\n", + " \n", + " if result < best_result:\n", + " best_result = result\n", + " best_model = model\n", + " results.append(result)\n", + " \n", + "model = best_model\n", + "print(\"Result over {} runs: {}\".format(num_runs, torch.tensor(results).mean().item()))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 1.182722, + "end_time": "2021-01-22T10:39:51.452048", + "exception": false, + "start_time": "2021-01-22T10:39:50.269326", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "### Max pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-22T10:39:53.834200Z", + "iopub.status.busy": "2021-01-22T10:39:53.833677Z", + "iopub.status.idle": "2021-01-22T15:49:42.680757Z", + "shell.execute_reply": "2021-01-22T15:49:42.681250Z" + }, + "papermill": { + "duration": 18590.047488, + "end_time": "2021-01-22T15:49:42.681409", + "exception": false, + "start_time": "2021-01-22T10:39:52.633921", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 72.08 | loss 86.32 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 73.16 | loss 36.04 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 72.00 | loss 22.79 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 72.66 | loss 20.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 75.38s | valid loss 24.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 82.91 | loss 15.21 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 82.50 | loss 14.74 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 72.67 | loss 13.10 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 80.22 | loss 11.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 81.06s | valid loss 14.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 95.32 | loss 10.55 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 83.19 | loss 9.93 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 81.77 | loss 10.86 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 81.88 | loss 9.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 85.56s | valid loss 15.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 82.64 | loss 10.04 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 81.90 | loss 9.06 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 90.40 | loss 8.04 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 93.90 | loss 9.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 87.64s | valid loss 13.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 81.38 | loss 8.12 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 94.85 | loss 8.50 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 85.75 | loss 7.72 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 93.31 | loss 8.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 86.35s | valid loss 18.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 89.47 | loss 7.11 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 89.94 | loss 7.31 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 86.24 | loss 6.52 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 78.66 | loss 6.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 86.18s | valid loss 7.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 81.92 | loss 7.41 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 84.70 | loss 6.43 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 81.75 | loss 6.40 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 96.19 | loss 6.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 86.44s | valid loss 9.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 94.98 | loss 5.65 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 82.42 | loss 6.50 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 81.89 | loss 6.45 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 81.85 | loss 7.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 85.90s | valid loss 13.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 91.98 | loss 6.28 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 96.31 | loss 6.37 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 95.93 | loss 5.61 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 96.44 | loss 6.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 95.59s | valid loss 7.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 77.07 | loss 5.98 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 80.25 | loss 5.97 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 91.74 | loss 5.45 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 80.53 | loss 5.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 81.59s | valid loss 7.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 92.31 | loss 5.56 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 87.05 | loss 5.58 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 82.32 | loss 5.93 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 77.75 | loss 4.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 83.04s | valid loss 8.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 71.76 | loss 5.93 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 79.99 | loss 5.12 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 81.53 | loss 5.88 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 85.55 | loss 5.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 82.37s | valid loss 7.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 82.00 | loss 4.71 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 81.65 | loss 4.33 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 75.21 | loss 5.95 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 72.07 | loss 4.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 79.82s | valid loss 8.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 91.70 | loss 4.37 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 85.28 | loss 4.53 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 79.33 | loss 6.12 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 93.34 | loss 4.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 90.44s | valid loss 7.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 89.37 | loss 4.18 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 81.52 | loss 5.00 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 81.37 | loss 5.16 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 83.17 | loss 4.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 84.69s | valid loss 6.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 82.06 | loss 4.20 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 84.66 | loss 4.59 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 80.68 | loss 4.55 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 80.78 | loss 4.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 83.02s | valid loss 6.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 85.46 | loss 4.12 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 81.70 | loss 4.15 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 81.71 | loss 4.52 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 78.96 | loss 4.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 83.17s | valid loss 7.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 82.18 | loss 3.84 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 74.89 | loss 5.01 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 71.21 | loss 4.14 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 81.40 | loss 4.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 78.46s | valid loss 7.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 82.14 | loss 4.47 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 81.90 | loss 4.01 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 85.07 | loss 4.11 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 81.79 | loss 4.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 83.78s | valid loss 6.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 82.27 | loss 4.04 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 74.51 | loss 3.81 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 81.62 | loss 3.08 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 81.69 | loss 3.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 81.64s | valid loss 6.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 81.64 | loss 3.15 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 81.37 | loss 3.36 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 81.05 | loss 3.66 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 81.06 | loss 2.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 82.47s | valid loss 7.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 92.42 | loss 3.28 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 70.80 | loss 2.97 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 87.86 | loss 3.12 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 80.35 | loss 3.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 82.62s | valid loss 6.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 74.09 | loss 2.69 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 82.76 | loss 3.37 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 81.45 | loss 2.81 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 93.85 | loss 3.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 84.31s | valid loss 6.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 81.81 | loss 3.35 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 85.01 | loss 3.12 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 93.38 | loss 2.27 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 89.22 | loss 3.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 88.96s | valid loss 7.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 81.87 | loss 2.96 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 80.93 | loss 2.68 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 80.57 | loss 2.15 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 79.83 | loss 3.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 83.78s | valid loss 7.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 81.59 | loss 1.87 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 81.12 | loss 2.48 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 82.44 | loss 2.54 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 85.92 | loss 3.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 84.18s | valid loss 7.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 81.19 | loss 3.41 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 80.91 | loss 2.77 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 81.16 | loss 2.18 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 84.16 | loss 2.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 83.75s | valid loss 7.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 89.04 | loss 1.93 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 81.23 | loss 2.74 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 75.67 | loss 2.17 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 80.16 | loss 2.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 82.63s | valid loss 7.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 85.25 | loss 2.60 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 82.50 | loss 3.10 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 82.67 | loss 2.84 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 81.23 | loss 2.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 83.84s | valid loss 7.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 90.94 | loss 2.07 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 88.53 | loss 2.68 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 73.57 | loss 2.72 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 73.42 | loss 1.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 82.17s | valid loss 8.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 31 | 200/ 938 batches | lr 0.00021 | ms/batch 81.69 | loss 2.36 |\n", + "| epoch 31 | 400/ 938 batches | lr 0.00021 | ms/batch 78.55 | loss 2.04 |\n", + "| epoch 31 | 600/ 938 batches | lr 0.00021 | ms/batch 71.10 | loss 2.25 |\n", + "| epoch 31 | 800/ 938 batches | lr 0.00021 | ms/batch 72.58 | loss 2.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 31 | time: 78.11s | valid loss 7.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 32 | 200/ 938 batches | lr 0.00020 | ms/batch 83.61 | loss 2.30 |\n", + "| epoch 32 | 400/ 938 batches | lr 0.00020 | ms/batch 86.53 | loss 1.71 |\n", + "| epoch 32 | 600/ 938 batches | lr 0.00020 | ms/batch 80.98 | loss 2.50 |\n", + "| epoch 32 | 800/ 938 batches | lr 0.00020 | ms/batch 81.30 | loss 2.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 32 | time: 83.21s | valid loss 6.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 33 | 200/ 938 batches | lr 0.00019 | ms/batch 85.28 | loss 2.42 |\n", + "| epoch 33 | 400/ 938 batches | lr 0.00019 | ms/batch 81.39 | loss 1.44 |\n", + "| epoch 33 | 600/ 938 batches | lr 0.00019 | ms/batch 80.21 | loss 1.69 |\n", + "| epoch 33 | 800/ 938 batches | lr 0.00019 | ms/batch 80.44 | loss 2.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 33 | time: 82.84s | valid loss 7.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 34 | 200/ 938 batches | lr 0.00018 | ms/batch 84.36 | loss 2.09 |\n", + "| epoch 34 | 400/ 938 batches | lr 0.00018 | ms/batch 81.29 | loss 2.04 |\n", + "| epoch 34 | 600/ 938 batches | lr 0.00018 | ms/batch 75.82 | loss 1.92 |\n", + "| epoch 34 | 800/ 938 batches | lr 0.00018 | ms/batch 70.46 | loss 1.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 34 | time: 77.43s | valid loss 8.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 35 | 200/ 938 batches | lr 0.00017 | ms/batch 87.75 | loss 1.74 |\n", + "| epoch 35 | 400/ 938 batches | lr 0.00017 | ms/batch 82.99 | loss 2.15 |\n", + "| epoch 35 | 600/ 938 batches | lr 0.00017 | ms/batch 80.77 | loss 1.53 |\n", + "| epoch 35 | 800/ 938 batches | lr 0.00017 | ms/batch 80.78 | loss 1.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 35 | time: 84.18s | valid loss 8.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.30 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 72.80 | loss 76.78 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 72.38 | loss 29.43 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 76.76 | loss 21.91 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 82.17 | loss 17.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 79.02s | valid loss 33.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 87.00 | loss 14.41 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 86.09 | loss 14.04 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 76.59 | loss 13.13 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 80.50 | loss 11.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 83.54s | valid loss 15.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 92.15 | loss 10.67 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 95.24 | loss 11.24 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 87.20 | loss 10.13 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 81.70 | loss 9.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 89.00s | valid loss 14.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 82.09 | loss 9.23 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 93.11 | loss 8.34 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 95.44 | loss 8.24 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 76.88 | loss 8.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 85.11s | valid loss 14.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 72.23 | loss 8.78 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 91.09 | loss 7.85 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 86.00 | loss 8.45 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 83.27 | loss 7.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 82.66s | valid loss 12.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 87.70 | loss 7.42 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 81.93 | loss 7.44 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 86.87 | loss 7.34 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 95.29 | loss 7.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 86.78s | valid loss 9.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 91.30 | loss 6.62 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 81.82 | loss 7.16 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 82.98 | loss 6.32 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 78.13 | loss 7.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 82.10s | valid loss 9.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 72.02 | loss 7.51 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 71.61 | loss 6.00 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 71.75 | loss 6.44 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 85.09 | loss 6.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 76.00s | valid loss 9.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 85.18 | loss 6.42 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 82.47 | loss 6.32 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 82.43 | loss 5.62 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 82.36 | loss 6.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 84.21s | valid loss 10.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 82.24 | loss 5.54 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 81.84 | loss 6.49 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 94.32 | loss 5.81 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 87.26 | loss 5.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 88.44s | valid loss 10.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 72.83 | loss 5.94 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 83.67 | loss 5.54 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 92.92 | loss 5.59 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 90.16 | loss 5.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 85.55s | valid loss 9.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 83.32 | loss 6.03 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 88.46 | loss 6.51 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 81.55 | loss 5.19 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 83.07 | loss 5.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 86.41s | valid loss 8.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 82.47 | loss 5.49 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 81.59 | loss 4.59 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 81.53 | loss 4.87 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 92.86 | loss 5.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 86.92s | valid loss 8.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 84.33 | loss 4.72 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 90.21 | loss 4.42 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 81.94 | loss 5.12 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 81.96 | loss 4.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 85.88s | valid loss 8.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 82.29 | loss 4.24 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 79.38 | loss 4.68 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 71.15 | loss 4.88 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 83.92 | loss 4.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 80.91s | valid loss 9.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 82.29 | loss 4.85 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 82.01 | loss 4.60 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 81.83 | loss 3.95 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 81.92 | loss 3.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 83.27s | valid loss 8.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 82.38 | loss 3.98 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 81.77 | loss 4.30 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 81.91 | loss 4.37 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 87.28 | loss 4.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 84.99s | valid loss 10.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 94.63 | loss 4.37 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 82.93 | loss 3.94 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 81.73 | loss 4.02 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 91.95 | loss 4.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 90.39s | valid loss 7.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 81.62 | loss 3.92 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 81.21 | loss 3.96 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 81.01 | loss 3.50 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 83.40 | loss 4.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 85.92s | valid loss 9.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 74.56 | loss 3.88 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 81.68 | loss 3.33 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 75.92 | loss 3.51 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 71.12 | loss 4.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 76.93s | valid loss 9.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 85.87 | loss 2.92 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 87.92 | loss 3.87 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 81.73 | loss 2.97 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 82.52 | loss 5.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 85.22s | valid loss 9.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 82.46 | loss 3.38 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 82.68 | loss 3.36 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 79.94 | loss 3.36 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 81.59 | loss 2.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 81.89s | valid loss 9.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 80.42 | loss 3.27 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 81.14 | loss 2.99 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 87.50 | loss 3.26 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 72.54 | loss 3.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 79.58s | valid loss 8.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 71.48 | loss 2.54 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 80.60 | loss 2.86 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 81.23 | loss 3.42 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 72.85 | loss 3.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 76.44s | valid loss 8.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 71.37 | loss 2.97 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 81.90 | loss 3.02 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 81.03 | loss 2.96 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 80.60 | loss 3.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 80.42s | valid loss 9.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 81.39 | loss 3.03 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 80.74 | loss 2.73 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 81.05 | loss 2.79 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 89.82 | loss 2.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 85.50s | valid loss 8.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 81.52 | loss 2.42 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 81.00 | loss 2.50 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 81.01 | loss 2.76 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 81.41 | loss 2.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 82.74s | valid loss 8.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 82.35 | loss 2.58 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 79.60 | loss 2.78 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 75.34 | loss 3.16 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 84.65 | loss 2.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 79.76s | valid loss 9.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 75.16 | loss 2.53 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 74.14 | loss 3.03 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 73.49 | loss 2.50 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 70.72 | loss 2.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 74.21s | valid loss 9.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 74.51 | loss 2.63 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 82.31 | loss 2.43 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 87.14 | loss 2.14 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 81.02 | loss 1.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 82.33s | valid loss 7.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 31 | 200/ 938 batches | lr 0.00021 | ms/batch 80.91 | loss 2.33 |\n", + "| epoch 31 | 400/ 938 batches | lr 0.00021 | ms/batch 80.44 | loss 1.78 |\n", + "| epoch 31 | 600/ 938 batches | lr 0.00021 | ms/batch 83.61 | loss 2.49 |\n", + "| epoch 31 | 800/ 938 batches | lr 0.00021 | ms/batch 94.01 | loss 1.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 31 | time: 85.75s | valid loss 8.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 32 | 200/ 938 batches | lr 0.00020 | ms/batch 88.65 | loss 1.67 |\n", + "| epoch 32 | 400/ 938 batches | lr 0.00020 | ms/batch 91.17 | loss 2.05 |\n", + "| epoch 32 | 600/ 938 batches | lr 0.00020 | ms/batch 90.08 | loss 2.05 |\n", + "| epoch 32 | 800/ 938 batches | lr 0.00020 | ms/batch 87.94 | loss 2.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 32 | time: 87.25s | valid loss 9.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 33 | 200/ 938 batches | lr 0.00019 | ms/batch 91.74 | loss 2.14 |\n", + "| epoch 33 | 400/ 938 batches | lr 0.00019 | ms/batch 89.47 | loss 2.37 |\n", + "| epoch 33 | 600/ 938 batches | lr 0.00019 | ms/batch 85.53 | loss 2.58 |\n", + "| epoch 33 | 800/ 938 batches | lr 0.00019 | ms/batch 82.75 | loss 1.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 33 | time: 87.30s | valid loss 8.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 34 | 200/ 938 batches | lr 0.00018 | ms/batch 85.59 | loss 1.94 |\n", + "| epoch 34 | 400/ 938 batches | lr 0.00018 | ms/batch 83.37 | loss 2.44 |\n", + "| epoch 34 | 600/ 938 batches | lr 0.00018 | ms/batch 85.07 | loss 2.02 |\n", + "| epoch 34 | 800/ 938 batches | lr 0.00018 | ms/batch 70.28 | loss 1.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 34 | time: 81.35s | valid loss 8.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 35 | 200/ 938 batches | lr 0.00017 | ms/batch 73.65 | loss 1.85 |\n", + "| epoch 35 | 400/ 938 batches | lr 0.00017 | ms/batch 70.85 | loss 1.52 |\n", + "| epoch 35 | 600/ 938 batches | lr 0.00017 | ms/batch 70.66 | loss 2.48 |\n", + "| epoch 35 | 800/ 938 batches | lr 0.00017 | ms/batch 89.96 | loss 1.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 35 | time: 81.27s | valid loss 7.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 36 | 200/ 938 batches | lr 0.00017 | ms/batch 76.67 | loss 1.85 |\n", + "| epoch 36 | 400/ 938 batches | lr 0.00017 | ms/batch 82.33 | loss 2.07 |\n", + "| epoch 36 | 600/ 938 batches | lr 0.00017 | ms/batch 80.41 | loss 2.15 |\n", + "| epoch 36 | 800/ 938 batches | lr 0.00017 | ms/batch 82.19 | loss 2.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 36 | time: 81.52s | valid loss 9.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 37 | 200/ 938 batches | lr 0.00016 | ms/batch 87.05 | loss 1.84 |\n", + "| epoch 37 | 400/ 938 batches | lr 0.00016 | ms/batch 93.42 | loss 1.80 |\n", + "| epoch 37 | 600/ 938 batches | lr 0.00016 | ms/batch 94.07 | loss 1.61 |\n", + "| epoch 37 | 800/ 938 batches | lr 0.00016 | ms/batch 86.75 | loss 2.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 37 | time: 89.78s | valid loss 8.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 38 | 200/ 938 batches | lr 0.00015 | ms/batch 80.63 | loss 1.41 |\n", + "| epoch 38 | 400/ 938 batches | lr 0.00015 | ms/batch 80.27 | loss 1.88 |\n", + "| epoch 38 | 600/ 938 batches | lr 0.00015 | ms/batch 76.42 | loss 1.39 |\n", + "| epoch 38 | 800/ 938 batches | lr 0.00015 | ms/batch 80.82 | loss 1.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 38 | time: 81.09s | valid loss 8.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 39 | 200/ 938 batches | lr 0.00014 | ms/batch 94.23 | loss 1.65 |\n", + "| epoch 39 | 400/ 938 batches | lr 0.00014 | ms/batch 84.92 | loss 1.91 |\n", + "| epoch 39 | 600/ 938 batches | lr 0.00014 | ms/batch 70.68 | loss 1.34 |\n", + "| epoch 39 | 800/ 938 batches | lr 0.00014 | ms/batch 70.80 | loss 2.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 39 | time: 80.34s | valid loss 8.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 40 | 200/ 938 batches | lr 0.00014 | ms/batch 83.11 | loss 1.28 |\n", + "| epoch 40 | 400/ 938 batches | lr 0.00014 | ms/batch 81.75 | loss 1.81 |\n", + "| epoch 40 | 600/ 938 batches | lr 0.00014 | ms/batch 92.85 | loss 1.23 |\n", + "| epoch 40 | 800/ 938 batches | lr 0.00014 | ms/batch 85.57 | loss 1.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 40 | time: 87.47s | valid loss 9.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 41 | 200/ 938 batches | lr 0.00013 | ms/batch 73.08 | loss 1.05 |\n", + "| epoch 41 | 400/ 938 batches | lr 0.00013 | ms/batch 70.70 | loss 1.51 |\n", + "| epoch 41 | 600/ 938 batches | lr 0.00013 | ms/batch 76.67 | loss 1.17 |\n", + "| epoch 41 | 800/ 938 batches | lr 0.00013 | ms/batch 87.61 | loss 1.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 41 | time: 77.67s | valid loss 9.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 42 | 200/ 938 batches | lr 0.00012 | ms/batch 80.96 | loss 1.09 |\n", + "| epoch 42 | 400/ 938 batches | lr 0.00012 | ms/batch 83.83 | loss 1.50 |\n", + "| epoch 42 | 600/ 938 batches | lr 0.00012 | ms/batch 83.58 | loss 1.21 |\n", + "| epoch 42 | 800/ 938 batches | lr 0.00012 | ms/batch 90.58 | loss 1.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 42 | time: 85.22s | valid loss 8.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 43 | 200/ 938 batches | lr 0.00012 | ms/batch 80.51 | loss 1.01 |\n", + "| epoch 43 | 400/ 938 batches | lr 0.00012 | ms/batch 85.15 | loss 1.50 |\n", + "| epoch 43 | 600/ 938 batches | lr 0.00012 | ms/batch 93.74 | loss 1.38 |\n", + "| epoch 43 | 800/ 938 batches | lr 0.00012 | ms/batch 72.84 | loss 1.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 43 | time: 82.20s | valid loss 8.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 44 | 200/ 938 batches | lr 0.00011 | ms/batch 80.97 | loss 1.22 |\n", + "| epoch 44 | 400/ 938 batches | lr 0.00011 | ms/batch 80.55 | loss 1.32 |\n", + "| epoch 44 | 600/ 938 batches | lr 0.00011 | ms/batch 80.62 | loss 1.48 |\n", + "| epoch 44 | 800/ 938 batches | lr 0.00011 | ms/batch 80.57 | loss 1.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 44 | time: 82.21s | valid loss 9.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 45 | 200/ 938 batches | lr 0.00010 | ms/batch 86.62 | loss 1.10 |\n", + "| epoch 45 | 400/ 938 batches | lr 0.00010 | ms/batch 92.71 | loss 1.64 |\n", + "| epoch 45 | 600/ 938 batches | lr 0.00010 | ms/batch 80.20 | loss 1.11 |\n", + "| epoch 45 | 800/ 938 batches | lr 0.00010 | ms/batch 80.05 | loss 1.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 45 | time: 84.92s | valid loss 9.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 46 | 200/ 938 batches | lr 0.00010 | ms/batch 78.89 | loss 1.40 |\n", + "| epoch 46 | 400/ 938 batches | lr 0.00010 | ms/batch 80.54 | loss 1.36 |\n", + "| epoch 46 | 600/ 938 batches | lr 0.00010 | ms/batch 81.38 | loss 1.49 |\n", + "| epoch 46 | 800/ 938 batches | lr 0.00010 | ms/batch 80.07 | loss 1.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 46 | time: 81.54s | valid loss 9.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 7.53 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 75.66 | loss 82.42 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 87.73 | loss 32.30 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 82.00 | loss 23.33 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 82.06 | loss 20.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 83.25s | valid loss 20.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 91.37 | loss 15.14 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 93.99 | loss 13.64 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 76.38 | loss 12.92 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 94.44 | loss 12.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 91.50s | valid loss 15.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 75.83 | loss 11.22 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 72.37 | loss 9.84 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 72.34 | loss 10.69 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 78.29 | loss 9.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 77.51s | valid loss 13.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 90.40 | loss 9.44 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 88.71 | loss 8.85 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 81.78 | loss 9.35 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 81.76 | loss 8.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 86.16s | valid loss 11.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 87.82 | loss 7.87 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 80.20 | loss 8.81 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 92.64 | loss 8.09 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 96.21 | loss 8.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 90.01s | valid loss 12.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 82.91 | loss 7.44 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 82.44 | loss 8.09 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 84.67 | loss 7.36 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 73.80 | loss 7.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 82.42s | valid loss 7.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 82.58 | loss 6.92 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 81.59 | loss 6.90 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 80.06 | loss 6.79 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 82.31 | loss 6.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 83.00s | valid loss 7.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 73.47 | loss 6.38 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 79.46 | loss 7.10 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 81.95 | loss 5.85 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 83.63 | loss 6.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 81.44s | valid loss 9.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 82.77 | loss 6.09 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 87.43 | loss 6.37 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 94.87 | loss 5.72 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 81.83 | loss 6.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 86.80s | valid loss 10.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 82.05 | loss 5.70 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 81.79 | loss 6.28 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 81.56 | loss 6.75 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 81.51 | loss 5.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 85.07s | valid loss 8.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 90.35 | loss 5.55 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 95.85 | loss 4.70 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 81.25 | loss 5.76 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 73.65 | loss 5.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 83.61s | valid loss 8.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 88.71 | loss 5.78 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 93.89 | loss 5.27 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 84.78 | loss 5.89 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 73.20 | loss 4.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 86.49s | valid loss 8.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 82.71 | loss 4.77 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 75.26 | loss 5.86 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 71.25 | loss 4.65 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 79.53 | loss 5.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 78.73s | valid loss 7.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 93.44 | loss 5.28 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 94.62 | loss 5.13 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 89.88 | loss 5.02 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 83.83 | loss 4.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 90.33s | valid loss 8.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 86.10 | loss 4.24 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 89.53 | loss 4.85 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 95.89 | loss 5.18 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 92.37 | loss 3.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 93.06s | valid loss 9.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 93.23 | loss 3.62 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 82.40 | loss 4.30 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 81.13 | loss 4.18 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 74.11 | loss 4.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 81.44s | valid loss 8.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 71.94 | loss 4.43 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 71.15 | loss 4.66 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 71.03 | loss 3.75 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 74.05 | loss 4.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 72.86s | valid loss 9.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 72.65 | loss 4.23 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 81.42 | loss 4.13 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 77.78 | loss 4.04 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 94.75 | loss 3.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 85.35s | valid loss 9.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 73.98 | loss 3.83 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 88.30 | loss 3.43 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 89.77 | loss 3.81 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 81.40 | loss 3.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 84.12s | valid loss 8.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 83.48 | loss 3.16 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 86.71 | loss 4.35 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 81.40 | loss 3.84 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 81.23 | loss 4.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 84.75s | valid loss 10.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 81.95 | loss 4.21 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 81.36 | loss 3.34 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 81.70 | loss 3.53 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 76.08 | loss 3.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 81.27s | valid loss 7.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 81.73 | loss 2.79 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 88.48 | loss 4.55 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 85.95 | loss 3.42 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 81.49 | loss 3.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 85.20s | valid loss 8.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 80.82 | loss 3.88 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 78.65 | loss 3.05 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 80.67 | loss 3.41 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 80.67 | loss 3.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 81.61s | valid loss 6.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 91.95 | loss 2.98 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 81.18 | loss 2.33 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 80.99 | loss 3.37 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 81.48 | loss 3.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 84.60s | valid loss 7.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 81.57 | loss 2.81 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 81.04 | loss 2.64 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 80.57 | loss 3.29 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 80.77 | loss 3.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 82.94s | valid loss 6.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 81.09 | loss 2.79 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 81.01 | loss 2.56 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 81.19 | loss 3.21 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 80.51 | loss 2.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 81.62s | valid loss 7.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 80.88 | loss 2.10 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 86.06 | loss 2.81 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 82.91 | loss 2.80 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 79.96 | loss 2.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 83.38s | valid loss 8.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 87.17 | loss 2.30 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 82.49 | loss 2.32 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 80.31 | loss 2.23 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 80.69 | loss 2.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 83.47s | valid loss 7.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 81.02 | loss 2.11 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 80.60 | loss 2.51 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 80.42 | loss 2.72 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 80.56 | loss 2.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 83.17s | valid loss 8.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 80.93 | loss 2.34 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 80.65 | loss 2.24 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 80.62 | loss 2.34 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 80.62 | loss 2.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 80.97s | valid loss 7.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 31 | 200/ 938 batches | lr 0.00021 | ms/batch 82.44 | loss 2.41 |\n", + "| epoch 31 | 400/ 938 batches | lr 0.00021 | ms/batch 80.48 | loss 1.96 |\n", + "| epoch 31 | 600/ 938 batches | lr 0.00021 | ms/batch 80.46 | loss 2.48 |\n", + "| epoch 31 | 800/ 938 batches | lr 0.00021 | ms/batch 80.54 | loss 2.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 31 | time: 82.30s | valid loss 7.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 32 | 200/ 938 batches | lr 0.00020 | ms/batch 80.98 | loss 2.68 |\n", + "| epoch 32 | 400/ 938 batches | lr 0.00020 | ms/batch 80.49 | loss 2.33 |\n", + "| epoch 32 | 600/ 938 batches | lr 0.00020 | ms/batch 80.53 | loss 2.56 |\n", + "| epoch 32 | 800/ 938 batches | lr 0.00020 | ms/batch 80.49 | loss 2.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 32 | time: 81.87s | valid loss 7.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 33 | 200/ 938 batches | lr 0.00019 | ms/batch 80.76 | loss 2.25 |\n", + "| epoch 33 | 400/ 938 batches | lr 0.00019 | ms/batch 80.29 | loss 1.99 |\n", + "| epoch 33 | 600/ 938 batches | lr 0.00019 | ms/batch 82.71 | loss 2.38 |\n", + "| epoch 33 | 800/ 938 batches | lr 0.00019 | ms/batch 81.72 | loss 2.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 33 | time: 82.52s | valid loss 6.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 34 | 200/ 938 batches | lr 0.00018 | ms/batch 81.24 | loss 1.79 |\n", + "| epoch 34 | 400/ 938 batches | lr 0.00018 | ms/batch 81.08 | loss 2.56 |\n", + "| epoch 34 | 600/ 938 batches | lr 0.00018 | ms/batch 80.80 | loss 2.01 |\n", + "| epoch 34 | 800/ 938 batches | lr 0.00018 | ms/batch 80.91 | loss 1.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 34 | time: 80.73s | valid loss 7.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 35 | 200/ 938 batches | lr 0.00017 | ms/batch 81.61 | loss 1.90 |\n", + "| epoch 35 | 400/ 938 batches | lr 0.00017 | ms/batch 80.34 | loss 2.06 |\n", + "| epoch 35 | 600/ 938 batches | lr 0.00017 | ms/batch 80.43 | loss 1.44 |\n", + "| epoch 35 | 800/ 938 batches | lr 0.00017 | ms/batch 80.21 | loss 2.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 35 | time: 79.83s | valid loss 6.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 36 | 200/ 938 batches | lr 0.00017 | ms/batch 80.37 | loss 1.72 |\n", + "| epoch 36 | 400/ 938 batches | lr 0.00017 | ms/batch 80.14 | loss 1.57 |\n", + "| epoch 36 | 600/ 938 batches | lr 0.00017 | ms/batch 80.74 | loss 1.95 |\n", + "| epoch 36 | 800/ 938 batches | lr 0.00017 | ms/batch 82.78 | loss 2.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 36 | time: 83.70s | valid loss 7.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 37 | 200/ 938 batches | lr 0.00016 | ms/batch 83.01 | loss 1.48 |\n", + "| epoch 37 | 400/ 938 batches | lr 0.00016 | ms/batch 78.34 | loss 2.07 |\n", + "| epoch 37 | 600/ 938 batches | lr 0.00016 | ms/batch 91.32 | loss 1.43 |\n", + "| epoch 37 | 800/ 938 batches | lr 0.00016 | ms/batch 90.81 | loss 1.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 37 | time: 86.97s | valid loss 8.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 38 | 200/ 938 batches | lr 0.00015 | ms/batch 81.22 | loss 1.69 |\n", + "| epoch 38 | 400/ 938 batches | lr 0.00015 | ms/batch 86.03 | loss 1.57 |\n", + "| epoch 38 | 600/ 938 batches | lr 0.00015 | ms/batch 83.36 | loss 1.83 |\n", + "| epoch 38 | 800/ 938 batches | lr 0.00015 | ms/batch 89.20 | loss 1.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 38 | time: 84.94s | valid loss 7.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 39 | 200/ 938 batches | lr 0.00014 | ms/batch 90.78 | loss 1.46 |\n", + "| epoch 39 | 400/ 938 batches | lr 0.00014 | ms/batch 80.15 | loss 1.23 |\n", + "| epoch 39 | 600/ 938 batches | lr 0.00014 | ms/batch 82.62 | loss 1.50 |\n", + "| epoch 39 | 800/ 938 batches | lr 0.00014 | ms/batch 84.69 | loss 2.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 39 | time: 84.56s | valid loss 8.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 40 | 200/ 938 batches | lr 0.00014 | ms/batch 80.03 | loss 1.23 |\n", + "| epoch 40 | 400/ 938 batches | lr 0.00014 | ms/batch 80.73 | loss 1.54 |\n", + "| epoch 40 | 600/ 938 batches | lr 0.00014 | ms/batch 80.69 | loss 2.03 |\n", + "| epoch 40 | 800/ 938 batches | lr 0.00014 | ms/batch 72.92 | loss 1.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 40 | time: 77.92s | valid loss 7.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 41 | 200/ 938 batches | lr 0.00013 | ms/batch 85.85 | loss 1.48 |\n", + "| epoch 41 | 400/ 938 batches | lr 0.00013 | ms/batch 75.80 | loss 1.39 |\n", + "| epoch 41 | 600/ 938 batches | lr 0.00013 | ms/batch 78.70 | loss 2.05 |\n", + "| epoch 41 | 800/ 938 batches | lr 0.00013 | ms/batch 78.48 | loss 1.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 41 | time: 81.00s | valid loss 8.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 42 | 200/ 938 batches | lr 0.00012 | ms/batch 80.48 | loss 1.59 |\n", + "| epoch 42 | 400/ 938 batches | lr 0.00012 | ms/batch 80.03 | loss 1.48 |\n", + "| epoch 42 | 600/ 938 batches | lr 0.00012 | ms/batch 80.03 | loss 1.39 |\n", + "| epoch 42 | 800/ 938 batches | lr 0.00012 | ms/batch 82.32 | loss 1.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 42 | time: 80.77s | valid loss 7.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 43 | 200/ 938 batches | lr 0.00012 | ms/batch 77.93 | loss 1.78 |\n", + "| epoch 43 | 400/ 938 batches | lr 0.00012 | ms/batch 84.22 | loss 1.72 |\n", + "| epoch 43 | 600/ 938 batches | lr 0.00012 | ms/batch 92.43 | loss 1.29 |\n", + "| epoch 43 | 800/ 938 batches | lr 0.00012 | ms/batch 92.88 | loss 1.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 43 | time: 86.86s | valid loss 9.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 44 | 200/ 938 batches | lr 0.00011 | ms/batch 83.34 | loss 1.68 |\n", + "| epoch 44 | 400/ 938 batches | lr 0.00011 | ms/batch 88.79 | loss 1.40 |\n", + "| epoch 44 | 600/ 938 batches | lr 0.00011 | ms/batch 92.62 | loss 1.22 |\n", + "| epoch 44 | 800/ 938 batches | lr 0.00011 | ms/batch 76.35 | loss 1.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 44 | time: 86.57s | valid loss 9.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 45 | 200/ 938 batches | lr 0.00010 | ms/batch 79.62 | loss 1.45 |\n", + "| epoch 45 | 400/ 938 batches | lr 0.00010 | ms/batch 79.82 | loss 1.08 |\n", + "| epoch 45 | 600/ 938 batches | lr 0.00010 | ms/batch 80.13 | loss 1.55 |\n", + "| epoch 45 | 800/ 938 batches | lr 0.00010 | ms/batch 80.01 | loss 1.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 45 | time: 81.36s | valid loss 8.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 46 | 200/ 938 batches | lr 0.00010 | ms/batch 80.63 | loss 1.37 |\n", + "| epoch 46 | 400/ 938 batches | lr 0.00010 | ms/batch 80.05 | loss 1.39 |\n", + "| epoch 46 | 600/ 938 batches | lr 0.00010 | ms/batch 80.70 | loss 1.01 |\n", + "| epoch 46 | 800/ 938 batches | lr 0.00010 | ms/batch 80.61 | loss 1.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 46 | time: 81.87s | valid loss 8.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 47 | 200/ 938 batches | lr 0.00009 | ms/batch 80.90 | loss 1.23 |\n", + "| epoch 47 | 400/ 938 batches | lr 0.00009 | ms/batch 80.54 | loss 1.31 |\n", + "| epoch 47 | 600/ 938 batches | lr 0.00009 | ms/batch 80.48 | loss 1.22 |\n", + "| epoch 47 | 800/ 938 batches | lr 0.00009 | ms/batch 80.05 | loss 1.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 47 | time: 81.74s | valid loss 7.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 48 | 200/ 938 batches | lr 0.00009 | ms/batch 80.55 | loss 1.35 |\n", + "| epoch 48 | 400/ 938 batches | lr 0.00009 | ms/batch 81.06 | loss 1.59 |\n", + "| epoch 48 | 600/ 938 batches | lr 0.00009 | ms/batch 80.26 | loss 1.46 |\n", + "| epoch 48 | 800/ 938 batches | lr 0.00009 | ms/batch 86.14 | loss 1.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 48 | time: 83.06s | valid loss 8.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 49 | 200/ 938 batches | lr 0.00009 | ms/batch 80.96 | loss 0.64 |\n", + "| epoch 49 | 400/ 938 batches | lr 0.00009 | ms/batch 80.69 | loss 1.21 |\n", + "| epoch 49 | 600/ 938 batches | lr 0.00009 | ms/batch 81.91 | loss 1.26 |\n", + "| epoch 49 | 800/ 938 batches | lr 0.00009 | ms/batch 89.15 | loss 1.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 49 | time: 84.71s | valid loss 8.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 50 | 200/ 938 batches | lr 0.00008 | ms/batch 81.76 | loss 1.03 |\n", + "| epoch 50 | 400/ 938 batches | lr 0.00008 | ms/batch 79.83 | loss 0.90 |\n", + "| epoch 50 | 600/ 938 batches | lr 0.00008 | ms/batch 80.00 | loss 1.29 |\n", + "| epoch 50 | 800/ 938 batches | lr 0.00008 | ms/batch 85.21 | loss 1.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 50 | time: 82.92s | valid loss 8.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 51 | 200/ 938 batches | lr 0.00008 | ms/batch 71.04 | loss 1.71 |\n", + "| epoch 51 | 400/ 938 batches | lr 0.00008 | ms/batch 70.70 | loss 1.34 |\n", + "| epoch 51 | 600/ 938 batches | lr 0.00008 | ms/batch 73.49 | loss 1.38 |\n", + "| epoch 51 | 800/ 938 batches | lr 0.00008 | ms/batch 87.79 | loss 1.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 51 | time: 77.98s | valid loss 8.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 52 | 200/ 938 batches | lr 0.00007 | ms/batch 80.80 | loss 1.44 |\n", + "| epoch 52 | 400/ 938 batches | lr 0.00007 | ms/batch 79.92 | loss 0.94 |\n", + "| epoch 52 | 600/ 938 batches | lr 0.00007 | ms/batch 80.14 | loss 1.31 |\n", + "| epoch 52 | 800/ 938 batches | lr 0.00007 | ms/batch 84.44 | loss 0.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 52 | time: 83.42s | valid loss 7.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 53 | 200/ 938 batches | lr 0.00007 | ms/batch 80.30 | loss 0.93 |\n", + "| epoch 53 | 400/ 938 batches | lr 0.00007 | ms/batch 79.84 | loss 0.74 |\n", + "| epoch 53 | 600/ 938 batches | lr 0.00007 | ms/batch 79.93 | loss 1.05 |\n", + "| epoch 53 | 800/ 938 batches | lr 0.00007 | ms/batch 79.78 | loss 0.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 53 | time: 79.52s | valid loss 8.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.70 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 84.30 | loss 93.78 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 78.78 | loss 39.99 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 88.50 | loss 22.88 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 83.12 | loss 19.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 84.57s | valid loss 39.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 82.34 | loss 14.61 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 85.03 | loss 13.66 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 95.59 | loss 13.33 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 86.51 | loss 12.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 87.15s | valid loss 22.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 87.83 | loss 11.04 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 83.48 | loss 11.20 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 82.56 | loss 10.88 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 82.59 | loss 9.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 84.91s | valid loss 17.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 82.39 | loss 9.10 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 90.08 | loss 8.96 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 81.27 | loss 8.81 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 84.72 | loss 8.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 85.57s | valid loss 11.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 88.07 | loss 7.65 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 92.22 | loss 8.56 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 95.29 | loss 7.67 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 82.16 | loss 8.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 88.40s | valid loss 10.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 82.32 | loss 6.85 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 81.91 | loss 8.02 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 81.85 | loss 6.56 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 81.89 | loss 7.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 83.26s | valid loss 9.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 88.19 | loss 6.60 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 87.55 | loss 7.12 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 81.83 | loss 6.99 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 83.52 | loss 7.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 85.78s | valid loss 7.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 82.32 | loss 6.95 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 81.89 | loss 6.34 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 81.88 | loss 6.61 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 82.00 | loss 6.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 84.04s | valid loss 8.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 89.99 | loss 5.64 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 95.07 | loss 6.44 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 85.68 | loss 5.50 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 85.12 | loss 6.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 88.84s | valid loss 8.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 93.69 | loss 5.76 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 87.72 | loss 5.83 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 94.07 | loss 5.75 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 80.30 | loss 6.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 86.73s | valid loss 9.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 93.81 | loss 5.67 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 92.22 | loss 5.63 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 87.86 | loss 5.20 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 83.23 | loss 5.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 91.06s | valid loss 8.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 86.07 | loss 4.87 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 90.26 | loss 5.62 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 77.02 | loss 5.22 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 79.47 | loss 6.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 84.24s | valid loss 9.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 82.70 | loss 4.72 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 80.97 | loss 4.52 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 84.85 | loss 5.88 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 81.67 | loss 4.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 83.67s | valid loss 8.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 89.83 | loss 3.96 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 76.88 | loss 5.18 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 76.07 | loss 5.25 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 91.64 | loss 5.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 82.07s | valid loss 8.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 89.90 | loss 4.32 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 93.20 | loss 4.43 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 92.04 | loss 5.03 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 82.10 | loss 4.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 87.01s | valid loss 7.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 81.44 | loss 4.24 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 81.25 | loss 3.85 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 81.40 | loss 4.99 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 74.28 | loss 4.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 78.80s | valid loss 7.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 73.46 | loss 4.53 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 81.92 | loss 4.45 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 88.82 | loss 3.66 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 79.79 | loss 5.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 82.30s | valid loss 7.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 81.45 | loss 3.88 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 81.15 | loss 4.49 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 83.45 | loss 4.01 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 81.10 | loss 3.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 82.94s | valid loss 8.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 91.59 | loss 3.88 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 88.52 | loss 3.48 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 70.91 | loss 4.05 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 71.03 | loss 4.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 82.79s | valid loss 7.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 82.35 | loss 3.91 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 72.33 | loss 3.99 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 76.54 | loss 3.63 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 71.37 | loss 3.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 79.83s | valid loss 8.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 92.48 | loss 3.66 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 81.97 | loss 3.46 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 80.86 | loss 3.82 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 72.40 | loss 3.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 80.73s | valid loss 8.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 71.58 | loss 2.57 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 83.69 | loss 2.55 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 93.80 | loss 3.67 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 88.18 | loss 4.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 86.63s | valid loss 7.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 81.43 | loss 3.19 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 80.89 | loss 3.48 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 81.03 | loss 2.94 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 81.69 | loss 2.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 84.83s | valid loss 8.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 71.36 | loss 2.46 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 77.81 | loss 2.40 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 90.82 | loss 3.24 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 80.64 | loss 3.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 81.39s | valid loss 8.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 86.91 | loss 3.06 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 89.35 | loss 2.94 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 81.28 | loss 2.88 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 80.74 | loss 3.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 84.78s | valid loss 8.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 71.94 | loss 2.38 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 70.76 | loss 3.04 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 75.02 | loss 3.50 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 80.57 | loss 2.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 75.38s | valid loss 6.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 74.95 | loss 2.42 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 73.46 | loss 2.82 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 70.79 | loss 2.65 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 74.79 | loss 2.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 76.19s | valid loss 8.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 81.09 | loss 2.79 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 80.61 | loss 2.79 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 80.89 | loss 2.80 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 84.73 | loss 2.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 85.60s | valid loss 9.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 87.82 | loss 2.74 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 88.97 | loss 2.36 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 87.65 | loss 1.40 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 92.05 | loss 2.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 86.63s | valid loss 8.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 87.12 | loss 2.01 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 80.90 | loss 2.64 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 80.86 | loss 1.58 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 90.76 | loss 2.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 86.55s | valid loss 7.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 31 | 200/ 938 batches | lr 0.00021 | ms/batch 81.07 | loss 2.23 |\n", + "| epoch 31 | 400/ 938 batches | lr 0.00021 | ms/batch 72.45 | loss 1.92 |\n", + "| epoch 31 | 600/ 938 batches | lr 0.00021 | ms/batch 71.10 | loss 2.03 |\n", + "| epoch 31 | 800/ 938 batches | lr 0.00021 | ms/batch 81.45 | loss 3.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 31 | time: 77.65s | valid loss 8.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 32 | 200/ 938 batches | lr 0.00020 | ms/batch 85.11 | loss 2.23 |\n", + "| epoch 32 | 400/ 938 batches | lr 0.00020 | ms/batch 86.41 | loss 1.58 |\n", + "| epoch 32 | 600/ 938 batches | lr 0.00020 | ms/batch 93.91 | loss 2.28 |\n", + "| epoch 32 | 800/ 938 batches | lr 0.00020 | ms/batch 90.03 | loss 1.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 32 | time: 88.12s | valid loss 9.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 33 | 200/ 938 batches | lr 0.00019 | ms/batch 87.71 | loss 1.74 |\n", + "| epoch 33 | 400/ 938 batches | lr 0.00019 | ms/batch 82.88 | loss 1.98 |\n", + "| epoch 33 | 600/ 938 batches | lr 0.00019 | ms/batch 75.83 | loss 1.77 |\n", + "| epoch 33 | 800/ 938 batches | lr 0.00019 | ms/batch 80.45 | loss 2.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 33 | time: 82.49s | valid loss 9.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 34 | 200/ 938 batches | lr 0.00018 | ms/batch 88.38 | loss 2.58 |\n", + "| epoch 34 | 400/ 938 batches | lr 0.00018 | ms/batch 90.55 | loss 1.76 |\n", + "| epoch 34 | 600/ 938 batches | lr 0.00018 | ms/batch 89.52 | loss 1.99 |\n", + "| epoch 34 | 800/ 938 batches | lr 0.00018 | ms/batch 73.92 | loss 1.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 34 | time: 86.41s | valid loss 8.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 35 | 200/ 938 batches | lr 0.00017 | ms/batch 82.87 | loss 2.33 |\n", + "| epoch 35 | 400/ 938 batches | lr 0.00017 | ms/batch 80.37 | loss 1.70 |\n", + "| epoch 35 | 600/ 938 batches | lr 0.00017 | ms/batch 89.30 | loss 1.94 |\n", + "| epoch 35 | 800/ 938 batches | lr 0.00017 | ms/batch 85.87 | loss 2.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 35 | time: 85.90s | valid loss 8.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 36 | 200/ 938 batches | lr 0.00017 | ms/batch 80.43 | loss 1.48 |\n", + "| epoch 36 | 400/ 938 batches | lr 0.00017 | ms/batch 80.25 | loss 1.57 |\n", + "| epoch 36 | 600/ 938 batches | lr 0.00017 | ms/batch 80.44 | loss 1.96 |\n", + "| epoch 36 | 800/ 938 batches | lr 0.00017 | ms/batch 80.34 | loss 1.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 36 | time: 81.67s | valid loss 8.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 37 | 200/ 938 batches | lr 0.00016 | ms/batch 81.41 | loss 1.88 |\n", + "| epoch 37 | 400/ 938 batches | lr 0.00016 | ms/batch 81.01 | loss 1.64 |\n", + "| epoch 37 | 600/ 938 batches | lr 0.00016 | ms/batch 80.71 | loss 1.65 |\n", + "| epoch 37 | 800/ 938 batches | lr 0.00016 | ms/batch 80.22 | loss 1.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 37 | time: 82.01s | valid loss 8.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 38 | 200/ 938 batches | lr 0.00015 | ms/batch 88.10 | loss 1.19 |\n", + "| epoch 38 | 400/ 938 batches | lr 0.00015 | ms/batch 85.57 | loss 1.68 |\n", + "| epoch 38 | 600/ 938 batches | lr 0.00015 | ms/batch 83.42 | loss 1.65 |\n", + "| epoch 38 | 800/ 938 batches | lr 0.00015 | ms/batch 92.75 | loss 1.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 38 | time: 88.07s | valid loss 8.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 39 | 200/ 938 batches | lr 0.00014 | ms/batch 81.13 | loss 1.62 |\n", + "| epoch 39 | 400/ 938 batches | lr 0.00014 | ms/batch 78.67 | loss 2.02 |\n", + "| epoch 39 | 600/ 938 batches | lr 0.00014 | ms/batch 70.24 | loss 1.33 |\n", + "| epoch 39 | 800/ 938 batches | lr 0.00014 | ms/batch 70.33 | loss 1.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 39 | time: 75.10s | valid loss 7.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 40 | 200/ 938 batches | lr 0.00014 | ms/batch 72.30 | loss 1.66 |\n", + "| epoch 40 | 400/ 938 batches | lr 0.00014 | ms/batch 80.48 | loss 1.51 |\n", + "| epoch 40 | 600/ 938 batches | lr 0.00014 | ms/batch 80.98 | loss 1.44 |\n", + "| epoch 40 | 800/ 938 batches | lr 0.00014 | ms/batch 80.76 | loss 1.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 40 | time: 80.36s | valid loss 8.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.86 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 82.96 | loss 85.19 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 82.52 | loss 33.80 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 82.49 | loss 24.31 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 82.61 | loss 18.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 85.08s | valid loss 35.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 93.74 | loss 15.67 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 81.52 | loss 14.29 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 82.52 | loss 12.62 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 83.02 | loss 12.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 86.09s | valid loss 17.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 82.80 | loss 11.12 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 91.02 | loss 10.29 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 87.26 | loss 10.56 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 83.11 | loss 10.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 86.34s | valid loss 15.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 73.79 | loss 9.08 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 71.80 | loss 8.75 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 72.21 | loss 8.56 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 89.68 | loss 8.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 81.22s | valid loss 10.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 79.63 | loss 7.91 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 79.39 | loss 8.93 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 93.66 | loss 7.86 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 82.46 | loss 7.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 84.96s | valid loss 12.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 80.04 | loss 7.57 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 80.57 | loss 7.61 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 81.85 | loss 7.41 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 95.10 | loss 7.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 86.37s | valid loss 13.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 90.50 | loss 6.35 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 84.02 | loss 7.40 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 87.36 | loss 7.00 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 81.94 | loss 6.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 86.36s | valid loss 9.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 83.03 | loss 6.67 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 84.90 | loss 6.45 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 96.41 | loss 6.24 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 84.09 | loss 6.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 87.34s | valid loss 9.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 82.24 | loss 5.81 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 81.88 | loss 5.58 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 81.81 | loss 6.18 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 82.04 | loss 5.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 83.53s | valid loss 10.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 86.91 | loss 5.91 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 89.13 | loss 5.43 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 86.63 | loss 6.22 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 82.30 | loss 6.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 87.56s | valid loss 11.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 83.95 | loss 5.66 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 81.80 | loss 5.96 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 81.80 | loss 5.53 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 81.73 | loss 5.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 83.48s | valid loss 7.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 94.02 | loss 5.38 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 75.71 | loss 5.22 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 79.44 | loss 5.68 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 71.79 | loss 5.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 80.45s | valid loss 9.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 82.48 | loss 4.76 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 82.08 | loss 5.20 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 82.00 | loss 5.22 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 81.55 | loss 4.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 83.39s | valid loss 8.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 82.28 | loss 4.25 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 81.81 | loss 4.81 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 84.87 | loss 5.37 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 71.79 | loss 5.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 80.64s | valid loss 7.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 74.74 | loss 4.73 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 71.87 | loss 4.36 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 79.53 | loss 3.69 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 89.86 | loss 4.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 80.49s | valid loss 8.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 85.97 | loss 4.33 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 87.88 | loss 5.17 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 81.80 | loss 4.49 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 81.71 | loss 4.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 85.08s | valid loss 9.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 76.06 | loss 4.42 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 84.92 | loss 4.09 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 94.27 | loss 4.21 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 82.87 | loss 4.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 84.96s | valid loss 7.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 79.64 | loss 4.23 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 81.15 | loss 3.46 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 83.47 | loss 4.05 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 71.53 | loss 4.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 78.40s | valid loss 8.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 73.30 | loss 4.92 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 92.50 | loss 3.90 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 71.78 | loss 4.89 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 76.88 | loss 3.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 80.38s | valid loss 8.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 81.28 | loss 3.67 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 88.44 | loss 3.48 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 87.26 | loss 4.00 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 80.98 | loss 3.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 85.82s | valid loss 8.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 81.46 | loss 3.71 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 86.56 | loss 3.22 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 93.28 | loss 3.53 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 75.38 | loss 2.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 82.53s | valid loss 7.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 71.85 | loss 3.50 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 79.31 | loss 3.53 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 80.84 | loss 3.23 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 80.91 | loss 3.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 80.04s | valid loss 8.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 81.07 | loss 2.86 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 81.08 | loss 3.34 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 81.53 | loss 4.16 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 81.54 | loss 3.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 82.62s | valid loss 7.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 83.61 | loss 2.75 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 94.76 | loss 3.36 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 89.32 | loss 2.70 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 78.06 | loss 3.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 86.61s | valid loss 8.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 81.02 | loss 2.70 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 83.73 | loss 2.68 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 81.83 | loss 3.19 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 82.78 | loss 3.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 81.77s | valid loss 8.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 79.07 | loss 2.92 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 70.69 | loss 2.00 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 70.74 | loss 2.62 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 70.57 | loss 2.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 76.14s | valid loss 7.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 80.85 | loss 2.61 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 80.27 | loss 1.85 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 80.49 | loss 3.06 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 72.65 | loss 2.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 79.90s | valid loss 8.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 80.78 | loss 2.28 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 88.33 | loss 2.07 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 71.98 | loss 2.47 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 80.44 | loss 2.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 81.79s | valid loss 6.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 85.48 | loss 2.24 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 81.91 | loss 2.13 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 81.44 | loss 2.38 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 82.49 | loss 2.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 84.55s | valid loss 7.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 74.29 | loss 2.11 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 70.43 | loss 2.09 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 70.52 | loss 2.27 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 73.79 | loss 2.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 75.48s | valid loss 7.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 31 | 200/ 938 batches | lr 0.00021 | ms/batch 80.65 | loss 1.73 |\n", + "| epoch 31 | 400/ 938 batches | lr 0.00021 | ms/batch 79.70 | loss 1.81 |\n", + "| epoch 31 | 600/ 938 batches | lr 0.00021 | ms/batch 80.32 | loss 1.94 |\n", + "| epoch 31 | 800/ 938 batches | lr 0.00021 | ms/batch 80.39 | loss 2.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 31 | time: 80.81s | valid loss 7.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 32 | 200/ 938 batches | lr 0.00020 | ms/batch 81.92 | loss 2.67 |\n", + "| epoch 32 | 400/ 938 batches | lr 0.00020 | ms/batch 73.73 | loss 1.72 |\n", + "| epoch 32 | 600/ 938 batches | lr 0.00020 | ms/batch 71.01 | loss 2.56 |\n", + "| epoch 32 | 800/ 938 batches | lr 0.00020 | ms/batch 70.95 | loss 2.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 32 | time: 77.09s | valid loss 6.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 33 | 200/ 938 batches | lr 0.00019 | ms/batch 81.48 | loss 1.67 |\n", + "| epoch 33 | 400/ 938 batches | lr 0.00019 | ms/batch 88.54 | loss 2.56 |\n", + "| epoch 33 | 600/ 938 batches | lr 0.00019 | ms/batch 82.65 | loss 2.25 |\n", + "| epoch 33 | 800/ 938 batches | lr 0.00019 | ms/batch 82.89 | loss 2.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 33 | time: 84.84s | valid loss 8.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 34 | 200/ 938 batches | lr 0.00018 | ms/batch 80.77 | loss 2.19 |\n", + "| epoch 34 | 400/ 938 batches | lr 0.00018 | ms/batch 80.54 | loss 2.47 |\n", + "| epoch 34 | 600/ 938 batches | lr 0.00018 | ms/batch 80.42 | loss 2.05 |\n", + "| epoch 34 | 800/ 938 batches | lr 0.00018 | ms/batch 80.53 | loss 2.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 34 | time: 81.95s | valid loss 7.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 35 | 200/ 938 batches | lr 0.00017 | ms/batch 79.13 | loss 2.44 |\n", + "| epoch 35 | 400/ 938 batches | lr 0.00017 | ms/batch 79.80 | loss 1.40 |\n", + "| epoch 35 | 600/ 938 batches | lr 0.00017 | ms/batch 86.83 | loss 2.11 |\n", + "| epoch 35 | 800/ 938 batches | lr 0.00017 | ms/batch 88.97 | loss 1.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 35 | time: 84.39s | valid loss 7.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 36 | 200/ 938 batches | lr 0.00017 | ms/batch 87.72 | loss 2.24 |\n", + "| epoch 36 | 400/ 938 batches | lr 0.00017 | ms/batch 92.57 | loss 2.20 |\n", + "| epoch 36 | 600/ 938 batches | lr 0.00017 | ms/batch 90.16 | loss 1.48 |\n", + "| epoch 36 | 800/ 938 batches | lr 0.00017 | ms/batch 87.66 | loss 1.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 36 | time: 88.11s | valid loss 6.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 37 | 200/ 938 batches | lr 0.00016 | ms/batch 76.40 | loss 1.31 |\n", + "| epoch 37 | 400/ 938 batches | lr 0.00016 | ms/batch 85.67 | loss 1.75 |\n", + "| epoch 37 | 600/ 938 batches | lr 0.00016 | ms/batch 79.19 | loss 2.10 |\n", + "| epoch 37 | 800/ 938 batches | lr 0.00016 | ms/batch 80.16 | loss 1.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 37 | time: 80.75s | valid loss 7.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 38 | 200/ 938 batches | lr 0.00015 | ms/batch 70.61 | loss 1.67 |\n", + "| epoch 38 | 400/ 938 batches | lr 0.00015 | ms/batch 70.38 | loss 2.05 |\n", + "| epoch 38 | 600/ 938 batches | lr 0.00015 | ms/batch 70.43 | loss 1.43 |\n", + "| epoch 38 | 800/ 938 batches | lr 0.00015 | ms/batch 78.38 | loss 2.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 38 | time: 75.95s | valid loss 8.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 39 | 200/ 938 batches | lr 0.00014 | ms/batch 73.86 | loss 2.07 |\n", + "| epoch 39 | 400/ 938 batches | lr 0.00014 | ms/batch 83.08 | loss 1.59 |\n", + "| epoch 39 | 600/ 938 batches | lr 0.00014 | ms/batch 70.76 | loss 1.41 |\n", + "| epoch 39 | 800/ 938 batches | lr 0.00014 | ms/batch 80.09 | loss 1.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 39 | time: 78.66s | valid loss 8.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 40 | 200/ 938 batches | lr 0.00014 | ms/batch 88.08 | loss 1.46 |\n", + "| epoch 40 | 400/ 938 batches | lr 0.00014 | ms/batch 83.79 | loss 0.96 |\n", + "| epoch 40 | 600/ 938 batches | lr 0.00014 | ms/batch 80.81 | loss 1.95 |\n", + "| epoch 40 | 800/ 938 batches | lr 0.00014 | ms/batch 80.82 | loss 1.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 40 | time: 83.50s | valid loss 8.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 41 | 200/ 938 batches | lr 0.00013 | ms/batch 80.26 | loss 1.09 |\n", + "| epoch 41 | 400/ 938 batches | lr 0.00013 | ms/batch 80.20 | loss 1.78 |\n", + "| epoch 41 | 600/ 938 batches | lr 0.00013 | ms/batch 83.35 | loss 1.45 |\n", + "| epoch 41 | 800/ 938 batches | lr 0.00013 | ms/batch 82.78 | loss 1.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 41 | time: 83.51s | valid loss 7.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 42 | 200/ 938 batches | lr 0.00012 | ms/batch 81.80 | loss 1.65 |\n", + "| epoch 42 | 400/ 938 batches | lr 0.00012 | ms/batch 79.96 | loss 1.30 |\n", + "| epoch 42 | 600/ 938 batches | lr 0.00012 | ms/batch 80.01 | loss 1.82 |\n", + "| epoch 42 | 800/ 938 batches | lr 0.00012 | ms/batch 79.96 | loss 1.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 42 | time: 81.72s | valid loss 7.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 43 | 200/ 938 batches | lr 0.00012 | ms/batch 88.52 | loss 1.21 |\n", + "| epoch 43 | 400/ 938 batches | lr 0.00012 | ms/batch 80.77 | loss 1.35 |\n", + "| epoch 43 | 600/ 938 batches | lr 0.00012 | ms/batch 80.43 | loss 1.77 |\n", + "| epoch 43 | 800/ 938 batches | lr 0.00012 | ms/batch 80.40 | loss 1.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 43 | time: 83.46s | valid loss 7.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 44 | 200/ 938 batches | lr 0.00011 | ms/batch 77.53 | loss 1.26 |\n", + "| epoch 44 | 400/ 938 batches | lr 0.00011 | ms/batch 70.19 | loss 0.90 |\n", + "| epoch 44 | 600/ 938 batches | lr 0.00011 | ms/batch 70.26 | loss 2.25 |\n", + "| epoch 44 | 800/ 938 batches | lr 0.00011 | ms/batch 70.26 | loss 1.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 44 | time: 73.96s | valid loss 8.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 45 | 200/ 938 batches | lr 0.00010 | ms/batch 91.70 | loss 1.93 |\n", + "| epoch 45 | 400/ 938 batches | lr 0.00010 | ms/batch 74.02 | loss 0.92 |\n", + "| epoch 45 | 600/ 938 batches | lr 0.00010 | ms/batch 70.60 | loss 1.26 |\n", + "| epoch 45 | 800/ 938 batches | lr 0.00010 | ms/batch 84.37 | loss 1.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 45 | time: 79.82s | valid loss 7.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 46 | 200/ 938 batches | lr 0.00010 | ms/batch 86.68 | loss 1.35 |\n", + "| epoch 46 | 400/ 938 batches | lr 0.00010 | ms/batch 86.35 | loss 1.51 |\n", + "| epoch 46 | 600/ 938 batches | lr 0.00010 | ms/batch 79.91 | loss 1.30 |\n", + "| epoch 46 | 800/ 938 batches | lr 0.00010 | ms/batch 80.04 | loss 1.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 46 | time: 82.94s | valid loss 7.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 47 | 200/ 938 batches | lr 0.00009 | ms/batch 79.99 | loss 0.92 |\n", + "| epoch 47 | 400/ 938 batches | lr 0.00009 | ms/batch 83.95 | loss 1.24 |\n", + "| epoch 47 | 600/ 938 batches | lr 0.00009 | ms/batch 89.50 | loss 1.33 |\n", + "| epoch 47 | 800/ 938 batches | lr 0.00009 | ms/batch 70.74 | loss 1.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 47 | time: 80.03s | valid loss 8.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 48 | 200/ 938 batches | lr 0.00009 | ms/batch 82.23 | loss 1.20 |\n", + "| epoch 48 | 400/ 938 batches | lr 0.00009 | ms/batch 75.30 | loss 1.29 |\n", + "| epoch 48 | 600/ 938 batches | lr 0.00009 | ms/batch 77.55 | loss 0.94 |\n", + "| epoch 48 | 800/ 938 batches | lr 0.00009 | ms/batch 85.70 | loss 1.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 48 | time: 81.64s | valid loss 8.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 49 | 200/ 938 batches | lr 0.00009 | ms/batch 83.12 | loss 1.07 |\n", + "| epoch 49 | 400/ 938 batches | lr 0.00009 | ms/batch 92.16 | loss 1.15 |\n", + "| epoch 49 | 600/ 938 batches | lr 0.00009 | ms/batch 81.79 | loss 0.82 |\n", + "| epoch 49 | 800/ 938 batches | lr 0.00009 | ms/batch 83.44 | loss 1.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 49 | time: 85.69s | valid loss 8.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.77 |\n", + "=========================================================================================\n", + "Result over 5 runs: 6.298228740692139\n" + ] + } + ], + "source": [ + "num_runs = 5\n", + "best_model = None\n", + "best_result = 1000000\n", + "results = []\n", + "\n", + "for i in range(num_runs):\n", + " features = nn.Sequential(nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(64),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(128),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(256),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=(0,1)),\n", + " nn.BatchNorm2d(512),\n", + " nn.ReLU()\n", + " )\n", + "\n", + " classifier = nn.Sequential(nn.Linear(512, 1028),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=1028, bias=True),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=10, bias=True))\n", + "\n", + " conv = nn.Sequential(nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.MaxPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.MaxPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.MaxPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.MaxPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.MaxPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.MaxPool1d(kernel_size=2, stride=2))\n", + "\n", + " model = ConvMNIST(features, conv, classifier)\n", + " model = model.to(device)\n", + "\n", + " optimizer = torch.optim.Adam(model.parameters())\n", + " scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "\n", + " epochs = 30\n", + " model, result = train_model(model, epochs, train_loader, test_loader, seq_len, digits_per_batch, optimizer, scheduler)\n", + " \n", + " if result < best_result:\n", + " best_result = result\n", + " best_model = model\n", + " results.append(result)\n", + " \n", + "model = best_model\n", + "print(\"Result over {} runs: {}\".format(num_runs, torch.tensor(results).mean().item()))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 1.506505, + "end_time": "2021-01-22T15:49:45.696504", + "exception": false, + "start_time": "2021-01-22T15:49:44.189999", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "### Smart pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-22T15:49:48.806476Z", + "iopub.status.busy": "2021-01-22T15:49:48.805965Z", + "iopub.status.idle": "2021-01-22T23:26:11.854855Z", + "shell.execute_reply": "2021-01-22T23:26:11.855474Z" + }, + "papermill": { + "duration": 27384.577647, + "end_time": "2021-01-22T23:26:11.855678", + "exception": false, + "start_time": "2021-01-22T15:49:47.278031", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 192.00 | loss 44.57 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 192.64 | loss 15.32 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 198.13 | loss 13.64 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 180.46 | loss 11.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 191.13s | valid loss 16.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 197.32 | loss 9.36 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 198.43 | loss 9.60 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 193.15 | loss 8.87 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 201.25 | loss 8.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 198.87s | valid loss 11.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 192.41 | loss 8.05 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 199.58 | loss 8.12 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 197.21 | loss 6.54 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 195.90 | loss 7.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 195.23s | valid loss 9.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 199.58 | loss 6.75 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 176.87 | loss 7.08 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 183.53 | loss 6.41 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 193.05 | loss 6.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 192.24s | valid loss 9.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 199.48 | loss 6.28 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 194.21 | loss 6.03 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 191.66 | loss 5.53 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 184.11 | loss 5.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 191.69s | valid loss 11.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 178.80 | loss 5.39 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 189.63 | loss 5.20 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 200.70 | loss 4.68 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 188.04 | loss 5.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 188.82s | valid loss 7.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 158.19 | loss 4.90 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 174.33 | loss 4.65 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 202.57 | loss 4.36 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 198.93 | loss 4.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 184.24s | valid loss 7.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 187.49 | loss 3.99 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 192.16 | loss 4.94 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 175.37 | loss 4.60 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 197.61 | loss 4.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 189.96s | valid loss 7.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 194.31 | loss 4.38 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 207.63 | loss 4.17 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 195.00 | loss 4.28 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 180.37 | loss 4.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 192.01s | valid loss 7.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 204.18 | loss 4.08 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 190.34 | loss 5.01 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 176.51 | loss 4.00 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 198.18 | loss 3.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 195.44s | valid loss 7.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 195.26 | loss 4.03 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 202.31 | loss 4.18 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 175.77 | loss 3.69 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 190.08 | loss 3.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 189.76s | valid loss 7.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 165.52 | loss 3.50 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 159.77 | loss 4.33 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 183.86 | loss 3.84 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 197.37 | loss 4.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 182.78s | valid loss 9.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 202.47 | loss 3.67 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 203.80 | loss 3.70 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 190.92 | loss 4.40 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 181.08 | loss 3.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 194.01s | valid loss 10.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 185.48 | loss 4.06 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 202.29 | loss 3.43 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 192.06 | loss 3.44 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 202.79 | loss 2.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 199.71s | valid loss 6.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 208.31 | loss 3.40 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 168.61 | loss 2.95 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 191.31 | loss 4.15 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 193.97 | loss 3.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 194.76s | valid loss 8.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 196.15 | loss 2.78 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 193.99 | loss 2.95 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 205.20 | loss 3.06 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 182.98 | loss 3.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 193.02s | valid loss 7.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 192.12 | loss 3.11 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 185.50 | loss 3.47 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 183.06 | loss 2.84 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 193.39 | loss 2.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 188.83s | valid loss 7.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 167.80 | loss 2.35 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 191.28 | loss 2.80 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 190.70 | loss 2.35 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 194.03 | loss 2.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 187.77s | valid loss 7.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 198.86 | loss 1.96 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 195.18 | loss 2.72 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 188.18 | loss 2.26 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 184.24 | loss 2.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 191.27s | valid loss 7.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 205.68 | loss 2.18 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 193.04 | loss 2.20 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 196.30 | loss 2.17 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 193.08 | loss 2.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 197.20s | valid loss 8.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 193.67 | loss 1.68 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 199.21 | loss 2.05 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 172.01 | loss 2.52 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 190.37 | loss 1.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 190.16s | valid loss 8.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 191.69 | loss 2.39 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 184.18 | loss 2.09 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 198.45 | loss 2.06 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 177.52 | loss 2.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 190.60s | valid loss 7.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 201.29 | loss 2.22 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 208.33 | loss 2.33 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 198.41 | loss 2.37 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 196.53 | loss 2.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 201.63s | valid loss 9.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 209.70 | loss 2.24 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 193.13 | loss 1.25 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 203.02 | loss 1.40 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 197.26 | loss 2.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 200.16s | valid loss 7.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 180.58 | loss 1.55 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 194.07 | loss 2.07 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 195.20 | loss 2.00 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 192.11 | loss 1.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 193.16s | valid loss 9.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 199.70 | loss 1.60 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 185.39 | loss 2.06 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 207.10 | loss 0.98 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 197.62 | loss 1.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 198.19s | valid loss 7.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 198.36 | loss 1.81 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 201.03 | loss 1.56 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 186.03 | loss 1.90 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 196.40 | loss 1.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 194.52s | valid loss 7.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 190.03 | loss 0.80 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 184.93 | loss 1.47 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 200.01 | loss 1.59 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 200.36 | loss 1.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 191.71s | valid loss 7.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 170.35 | loss 2.11 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 154.04 | loss 1.62 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 165.71 | loss 1.01 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 195.06 | loss 1.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 171.62s | valid loss 9.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 191.77 | loss 1.29 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 191.72 | loss 1.73 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 186.27 | loss 1.30 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 178.34 | loss 1.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 190.86s | valid loss 8.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.59 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 183.96 | loss 50.13 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 202.16 | loss 16.40 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 183.57 | loss 12.98 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 205.68 | loss 10.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 196.21s | valid loss 14.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 193.82 | loss 9.15 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 181.83 | loss 9.31 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 189.95 | loss 10.70 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 166.29 | loss 8.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 182.16s | valid loss 12.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 193.03 | loss 7.49 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 191.13 | loss 6.75 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 199.80 | loss 8.15 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 186.62 | loss 6.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 189.34s | valid loss 8.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 195.65 | loss 5.96 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 160.88 | loss 6.81 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 180.58 | loss 6.65 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 190.45 | loss 5.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 185.26s | valid loss 11.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 198.88 | loss 5.95 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 204.35 | loss 5.36 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 198.61 | loss 5.28 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 196.22 | loss 5.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 199.26s | valid loss 8.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 168.30 | loss 5.39 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 182.13 | loss 5.16 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 201.16 | loss 4.86 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 205.43 | loss 5.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 184.26s | valid loss 7.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 193.50 | loss 4.68 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 194.27 | loss 4.20 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 197.26 | loss 4.88 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 179.82 | loss 5.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 190.22s | valid loss 9.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 184.53 | loss 4.89 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 184.02 | loss 4.10 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 197.27 | loss 4.68 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 185.12 | loss 3.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 192.30s | valid loss 9.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 196.10 | loss 4.05 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 201.22 | loss 4.27 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 197.93 | loss 4.12 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 171.15 | loss 4.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 194.62s | valid loss 7.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 184.70 | loss 4.32 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 185.89 | loss 3.11 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 195.44 | loss 5.05 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 189.94 | loss 4.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 187.87s | valid loss 10.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 184.61 | loss 4.38 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 183.44 | loss 4.32 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 191.94 | loss 3.97 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 192.34 | loss 3.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 186.72s | valid loss 9.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 186.77 | loss 3.43 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 190.85 | loss 3.84 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 195.70 | loss 3.08 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 199.19 | loss 3.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 197.05s | valid loss 7.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 194.27 | loss 4.17 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 202.00 | loss 3.49 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 199.70 | loss 4.40 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 177.60 | loss 4.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 192.34s | valid loss 7.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 201.15 | loss 3.01 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 192.63 | loss 2.93 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 187.68 | loss 3.31 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 160.63 | loss 3.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 187.73s | valid loss 8.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 191.80 | loss 3.12 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 207.02 | loss 3.30 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 181.67 | loss 2.81 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 187.87 | loss 3.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 196.19s | valid loss 7.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 194.50 | loss 2.55 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 191.24 | loss 3.03 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 204.68 | loss 2.47 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 192.81 | loss 2.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 196.14s | valid loss 7.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 190.89 | loss 2.78 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 183.94 | loss 2.59 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 193.28 | loss 2.71 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 195.69 | loss 3.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 187.93s | valid loss 7.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 155.26 | loss 2.75 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 154.49 | loss 2.93 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 187.79 | loss 2.60 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 186.26 | loss 2.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 177.91s | valid loss 6.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 204.27 | loss 2.01 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 182.71 | loss 2.72 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 194.10 | loss 2.49 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 205.64 | loss 2.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 195.72s | valid loss 6.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 196.02 | loss 2.74 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 184.85 | loss 2.53 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 188.47 | loss 2.54 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 194.70 | loss 2.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 192.71s | valid loss 7.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 204.25 | loss 2.58 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 184.59 | loss 2.35 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 200.63 | loss 2.18 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 203.01 | loss 1.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 196.44s | valid loss 8.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 196.67 | loss 2.43 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 181.91 | loss 2.65 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 162.48 | loss 1.32 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 197.64 | loss 1.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 187.62s | valid loss 9.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 199.84 | loss 1.78 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 191.73 | loss 2.09 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 198.51 | loss 2.06 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 202.83 | loss 2.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 197.68s | valid loss 8.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 202.11 | loss 2.19 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 199.24 | loss 1.16 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 200.77 | loss 2.43 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 177.54 | loss 1.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 195.37s | valid loss 7.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 172.02 | loss 1.90 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 200.37 | loss 1.75 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 194.22 | loss 1.76 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 194.53 | loss 2.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 191.88s | valid loss 8.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 191.74 | loss 2.07 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 196.35 | loss 1.58 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 180.25 | loss 1.47 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 162.24 | loss 2.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 185.79s | valid loss 8.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 179.43 | loss 1.38 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 179.70 | loss 1.79 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 186.68 | loss 1.41 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 197.45 | loss 2.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 188.34s | valid loss 8.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 208.13 | loss 1.67 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 200.70 | loss 2.26 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 195.01 | loss 2.02 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 200.18 | loss 1.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 200.31s | valid loss 8.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 200.10 | loss 1.69 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 190.75 | loss 2.22 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 169.06 | loss 1.33 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 195.76 | loss 1.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 186.49s | valid loss 8.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 174.56 | loss 1.10 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 197.22 | loss 1.38 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 179.32 | loss 1.23 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 188.59 | loss 1.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 186.53s | valid loss 9.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.80 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 190.18 | loss 48.85 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 188.25 | loss 16.87 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 185.72 | loss 13.42 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 190.35 | loss 12.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 188.03s | valid loss 15.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 194.10 | loss 10.01 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 199.98 | loss 9.30 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 176.19 | loss 8.41 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 188.74 | loss 7.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 191.02s | valid loss 17.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 189.58 | loss 7.45 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 198.27 | loss 7.89 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 177.63 | loss 7.31 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 201.48 | loss 7.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 193.07s | valid loss 7.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 193.28 | loss 6.40 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 196.73 | loss 6.20 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 181.56 | loss 6.70 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 186.50 | loss 5.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 192.50s | valid loss 8.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 208.80 | loss 5.76 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 178.82 | loss 5.74 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 199.45 | loss 6.36 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 204.18 | loss 5.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 196.82s | valid loss 7.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 180.98 | loss 5.23 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 195.21 | loss 5.22 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 188.59 | loss 4.74 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 175.64 | loss 5.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 188.26s | valid loss 8.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 197.72 | loss 4.67 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 203.11 | loss 4.92 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 193.94 | loss 4.51 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 181.59 | loss 5.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 193.76s | valid loss 7.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 190.37 | loss 4.60 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 201.31 | loss 4.63 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 192.33 | loss 4.90 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 166.74 | loss 4.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 182.03s | valid loss 7.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 184.93 | loss 4.52 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 206.59 | loss 4.11 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 188.20 | loss 4.35 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 198.57 | loss 4.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 196.93s | valid loss 7.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 194.03 | loss 4.29 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 202.53 | loss 4.44 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 168.84 | loss 3.80 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 194.59 | loss 4.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 190.31s | valid loss 7.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 193.89 | loss 3.77 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 191.97 | loss 4.04 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 175.91 | loss 3.50 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 168.07 | loss 3.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 178.03s | valid loss 8.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 199.05 | loss 4.00 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 193.20 | loss 4.34 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 189.41 | loss 3.10 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 186.07 | loss 3.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 193.95s | valid loss 7.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 196.93 | loss 3.28 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 190.90 | loss 3.09 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 208.11 | loss 3.65 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 196.33 | loss 3.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 198.90s | valid loss 6.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 198.27 | loss 2.94 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 175.90 | loss 3.34 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 192.50 | loss 3.07 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 193.29 | loss 3.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 193.66s | valid loss 7.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 186.94 | loss 3.25 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 181.03 | loss 3.43 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 206.26 | loss 2.94 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 208.30 | loss 2.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 191.60s | valid loss 7.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 201.58 | loss 2.80 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 191.81 | loss 2.73 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 191.37 | loss 3.33 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 198.62 | loss 2.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 194.16s | valid loss 8.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 169.01 | loss 3.41 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 168.23 | loss 2.90 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 182.74 | loss 2.59 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 199.68 | loss 3.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 183.04s | valid loss 6.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 184.61 | loss 2.61 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 209.15 | loss 2.18 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 191.33 | loss 2.62 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 192.28 | loss 2.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 194.88s | valid loss 9.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 175.44 | loss 3.03 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 192.01 | loss 2.65 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 193.38 | loss 2.04 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 191.10 | loss 2.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 190.05s | valid loss 6.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 171.21 | loss 2.18 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 196.20 | loss 2.76 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 164.82 | loss 2.61 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 190.89 | loss 2.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 181.77s | valid loss 8.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 200.61 | loss 2.26 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 186.15 | loss 2.67 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 177.38 | loss 2.31 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 197.84 | loss 2.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 194.33s | valid loss 7.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 209.57 | loss 1.99 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 195.93 | loss 1.95 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 191.42 | loss 1.94 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 181.04 | loss 2.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 195.44s | valid loss 8.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 190.90 | loss 2.55 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 191.19 | loss 2.04 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 195.83 | loss 2.28 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 172.45 | loss 2.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 183.29s | valid loss 7.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 184.51 | loss 1.74 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 194.83 | loss 2.41 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 190.63 | loss 2.23 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 204.75 | loss 1.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 194.95s | valid loss 7.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 193.16 | loss 1.28 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 190.56 | loss 1.88 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 195.80 | loss 1.93 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 191.42 | loss 1.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 196.75s | valid loss 7.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 182.01 | loss 1.87 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 177.20 | loss 1.53 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 196.95 | loss 1.51 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 196.91 | loss 2.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 188.36s | valid loss 7.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 193.75 | loss 1.67 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 182.43 | loss 2.13 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 188.79 | loss 1.66 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 176.96 | loss 1.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 184.59s | valid loss 7.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 188.63 | loss 1.65 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 202.29 | loss 1.81 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 177.07 | loss 1.90 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 202.77 | loss 2.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 191.60s | valid loss 7.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 191.81 | loss 1.64 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 190.95 | loss 1.60 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 191.89 | loss 1.90 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 183.54 | loss 1.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 191.50s | valid loss 8.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 204.10 | loss 1.86 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 202.88 | loss 1.80 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 187.65 | loss 1.89 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 196.06 | loss 2.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 197.67s | valid loss 7.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.21 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 201.39 | loss 48.47 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 192.22 | loss 15.16 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 201.79 | loss 12.21 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 195.63 | loss 11.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 198.24s | valid loss 14.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 206.19 | loss 10.34 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 201.95 | loss 9.61 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 169.77 | loss 8.05 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 188.56 | loss 8.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 192.79s | valid loss 12.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 192.70 | loss 7.18 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 193.11 | loss 6.79 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 197.72 | loss 7.03 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 178.99 | loss 6.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 193.58s | valid loss 7.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 188.88 | loss 6.24 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 204.38 | loss 6.01 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 180.48 | loss 6.14 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 195.01 | loss 5.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 194.09s | valid loss 7.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 181.29 | loss 4.97 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 206.80 | loss 4.67 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 192.73 | loss 5.90 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 198.84 | loss 4.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 190.82s | valid loss 8.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 196.25 | loss 4.86 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 186.26 | loss 5.24 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 194.92 | loss 4.82 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 205.89 | loss 5.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 197.54s | valid loss 8.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 182.27 | loss 5.04 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 187.30 | loss 4.12 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 179.56 | loss 4.76 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 179.64 | loss 4.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 185.44s | valid loss 9.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 194.32 | loss 4.14 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 190.08 | loss 4.89 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 163.69 | loss 5.44 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 183.47 | loss 4.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 185.52s | valid loss 7.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 199.72 | loss 4.09 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 193.36 | loss 3.69 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 198.43 | loss 4.19 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 191.96 | loss 3.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 194.70s | valid loss 9.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 207.95 | loss 4.22 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 189.53 | loss 3.73 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 165.60 | loss 3.15 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 186.23 | loss 3.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 187.67s | valid loss 7.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 191.44 | loss 3.46 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 200.20 | loss 4.20 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 183.88 | loss 3.29 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 185.54 | loss 4.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 193.69s | valid loss 9.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 177.71 | loss 4.15 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 174.27 | loss 4.22 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 196.24 | loss 3.55 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 198.66 | loss 3.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 185.05s | valid loss 8.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 189.91 | loss 2.76 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 171.20 | loss 2.99 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 185.79 | loss 3.45 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 182.70 | loss 3.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 182.65s | valid loss 6.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 193.53 | loss 3.06 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 174.16 | loss 2.73 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 198.19 | loss 3.18 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 199.38 | loss 2.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 189.92s | valid loss 6.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 200.66 | loss 2.81 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 187.89 | loss 3.44 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 171.90 | loss 3.83 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 178.68 | loss 2.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 187.77s | valid loss 6.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 197.81 | loss 2.29 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 194.41 | loss 3.77 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 193.74 | loss 2.22 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 159.00 | loss 3.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 190.55s | valid loss 6.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 174.33 | loss 2.95 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 203.01 | loss 3.30 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 190.69 | loss 2.93 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 170.86 | loss 2.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 188.09s | valid loss 7.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 175.41 | loss 2.74 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 196.25 | loss 3.24 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 202.05 | loss 2.89 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 191.64 | loss 2.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 189.99s | valid loss 6.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 172.05 | loss 1.69 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 199.81 | loss 2.82 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 192.44 | loss 2.45 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 172.67 | loss 2.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 187.35s | valid loss 6.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 192.33 | loss 2.35 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 177.40 | loss 1.95 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 179.47 | loss 2.80 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 193.77 | loss 2.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 188.00s | valid loss 7.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 184.47 | loss 2.04 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 180.16 | loss 1.95 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 194.86 | loss 1.99 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 193.78 | loss 2.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 191.84s | valid loss 7.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 195.54 | loss 2.02 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 208.99 | loss 2.15 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 177.28 | loss 1.93 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 195.33 | loss 2.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 195.24s | valid loss 7.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 186.50 | loss 1.86 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 191.65 | loss 1.50 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 179.81 | loss 1.65 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 165.38 | loss 2.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 181.55s | valid loss 9.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 193.87 | loss 1.56 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 197.87 | loss 1.52 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 193.77 | loss 2.11 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 189.63 | loss 1.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 194.44s | valid loss 6.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 175.90 | loss 1.24 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 155.56 | loss 2.08 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 151.30 | loss 1.98 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 151.32 | loss 1.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 158.11s | valid loss 6.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 151.98 | loss 1.49 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 151.16 | loss 1.23 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 151.28 | loss 1.63 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 151.18 | loss 1.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 152.40s | valid loss 7.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 151.88 | loss 1.34 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 158.73 | loss 1.62 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 165.58 | loss 1.61 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 154.81 | loss 1.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 157.51s | valid loss 7.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 151.93 | loss 1.21 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 151.17 | loss 1.34 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 151.16 | loss 1.10 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 151.20 | loss 1.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 152.41s | valid loss 7.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 151.99 | loss 1.49 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 151.18 | loss 1.30 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 151.19 | loss 1.78 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 151.23 | loss 1.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 152.44s | valid loss 6.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 152.01 | loss 1.23 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 151.17 | loss 1.73 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 151.31 | loss 1.73 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 151.12 | loss 1.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 152.43s | valid loss 7.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.02 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 152.61 | loss 43.21 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 151.83 | loss 14.54 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 151.85 | loss 12.57 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 151.82 | loss 10.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 153.00s | valid loss 15.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 152.49 | loss 9.57 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 151.78 | loss 9.07 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 151.79 | loss 8.30 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 151.80 | loss 8.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 152.95s | valid loss 11.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 152.68 | loss 7.95 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 155.20 | loss 7.06 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 155.56 | loss 7.52 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 151.85 | loss 6.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 154.42s | valid loss 9.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 152.52 | loss 6.13 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 163.36 | loss 6.82 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 151.83 | loss 5.63 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 151.77 | loss 6.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 155.28s | valid loss 12.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 152.46 | loss 5.40 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 151.80 | loss 5.17 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 151.74 | loss 5.86 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 151.83 | loss 5.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 152.94s | valid loss 17.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 152.64 | loss 5.69 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 151.77 | loss 5.68 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 151.81 | loss 5.19 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 151.73 | loss 4.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 152.98s | valid loss 7.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 152.55 | loss 4.34 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 151.94 | loss 4.79 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 151.80 | loss 5.82 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 151.89 | loss 5.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 153.01s | valid loss 8.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 152.60 | loss 4.42 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 157.38 | loss 4.53 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 154.19 | loss 4.39 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 151.41 | loss 4.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 154.40s | valid loss 7.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 152.17 | loss 4.64 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 151.41 | loss 4.40 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 151.37 | loss 4.55 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 151.43 | loss 4.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 152.68s | valid loss 8.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 155.57 | loss 3.77 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 151.37 | loss 5.58 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 151.35 | loss 3.83 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 151.25 | loss 3.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 153.18s | valid loss 8.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 152.07 | loss 3.90 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 151.40 | loss 4.09 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 151.36 | loss 4.21 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 151.36 | loss 3.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 152.53s | valid loss 5.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 152.17 | loss 3.55 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 151.28 | loss 3.41 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 151.41 | loss 41.85 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 151.50 | loss 29.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 152.59s | valid loss 46.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 152.25 | loss 25.21 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 151.54 | loss 24.48 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 151.48 | loss 23.75 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 151.52 | loss 23.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 152.67s | valid loss 42.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 152.31 | loss 22.41 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 151.47 | loss 21.29 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 151.53 | loss 20.84 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 151.47 | loss 20.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 152.67s | valid loss 37.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 152.25 | loss 20.35 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 151.53 | loss 20.19 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 151.50 | loss 19.70 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 151.53 | loss 19.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 152.66s | valid loss 36.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 152.29 | loss 18.65 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 151.49 | loss 18.38 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 151.52 | loss 18.61 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 151.47 | loss 18.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 152.66s | valid loss 34.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 152.33 | loss 18.34 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 151.52 | loss 18.44 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 151.49 | loss 17.08 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 151.55 | loss 17.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 152.68s | valid loss 32.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 152.37 | loss 16.57 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 151.52 | loss 16.97 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 151.57 | loss 16.46 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 151.51 | loss 17.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 152.74s | valid loss 33.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 152.28 | loss 17.41 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 151.60 | loss 16.88 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 151.56 | loss 16.32 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 151.60 | loss 16.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 152.73s | valid loss 32.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 152.33 | loss 15.94 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 151.52 | loss 15.84 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 151.56 | loss 16.76 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 151.50 | loss 15.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 152.70s | valid loss 30.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 152.23 | loss 15.39 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 151.56 | loss 15.21 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 151.50 | loss 15.02 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 151.55 | loss 14.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 152.68s | valid loss 29.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 156.13 | loss 15.30 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 153.87 | loss 15.99 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 153.93 | loss 14.44 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 153.89 | loss 14.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 155.38s | valid loss 29.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 154.64 | loss 16.19 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 153.95 | loss 14.88 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 153.88 | loss 14.81 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 153.92 | loss 14.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 155.08s | valid loss 29.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 154.66 | loss 15.26 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 153.86 | loss 14.72 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 153.89 | loss 15.04 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 153.85 | loss 13.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 155.07s | valid loss 28.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 154.67 | loss 14.62 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 153.95 | loss 13.66 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 153.86 | loss 13.34 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 153.95 | loss 14.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 155.09s | valid loss 28.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 154.70 | loss 14.49 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 153.86 | loss 14.69 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 153.89 | loss 14.07 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 153.87 | loss 14.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 155.06s | valid loss 27.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 154.62 | loss 13.79 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 153.94 | loss 14.14 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 153.90 | loss 14.19 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 153.94 | loss 14.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 155.09s | valid loss 27.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 154.80 | loss 13.92 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 153.95 | loss 13.41 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 153.96 | loss 13.33 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 153.92 | loss 13.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 155.14s | valid loss 27.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 154.65 | loss 13.48 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 153.93 | loss 14.03 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 153.85 | loss 13.50 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 153.91 | loss 13.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 155.06s | valid loss 27.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 154.65 | loss 12.97 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 153.89 | loss 13.52 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 154.00 | loss 12.58 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 153.92 | loss 13.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 155.10s | valid loss 26.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.87 |\n", + "=========================================================================================\n", + "Result over 5 runs: 6.170612335205078\n" + ] + } + ], + "source": [ + "num_runs = 5\n", + "best_model = None\n", + "best_result = 1000000\n", + "results = []\n", + "\n", + "for i in range(num_runs):\n", + " features = nn.Sequential(nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(64),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(128),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(256),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=(0,1)),\n", + " nn.BatchNorm2d(512),\n", + " nn.ReLU()\n", + " )\n", + "\n", + " classifier = nn.Sequential(nn.Linear(512, 1028),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=1028, bias=True),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=10, bias=True))\n", + "\n", + " model = DoXTimes(Smartpool(divider, 0.3), classifier, features=features)\n", + " model = model.to(device)\n", + "\n", + " optimizer = torch.optim.Adam(model.parameters())\n", + " scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "\n", + " epochs = 30\n", + " model, result = train_model(model, epochs, train_loader, test_loader, seq_len, digits_per_batch, optimizer, scheduler)\n", + " \n", + " if result < best_result:\n", + " best_result = result\n", + " best_model = model\n", + " results.append(result)\n", + " \n", + "model = best_model\n", + "print(\"Result over {} runs: {}\".format(num_runs, torch.tensor(results).mean().item()))\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-22T23:26:15.158129Z", + "iopub.status.busy": "2021-01-22T23:26:15.157354Z", + "iopub.status.idle": "2021-01-22T23:26:15.492197Z", + "shell.execute_reply": "2021-01-22T23:26:15.492601Z" + }, + "papermill": { + "duration": 1.993199, + "end_time": "2021-01-22T23:26:15.492752", + "exception": false, + "start_time": "2021-01-22T23:26:13.499553", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQcAAAMwCAYAAAAtS6WbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAEAAElEQVR4nOz9d5Rk13Uein/n3sqhqzp3T4fJGZOAwSBnMGdKlEhRMinJovysTDnJ8nN47ye/934Oz5JtSZYoiVQgKQYJYgZBgMhxZjA5h57p6dxdXTnXPe+Pb59b03ABpEgCaK91vrV6VfUNJ91bO+99lNYaFhYWFq+E82YPwMLCYnXCEgcLC4uOsMTBwsKiIyxxsLCw6AhLHCwsLDrCEgcLC4uOWBXEQSn1dqXUWaXUBaXUv3gD+x1TSn1XKXVaKXVSKfVrcrxHKfWIUuq8fHa/gWNylVIvK6W+9maORSmVVkp9SSl1RtbntjdxLL8hz+eEUupzSqnIGzUWpdSfKqXmlVInrjv2qn0rpX5L3uOzSqm3vQFj+Q/yjI4ppf5WKZX+UY3lTScOSikXwH8H8A4AOwB8RCm14w3qvgngN7XW2wHcCuCXpO9/AeBRrfVmAI/K/28Ufg3A6ev+f7PG8rsAvqW13gZgj4zpDR+LUmoEwK8C2K+1vgGAC+DDb+BYPg3g7a841rFveXc+DGCn3PP78n6/nmN5BMANWuvdAM4B+K0f2Vi01m/qH4DbADx83f+/BeC33qSx/B2AtwA4C2BYjg0DOPsG9T8Kvmz3A/iaHHvDxwKgC8BlAOoVx9+MsYwAmATQAyAA4GsA3vpGjgXAOgAnvtc6vPLdBfAwgNtez7G84twHAPzVj2osb7rkgPbDN7gmx95QKKXWAdgH4AUAg1rrGQCQz4E3aBj/BcA/A+Bdd+zNGMsGAAsA/kxUnE8ppeJvxli01lMA/iOAqwBmAOS01t9+M8ZyHV6t7zf7Xf45AN/8UY1lNRAH1eHYGxrTrZRKAPgygF/XWuffyL6vG8O7AcxrrQ+9Gf2/AgEANwL4A631PgAlvLGqlQ/R598HYD2ANQDiSqmffjPG8n3gTXuXlVK/DarJf/WjGstqIA7XAIxd9/8ogOk3qnOlVBAkDH+ltf4bOTynlBqW88MA5t+AodwB4L1KqQkAnwdwv1LqL9+ksVwDcE1r/YL8/yWQWLwZY3kQwGWt9YLWugHgbwDc/iaNxeDV+n5T3mWl1McAvBvAR7XoED+KsawG4vASgM1KqfVKqRBoRPnKG9GxUkoB+BMAp7XW//m6U18B8DH5/jHQFvG6Qmv9W1rrUa31OnANHtNa//SbNJZZAJNKqa1y6AEAp96MsYDqxK1KqZg8rwdA4+ibMRaDV+v7KwA+rJQKK6XWA9gM4MXXcyBKqbcD+OcA3qu1Lr9ijD/cWF5vg9L3aWR5J2hpvQjgt9/Afu8ERa1jAI7I3zsB9IKGwfPy2fMGr8e9aBsk35SxANgL4KCszUMAut/Esfw7AGcAnADwFwDCb9RYAHwOtHU0QG7886/VN4Dflvf4LIB3vAFjuQDaFsz7+4c/qrEoacTCwsJiBVaDWmFhYbEKYYmDhYVFR1jiYGFh0RGWOFhYWHSEJQ4WFhYd8boRh79vpqVS6hOv11j+vrBj6Qw7ls5YLWP5UY/jdSEOP2Cm5apYYIEdS2fYsXTGahnL6icOAA4AuKC1vqS1roPhwO97nfqysLB4HRB4ndrtlBF2y6td7MbjOpDqRmR0TGtJF1EhJiY6RdIv7QBeUM41+Rmo8bMZBlRLzkk+oxay5zSBVlSOuVruZydOrIlWbeUShHIa4UgakZEx7Ug/rZCcy/P+ekrBqbfbBwDPNX0ArpxrxmRMMjYongcAFZCBVsxAAbcq90Xa17vd3QiPj2kzZy3DdRpyjQa8VzxFswaBCtBIyvfyynXxAoCS+DcvIutSV/79/jqa5+EBwUQ34n1juiXjM2Pywrzfrar/KbXHrE8jAX/NzPWO9KdVe82cBs81+qShmgNHnrMZbysEBFLdiIyMab+/V6QZqVb7mHk2bs30f90z0SvXRenr1hbt/gBAhzRUwyyItBltIjTQhejwmD9z/YqxaLc9d3OqZda8odosWloIVPjZSGk4Zd4RqPFkM9peM8fMQRBMdiM2KL+hTusBPo9GYmV/9WvXFrXW/XgFXi/i8D0zwkQ/+gQABLu6sfkT/ztCBY1KP2+NzvPywgZef+Du03j2MEP941f5tCOLvGbpJg+hRR4Llnh9rZvnQtn2UMwL0IzzXCuioTyeT53Fiv7gAYOHuKLz+7hMtTE+YScbQKAiBEZe8OSEPGwPyLyDT7e1EJG22bFb08htgd83ACSu8Fwop1HrZVvFrexHFdlveMlB+rY5AEDm8MCK1QwWFYKSR+rJS9wUYthIagw/25JzbLvSy/6yWzW8KBuJT3Dt6in+H1lSGHl4EQAw+e4+WUeeK69R6H+Zv/j5m3if+THU+1sIz3HMIRmTeUblYdUmJkLkzTXlYY16D8cZv8r7G/uKXNdH4v6Pup5kR7lbSUUHvxlCeYDzMT/o0ij788IaXh/XsffxMAAgL8+2MV5D/6M8VlhrKAg/wstAcZxtRBd4zhCHrgkPs/fwwmCGc3drvKY62ER0hmNPn/Ok7fbYKgNsM1CU/uRdjE1rlMZ4LLIg63E35z70lxEsb2abhZ2cS3SCg6msr2P42zy3cCPvj09KO8secpvYgRdgv0MvcH2nPtxAq8T7Ahl+Xvrnv3kFHfB6EYfvmRGmtf4jAH8EANHBMe00gFZI+RQuWBIOIi/N889uA0Lyoz7At6r6QhcvbgEjT/INKoxy8UI5vyO/LbNgrvywYzPK57qhEh9o31H+n1/rYPJ97HvtF+SH/5ySPhQy+3h99Bpfkux20x8QeZkig/mR1lP8jM6zTwBo3FkAADgXOIdWRKEq3DKwyF9PONMmbPknB3lODpkfZGltE13nOInus/yFLO3g/W5VIb+O58pDK7l1ZFGh2stjxW1CjEpCJHo0vACJQniJ12Tu4jXdz4WweEPAnys7aktktTUcQ/IKr8ns4rnYjEJ8mmvWSLS5HwAEtueBU1yHygCv0fNcPKcJLIu1KsDfDMKXSHQzO7RPOF4pQQw+r5Ffy+vicxyTW+f85oYdLO3idWueIcXKyTqVRrX/w63L62WkqNn7WoAwk/i1lfyvutZDK2QYhLynQlSK65v++JKXOYb4vPxY73WQmJD3cYEdVU+StRdGNNx7MgCA9MM9AIDyPVyE6IkElreYuQtBNES36CE2zTYNQVzcJYzmRAC1XiFwxZVzeCVeL5vDm5ZpaWFh8aPB6yI5aK2bSqlfBktTuQD+VGt98lVvcIBWBChtaqDnIIc0f4Cnxr9C6nb1g00kT5IU62sk6cGiiMFzri/iGj0/vMTPWi8w/DS53vIO3l8fJycJngghv1VkXSUcdg3b1K6HxCmKnrO38hKjCkTnFHSUlL8VXUlfm0kP0TmORQnniIponbm7hjVfJXnPuSJ5LPJz6b0VpB8xEoeoLKJ/l4cUAmKPMKK/EcO7zgX8OdS7gnK/6KfdTTRjnFdEpJDSDrLaoW8G0Qpz7C0yJfQd5v9LezX6jnON5vexze5nOZnM/ibWSdWL6bvYdrDAtsMZF2uepHHjwk+Q+xm1zWkAc7dxXIPP8/7MdrnP9eCJHcjYHoa/KxKE20KgzH5GnmDbxXFKBLUuhewurkNoSdQzUSPLgy6Km7gu0QVRVeI8t+VTdVz4Jd5XGhADinD25mgNkRgHEQzwmvesYz3XE7k12NpF9e5LVb4UXpzXOPkA6iNcs0JWHrxg5FGFzFa+E9W3UaRtPU5xsus8UJP1z6/jHJJX5T1bbCHzjJwUO0F9meMN7ixCT8S5ZiI9Gckzi4D/zjW7ZH1yXAPltdXrVvS1ky5fL7UCWutvAPjG69W+hYXF64vXjTj8feCFNUobG4hdDiKyTI4RnyJ5m7+Jn8E5jfQFcgKj89alIl5iUmN5p+hReV5fT4mVtwjM3kYJoBWRa2Kk8OFM0B+DoaaJq0YPUyiKcas5wOsjYgyKZDSaF0WKEUt4Yw25TfpICHXxEGhpvjwobRaCaIik4b2YBgBkN/PUwJcjmLlddMcU+xv7qjFouVh+kKJD+AR18WCBa5Df0UDsEjvqusL5zd8sc7kQ9D0sy8Jho+e4FrkNQESkq0aC9xtjcM9RYOL9ogf3k9Nl5uMyKWD6Tvbt22vEWJmaaGL6Lk4+Qnsmek/ymYWX69AuJaOqMEMjiYW+mga4HL6hd+Y2Lmwor3yjptHlcxu5LqkLni8x1AY4v97H+P/srW2vVLAihus9YndpxRE+K+9Hlccqb6Ud6/Y113BqkfadTT2cxMPXaFD69c2P4uuLu9lmnvfXZFkGXgAWRcoq7hYxTySI+bjjr0fzMCWG2//BYQDA4d/d60twsXmRInex7eaE69uDgvNs2xHv1qbfrWD+TnZuJOjMDvE2tYCFm8TgPWFsKVyfvkMOtMM2jAfr1WDDpy0sLDpiVUgOTk0hdjmIRpdGVjwKxs1lfP7BvEJmG4dbWUvOmjhHaprZ10TiotgMho3kwfuqfQq9J0k1Z24Tb8VZUtx6FzD4FI8t7BcdV8ilW1OIzor7T7Ef3+aw2EJpmP0ZHTm4IJS9rlHezPF1HSfnMFbr2DUX8/dTQex9hgcb787y83Lad+RHroptJMExeSEg/V0uRGG9WP/FGh2dCKKeFunA+MBDJkgBCL6P/rGeL9ONnbmRaxGec1FYK65MaashXLC0RkHVuBDpz1PZ9XaQk4czbT95eQPnuZzm/41E0PcWhEWauPou/h/IRX1OFRSvQyvBsdR6An48hFl/46UKlAO+RDN3CwdYFSmhMu4heYbrHp3l82h9XMo5nu1DdIpjLtPxguQEP5d3t3wbw/C9MwAAVeeab47P48f7DwIADpbWAwByNUpr/+qxH8PNuy5yzfZR7KovUVKq9gbRinFcaoltrXla7FdKY3kLx2LeoW8eprsktFX567G4j59K3LJuDUgfZFvhvHgyerhAlz7S579XqQtcn3q/9L+1AszGpG9ZV3FpVvqV78mrp/GaWBXEARoIVAEvrPz4BOM/NgtX2FGHEv8sRFxMiGssknHh1vm9FeLilYfFFZrwsLxZDITi8kmf5bW1tEIzxra6xVxqXKn5t5fgTPJXYH6IxuAze5uLeq8YuyY5pmbMxE44QHWl770hRtJgGQhP8on2P78MAFgAN0vKbgVCOd5XHWbb1XUcTPpQGFVKuqgPi9uxKcSlywP6SXBaTel3iT+YQBUI/wFleJXk+LrOirsyCQQkwCYxJeuRMm8SoORXatxl1weZmR9+c55zj93AuXinenzikLmV44xcphqTuKYRqLIfI/7Gxd1Z3lPBwNd5nTHGtiI8px34xtjSfsaPbPhjEem7g5g7IKpJTmJETpAIus32u1OSuIWIxC2ohvKDmSZmegEAv3PgIX5+6iP4zKgwiiSfQ/oQ19q5rYJ7es5x/Vr0r+YKfLi57S2kTpl4EfY78+Okhl4mDB0VV/BBIWaiJkSWPVQkvqX7BD8Xb2e/5aEgKlv4bEPCMFLnOZdaj/Lf59IaibU5Z97FJELiVq0MCqMwwW/BtgG0vl2irV4FVq2wsLDoiNUhOYAcInFVIyv1jmsiKvcfJeUrrg34RiAlVsC5W4XChzyseVSMaWdEbM4K9+0NotIjhiiROBb3Sp+uRs+JlW7DZXGvpb4ZR+UVAaUh47Jb0vAk+KlGxo/0GeFcYxrpk1zW3BaOr3sDA1l+buNzeC67EQDwYoBGrt4TJoBF+aJfIM+2u4UTldYALZFMopclQk6ki65zAQROR2R84vocFAmkD77V0K2ZEGfpr1chuDvL9XCE1ZkI0rVVBC8awyePGdUlNu8hdZKSwuWfoLxee4msKJHTWLyRc3ZFtal3i1RSdJAXjhVeFkms38QuA9kt4laV0GrtiCQW89Aq81z0KMd06cf5jHuOOn5QWXt+8hxuqqBWj5rmAbTDvd2qQqNHDKVhfv5fp7nLXCvUDvnuekkMtSK1uYEWvjpLg+TlBUocyacpObh17Yv5VQnk6k0xXHcxE0biDE8u3yjGcImmLN5WhXuJ4yzskyjIC8aArn0DcnkDzy2Ja9qttI3hySvyTKX/0phGKLtSenWyItWktS8FRkSaeTVYycHCwqIjVoXk4EU13T/HIghsIKvq+QIp8uJuSbyKNeFJkpQJN22sF0p7Oor8Ora18d00GE3+FeNGC+uAmBgWjYtJSxitG2uiIVS7eh8V1K5v0s5QHlSILK3U6SIUAJDd1g49DWV5LLdV7BmnFPqOcA6/+EuPAQCu1shl/vPhB7FzjAawj7zrSQDA7IMM6Hrk1A6sHaG/azbLYzmJ+x/tzWJTFw2L33l6DwDg5t2c50vY6OcjFGQN+o+Qc5UbDhIzZBP5tWJQzBpJRaN+ihKDWzdBTLy/7EVQ75XgGZFijE4/e5vC8hZKDH6wVUoMhOs9DDxBbjt/H59b/1FxGVY8uPdwscrHKW410rwvcj6K8W9z/S+/V6yiAtVSvhHPc9lW+kQ7Gc/YgUyoutHDw+ei/n1dl3nOuF7dikLwMsfphfkZEu5b2N8C4hQdstL22BfFdX5LCSGXY/7VXd8FAPzXM+8BwMC4cI7rbvInsse5TpGi8oPKqr3C+UXCSX0rioWbeV//YyIVigG1muBzAoDwFM/57kcFJM7wvtIak7vDU7GZtu3O2ICM3Ux57WS8xskuvBas5GBhYdERq2LfiujwmF7/s59Eedjzdfdqr9EleU3qcgu1JGnZ4l2kwpv+jFT8ws+6cMVCn5hs6/4AkLyi0KLa5rt1UhO8b+5mB+PfpvQxf6NY1afaLs3SMPsrjfP6/pdMpls7ocWEDjduJOdrNV0/UMm45YzrNdpTQSTE75XDlCbe8u6X2KZ2cK2cZn/NleG3APChNdxC8w/P3wUACAfJ3f5yx2fwjs/+UwBAs4tjH3yaY1rarfwkM5MmbSQCuBrpI+0gMKBtxfaC2l8so58a91fv6TquvpXcr+e49PMWSmRDXwlj6cepZ7tHyJ7qXW3pInUqsKLNqiSYdl1sv4PGbVkaE5f0pOOvo3kOJiu31qf9cGmTxRl7ipJfrRuoimsvIVmnRievdWs/fT8666w451aBwhYTUi+u3n6y65/adBBBMUh85hzDp/VLlL7q3RrNHj7b0OxK13d8ykFsjvOZvUsS/F4UN+uwQjgjz8YE7ol00Iq2pR2TSeyHh494SF7i2I0LujzOsfWPL2NZpBYj8TUT4r3r8jD2bX6/9gDvn/i1f3JIa70fr8CqIA6JnjG9+4FfQ6XPQeZmsaC0hDgkxGV4NIrSWj7sNZTokP0pvhDBR1MwNQZMTYT8JjEKbVnC4iW+9fFJPpDiOrYZWnJ9d55ZWBMvESi3XVLxGYnMk2hGzwWi8+2cAYAGKUDqTkjqpEkfjk8Z0bq91nm60P0fNAAMPC/3DYkB7j6qEs7nen0Xn/nc9CG61K7mu1F6mpbTtV9m3P/pT5Lw9I1mUX5WjIY38I0LHxf/d6D94wlIzQzzAo4/3EJhTAxmkltrXMu1bs+XN9OnJA5EftCNpEZrjcRxSJp0rVuuGdRIned9raBcb5Jqw0B0zry8ojpc4NiKwy7ym7lG4VE+757P89egFbAgEbRmZY0KEagA+Q2yZvIjHf0uH9bEBxz0PyfqkmTjTj0gKshwAc53qPYYI/XHf5VZAMeKozA4OMuFqR3ku9V7soXCiGRcvmsWAJB5doh9FP7nGg/RBelvoorL7+NaRRbNj53nek9ozL+X6+kt8pq+wyYHRWNxn6i2YuA1KkRkSfuuZFQ5JkOM6mnlE1tDsF76i87EwaoVFhYWHbE6JIfuMb3ngV9DM6x8ami4jAnM6brSQiRD7j5zK6loZUSkiqkAus9Jlt3QykCUUK7N1Q3hb21k8EfgfAyJa2J0HBHOKJF5o49qNGKixuyRMVzi/dph/QWgbQgzGW6x6XbBGFOPwXDBmdsVui6yzcI6cdUleS55PoBQTnIULpHqX/oJMSLOBXwXnzdC0ehv7/oDAMB/nHkrjnzxBo5Fgq2qw2xThzw/7yJ1WfJKhFMu7Qj4KptveJU1iCxptMJiVNtDbhtYoiQRnVcobBY34IK7Yg2U1845MAjT64nKIJAS9cEEhRXX8tOtKl8CM1WQSiY7NqjhhUW6CvIzdok6QN+xJgpj4u4dbbfFT/ic1BgtjSE7fbGO5S2SGyPsMb9HgrauhFBbz4WJJrnWAcmgvWPkMr59fhsAIP4CJ2HyGpwm6ygAwMJek13JtssDCukLUgBm3LhsZWyD7RoRZu2My70Z134tiVcWytFjFWCG4rKR4MwzK2zw/Nwkk7EbvdaWiE19k8w+vidXf/GfWcnBwsLi+8eqcGW2IkB2kwu3CjQlZDUggS9Gcljc66D7tASlbCSVD0g+Q63Xw9IN5CCNFEltYoL3h7Ma+sOSEneY+rdzhawunAHKb5OKTIeoADu9bDt+sYJz/1yqEU2RQtdFHy5ubSC4KJxU3KSO2Ejymz0o+W706HIfxzL8rIeFPSY8nEOKzps6Eh4aCSNVkKulj0nb71gEvko7QlYyUT/68s9x7rUAEmW2WTU5BBe4FpV+x68zOHM31yV1mv2tebKE+Zupu5us0Wq/hKBHHFSH+ByciNhiJrnW2b0NuDEeC14S1/JVw4k8OA0x8AmHCwgnT17Rfp5MfrNISJIZubynBR1ZWRCx50U2UE86gGQRViWYzQRmLe4K+POLT/Jc33HaVq7dF/O57uCL/JLbwHWZ+fkaGsJJjZ5v3iW3CqS6aVTNZSkd/Ptb/xYA8OfTtyH+otTcEOmnIIFug88olPvZ1sBhsWO8n+JQvKuKmTEaSh0xNhpjbjPuILwk75XUnxh9mP/P73fa5fMku9K4RIvDUeTfwnEuxbmwMQlHD5QcP5PVqa/8ide72oF7Jtju1WAlBwsLi45YFZKDF/VQ3lWBVwoiIdyokVhpC+m6AMzfInr9BXJWP6ilpnxXFPpEX3yJSl338SzO3EcDREC4mbEd5O6pIPQyJYbyenE3nqa0cOYfBxG6Qsrac0okALHKq6CHZlx0yE1sKyG1Ab2+OoKT7HvxVpNqKNb5SwF4ps7gK0w9gaLjh8FmJdnJeDt6Pt+D+bdxXjvGaAk/eXkN5/diyC8ea8Kvje2illZY8yQt/MvzxsIvCTlDkf+pOrYeYB/hK1HfsxOnAwQS9Y3AcgA6K0FBJqSaQ0LyggtXPDJ5uX74nVS8z10a9jM9gxLaazJao9dceEEpGiw1Jnz7yYCGs1aqBl/mHCSmDNE5jbhY3EtSaPbCL/Izcq49ryXJKE1d5LWLA3Ffotn8NgaTLf3uOgDA3H4HG9JZAMCHN9B9/J8vPMg1e7wfuK7YLNCuBJbZwbBzAChdJScPzPA9KKGt85tw8iXJAo5PuKgM8fu6h7h2U3dzvIMveZj7SbbZmqXkYSpzAUAkwne22eA7a6S18DLgtMTDdoDPVNekAHMm4Ht2+g/x2yV0xqogDmg4wEIYiVkHQb84ibivpKp0YFojMi8TFLEyt5uLM/BUwC8Y6knmZi0tL8tPdWPTn9LKNXMHn+zP/BpdU392/jZURCQbeVjUGPHL13odP8W3Km2VR8Q1eSqM4hZRP4RQ+W7KYhD1ARG7u/hg1vwFX5KpuzSaYoB0pZirX6reaxu0WuKuTF5mv3P3N+CKXz4gsnJgjv3mNwBxKd0bmzKxIZLPMNbA0g38QRljo3k53aEKnHM8Z4hs76OmZJr2i7qGxUhmytkFywrxWTGw3mGMvyY2Rfup8oZQzT00znb6dTunQgyEC5J9mDgX9NO/mwkOJilvbGRBoRQmpTAva1XWtxVxsSxq2tCTEtfyMucQzmgsHuCYjevbILLgorSebZx6htbj5n289t++5Yv4rlQLzomltfIEXcWVIQ/xa3wmxVFJaZcM37kHmlBS3MW4FpPXeG6m2/Wfs4l4RFIK8V5VqN1EXaM8wOcRETUjs81F8rs8ZvJ8YvNt4hv7aprrlzIRvHJuvfLjhGKnIrJm7ejNdun913ZGWLXCwsKiI1aF5KA85iow0ksCOzISmCMBLEu7FBopybQUA5oxCla7FVppMf5caOfJA0BxXGFxLzlPTAyEf/S5dwKQgKWtlCrKIgoa91qgqJDdLpKCRF0GJdqsPOwhPM1+ymvIRespjjd+2UVAAk8isrfBguSHOE3tl+3yNzARiTmkte+GDYirqf993BcoPzmAX9zDXIw/P79ybyClgf4jFD3nbuYcTFZmZCKEzF6pCXHSlFMTY+DDcVRkHcs7eH8WUlMh3s5WzW5auR9IaVSjGZXCKhETwGUCwhSyN4gbVQyM9bQUrunxoJWUQ5P6D64EEJX3VhCWLFCjruVFXfOCGm4/x+eekwAucEyxKQVMS16JBGvVxGhZHgICUjIws22lezU6r9FIsI2Bl4W738FrXixsRK218mdRkexR1VKo3UaxtTFFjh4SyWrw0QBKI/LOSjEcY0wfeDqAMuOhEMqJsfgWzmnmgRacGc7L7MthNsAprmshKe+eeV+MypmcUKiKNGECAI17few7FczcLkZ3yaVJixu52q39ADwTbPdqsJKDhYVFR6wKyQEAoBW0q/0KTrpHyKfocb1HlF9m3VBRk2df2NhCSFyCxhhn7ASp89oP9kieFz1fmqkONhE5RwpbklyMppRmS590UJc2+o7RdlDtk7J0AQexWQlikuKgxqVZ2NQuQZaVMmhmkxo1XgIOiUtLbJXXb3hTHuZ3o3NePkK/5fiuWfzhy3fzupMcb1T0xsKmJi6/TyoV1c0c2C/dlqY8Ga833DS20EJeXHvJQzwZWRZbwM3aD0YypcSMq1Y1gdQlDr4gpdRNHQPVbLcfPy35D3J/4rLrhxC7X6MvrSIhvtGzUT9k2FTDqqxhH70HXTRjUphWXLWmCG1lQCOyKLkjYowLbiRnb51N+lmj0TmRkGRjn+rmGhLHzJ6DxKZ9lNJu67qA/3bpXgBApsZ+k5fFhjDdROMIJYaiZEIaA2+o5GHRbEbmmE+xBSwobmwDICaVwyIvsZ1Ysx1EFpONbqKLEnhWCSO3sR0QBQDBkvLX1RQUzuwUCeWQkdaCfn6Gsd0tSxrC2s87KA+I1DtsbQ4WFhY/AFaF5KBdoJ72EJ1z/ACipujGEA5Z7Ve+e3PgFvrOgp+jIlfLun64bWVIrOvC2aMZD3FxSea3y2Y2EgocTNeQflI8EaKnx6bNHogaow8zFXHqQbL34gZJ2MooVAZMmLdxG8pcgtovWtuMSqly2QvRm4ijJHUhg5mVdHnp5pbvxjXbsLW62d+/3fgV/EblJwAABfHK9H2D9gEvEGhbotdLZqgUF23G4ZP/vIQ89x2U7fs2BtCK8j5TR3Nho3gyyg6iC2JNn5AMQ5EOEjdkUDsliUl1w9Xk3KT2pQNTCNdkhRZ21pF6mY0s3cK2TW3G/BYP0Rlxb0rinHGhpCZqmJGtBYybM7dNAuWKrq9TV0TqCr3ExavvKcOZXKl3F0wy3hNh5CSAqLCNY/hINyWHllbYnGbC26GvMiy9vFs2DUoHEL+D5yo5CSCbk8K/61w/sM1prMyuVB43tgGA2du9FeuSOg+UJLBtToLlhp7nmpdGlL/Xpb+ZsPwegmUGuQFA1wVeM/k2sVlUHL82x5rH+bmwj/OcvgtwTU2I7yEarArioFpAeNlBeVMdSuLY3UnZ59Ds+bepBS3VfScv0xITYJg7khNtN5lJkzU/3vwmhfRpIz7JoksN/9RTcX8PBSPahaWMWm4rEJcfmYm0S1wWQ9yBPMKP8yXMbpOHJw8ttOT6bkDjWmxuooze/Z0ICuvFMCh7Sg49Jq7Xqba7q7yb1wevcQ0+8eLPwJuSaM2GESvNfIGk5Gtko7wmfs0QOKAkGajBZYnDkCfuheG75coH+LaoefbXfQqYfFCK1K4lwfEuUh1yD/aisFvcqiLimhwXtxJAcQdVhdGvGiIkBs25oL+OA0+y7aU97bRsE9G3cAfbSh3nNbMHwojewgjXzLU0gPZzqHe1ywr6xuIR2Z8hVULzKa5H7f1ZAIBekqrjSRfBY/w+9sA1AMCxHH+he+NX8MJjOzmGafPeSBTr+gZSf84gi7ik85u9Jhb3Ako04dj0yliP1qiHVljiRiTfwqSkV/scXx00xs2lne2Yj+w+IaSHV+aC6AD8zX9NP3617XUNJI9KKcWbJVJYfh/1Lu0X7I3P4DVh1QoLC4uOWBWSg1YsOTb+Nw4y21dGP5qty8MZF93PivgrmZtNKQ9fKbcNMFpKiTV3ym7EhxPI7jD5FhJJGDT5DRpLNwmnEXHbBI9oaEzfI3s03sTInqlDjEpsFMLQInGseUqCYG4W99qM8gvLFtetnOfyDsAbMKXGZQt4UTkSUx6ym/m9t5tj/9iN3wEA/OWVAwh+PryiH3/n8B0eahtFXQpyLoNfMlmgYQSksGjPCWNs5FhCWYWkuHaDj5HDLt3C9Wwkg9CSATn0Z5QmlreZbEeNYsS0wU8vIC7YMvxCqjnJTDWSxLq/Bq7dJ/kLUhvC812h7Z2nUifEeCunRh8t4FKa3HroyMrs2vpIHe4i+6v18YaBF9h2+NvduCY1GnofSrMPkeicVnvPjVt6JwAAdyRZH+OeSBb/hxTwWbhTtgGQYrkqF/TfPbNHRWGvBHIdD/vqoCmaM3CQ19zwT4/hiexeAMDb38XiPg9fYqBVI5/0M0mN6ubv2RIF0qKKmcLAJuO3FWqvn4k4NUbuxPkglkS6M0VtTBRrK6xQGjXqpHVlWlhY/ABYFZIDFMNtl7cEUNolRWDFMOkIB/OqLvRZ2URF3IZB2cSj1t02OpVkJ2V1kTpldF771y/tNuG0oqNvVL4RsMgoX7Qk9yG8JY/gYzRETvSLjUOGO/idIJyWZN5JIdW47LFZHNfY8BA5zsJuMVZFSP17zgFLSckdkTDmUIFcN7fBQUx2UYoEyAIeXaRRpfaVAdTWrFwyE66dPOeiKoYs46LNGV03opG6wO/pczTA5dclZQ01FqTcXvoIx2d2j4rPtuBKHkR2k9SDEPfl8uYANn2Gu0rN3cN1MaXnlBfA2HdoL5l8kNJIjxjXMjuU72qbvV35Y+eNQEXmkNvOfpLnOZmLPxH3DX0LUnHA6Ojpl8K+q9W4hA33nbrHhSfZo8vb2dbwM+x/YU8Aax/idR9/63MAgC/kbgIAPJaLob6Ha/Xfbv5rAMD5Gg3fv/f42/Cz72TR4L84y23ga9Pi2txVw5P3/y4A4A8ztwEA/uo5frrLQ7jn7UcAAM/MkPXvGmbM+5HWCMIHpajxXqkzcphr13uqgeav8J1Yeo5jGHyx/RzgSGWzTXyOSvy5qtW2dxkbVajAuVeHNdIn2r+b14KVHCwsLDpiVVSCig6N6U0f/SQacSA2Y8JfJUlnM9lGYDaE4adJ/RZ3ibV6Fw0N0UMx3/o78CLbnJX6Bf3PuygNi1V9q+j7V6i/x2Y0lm+SLc+k2KrRG5W+bgs4+ew5TQo9e2vQDxIy4c+5XUK9aw5iYjU21XvKa0R/f74dPJO+KIFEoxKYpRRi76ViOJeRQUxT8ui6oPy6Baamo/GudF1sB9sYy3ZEai80tlaQeH5lxl5UvALZre3S8jDZmGfl2gBQG5L9SM+L5CBFeQtjrl8QtTiuVqyBF2qHL5sqWCXJdg0tBDBwSKRAqSFZk/oYmbtr6HqJczU2Jk+qdSWejSG3n+NLHpHNXqSKUjOu/d21+5+XilDyrBtd2rdtxKfEPiMSVfJK29sTuZ2ekM2yo3a2FsWkbP55zzgzNj/e9xQA4KNf+FV8/cP/EQCwJUiJ4c5jH2S/tRA88eM+tO+PAQDvf/kXAAC3rpnAd85RCgxJ0J3aTcNCdSrhb3OQvMzx5SR0PHWRwWoAMHuL2HXk2v6jTdT+Mf2VoT+ikcP5x5ToZl4aRr1PNj06vbKIsFvV/jYKpubJif/yydVbYDY6NKY3/vQnUU8B9T4uRvdRiYu/jSJ64nTYNzpm5YfoyP4CoXNR35VoIslMCrdqAekz/F77QJbnnqE8Fcq3IwH9jXRFrUidDPhpw8PPsOOJ9/LBtsLaLxJSFUNYeNnESWgkJzmuq+/gsdCySRnWfvy8iYtoyB6Wv/7er+GpZVawfeGcBL9XeN/odxSWtvP7+r+kP/7sr9D1Fs44foVn465qdLV/oMY9an7Aplirl2gCdSl0IqpN73Exeo07PkE0hmHTTmQRKN3FxuJP8QfSeitzJaqn0m1jo/jnTXp8cVwhciNf5uwMiV/ighQnqbCIDAD0PcdjGTGo9RxTyNwnvjcpsmoMyk7F8aNJzQ5pXTsphudP9vpFgLuuyA/stnaOhb9hssRqeON8Ebymg0QXvxcynN9P7OPGun93fhfuGOcv+LvnmOQw2M9YmGI1jPIE5xWdE1VTXJrzt7fgSPEiE92ppBRc+rzn7zthXJOtqBkk438AYIBDQHxGUrAdhYs/Je5RyScyDKAy0vT3AjWMyriKC+vaJQ3DS+z3zL/vTBysWmFhYdERq8Ig6YU1CpubCBRcRCWAJ7tdOLK4/OopjZYUNUlclPJw6YB/f+qSiVQUQ9oN5N4q0kK+IbkHL0lM/0Y511AISi7AwItse+4BiSQ83sSlD7KfqXsoQnSfFJXnfQVUPBr2IvOSXyDBV82owtwBEcXPQs7xszyo/O+1LeROWnJHDufX4v19LwMAJv6AXGlOoukCZQ9Ok+uydNfIirULFtsl3g23N6JncVcNRjAckN2UnAFyYTUdwZqnTUFUXr8sAUX14bqfD2K4dHgdcxYKl7qQFInBuOwq8zSopadUuxy/ZBOaSMnwcrtQa3RKgslupWgdeLwLrmQrmmhN1WoHEqWfkpoEfRLIk5K8loj23alGEitVwrJmyuek8/ulyIysCzy0i8/KnqOQYijJ42GU14j7UK75m0doWGwmWjjzRQZIpXul+PA6JlQ4jfaPKX1echyk7N/gUw4qAybLmNeU10qG8VIA/UcoHc/dxH5HHue7sbwtAi/I+5Y+QPd27gjXujLkYWyclXiWLjE81Eh3g087KKzlfSVxc6sWFyM5oVFYJ+rWjM2tsLCw+AGwKmwO8f4xve19v4HiO4poXSBlNNmRjmRnRo9EURojRTZFYI2xza1rP3jJBIL4ZcaGmojJXpJqP/VD46IEuEENT/LDGNS029ZLjS5fGeSYui4on0ubMG2T99FMtxCdfEVRzx7el7zswHuQ+nleQnkHh7MAgGwxirvWUkH/zsntcqPYMa4G/MCY3BapJFU1Jc41tuyiHWLuyyxGUb+f86xMJ/wNa1pjUmb9ClnX+oeKWLiRa23yIUw9gEBF+5WfBn6NY5r8NK1kmd0a6dNiLO4xbjIZbhKobGc/0TNiTJWS+E4LmHknuVjXIXL3kFT9Wryvjr7H25WtOD9pO9uuwJU2kpgYJKGBpVvFcCrBV3XZIzKUVSiKhGhsG8YOFc5oXwpJSGFak/+SmNRYuFcCtz4v8/x1PrOlowN+rQVTRj59SVyEKQcNsSeVb6NNprnINVjzODB3y0q7gm978IBmbKUR19iQ9KYyhv+Kk83s4BxKIyJRZxw/UMy4ek1aQLDg+G2arRAC5fbv3MzV2ERe/qPftDYHCwuL7x+rQnKIrBnTaz/xSYQKQF72KTTBSYaT97+sURPKbCi0cTt2TWhUu1dmCBrKGSy0dc8g1Tb/PqfZ3pbM7FBtqHct3a4VYDYuKQ+rFe0AbZeiKZYan/aQX79y45JuqTPY9dnnce5PSKCTveQuAYfnfn3Lo/g3j9MtFhD9e/xhNjpzR5ibnwAYeE7Cu8V9WU8qX+rJbZNdq+d4//BzNUx8XEJ/5zgYf3/LfRoJqVHZkMpFRgLIb/T8PSSN96froqxdSqE4LjUvpLS8ST7zop5vT6h1tzkcwLqfdQm6qQ6T23cf4bXZW+qInpXFkrkYaaayqYZIgusQepp2HiNJ9B7T/n6m7RofYvPYnEXXZ/igZ2433FPWrLvlZ/96shWCK0VzU+eVnyVp3huTfVobaCIyK/MTF6ojEsDIEx5y63kuf6MUOT4Xbs9Fhmm8D/07mN1Z+s6gv9dlTZ6xCXlOXAj4nqeY1Ak19UUrg9r3CCUnVm4HEMppf4tCk+VqbF19xxrIbpZqaRmO5dW2w1sVxCG8blQP/atfReJc0FcHovvpkiq/zLh65bV/ZEtS3CImRTwqA9qv/W9eQFOENlDWWN4jRVekTn/8mvzAVFsNMca1tmFR+25N88KarLfK1ir6v8MHn18vJ+Xhp2+Zw9IRGqluvfckAODpg1QTnJqCkt2NWlIObf8ehjB2h8p44S/3AQASUxxvWYxYtXTbJVnrlchKcZ1WpSgKJy0pyc9JRmVFY3mHrIvZQFeuGfuKA/xjvqDXznG8UUlXD1SA0kibuAJ0YQIsE2ciMY1qhfUkdLFnEu11lB26TIRm95kypu8SVeol/nim7+QaJq9oLD4gLroyGzf7NADtOIX5OznXocflOc7UceWdbMMUdDEEuZ7S/iaypkp2K8a5xyZdXw1MXuH1xhDaCgO9J0i85m5emefRNeEht0GKCF2SYsAhY0C9buNkUTXLIuZHZl1/FzLzDpmI3lZI+QblsKSk5yU3JzLn+HEp0SWTwyNZlksKQy/RcGnK9mW2cbz5nQ2E5njMPCsTMZnb3vSLNjfWk3JM/PRvW7XCwsLi+8eqcGUqVyOYrMNpBX1OqI+QBZlS3q2oxuztwo3OGpGX9wdKCp6JmhPO33NSMiNHXV/kNDn/xVsZ1DT0N2F/d+UNXyRlz22UgiQHWkiclR22RFT1oyKnw76KYQJKEsKBCo8PQu9h+y88SrcXxskVex4N+1yotIbU+yBo6Nux86pftqsZM9uze36/jTTHt/YrwkVvFFE56yKcbXMv3s/PWreCJyJqWDiJkajy40BxiYZZI8YazucF4EdkGlUjKAat7jMslAsASzeIAfNEwr/flL43NSYSU1QJMttjKG/nOlzp4xpHZU+MzG6N0IRIAPMmiIrnGkkgt4nHBp4RF6gYFifeE8LGL1JqmXh3XOZn9BL4+1PO3C2qxoREUe6uInypXYafa9x+xiafpDq2cp/QZlT5KqUpDpS+0JT1DCB3g+SqHOX9ISkw3Ixr/x00EoMp5++5bdWtMiYqToLtpJ6PoChFa80emyY/JL82gMkHOIfaGNc4KFm2gaWAH0xosnJDBZ4beL5db8RExL4arORgYWHREavC5hAZHdOjv/wbb/YwfmRodnnf+6IfEQJ553/qzxR5tSCaSVkf9drXvZ5Yzc/k4m9ZV6aFhcXfA5Y4WFhYdIQlDhYWFh1hiYOFhUVHWOJgYWHREZY4WFhYdIQlDhYWFh1hiYOFhUVHWOJgYWHREZY4WFhYdIQlDhYWFh1hiYOFhUVHWOJgYWHREZY4WFhYdIQlDhYWFh1hiYOFhUVHWOJgYWHREZY4WFhYdIQlDhYWFh1hiYOFhUVHWOJgYWHREZY4WFhYdIQlDhYWFh1hiYOFhUVHWOJgYWHREZY4WFhYdIQlDhYWFh1hiYOFhUVHWOJgYWHREZY4WFhYdIQlDhYWFh1hiYOFhUVHWOJgYWHREZY4WFhYdIQlDhYWFh0R+GFuVkpNACgAaAFoaq33K6V6APw1gHUAJgD8hNZ6+YcbpoWFxRuNH4XkcJ/Weq/Wer/8/y8APKq13gzgUfnfwsLifzG8HmrF+wB8Rr5/BsD7X4c+LCwsXmf8sMRBA/i2UuqQUuoTcmxQaz0DAPI58EP2YWFh8Sbgh7I5ALhDaz2tlBoA8IhS6sz3e6MQk08AQCDd/UMOw8LC4keNH0py0FpPy+c8gL8FcADAnFJqGADkc/5V7v0jrfV+rfV+Nx7/YYZhYWHxOuAHJg5KqbhSKmm+A3grgBMAvgLgY3LZxwD83Q87SAsLizceP4xaMQjgb5VSpp3Paq2/pZR6CcAXlFI/D+AqgA/98MO0sLB4o/EDEwet9SUAezocXwLwwA8zKAsLizcfNkLSwsKiIyxxsLCw6AhLHCwsLDrCEgcLC4uO+GGDoH4k0C7Q7PIQmXURLPBYI8HP2rYKAKDvkQhqacVjaZ5rJjQAQDUBpaUxTz63lPjvRBzKHLuuPwAYfNFDYZT/rP/ARQDA8UPreV93A4G5EABg3derAIB6KggAmLs5gACHhcoQGx//ZhMAkNkWgrfMNoubGgCA2ATvCy9rdF3hdfM38ZhblXn2aMRmOb/ybRy7uhgDAAR35IEXUgCA0ka2Gb/I+2OzGos3s43ew6T1zSjb8YJA19UWACCUZb/LW0JyTsFpcNGyt9XYQJFt9h50EKjy3OJethWbls8FD7n17OeV61re0EBolq9UawMnFj8Y5ZgiQCPJNrsu8frSiDzP3hbcCtvUAV7T6uK4gwsBNHr4PXGBbVf7eU3fUY3MDrYRv8Y2A+9ZBADkDvcheUWmNS48UPPaetpD10UeK47Ji+Pws/uE8t+lhXvqAID0wTDHO6bRdUEmy6bQCvNLtQeILvDG3B2ce/BSBADQ/7KHwiivy+/nufTzbLMRV2hKmE9tgPPUEX72Ph9EUe5rJLnYsWmOO32phan38pl2P8tnuryT/ScnHP93ULiR/SWOcSyF7XUkzoZkHcyPpjNWBXEIlIDeQw5q3UBdgiVDWX42A5yl09D+g2jFZFLykbgK4J0ZAID6eg8AYHmcU+u6phCoyIP/CN+gy4dHAQD5tS6Ka/kgzn9rIwAgKm1WVBAp0gtceQcX1q2xfx3Q/vfhp+QHtpELDgcoSZsj3+aDnL2V1zhNhel7OK7kZR4rrJP5bs2j4nXxn4mYrAv7iHyjC9ltXIe1EjUy+SD/r/YDqqlWtJW4yrZraYWZ23kuWODLGJ/iObeukdktL9PhiH8MADIPVOFORlasceEmvmShniLSf8k1njvAc8kJzjMyFUT6HMeVL5MoVAaEgHvwf6yu0KJWhOfCiy7Wf3YGAFDe2sf+RrlOmVsacEJcz+IOth29yLWee2sdoatC7EKcZ+5wn/ShoDy2H53nuexO/pgQ0FCeECOX1ySu8P/Fu2tIHeZaDTzGtpfeUQYAOFeiaMSFgN/CY9GDfFbV9TU4zfCKNQvuygEAprqTSJ+UdciwTfMu11PaJzQ9RziGzC7+n7+/jMhBUo6GvBrFzWQO9e4AdJVMqDzMBgyjqfZpuGUeG/k7EvySPIeNf+VhaYeMr4jXhFUrLCwsOmJVSA7NpMbSPXW4cyF4QVI4I8Z610iZF27S6D7FcyESZOTJ7BHOazQfIjdbvJXcQYgxtAtk9pDjFJ8e4/0kvhTlp0h9jYgVLPDO/oNAYa2IrFO8vjwkXOYqkLzGRrIbSJmNGB7JamjFNkvDvK9HuIZWGk5duPxaHht+llwxN51CLN/m+AAQWeb/y9uBVoJzULwcjkguqQtAeZDfw7Iu3Wep89RTUUQuitjbK+sh7CD/YAnxl6i7VQZFmqjKmlddrH+Y7H3iPeR03c+QKy7uDcB9L1lU7GU+m/wWGVRXAwtxXmdUlvRZWetuhSYv91XG3mOckxcAFu8cAgBkdso45T2InwshdPsS17PCtquDXF+VDSKc4ZhDOVFHhCNDA7lN/Bqglob+5125BlDaPG8uSHmNPNuTYZTvIEttHeZAWwU+Y2ddGdGTlIjK10RdEpUgdj6MyhYRifJcM32C14yeafpqRf/WBQBA/Vw/709oOINcz9R3Ob5Kv8yzFfWfd2TeqHL8DFTgv2epCyKtbeC56kALTo3nFm/gZ3WcKlJ+cwh9h3l9btNrywZWcrCwsOiIVSE5qJpC5EIYAy83sPwLpNq1eRof4tO8xgsonxPXe0j5+g/x/7nbNIae4Xc3T0oZWia1b0WAQIk00Ohk/UfI9afuDaC0lRQ1OCsSgHCZwloHDk8hssz+SkL9l3d7yEhpm4RwRu3wXGaXBx2g9BJe4PJmd5D8p08F0IrKOIXzz9/Ia0aerGLyAXKMwYO8f/oOzsVpAalTAblexkRmiuIYEJsh12uKPnz17bQXJCc06kkeq4ywzXq32AdeTvgcPFgUPXqc17jLQUz8AhfLE+mlMsD7UqcDKI2IRVeYdHiB/8ePOCit4UFjsIVw6Eq/htPgOd8usY5thgptzm8QWpZxLmk0HqPY48p46+Mi+kVa0GIbqXeJPUpMP82kRnInFykzQ2Ou8viMyxsaCGQCMnde7wmHbt5SgDfBjsprOM413xHpYjCOao9MS6yWYalxVri9jOSLFI16T/HFKQ2JzekWF420Wdskx7lHDK7LLkY/zbEs7eTgKyM855QdeDKfgfspvl45xR+B0wAGDsrY5VfsypoPPaOQo10d1X6xTHrysIarKIxznN1nXmFRfgWs5GBhYdERq0JycFpAKA8s7A4i9XlS+fn9QtXWCLufiUAPUqdLvET2O/sgKXRgPghPrM7dp4S7P0Aymng+ivKoeDzqpIXer9PdpZaT6HqeXCIk+r7Rlest5XOzWoptRudEJ8+4aCTZ1jd/9f8PAHjbS78IAEg9lkJ+M/trxsW6Ps1lzm/yfKt4aS05STDNOV0cDcEpsL+S6NSJq8ofW+YGnvNibDN1Styl65uop9lmeLntwgSApX0tv29jQVct8bgcyEG3hCOKJ8O4uEo7r/NWRI0NqO11SEwaD4i47jbz0uJ42yZST5nrxRU3XEPvM2x/4SZe03ZpahTpQMKaZ9hAKyj3xRwU7uQ70Pddcf8lZE5OAKURrkd8Sp5tWOxSywq5Ajmkcfsa12v8QhAlcTO3BuVgjtekvpFAOM9j0+/jNZntXIuxR0pYuCku68jbjC2n7xsRX6rwAiKt9fOz2dVCaInPK3CNYxevKuo9HoprgiuOdZ3htbkdTRT72FHsL9bwpPwuWj0tzB2QtkL+w+V6jgNhOu+w8Uv8HXghtnnxJ0IIinS8tEs6/Dw6YlUQBy9Ig5BbVb57LHGVD7u1KFYsDWCeRMETj1HsPF+25KTnG6KM2zIshqPczibiEyJC5nnftRk+RWcx6BvjmjF5oENc4K4LLrJbeH1zvfiKxWevXaAhxz6dpX4RC/NF8poafYfF6DgmL0ecfaz9WgPVXnlRmxxTJMN5LtwEhHL8nr2TD3TTf+dYrt2XQICeM3Sd5DWFtfw/mHUx9KK8qZovztQ9YrQquL7bUDXEdSf+/LHuLD44/DLHLG/l7155H9dlIeSrDOZHVxrh/15YIzkhP8Ai+wtl+eKFsxqRrLygQzyW38BrgnMh30CYOi8+/w1CeGaV/2ObuU1UlGvKX+voGaM68JpmL9c6NB9A+owhiBJDcYX9Ld3gInyc746JW6ju4yK6F6IYeEpiJnrFYNsnTCGtfMNeaEIMg2Oiht4XR2SJ1w29yH4Wd4k7cdDxCVWtm8/YGAFV2fWNvWue4rOdOyD6ZcZB/p0M7un6OhmVMbS7JQchISbLO8TgLSqct+z6LuvSyMq4k2ZCo9bfkn7isj48FyhqX510msZs3xlWrbCwsOiIVSE5qCYQWaAY39jOY8YQY4xti/s99B0kLTPGp7oEdmQ3O75hqDjOc3vewop1Jx/ahvIeUuvoYyIqFzjt6KyD6KKI68J5Rp4gq73yjijS5+TyulB54UCeC+gcB3hwmSzc/TylkfIH8lAvUjUywVqNPqoQxdE29zSSQGrCcN8AWiLCj32O47v4IXKuVlcDsUsk/Yv3c3yGm6Yuakz9GDmbGZNhla2wRqtoVBSuXfzBOQDAPx77Ln77xPs55xR9oGvuliCxqT50HWL7uR3iGhZjYnjRRXaLcM/n2Z1Ryap9Co2EuAtlqf0AqSUPc3dyrvrtFOG6/45G5+IYUB3lHFSZ93efZ79z+4MYuJtW6epn6O70znAtkpMe5m/iuByJb6r2GMlDozwkEZhGxZngc4xPAQu3cCzpEyJliVHWCwChPL8npnjNbL9RKzXKYnDN7BMJdb6tPpmgNfOeuFmRDucd34U8fyPHUO1tu85jj4kBVMZr3o162vPZd/dJfi7cKiJWC4gc59fcFl7Ud1Se+0eXsJRJSD8SpXlE1JGIi/oN7KC1JCL4q8BKDhYWFh2xOiQHzZDaxFQTjTi5QnEdKZ0xsjk1hcUDor+Km8tIF9FZIH8D9bvEGR586QUaDPR4Cz3fJfWs9rCtrvPkLoV1Hgx9LO4kRy4PmNDltruqsl4Mnyck9DWiMbiRRs393YwJ/twYFcXKTAJR8fSZENbgkhgkN7SDrAbfQi69vExl3mkCqmTcm2KXaIlOfjno54MkhaObcOb4VBjxl8mNjM6Z38O59D4dQmaviZrix0dHjwEA/vvV+/HAOEWjmPhsD2XG2Y6jUThAaWvr75IlX/ogOVF0TqM2wI7mDkicv7gM05/uRrmfx/K7RRIQKa2WdhDMSRizovHAuY9zULMRjHxLOPgvzgIAygfpsnPrQO4hGuMGz1DCqfRRMmtEFdb/XVnGxzUwEoBqtaUJg5TkRcQWWnAaXFATzDZ7C59tMKt9Q+Ls3Zxn70tiP9kM9OxmSdTKFKWeeloWXbuoD0qHIjl0H+HcN330LA69SKtteJy+09BBrkEoq30bQ0hsYi1jYNTKd086TTkmjzN5ycX8TWIvG+Dzm30LzwVO9EHs84gsiS3t4xStGyd74YmReONfi5SMzrCSg4WFRUcorV87M+uNQHR4TK//+CdRWt/0qe7Qk5III1mBXReB+DzJ5sytpOTNtGSvHXSRvZ8kdu2f8NzEu8h902fblnAToJO6KPpXWKE41tYngXYCT3GsnSU5cJgcbuoecu3xu65ia4q6+7e+TW+FsYjHr9GlBwApsVksPyhc/qV2OKw2MpvcVxrzENtIzlg7mQYA1MVWEcgGfGu3I/E/aQmZrX00A/2N3hXrubxXAm7yAey/nVFa2xPkyC9nGUJe91z85vjDnENuNwDgcIbnooEGhqNkY48+zyygyEKbj6QuieRwC/8Pi8clUGoHDo09KmHh6yX7tKb9MG9j8TdBZnAYPAYAoawj50zoukbtfVkAQKVMHXnoi/zMbHPhyHr2nuDCTP6UBKCdifpra/qri0u6FW4noDXEfmVC48cfriO7iVJE+S0SkFfi/yobROqsuMoPrMzYbIWBwnaOYfSbkkC1le9irddDdE68Pus44J6XxRsXUWhICHZykmuwvE28XFFg+Dlxq94pv4MLvK8y2PY8GYkldrntsu0+z34W9riyPpKdm2/588tt47Erv/xPD123Y52PVaFWeAGg1quRuBTw/eP5dZIizHcaxTGNiLw4oZxxwXDikayH5NNUBy7/jBjszkmceQ9Q65PYAPmxzt0jvuPLQT+Sz2Q0hrP8dFpA41a6mLJZEalnRc1oBnG1RAOk8atHZ8QNuc2D209isBQjMQld5Gd+Vx3BeROJKXMwH02F0gTFZT0qc7jA+4J5oDQqacpH+Dn7dr6IwWO9SIvfelmy7dJH2cfIT1zGvxz5BgDgd2cf5LxE97ixexqP5pnI8Nw8w+la4tLMViLYm6ba8+47GIb6xF8yL7w01k7Z1jGOwZkVNS/YdsdCc41NJuTQU44fIVkkDUJc8meyuxpInRR1UlKonZD22yws89m6i/JSrzfZsYBJhKylxZ0qxkodALCTz694Xp6fbJJQ69a+Slpcy/UwxHf+xrDv9nMOM5pRj4lRVrddn8nj7Di7TyJsE3UkJBejKCEJhnDFph20ZJxBeYdNbEhkCUhdllyH9YYAy7M+qpHdxHmZ556XyEflAYEKx9L3HV6zZPJSHI3ZH+c7ZFSImV5O2C0FEWF6B8a+zTatWmFhYfH3wqqQHJQHuBWF8hoPicsiNomYZ3IWmIvQ5ibXw2lqBEtyveS4t/aSawx355H7Mkm5iVxTUkwjWAz6Yn60dV1+PYBAQSH8BDmHJ1ysJSrHYiGOdV00whkJoOeMuCs3ArGXyOlSl3hs8WfEN1UIw62sjGKM0K6JRgIIi+ieepFUvhFti8N6mNJIeYaGt4RwrlBO+2KpX+BjI7/c3H0FH3j6fwMA9HZTRN7TR7fg16/sRL3JtXKfEQMfp4vA3iwulJg1+OGBFwEA3xym1Jm87Pjc3bhXe85yEfNjLsrdPLewh+eGnpJgod0KEYna6zsmEphw4a4zQXRdlfUbkyAfMcBmbmoC8mxMQZbKoPzbBOrDomedlUA3KRbk1jSWR0TyMizwXhrlWpNdCIlhuPdlkWbEBV7cXsfIN9n59H28LX2SbXuhdlSoCZrrfZ7zrPWEUNxObu1IXo8JMuo5Dizs533bdl8FAFxaoCoYuJJEPSHqlrjVW2JULY4oP2/CuO9N4NPSTR4ic5Jx2W0yU3ltI6HgnOU7aNRQ86m8drGc2Zsl+vgr6AgrOVhYWHTEqjBIRkbH9Oiv/AYiC8rXzdJiNJy/sR3iOXCIx8r9EugyR4419e4m0od4Y06oYWDKcNZ2HQEjJdR6Rc+sKISzQrUXTHw6P3pOVzD5YGzFsZqETL9zx0l8+8I2AECzJoEuF2XgXjtc2tQaMJQ6UFJIn2fflT4p6Waiwx2guoMGkOSLlA5i82L4uxXwIpJDIKHgJvDIuN2AdqZm4B0URzwN/N5OBs7/nxPvAQAU6hznr214FL/3Lz/M9vebqki8PzGp8IGffxwAcEOUtoezVboWP3/pRnjPd0t/wukkDyJU1Ci8l6w79jDFEGNohG4HCTXFphIWW0yw2A6NNqHZmT38TF50fHuLud9Icm5N+cblrNQmqEkuQuqMi/Qlsssr75Ss3HI7B6UtubGtwgb+r1pAY4h2hNg5rlV5A9tJvxxEjVP38xOG30uNffKRtX7JQDNOYxurdbd/Y6ZORUiMuJVNNUQusx9TJs4EZpksWwC+dFhdI+7ScAsqz0kMvMRDxpYGAEOPidE+wjaW9soalhUSYmQwBtrT//cnOxokreRgYWHREavC5uDUgfikQmWwzTmyG8XVEyU1Tl52UPmZLACg/pzoa1KUtPfZEDZ9nOHSL5yjOdfotXO3aow8Lsk42zndnhNiJd+m0f8yuYLJpTd6d/eZdmZbSXTsUFSSq6DgmRqEwhRM1mJlQPkSTmYHqfeap0XC+XADWUgFoYRxnYqUseRgw6fY1tW3mPBn9tF1ASiNtYN7gHaAj9JA4Cbq0vkiOXGv+FW7wnX8yomPAADWpTmZiy/Sz/rvSu9GQOoymDU3bbp1jT//7t0cX5IH/9+7KYHcNDSAJ4Zpo4hPSYLZg7RnBL4VB16mCGCyZA37SVxy/JDqpnC86gjXM/FcwOfIpXdR8uj9Gh/E8jbtB4654vqsGWmrDuTXsoNGimtuAtzqqXbVq9FHOZapD1Hyi5yPoPwO9lO4wH7Sp3ltdhuQSFOCc4uSrXpeaoO8dRnqBQ60sJfST+E4XS9qoO2l8iTc3qxna7iGQFjcjc/Ro1GQ6lmB+ZCfHBe7Jpm260ycvsb4t19RkeuIVMHSLrJbjfQoklG2HUqe3SLvi4whPtm2ddX5+Hy386thVRAHL+mhdFcJzvnYisrJABBZNNmEwPISF3bDC1yw+RtF7BvxcOKrFPO3PMaHfvEnJf7+koMpceuMfU5SjNeaFFkNT6xVRoRfvInXzlZivnEyspmWnttHLgMAHru8GWqKb7pZwKWb2yJd15X2mAEgdpkxA8lnevx0buPHNxF9bg24dg9fKuNyM7735uYi+r5EudI89EqwbcRqPsUXds0ljqH2D3n/24dO4ctX93Je/4lEs/sTVDkWrnSjS8ZnRGrzA2kFAXdQfLxiAJ2st2MpwqYQi1S0Dkm5OSiNxDWO2Rj4TDSrdtoGs3qKx3peEPE5Cj+vJPK0RE82pSyeVkhMipHzNvHVd8uP/MmEbxw1uTG1u/n8a+UgPDFAOr/HorOpZ/jMwnkPGfnh90qxnub7eW3wxR7UzvLXk5D0/bK4JptXUghKvkzyKO83adZKA/kE36uEpPab4kKJoxGfMBpDuysFiIJF5asdpuiOUUcbCY0r72A/plyAUbGic8pXW0xmsbmm73gdV98hRtQEn9Gm9YzLKX5qBHPvECpbeIVl/xWwaoWFhUVHrArJQZUdBI/F4TSB8jCpYK8UZTVlzlphhcHvkNIt7JUoOlNpvNw2ZF78NU5Jicib39LOrZjfuzJ/fe03GrjyMVFbxAjoXJUSa1faItfcWja+UE34x0y9AdNvs8x+G3GNudsl6GqMovx0lgFTlSEN9FEyST0h+R59RoJo1xRIXRSOJSXH1bEkmlLGvUsCZjI3yH0V5edUZLaLRbFKEXSxkcDCPDlxai3Hl7vaLf25cCsrDX2hogQubXIRfYGSSukA3bB/fP52AMAN/bOoDnHhZ28REX5A6iscCiAmJfVCpziUwri4lkNAdg+vC0i2oinpVu1TiCysXM/MTpE8Cgol4dw+LogUtacBNyel/i9KMZWTfEZOj8K1LH2eXRtN8BznO3M3kH6BnedFago+y2eUmNVYlmCimrgITYm3QKoOXZGaHsJWjUG5sqYJVZWsYZE4jdEyWASgZI2zJvCLbTp115cAyutEzRI1ZuBwy1eF611s0+zPUU+1pa2GRAoH87y2NBxEQMrf1cWoOf8Vqj+JZguR85LRK1G2NgjKwsLi74VVITk4De7c1EgqNNaKzs8yCf6GK4W9NbiyA5UpClodJMWMT7h+XYa8y+uNO7H3kIPFW0RXXRRO18v/J7tCcCfZlskCXf+3VBQvfijix/nHjpBbXOtO8/75GLSUL9eXyMWMwSfWAlpSkqtQIZcOirGtOVxHTMqVL91KvS95mnOq9rV11Jxwuu7TUiehp11ENFQUrpSXEm/DGgOHeGxxt4QVS8ahozSCs2y/sF50eCmV10y0UFznSlts24TqrvvraZT+kMcaeXLibX00hLxweR3Sx0RKEvdj1yWRSjYBhfXGDSf1EmQ7gdKIRkgK7hp7krF1aEf7erZxB5pcjnqXRng7jRXBKm8c/CbFi2v3uxh6wbgyxXV3NydTnosjNinZsBIUZlx+bsnxDbsmfyYnRYC9m0vwriVkjWTN5b1Z/ydNv8RdWYodm1Dk9DnHz5zMblb+nNmOhnZMCL/UqzgqhsV3LaH6LG0i5tmEpajvtQ82EZPCwvU1EqpeD8n6al9yDkl1KDMnL9DOZm6MSS6H2FaufrSJruekpkT3a8sGVnKwsLDoiFUhOTQTGot3NtD7bBB9j5ErmE1i8iJBOEtB33VmdOwqCwOh68FZZJ7jPybbriZUMb9RI5ATai2ceO4O6be7CSdG8muo6dIufkbmFSqScNO1n5bsn1v/LADgPx99L5p1KptR0RcNF63vKaM1yzZMBp+xMKdfCPn7fAYTlBzCEmpbGQCSdIZg+UZyiZoESrkVBzExizfeQ84Yf4gN1XZW4Ynl/D/85GcAAJ+aphvyA6lDeKh5GwDAi0mAjSRl5TdzjgCT3tiPBMX86178i7FvAgD++OKdAIALGXK337n5IfyrGQZPRRYlnPwWrlP6eNCfn1HKtSu2kZLyXXZpyRicu4Xnxh5pYvJjwganxRYjY9KuRuARNloRCeDaj0ltxsUQisOyEcwN9K6s+SvZiGaPi+i8CcEXKUvCrpvpJjK3im1kiusRWhK38bMJf7OX+qaVQWmXfjzpJ+8ZVPvks9/xQ+FNJSfjeYtPKvQd5eSXdkp4uIRfFyfTSIkknHxe3hexNSUPR/xwbYASg7E5Zbc4aN0gm+9cpfRqXL3aBYJGwpzku1GR9fTKQGE9v/ecxGtiVRAHVVeIToSQ36x941hM9qswD7i+t4JaXhZBXjJTgTn9H1uY+SfiN65IpqYp/+YqP95+iZnJfqRc90nlFwM18Q0mf6L/aBNzUrWl1M8H83unGGyfutAmPgYm/bh1Ng6I8dAQMWNwSl5rwgtyycOPcy4mRr/rUnsvULfAfqOzbTegMXzhReooNSlK6lxIYP6d1Efuj9IAWhii/Pwrpz6CHhHri1JdufsCB1pPhVA2lZsnJVrvVr5sw+kiaiL7DybYT6nBNfj9iXt9kdUQlcQFyS9IAZEDDNNcnqY7MC+xGqG8066AbWy9QvAyO0IISBtmdzGzJ2V4GViSojD9j0gU7CZjUNMoSHp88gX+gOcOCEFQ7ZJuJufEVyWuBHzVNH2xIfcJ0dyg/DwESBm1/LZ2abb8elNoaGVErWq1x2x2TzO7tpWHNSYflEJBcSkzIJseB8oB9B0nEZq6UyJjpXxAeVD578TAQb70lz8s78vxEOqyG5we4vo4i7w/c3MDg9/le+bWpJ+foSuz528Hkd1mMjytWmFhYfEDYFVIDjoAVPs8DD3XruNgKKbJwGvkwgiJwc4Y52ILIma+bwTxCR4zksaSxOaP75qG/i8DAID5veQOJthk8UATQVE5TABJ7wkpbHrARWsDKfpPbXoZAPD0Iut55RvJdiakybuQuPiB5xQKsuW72QPDiHilQRdRGbMXNJl3vL+8rolhiYc3u2Hh3ZRT60/2oTwqxtcrvGbgT8k9Z29RiMo+Hu8eoLjfkOjN/33r1/Bvej7OtkyB030SCZrSPvdr3ElV5e3raLW6PXkB//pl5mL8y73fAgD8uyffBwBIngkiJgYzk8thdgBHfw1KoleD4s4zrrvS2iZUlHOobOe5gW+Ly3Wv9utABMqGq4nrblYjFCErNxKDKSZc7VN+0RtTArAp7r2ui46fr2FcirUeEadPaZRlB6/iiJSx6xGj9bKLgEgOOsDr+yVYK7dR+btEGa5rXK+tsPal1Rq9or7EEZt00f8yJbbCKN9Bkzsy9u0mZm6TEncSd2ZyHmLzGg2RLCfez3GmJUKy2gN/HxejEkcyUgCoN9iWNOXRLD1LtTtR10heMqEAr51XZSUHCwuLjlgVkoNbBVJnHdQTGiXJgAvPiL9L8gSCyy5Km41iL9mOA6SiQ896WJKgoOIGUuvEZU7tijuMxFZxryVFF6T6hdTpQLtw61XZn7DZDkW965202OyLTQAAPn2ZgUCBne1xqbVkS/FDNIQFSy2/zdLIylDwQLm9g7bJ1EwwvR+N5QAyO8TomJTCpl8mF65v0kjLZjbZnWJbqQb9axtbKVL9xjgNpp++ynH+x0tv84NtDAd5637WMy81w9gYpx/uaoWs7vgyo41Gwll8dDs3YvzDSzRujn1DgpLyNUx+Qp7RQc5Z95KDBa62w4QNBzdGzzWPKczfLG64hLgNXRMCrvydtBuLHIuxayzuBqLP0yBknq3JVmzFPXgBPmfjAjVGVWigW+pMLEu5NlPtaXF3u5irCXQyVaqcRjs4a/wbbHPmDq796GM1zO2nqODnuIiUUV9bRzXLc36uimwklLu/jOUyWbnh7mnZmS23IegXkTV2L1NCsLCxhbFv87uxVeUOSNZxsIWEBKqZHcJzsgmTarUDufoO8zMrladym9s1L4ZeNHX6OsNKDhYWFh2xKiSHVgTIbfWgwx4c2YTFUGRTCSoxo1GRsGDjYoqfExtCuYnovASLSH6Q0bnCy46/RZ4JMzW1JysD2t/sw2RQVsYlWChUxUCIJHZzkLp/NE2qXV8O+h6T4JHEiv5mb3Ggx2R/wizHa0rcD39F4+rbZRdwCbByxWXbe1Qju1Us0VJEdPFedjL0zSCKozympLqQSThSLYWuJ6hz/m78fgDA/7XzbwEAS60Edm6h2+dnj32M45QSSyPRLLZHeO6L5/cBAH56C70cV6q9qLXE2i3BO60Q+y9sC8A5J3tq7uJ6pJ6nuFAe0e1KReIGbiR47fJWFxFJKGtJmK/JxFQekPp9GgiMy814EypD2q8gVevhGIy0Ve8KoLyXa939uIxh0NRQAOopeaaytUD/U3xfItkWFnZLHQ6xm6imZHd2aTT7ZR/N4+ZZsc3pu8J+yPH8bRLqflIC6xZCKG+Tuo0Jfib/jqKAeyGK+Ky4Tn9hBgCQe4rGpmZMIzbTzqYEWIcD4B6rk++SWhJHeCx+gvMsbqsDknBlsoBz60zGZjtU3BRVNm7ORkKjKcljU/eIiPQddMSqIA6BMtB7RCG31fUzAwtMIvSz7QrrAW+YL4LOc1LGfdiMOn7157BEs5kXMburCS8oBKdi/N0SSRjQmL6X1/UdNAY0vkB3f+wIFmVTwf+xRNFaH+MLHK22+zaFSuvdfEDdJxUc2Y3KqBAmOjG3Tvlintl0tfg2/grcl5N+zkKX5AlEz1JMnXl7zd//waQf58dl164lz4+orF6h+zByA1+o5/Kb8Kc5BnV8Zs+nAQD/7NKPweBPMoxhWNfLX9+Ti9yO6+Mjz+K3Hv9xAPCNpHMMl0B0tm3QHfwix7Cwh+eSl9tRj9FLEvk5LEa5qy6KG008hBBySR1WGsit47HYYrusHEBVwOQxmChK4xMtbq8jIdGrRhUwP4LUpRaWt0hKei9Vv+IYqVHmhnY0Y0AiZI2aMX9rC7ELbGz2FuOuNAZYhflbhSh3saPqgOzfGtJQErMSfU4I4rvYbyMbQWlIcn7+Bw2D/U2uRbnfRVD2HDXxFebdUi3AKYk7XWJlzA87dj6EuJQJMLETxkXcSGg0EitTtq8vihO8kRbd5ktCnV8FVq2wsLDoiFUhOWjHRJNpFNabXHape2B2AvaA4b+T0uQbZAeiTUJxd7UQSMjeEM9Tvs/ulnLidYW+I2Y/AMncE5dodAFoxnisXc6M1Ph0bgipECWVDwzQlflomVuAh+5ZROZaGgCNmgDghUy2pOO7IlviujOFQCNZDznZwdns/eDKPomxhRZaUcMBTGSfUP9CAIGiiPWjYsTLtPeOGHjJRBXx/l/61D8CAFSGW/jTd/0xAOBrebL3alOCjQI1f36XTdboiwz3+5eDo37uRqAmcxC1TTtAVVyCC3tXvj6VAeXnVLQkEAxxPodW2AXC8rx2miQAfoSHylhekmzHY2aLelEFBxtwT0gWZ9gEk/Fc90tB5La2JTYA8OSa+R+vQk/yXQhJgRbcRPEg9UjSL9hqciR8bh1voizvlSqafTf52YgDWuaTPCy7jJlNrnaUoE5QjTDqUnNBigFfcVGQ3cbDZnuFgvTRwnWub95ngqj6983Be4KShpGefNfwuiYc2anduHZz+6jOrP2Cg/mfl4I1L1DaNbt3hZYUCvN859JSX+PVYCUHCwuLjlgVkgM0XYiRbVnUj5LsmqCYmujyiSsOlsRoWJNsTEfCoEMjFbROk2obPdZwpcSEi7lbTBs8ltsuuyItBHx90mTsLewTjtII4sIEI7COvEhdPCkup8FEEfmCBPtIgFP6RHsplUgfFXG1mkpGpUHHL25bkkpAZlOVwrq23cQYH7WU1w+dTvil0op3MHDfe5FcMXkZqIgRtpFYWZ8BDvCz32ZZqHv30ZizNU0/7kCogG+cvQEA0Cpy7IkbswCAnq+l/FoSC3t4zrgmowvtHAnjzvPDoRVtR0B705aYlNAv31hB5LxUupLNgeJzvHH2lgSC0obJjjS2o4HvXhfQI9Mym7/U0xrxa2JHkjybxg6uT/hIHC3ZHH3kSR5bzMomNQ6Qusx+5m9yVsyl62DE3ycUwpmLm6QOxXLAL+qal92tul+W53466dsA4rNsrNorAUsDGmPflipdYiQ1QVheUPlBWuadNZmec+MpmH2w/QxMKSxccrQfDFgaZ9t9T1Gy7vuX5zH9NB9AUKSRLindn7+hjuBcO9z9tWAlBwsLi45YFaXpw2tH9dBv/xpUQ8GpmloGPGeSWWq97SKuJgimIRylmdT+Ds4mCMeEviYvtesZGs5cHaWCGb0c8r0MJnhGXbccZlfmDBmsn0jjhYDysMnm4YepCdlznMk7wHUbkNxoSpY77V2gjYlDAqbC+zOIfi4NAJiVGn+hK+QbjYRGWLIG6z2i0/dI1aBTYT9zL7vDJFJJbYNoO4nHVDwyIdNdFxw/VNnsF2qyCr0Q/E2CTPhuXjwg0YmgP/bh52m8ufI2jjM+3d6EpbhRqhOZHdE3VBA+KjuYC2POSZsDTwewvG3l+AxCOccP2jHcM7uDX6LTAT9Lce5uCX6TKkpOA8hvFbuT2K2CUhcz9lQCVZG2quNc67i4Z91a2/5g9mZd2CdBULdMYekro/7aAkB9L9l+7Nm4X8vRrI+Srei0AvqOie0mwzlnN3LNwnmNpZ3OivuMDUErbs4DADpgwsNlLepAfgfbWvc3vCazjXPoOdvAtfvEdiOFfltiw+s6HUDzbhobygsMorr6i/9s9e6VGSgqDD7lwAso7PulIwCAs4/zF7m8SR52vf2CxyVj07xkXkj5+w0WRk2pLp5bvreK0EWuqEnn9oJsM7wMhEV9qQxI9qfZ06LkYP52U0yGy2QKvsanFbw0H0z8jKTE7uJg5ruC/s5MtR4xaEk+hNNqu9yMKFlYy2vDj/Rgaae8XI7JLOQ1OqChJcoyWJCYfnGFBkpAzxnK+cs3itolGaOtmGY2JNpFawtb+IMpjSk/n6S2QWL0L0veRaJtaDNl+0YeNrkPDczczvWYvl0osYkWbQLFTVwzVTfpivzo/XoE0UX+6q7dLy6/Hq5Z5YMtKNk82ESTajFeVlItaEmrNm48V9x7rbD2N52NXpOK1gNt4mKIgol+jD4tMSkRoNZrMi1XZlKGt+WQn5M9L3NGLeQ1Vw+OwBkwxmLeHjjFH9jgCwXM3k61xRAx48Iee6SF5c0c35RUFnck9sYLexh8hteb0n9mg1sVayGZpkoU+Ruuj9mnxGm2VZplKTpsiGFs3sXmA9ShLzy3FgAw/iW2ubQzAEfer9Fvy7zQGVatsLCw6IhVITm0wkB+A0t3PfbYXgBATEptGbFKecqPjR9+MgsAOPPL4qqaDWJWgnT8vQwTbQ4SWjb98H5TIj13axV9vZRZcxfJigee5TXL24GolBkL3sYgofJEGgCQ3+xhzTfJCabfIbs3XSIXdeoK4w+zzbP/iFJFcUM7ujEiJb1CMyKFTJkMPA952j2h5kWdSLXVEWOkCsn8GlIGPVTQmDvAvvufXakmJK+0C3skL/H+2MRKLgwA7jwlhvk7yXkGn3T9wr6hvEgXUsc+u8n1RVXXGLuO8/+521uIzsiamQKnkhk5d2cLqVNS00BcvPEnebK0RiO2M8t5XZNAM1l7p9nm6mbL+dHviri/O+iXwjfG2OoA12zbf1vClQ8yG9eobhLThkZKI7QsUpaUUTPrMfgXSUTSElTW2462BJgDUh6VMnbzRiITSaDc8EV+I9Ga91V/cgGRz9BiWkhw0XRTAvlqjr8rlVFbcyL1pg6G0QrxXci8nRKEKWQcjDdQP0KpxVeFRUorrHWw/OQ6timq4rUHpMRiHqjkZO+UG2RhvoyOsJKDhYVFR6wKg2RkzZhe9wufROCmZYS+muZBUydBQpDrKe3vjGwodO8Jo0e5vjRgdE6jZ1bX1tAtu1Ybd5UJLa32azTXyIY3Egq89HM0BjjPpBCfJpdYMoVb15IlBM7F0NhCSt73dXLtzC4Z25oaVIb9Gbdediv7DS8p1LtXhsEaA2UkoxDKCocUgm70y8pIC2GROEyItTF2Ra+57VLosj6Ga+d3131Dm6k7YIKvGrG2S9JsFpM8J6XNxz1EZ0yZN15jro0uaCzcJOOU3cqjk2ZX6bYuPvQ8b5i5gx079XZbRid3bqdIl8/EsUZ2ts6vN0VvjTtY+dc3JKs2KNJMdF6jPKRWjNMEuLkV5npcv8ZGmhz5bsOvq2DehabU5wgva1Tu5wLWyrL5kax119Gwb8uqjkrOg5TdK45rtCQArPcgF2H5BnH/zjm+DccYauMTHHDPmSbm9pvCu8b4yGu7T9E+AgBFmg780nyNBNDYynfQkfyX+FPtPJ/EVLucHNC2sbhlxzd8GiP6+X9t98q0sLD4e2B1SA6jY3r0l3/jTR2DvzFI1v2fjv19cX0bP2r8oGN6PfF6ztfi9cfF3/pNKzlYWFh8/7DEwcLCoiMscbCwsOgISxwsLCw6whIHCwuLjrDEwcLCoiMscbCwsOgISxwsLCw6whIHCwuLjrDEwcLCoiMscbCwsOgISxwsLCw6whIHCwuLjrDEwcLCoiMscbCwsOgISxwsLCw6whIHCwuLjrDEwcLCoiMscbCwsOgISxwsLCw6whIHCwuLjrDEwcLCoiMscbCwsOgISxwsLCw64nsSB6XUnyql5pVSJ6471qOUekQpdV4+u68791tKqQtKqbNKqbe9XgO3sLB4ffH9SA6fBvD2Vxz7FwAe1VpvBvCo/A+l1A4AHwawU+75faWU3Q7JwuJ/QXxP4qC1fhJA5hWH3wfgM/L9MwDef93xz2uta1rrywAuADjwoxmqhYXFG4kf1OYwqLWeAQD5HJDjIwAmr7vumhyzsLD4Xww/aoOk6nCs4069SqlPKKUOKqUOtkqlH/EwLCwsflj8oMRhTik1DADyOS/HrwEYu+66UQDTnRrQWv+R1nq/1nq/G4//gMOwsLB4vfCDEoevAPiYfP8YgL+77viHlVJhpdR6AJsBvPjDDdHCwuLNQOB7XaCU+hyAewH0KaWuAfg3AP5vAF9QSv08gKsAPgQAWuuTSqkvADgFoAngl7TWrddp7BYWFq8jvidx0Fp/5FVOPfAq1/8OgN/5YQZlYWHx5sNGSFpYWHSEJQ4WFhYdYYmDhYVFR1jiYGFh0RGWOFhYWHSEJQ4WFhYdYYmDhYVFR1jiYGFh0RGWOFhYWHSEJQ4WFhYdYYmDhYVFR1jiYGFh0RGWOFhYWHSEJQ4WFhYd8T1Ttt8QaMBpKOhtRdSzYQDA1j8qAwDmD3QBACr9CsEiLy/srAMAYhdCAID4tMbiAY/HrrLYda2b1elCOYVaL88NPc9j0/dKt9EWel4MAgC8KVkKKXRX6wbCyzxW3Mr+IpPsr9bjIZTjhfV1NQBA8ArHHSgrJK+yv9IwaW89xX4HD7UwdS+PxSf52Yyyv+RVjflbWfoi1FsFAIRfSAAAyms8xGZ4fTjDMSWm2UdxxEH/UY5v8n7OJZTn2GKzGuVBfm/ulcU7x6pbXhgILfPcwGHeX0vLfEcdFDY1AQC9h7ierRCvze5qItxbAQA0pqUtdov1X6ti8QZOqDTGOQdlLI2kRjPprVhjv4CgBiLz7KfrMq+Zv7VdXbDnZc49u43/9x7jZ61bIb+fazX2xcCKcVa7HXjySGs9PKalDrrTANLnudaFUR6Mz7LfSr8Dbd6BXn6GcjJMBfQd4/O+/GO8b+hJjm1xn0JkkTeWRthWdLTAsVxNIjLvSJs8p5q8tve4hsOl9sdeS/OzMqjRSHsr1qCR5Dnnvgyy0/xtxK7K3CPSzuYy9Az/SZ+RMa1Rco1Go7/B8V0J4bVgJQcLC4uOWBWSQ6AM9B9tYd6NI1YmhbvynhQAoC6UMz6p0JRSkwNPkFUpj+dymxyoLnK/6DwpZmWQnKeRBJKXhepGecwVJhpYcFHph9wnkkaRn4V1QPeUUPL54IprVMtBeJn3dZ8m9Z17G/vv+3oAlV72p2V1xx4jt5l8IIzYNbaZvkB2MfkOXhMsBhDIkxs1NLlvjLfBi2gUbuA/kQlKKAvvJsfs/VYU1+7j+BKTbLuwnuvSjCs0o/wePEMppDbGcXa/GIRh3bO3cg5NWR+npdF1xnBijqE4znMDz7qYv4trnJjmPIvrOJcLP+MifZjXh5c4loDUDm50AZEh/qOPC8ebYZuVQYXKWEPu41y6j/NcrVuh2sc2Iov8rKX56dY0VEakOb4uqPRzTIGyRv+HWAh9+uFxAEBpDaWF6JSLUIHflcc1X9zL8UbngaZw4LVfJ+e//D6unRcEJj7AdYlf4X3zt4gk0NL+O2fet3qGg1r/RBWXfpZtmvGueZr9T93jYPgZ3re8wUiHvDY+BVRrIlFdqcn1vL+RiyGY5TlPnlE4y8/B/6ox8V7Ox+VrgkZC3u+8QuQUb6inO9Z+9mElBwsLi45YFZJDKwJkN7mo9zQRynJI9S7h8lVSwOSkh8IoaVlmJ+9z6qJ/bSsg/Qipe2VgpUIbzijkN5O6h7K8362LPlwEqvvJzaotiiWtMO8PFgC3zvuchkgeIrnUU9qnupV7eP/wF2Mc2w4XSqpmNmO8ZvIfkbNGn4mgNMJjdbEhINiUPuDr4pEZcoT0RXJTtxZAaURsGlT3UavzmqVd7WOGc7k1sYcMNAHRbVFgf+N/w8/8uEJhA+eXOi+XrIOsgUZxHdvqPyRzzouUMAb0vmheG+mvwv/LI9qXvLygkWLMWiu0ziQBAGte4LzmbqaUUO3zEJnm90CV1y/tE137iOM/0/ByW9Lg/wpemIud28AxmLUvDwHOX7IQuh6E3xbnBzRj/F7tZ5vpszIjRyO/1XB1jnf0u2Jn+EAAbtG8C7wvMSHrubWJUIbPJL9Nnvc1jim7OQxoSmzhDK9f3iLSZaiF5c1i97hm1o5jKY21jxXGye37j4qtaSGC9PmV6xjK8drzPxNGfCjPvluU0gJrKS4P/kkYC/uM5CA2oFeBlRwsLCw6YlVIDgCgHSCUceGSwMLdQEpXn4vJBW0dyQvLp+ha458KY/pufo8s8LP7NLlLZm/Lt/Qaa252BzlD4oqL0gz1+3he+q2athUK47wvJOeMZyE+pVAe5vfwc+QuCz/G8QZOJlAdEPYl1vH+r1HkiM3XEcqRymeFcyRPhqQPjUCFY/Z2U9fNLVEaWr6xia6TvC+/ndwidYiSRL0LUMIAYrMce7WP7Qw/DUy+RfTYPTSSLNa453FtZwXJ5zih/EYjAYhnI+nBLQtHXNeWpACgFW1z99IQrwmWRDrIKTQjInEco7JbEo7XisL3Akx8gJ+qLjaAGRfxO/jgCh6NQAPPi3U9DDTE2xMsKVkr3h+f9VDpF64r9ovsVuHoVxy0ZCxiVkAjLnp4TSNxLgsAmLmNLonF/VxEt+zAkbl3n6cEMH8j13rwGQ8LN7HN1AXx3mzj/93HXN/rsDQg9i7R86sNheg5tlHZxnXpOsiXceBZhcxOXlce43rcvPsiAODSn23x13ruTo5v819QiolNAeURtmG8KZV+jilxOQDnHO0d5Q1sM/kM39PiryzCfaRP1ui1t7FdFcTBC3Fh+l5ysCjiZPwgJ+PK+Gsp7f+4Y1MrH57T1L76UV7DxWwNiTVvOYTIshjoIrym5ygbbcaVbzALFeRB9rZfSvMSGqOlURNCeUU1AEBxXF6qy6KWRLTvporKOOfvIcUbeCIIp9VuAwAK29hQbN5F92meKxY498Ja/t91Kuj3kz4mxGUX5x6eCyB9nucW93JM6x+injF/cwzo5/fCNYqXaRHNhz+lAPBFrSe5sNVBccFNuWgkjfGVbZZvoWvZm4sgLoZa4/6bu52fA8+5qHXz3MWf5KsVEaNlfEb759wpzsEQ28iiRvUxLnJLxHxX1jf0D2bRemiI6yLrEZP+Z+7SgCNq2jKfqZb/3bpGfr20IT8ew0yS75jDuY3UNYwRN7ws4v47y3AnyJDyazkHI+Yvf6CEyCE+m8X9XJj+F0XV3QUEivJ+vcT7Mjfymq7LLoq0iSL1PNc6Ps9zC3sd38WauMT7Tg5yvuXbGohOcNCuvKeTD/I9i2TIGIC2YXdZ3MfRecdXX40qXb+T1D0/24WEzKfebQ2SFhYWPwBWheTg1ID4hIul3Z4fLFLcSNax4a/FXbk+hPQZXm8MhZPvETdSQCNygefMNcWyBIGc85DZTtJcHRT3lXBD1dSIzgrlv0Go7izPJa96iCzzwtIaCTy6KsEpAxqJq7wuIxxEB9hO6qzrB7G0wjKml0n99YcXUBEOqe/MAgBCJyj+5dcrXwztOSEcUtSE7PuKGP5rcrP5/WJolaCh2IxGZif7W/sNSijlNRG//57H+L3eJW5ZahW4/L4QElckMKeHbUbmhFfsz8G7TA6pPFE1CmQ3XVcdFN5HkaoyzzElLnN9lvZ4iAsnNsFBs3dRwuk5rdBIcMzGELrmSTE+bncRErUlLq7e2dtEJbjcj4QE/hgXn3l+wbyD2IxIjMOiTkhwWbDoQYkeY9Su8rBIkM8PISXc1nkP/aPlJyhqB07FEb9FjuV5zBFVVx9NotrPNnpFVZ2/zbxMgNEjjRQKMZaWhwMIL/GQkX7+4S9/AwDwmcu3ovk19hPJyPt8hepkLOH40kE9zc/+o1zPwkjAl+5CF/jZd4TX5Da0JSKD9Kf5HszvDaLWt/I9ezVYycHCwqIjVoXk4NaB1KUWoF1EF41BkJzq0k9KqOekQnMnFa/gMdG7ropLpttDYzt14sIuUmv3CDlfboODpnBkHSFlHn6EFH5hn0JxIylxt9ghTNhvM+qgJe44pyESw5Bwm5pCg8QdwUUuoQkNzu6tI5Qkq9GXOM7NHzkHALi/5wz+4uq72fdpSgyiIqOR1GhJwJLh1pEs5xKL1DH1Looh3aLj1kU6qfUAxqW4uIfXGMNf74kmFvZyfL5BUYx06K0heILcxBugfWb8L2mfuKq6oXfwe73Oa7pfZjs6AOhjZGdKJDGjk3thja5JHpu5XdYuKe62AxG0kmKxE46a3cjxBsv+FPxgq57jEnadCPjcM8BHjOx+rm/idAiOuKW1sDnzXLRykD4jhtM1PBZZEEnphgqKQc4r9SUaJEsiAYbnXVRFihg6JIFH9/I9c+rKd59n7qO9JhTifc7JBKrreX15iPMKX+VnrbflS2kf/vhjnKcY0CKBJmbFHlFYz7bXfZXtzL0/7Nsxuk8qGQvbUS3PTwcwtjQTrh8sAvkt7fkAwORb+Kka2rcZLe3Ga8JKDhYWFh2xKiSHRpfG1Ns9JM67KA2vdFeF8u0ADy06bvZucjVPAoFUycWaz5G6z9zGY+OPU8q4/P4YvJBYsCU8eX6/uHwmFFrbyVJbZ6mMa1cCbTbWUU9L2LS4Rx0JLmrGgcIWckFjYcZ6chLnfAyBGzi+lrgGHcU2Xy6O+8k1XZd529Jt5Kxdx0LwghxfoEaqf+0+CY/NJPCe3UcBAI8kmX0UeIGSkeMBWrFN49EY+44J4gkgtJGm+uopst+uS+y3lo0iMc2+81PiZpMkpFAWCByiK6GwXhKU1pOPxGbaOv/w4/w0QWmhwTKm7uczGn6Cc17YRw7ZSLeghOtqJXalHeSQ8TNhaPFERRYkcUpCphPXPOS3mtBtY0MQC35N+wFRjUFKE6Gz7WSi5R3SxhV+Zvdxvr3fjWDgozw4O7kWQNvrMH9fDfEzXI+rb2Vb3WLHWry3hq7D4kIucX2MBySUAyCh7cG88RoYl6bjJz49sbAZAHDuCj0SXT0lrP0G351LH+D9Fz/ERnuOK9/7YrxoxibTjAKLu9h3M8HnHpbwci8KhCTYythy+o/xmsKagO8eNd6+S+iMVUEcnKpC4lwQzVjbeFQcMjkVnGRlQPmRgJFTfDDGIBPcksesZG82evjmLm/lS2oWFwB6ttEqVHmcRsFmHIj+bRoAI+oAICkr5ZwNtqMtZUy1XhHj4hqxK5LBKHkFqWepQuR2N+C8TJWhtokP/eODzwAAfukrPwt3l2RMimjX+4zkiWiN/Ab2k1/LH+nYd/gyj/zriziyNAoA6Poq5ebC+HXrJ24/s1ZzN/EzOgsUkhxXVH6Yi3fyR5Q8GUI9Kf08ymMLvyKq2VIcXSfkrU+xcVXii1vY1EIwx/YXpB+sY4xH6FASQVmrksSBNFN8HpG5ABpb5AHOsK2e4xJBuFX7qltkmmucuYX91pNBRK+tjErM7+S5yLUgBg5z/Rvnpa1N7CKyqFGRyEiqXkC8l/NbviGJ5Ze5gGNXeb+J6IxcDqMp/SSFqFQlq3PgkSDmHhAiNMPnFizwXOpKEzn5ORmCkRsTNaavhXffyqSTw4uM2vyDu/8CAPCrX/g5ZEkv/KzjoReEuN/l+EbwomTJRmbZR/cZD7N3iEoV5/XNqrxLTeXPoTbI+4J53pfb2cLgM+JGH33tn79VKywsLDpiVUgO2mUuRaig/Ph5t0K6ZYxd+sY8CmWS5KFv8HP+JhEpz3ehMUJukugmd3Ba5N6hrIPKWp4rvESJISzxUaGcRm6LGYMYQgMi1g61/Fh5V66PzhsDJbC8UwJczokEIYY0tBQqw6TWxpn0e5MPsO2IB08CPosS3WXE/NIoENhGXSr51xLNJi7UQ1Nj+LObPg0A+IXBX+GcJfckvKQgWotvlItPG6OqQt9L7McYN7MVyepLwI8AnbtV8hqOpznfmIYS22FQjGotydiMTrl+MFpjRNSCg5Rmqv0aoazMWj7Sx427FIgfpMRX5/SwvN0YnzVCMvd6ic9NFSVDNe1h42fpw2yleH9lMCFr56H5S5SlF45RTAhJtuzS7Q2Mfo1t5NZLFOWX2HHpvhYCUqtj6Rckuug5zj110fMjKZtxYxTlJYtrgN7nuFbG9W3qLVQGHcSmeZ1xV3ZJFGVjcx0DIt/PnBngumzhexooKZRGeF+tj4seO8s5pYaGkNkj7ncJrGvIfYVyDBFxu1fWcgzeekpmfV+NYO5WngvKPFtRUckayo9srfZZV6aFhcUPgFUhOUABOkhDV3G0XbEGaHPWzGLMj3mvJ1YGeLgVhYjkKFT7Sdkb76YC5xxP+obIgOh0ub3UG3teCPoutLjodia8GY7rh0sbjmyChVLnFMYfJrW+KvUYkhclI28LoFps6/+89e8AAP/P//hJAEB3RaNE04Ef798SN1Rk5zLwBI2i0/eRg6RFJw8EWvg/rryHc5f8kmaXuNAarj+HcEaknl6TQwBkt8r4JHvQ6K6hJReuVIIKL4qxUYKu8psV0u8hGywepzHGuOJyO5pIHzNuTa552VQ+mnXQd4xS2vIWPgcz30C5bTANLxnjppEyFIr9YmSU+fW/xGt6ji5j6q2U+FITMnYJh17ao9DzWUoMgfWG2xsxCqgn2EZhM++r9XDRE2eDvjQYOklJpSX5OoW1jm8MH3qOL8zEu+P+GizdzhuTJ8Ir2lZaIXAnJZzYMzRyGGlyqDeH5QZFxvtvOw4A+Ogz/xAA4I2JiAb4tq1LP0ODTfqch6hk6FbEBpd6XGLOlUZxDP5cASDxDPuopbRvv2ik5L5LEuZfC6Aixt7UBbwmrORgYWHREatCcgiUgf5DHmbublfYic3wnAldTZ4L+CGypi5iK0Gq69ZclG+kZ0DPSSDQZSqK9T4PXRckK/P+LABg3f8gJyiMsCYAwJqKPMY+xr9Vx+RbV9bYC+VMTQONwjrh6q/I5oxeC+Anf+JxAMDnZg4AaNchKI61E722v4OBUcemGKETeq4b1f3UGbufplK/5afpQ/upgefxa4/+NNuvGG4rUo0CeiUMdvkDZBetCc692V9H8nhY1khCbZek9uFzLVz7MLl8/GVyI0dclIkJYDJCiaFbahC2xPYTnwj4XoD+l8mNFvaKfSgEzNzOdWl0SQUqCeJp7ikCM+RsxhXafaEtZRgPlPYlKo732tt6fC5vrOsmKGr00QbmDvAZRSQ8WTuiW2+podbNc6YOZlTCk7MbXT+gzWQBl8WF3gppv8rX5ffyPek5JWHeu+C7ObvPNWW8JshMo/vrvL4kYdpGgptZSGHz+BwA4L+fvYdrLK7iYEkjJx4WEwRnMH+gXaPBvMPGnhHMK9SHZPDySmRv4v9rv6hQMPUt0jw2d4D9rf+DC7j4yxsBtLNrXw2rgji0wiz2ot2mn+lXS5nirLzGrQEY4Y+nJg/SmeeDUi2FgBRNaUlEnw4Yg52DwkYp9nI0DQAomCjIiEJTDITROR47cBurfhyqbfONcpEdWQBA8SrfytQZ1y9iYkR5UwjmQz/xBJ5e5OJfPMMfflTGG1mE39/yv6F/PSpRjSNfm8XiHH91C3ew41ML/P874Z1QMTFWiZjpituqMuRh9l7+2nofllJw4nqrOEE/89EUJu26KOLzJ3IIHKX4a1yhRgVRzXaGoVF/TMm7cEajvo7f5/dLkdw1Imo/FfbdsRFRVUzbpXIQPZJGn9skOSMbJQ19axOhBcmqlP5M+nGtx4MnhCI6zZOpi3ye03cGEeNvDrkDEmeyQIIQfjnpl8tzh2nEy0/wIamGRkSMy37uwkaJnam5CJ7muAzxy25iv05d+z/W3HqJjJXnOfJYAfM30+AZFAawYT/L1GUqMRySSjo/veklAMCfHXsL5+soYIzj60tJBPBnGLU5fa+HiETgmrT41nLbKJ46FloxBvOsl3YBRtfwysLEZExLb9voG5QXbuQn/godYdUKCwuLjlgVkoMXAiqjLcSuBVAWkdUU+TQFNLwQED0sYqlIX8bFFJ/WyIf5T2uYXCx8UWoUjDb8QimFjWLEk6KdrQjQkJwIT7Iqj35tOwCgvqmO6CVS5sTnRHwZMeXBgdjsSjdXQsp5Pbu4AVcWaFgcfUS49M/SUBV4qNs30F39hxK4UqnLmIZ88dAE67xj7WkAwMGlcXQ/QwkjJwEzzX4JxpkNwguJsVDcsjGTWTrhtPMe5LMpLq3sdAobHpfgp71s2y+tPurBGZTy+Ce5jrP3cLypUwFfpTE1NFpSNHV5h/ZzD0zBG2NQCyyE/EIsXoxrXk/y9es6HUB+u/GdmoKtHPDgCxq5De6KOdRSEmkZALK7KZqs+ZpEyN4pUmFMQ/dwfmv+XIryShHZrgkP2S38bnJOgpcksK7Lw+JNK93pjW6R2q4EULibzyZ6iO9idVzE9lu6/KjOwATvP3eekuNNOy/hapHvxMG/2MN1lbm0wkBrmeMrnmKblQc5p+TZIHK7ZRuGi5yfyT3K3FEHSly/5DmuT+8JjjM/HoCWgjOQPB0jOczfV0doWrKE3ZWG/VfCSg4WFhYdsSokB7QAt+Cg53TLD1gxxjGjHwFt/bW8idQ0fo4UMLsVaPUJJ5WgnZ4zEkteCiK/VbhSSFxukp1XGlUISCn8brk+NsdOWkcczNwuBtAB0Z+v8zqVxsTgJuGzJq//ff3n8CfH7gMAVIXDZWeo2PY6QHKC9y8nwzKndi6BKXVWabK/IxmKGdFAA5ckGAaSWdp13GQKAoMvkpud/4gYPrx2PQlTdHboed6X2So2i5Ljh2kb6acqG/QkToYRPklOuriP53pe5rWZfc22/UNKnRlpJJRViM0ZI5cJe+d/gy94qKbFtbhOCuheEPtSWvlhwX1HRaISO8PibsdPXY1PytrtFRvLSy6qUnavJqH0WrJAGymNyDmOz9iYjEt6YZ+CN0DJKCCSkTF8e2HPL/neEld2//NSvn6fh+BFrospTBwPm2pkQUByXBZv5H1bt0xxXZwWbu+jT/6zQ/Q/mude7/aQOin9ic3AFAWu9GskRIqsLsrmS5KbE5gJtXNGtotNTVz2eFsG3kSa3yW4zxQMTh8KI79Jrl9+bdnASg4WFhYdsTokh5AHPV7BVCzsF4g1noKa5Minnw+hLHn5xhZQWk8uMfS4CyUK6Zxs9jLdJ4kxWYWQ1Bd0Gpyuqc0YLAESZY3sZvGOdMkWe7MtaL9EOK83yVKlUc+vY2mCigyXeHppI/bcxAKhZyUDz9SKaEUUqrLFWnxSglukbmOjy0NtH/2inngiUmFa0JdrMYx9m/dN3cM5pCWjbupeB17IVG2SSkdivU5Oeijt5fot7CWHrK0hi0z3F1Es0FthgooGvyChzt0eFvcY6UMCozZIotghF16AY1/eI5mpFyWEfH0TQdGDTWiuqcGwuMdBc6O4m8XLNHO/hAaHPcRO85jxDJjAKuNVANocsu8FqSi1HjBW+cJbyclHviBFc9e5/jqYxCkT3NR/tIX5G0UkFfuVkVS7j7noPVmVsfCaxZt50fqtM8h+gbHORYeNu5JFGix5fohz4qqUnb9B6jPElvC1yRt4/TYaOdwXKU1G5xRii+y8EZWiwxKQl92kUBIP2ehzvGZpB9e31ueh2sd1SF6S5CwJivJOdCNqXMgSMGgqpJVGgL7DJlgOr4nVQRxaCq1sCKnzbrtYh4wsJHtQesF2CTelJY1VxP2F/UDPMU545MsmM43XTt2r/YhFs/uTKSZaT2mkGG7gi79m34q5/a5f8syIYQHZPSh1XqEgYlpkmgM9F6UlNRxr4J0bTgIAjmzl06qJey20rNBzOyeRe5xxBEa8DGwqoNkUkT/A/s4sclCDySLKvYbAcQyZbWafBu3Hf7QkdbclqtLiPsARsdK4CCPXOJZ8Po2+S+1YCaAdo9CMa/S9zGPGcFcf4oKWB4LtPScr7X0gACCQd5GY4ksclP0d/OzMjVV0SfxGSFKaF/azob5n2lF7fgyEFDfpOVlCdrMY6qR4sJb8l0BZobqb1Kf7YapU2U3Kb6fWLcZmMYD67lIV8JmQiW/Jb5CYmWrAJwpLNwqBmpXiQJdG4QhNScra5Te1iwNFF8Tl+gH6V29IM8r0ga6T+PI37gDQZjDJafY3d7OLZoztF8VgblLbg3lAiyo8dY8jc2YffQcdeEGJ2P0wGdP0U1RD670tfy8Rs6fr0l5Zi74aGnGppn3wOj25A6xaYWFh0RGrQnJwqgpdZwOop9rx/cb4V5E9GGrdQFWyHZ0qaVpkTtxkVfhUdPYWs3uQiMXxpr/vY0EkgLrEmw+81N57wcTkR+fZb25vA3XZd9HkdBQYt4RWRMMxO0kJee15SgqmvKOGvztDd9U7d1CCOLRIil6qhfAz4y8AAF7+IOsJDIUp654vDqAnRC74D/qeBgAcEH/X5r/839C8WUrGXV0ptjvN9h6XXWfb0XoAM0yXu9hGXErhm3WtdwFVKRVv3IimRkWgrJC6SDG9kSDX9kI857SAoKghccmNWLpV3G3pCmZdisE9nDpiUvCkMhb0DZdG/DUqw/ztTT+K0bhqzV6bs7fGfde18g11bZek53J8i3dwDIPDWa7Bp3oRyRjppS0RATRyG0O3UT1Mdm29C4jLO2jeL/NOpi7XkZd9OMpDUpdDMjHLI9o3SGauUV07FqSYl23EfLdv7hYJ1hLXdLCgUL6J6uO6z4iq2cexzN/sIZATN66od8agvXBbC26J81uaoIQZN5LtyYC/ZqZEffcJyaMpBDEvwU/Td8nL+xA6wkoOFhYWHbEqJAftMrCosq6OuOyZ6Bmbg+inpTHtc00Ta28q7hTHPV8XN3sRlkZ5X88RF5mbJQvzRbZtirNmdmqfwprdnszuT4kzIdTFPdZ9VtxqFAiQOq/8KlF+WKpsYJJ6IoWKFPd87ArZoDE+FTc08R8uMbsyOMobBz5LA9rkuzQistlL/m3kKuUmJ6i8dgUgk13n/QPm/Af+pBfLW0x5N+1fD3DHcFXjepjdlAKy52X8muNLH9Ep4Zqys1QopzB/E3X4QEXCxCVst7GviOCTNAxlhFuPfFUMkiMp9F1uB+IA19W5cDQKW/mQTC0Fk9fQdSbgF4E19RgiS+24f7O23cclq3KGfeTWBpCclPFlxUgdJxet3azRjK/MVQj0k7W2zsT998WUADSZjbUNNZTyYTknRtWqyUupoH4DzzX2MI8lIfkUtW6FygjHOTLKoLcLVzmW22+8hKAYQ1vXeH//YRomp+7twsBXeWzq5yk5BmUrg+Sldth0TmwppRGRAOYC/q7jxVuNnYf/N2Nt242xaRVF6i04DqJiRDUBa68GKzlYWFh0xKqRHOppD+n+ItxaGkC7IpMJmUW6jsAEFUWzX2T6KE+GQ0677JJwAKPj1VPt6j2G6iaE2xQ3NwGhrAPPkSuZzV9a4Xbf8weEw84Y24NGZGsWABD4DsdryGxlQCPQK3rlwS5pk/f3vuT6BVGDsmnM9Ieob6psCB/9IMuWnyiQjSaDFJEGX2hvzDMr4cG9noxla3uPxuWdK70W0O3NZYy+X5fArMSUB6VNHQfJrrzRZHwCPWfJ1qs9RgLgqcZc1E9WQoHnZn9MxvlQBLMHzMYuvMSEZHc/p3wPS3k71yf9PP+f+rEGeh+VzXeSJuGqndA29JTMdVy8TuJtKq7z0IyL21DWf/hZ2RH7xwLYu4uBR2ceoQQXuCJc/kARzhFKP2UJEmv0yGY1T4T8ykzGdW62CsivT6MyIu7bQyI97TLSmsaW7dcAAP9pw5cAAD9V/TkAwJ+/cDvCUsfSeCuKa2Usfdrf21TJVgYtkYibMY3yHRJs9aSptsVz4SyQ2yeh9wviBt4vInXVxfb/RJHo8kfpLjLFbmtp5dukzDvxalgVxEGFPATGS3C+0e3X0jdikYlgDEyF/Yy41HH+arf+FJ23F/50KzK7JYpOipH23suc7+JDQ8iIG8eUfcs8wJdTLYeQPsnrs9ukP1ETgsXrxHOzD6PZd9QBijN8WMGxlWmvXgBIPkErl0kRrskLX+1XKG+QByo/rPhhqhW1A0V8/rP3AwD6H6Br6v+38W8BAB/78U0ISK6I2QI+k+WL1H/Vw9xdXKytf2z2yJQ9O7ZqRDJ8mWfuEEIqqc3lfsf/YdS7pFKxGBqbMY1yv/jTJTvW3OcF2vuABAqm8KsQiVvbrlYTP2DE4sl3aqSPyLFJvsyLslNX6EIAnmTRmihB8xzqNxVRm5F9SiSvYGGfvNRa+7ELZUnBnr5TDH1ZjfNfJVGorTGGaF6beCaB3Dauy+CzUirvXrMHh+O/e8nL/DQ7mGkHUBHZF6Um5dckYjV1xsXlJinoT+VJFOoSV9NzKAC8mzpA5UX6bFtvlYFXg9DXxOgr75cxrrdiHlAQFWeJ/eRvIwHofSiEvFT97t3Mh7N8gm33HtO48A/pKm+uESslJJo1rv0dr1yJQXk1WLXCwsKiI1aF5KCbCrWlKEIJhb7dDCDJP8WgolqvBPbEPSzK/olOhFT/2LfI7iu31f1cA2OkbP4Z7+/6uRnUvkv5MH2BVD8j7Kn7rOfvJ4BNYs0RcdOtar+MlokSNCpLdVMdcYnoC4sRMDZvAq00iiPGKCr3byBHD56NISAx8mEpxFqTsmh9qSJmtvHcf9v8eQDAL5//MAAgciyKwYPkGJc+urKWxdI7q0COk77yz0VCWZBgmoaD3AbJPRgTd+OM5KPcWkPv47IjExMG/TJ4oZzyDcIGRamNoJpt6aywWTIvJWMwNq+RkOCehT3styhReIHlgM+5zQ7jRsrIb236UW/mmDGEpr8Wx/w9Uh6/KqzVaD+pOvKKc+g9KgY7Y9gsKF96iU2bACL+X9jg+RJDaY3wR4kEqwwoVMZlS4DP8PP8z8uO4RMhxE9K1JS8CybrtHBn2d+fZNcApdYXn2SGr1MHSi/IXpxSnb96Js3/i8rfJbsh0YxmXbvOB1CU4rEz90pA1nm+u82o52dqdn9K9hh5u0RWbgVGvyPZojdTYjABfNWxBmISYVzZImrIq8BKDhYWFh2htH7tUlFvBKJDY3rTRz8pIa88ZvQ+o2cWx9vuNBN3HyhJKOoloBkzIbXi+pRCtT2nWn4/Jn5+fp/o0cvK55amPgPeQjeU91gPSuPtYCkACFQk03M0gKYkQFb2kh2FT8gmOm67xFkwbzIuxT06qZDfTxLe95jUZ5ASYW5FoS5GsegG6qMf3HAUAPD1/3q3H4Blxpnb0Z5X+oSEMZs9EyXMOLqg/GxDE+Js1rUVaXOxxDWZ11ox7t2aQ+tYSsYlncjy1NIaruz8ZQJzmhKeHFlwfX25LhJfz5G2wdDYcHpOVWScYX8OJmzdFA8uSEWp5OV2iLLXS04evtzm3rV+2RNyUfJXZEyhvPILBJtSdabAsPI0liXLtfuYGKLFUOtWgIYYXP38np62O3btHkY9XTo/tGJdItMBPzT+HWtOAQD+/OQtHOaVmJ+9a4zHxoCqA8DAYdl5bIwSSkUCrJwa0HVFSs718AZTjnD5Bg0vJq7Th8UOta1tGzPPIS5BWub90QHtFwvOi4Ry5Vf+6SGt9X68AlZysLCw6IhVYXOAYuUnLwy4ohuZgBxXKJ/TUChuJxuMTEgZ+jVSN3LmunBRkRhMgdPsJte3rpv894AEPEUXte+mMpWnGkdFdOkBhp4V67gUUK2b5KB4FZA9PAe/Ju7VdTzXCmvffWdcfmZLs/y2Fsa+wCW/9mFONHLSlBoHujYzAmhTDy3bT85TrCgPKYw+Tv1w4j3sd/xrnN/8TUG/AlEoy08TJAbN4DKgnVxlthDsOdHeBiBUIAfxueehFFqyZmZrgIUDkrS24KLWy+t7jgrH2strYtPa3zwlsmjCoU3il/b3GtWO6M0i7RX2VaHFnhDISqhzwngFXAy8KGO4UTJmr9u0x9QwMNKZv6/lXg+p8yJJ3SsBSw9Rupu7Q/sh+MZ2YGws7n3L6P8TLlpmqynBL2t2TKF0mEaNwY/yGWVfkqCrwRb29PJlTchLHI3yfS1Eo37gUWGL7I162gTkAYs3mFoNHENd3t2ui45f9Sp5TQK/pHZl7JqCF5QKYOt5nwnaWt7t+YWaTTHgrssi0SUd5Hau3O381bAqiIMX1iivbyB6JeiLv31HJCpRSnt1n9bIbqvL9VxMEzFZWO/5cQ3GCNWSl6sS0ohdMz5I47oTV1GQcQlmDMB1BOFGB9Pvkf0jXhCDm0QpFte6fl7AgmTu6SEjfwONGl9+I5aWRczvf8FBfp3EGRyUKs2Shdp1+zzmL9F6d+QkneImarPnnIfJt4jBVQqtlIbMTkbaf7Er4zwXWDLiqee7X83+lqZIb64W9fMCDOrdXLPx54GcVNeev0M2n5XxLt3Y9IuhNLpW+smzO9o7Xpl9Lc0cQjmFgqS3Z9K83riIR/8m4G+AbPbjMBsUL95eRzAhhXyOSVVtoafl24sIHuUxk3Mwf7tEgmZdvxJ191dIFJZ2G/VCodHPeZXXmC3V+FE9kca8uEprUuouPiH5DSGNsuTiVC7zWQXkvQnkHTz9eSYtXH0PfaClEhlH38sKC7KOA0+19wcFgMRVILtN3kFxiyYvtDNus7IrWLVXXMuirunuBlSWY+86zzEt7+K58W96flEfUzxn6gPsP3omgvC85B/ty3K86AyrVlhYWHTEqpAc3LJC9+EAlg/UETsvhUKlEEh4Vkqw9wHx79IKWFzH+4xxx2ko6A3ip5oglxj/Gqn/5NuBar+4Q8MSLSiGreBRF12XjBHPBAJJmfXLIWRSIplIf6awqgnMAtp1C2oS25+47PruSVNmrJWWgZ4O+lGXAQkOKmwWTlCK+gFOXawV45e/Tx2eg/IovtYTARmviO87l1GYov4SnpEc/qxxqypfchg8xDlfkZ2lQjXl72Uxf6NwIKltce0nK3ClGM3Q4/zMfpCiee+3En5hlXyKnDF12uS8aD/bsTogxtV5I7XBL9PX85K4VyW4aH5fAC3ZKdq0ZQzL6C2jforGUZM74KtImQh6JqVYjqgoRuKIzWjM3/GK8oCXOfdgHgjmRYaXbsqy81TiUsB/r4xkVNzMtRt92EFUgsGaMTEQbmcoqDrc5Uc2Lv8+rX/RdWKMVRqqLDUhbhHjrZTZz+xp+hJU+rAYR0UVzN5eQ/yYuCLl9a5KsFdwKoTEbg6mUBYpRoLSlna4vkRc2CrG+2lxvS+3d3PPnZOwzVeBlRwsLCw6YlVIDtplVlvP80HflRmbkJ2fRZUPL2ss7ROj03FS9N6TZCXX7k8g9RAV0bn30Bg0+XbZCWmwhNjDsmu1hDq7orcv7Qr4UoAx4uXX8b5aWvn1A6JzK91QiamWv6fm1N0ra0toB+g7RmptsiXdWckYHIa/l+T0XbL00kf84YTvijTlu8yORlc+NOyvlXHHVtbwy/r/GoPeu9IoZ9xrbgOoie4/+Rbh4FIQteuSh7kH+D08yfEVpa5A4lDU1+vnbhfJJivh2/0K9QKvH36Cc5+XMmprnvbgVkyhXrFRmN2oezwkzkrwjeQHGOOoarV3OjPP35RND389BcdsyHOV4518F+/btGEWs5dZHMLYXYx04QWAiEhSxq1nzhVuqiI0IWHJ4hqud4tBtNzeL9XsL2qCtjIfzaMqu5QbQ3LzDN8tJwD0H+P46pLvEZSM4sVbWug+YkrbSbDVRrGjTAfRc1LsXDdxfD0n+InlkF8PQ7ba9I2jrYhGZl5Kzck7bAr2ztzb8iWoYty8321X/+CLvK40fJ1U1wFWcrCwsOiIVSM5NJIasdl2oIzRlfO7JOz3QghenJTZ7AOY2UvqPfS0h1BRXF+TYiE+Sgqb3dyFpX28z2T35StSXnzQ863cZq/EntPknhPvifrBIoYD5KVKUSjnYOEW2SAnSZEjeEX2m2wCM3fzuqCEVht3YmxWIfxPGFob/zKTdEqS7bi0v4XgctsqDgB9z0h9h/XXBR6JVBAdJhtc2JtC5H5m4LnfIEs2AVqhrEZDOAfERTz6qHD7fY4vMYgpBbHjwg0j7d28TbKaWzP1F4Eu2dF85q18Nqbgb37c8e0YxnKuxV3WdSqI5FV+n/kA79NL5N6BsvKDs2p9Evq9i3aN8NeS0PfQxXttPTllQPb7XDg+Bk/malzfRuLQAeXvedkKtbkmwC3zBl9kP9nNkhwl3Hppl/bXwxgkzB6W0YMpdC8Y7wGviDNHDuGc52edGldrfqNcczngFyk2FcpyIj31nNSYfSulyZ7nxEUve2wOPasxe5f0PSW1LEQiqKU1PLFjDN/LbNArO6Rg8OWoX70qLqHjZrMoL8h6lwDQLZm3r4ZVESEZGR3To7/8G2/2MP5eaKRf20f8RsK4Fi3eXPyo3ok3+nle/K3ftBGSFhYW3z8scbCwsOgISxwsLCw6whIHCwuLjrDEwcLCoiMscbCwsOgISxwsLCw6whIHCwuLjrDEwcLCoiMscbCwsOgISxwsLCw6whIHCwuLjrDEwcLCoiMscbCwsOgISxwsLCw64nsSB6XUnyql5pVSJ6479m+VUlNKqSPy987rzv2WUuqCUuqsUuptr9fALSwsXl98P5LDpwG8vcPx/1drvVf+vgEASqkdAD4MYKfc8/tKKVuJxMLif0F8T+KgtX4SQOb7bO99AD6vta5prS8DuADgwA8xPgsLizcJP4zN4ZeVUsdE7ZCawRgBMHndNdfkmIWFxf9i+EGJwx8A2AhgL4AZAP9JjqsO13YsUqmU+oRS6qBS6mCrVPoBh2FhYfF64QciDlrrOa11S2vtAfhjtFWHawDGrrt0FMD0K++XNv5Ia71fa73fjcd/kGFYWFi8jviBiINSavi6fz8AwHgyvgLgw0qpsFJqPYDNAF784YZoYWHxZuB77luhlPocgHsB9CmlrgH4NwDuVUrtBVWGCQC/CABa65NKqS8AOAWgCeCXtNarp4a7hYXF943vSRy01h/pcPhPXuP63wHwOz/MoCwsLN582AhJCwuLjrDEwcLCoiMscbCwsOgISxwsLCw6whIHCwuLjrDEwcLCoiMscbCwsOgISxwsLCw6whIHCwuLjrDEwcLCoiMscbCwsOgISxwsLCw6whIHCwuLjrDEwcLCoiMscbCwsOgISxwsLCw6whIHCwuLjrDEwcLCoiMscbCwsOgISxwsLCw6whIHCwuLjrDEwcLCoiMscbCwsOgISxwsLCw64ntuavOGwAPcqkJsTqPexb14K0PeiktS5xS0bNNbkn27g0Ue0AdyCDyVAgA0ktLkrgIAIPp4Em6de/m2Qrw+nGPbThOYf2+N54pciv5n+ZndBn9bYLfKL0E2icLmJgJZFwAQyfBcfIptaqe9l7CZS7DE/jM7gegcj9V6eEzLE+g7ojF7J9tIn+RBt8ZrcpuBZko2DlM8FpsIAgCaCQ3PlTU6z8/4HK/NrQ8gt6fOucr83IqsmQt4QbaVuEoeEZtl/5kbFLz1FQBAzzejAIDycHvcoWX53it7JMujCuUUmnEe67rIYwGZw9IuhWZCLpTb0qfYrxdU8DgdlNfwmuic0+5DrleyBP0v85qpd7YwtGYZAND40gA/Zc3zW5tQEd7gzoU4vmz72cgyonmjvCdP8MWp9gKRJZ6r9vOzIXMKFhRqvew7tS4LAMguJNj2TBCxycCKtkf+n+cAAOf+x34ElwIr5m7mkNnhYu3X85zP/V0AgPg0z7WCCqU1HHMkwxsDZd7v/OQC5q72cFzyLqZP81x2e3uuYbnPafJcbhMQlnPFdU28FqzkYGFh0RFKa/1mjwHhsTE98hu/DreqoIVcmc9Q3nAsD06T35OXeK6e5v+JSQ+Fcd4QXeB8GnHh6HMeGjF+X97G+wJl/u8FNdLneGzxrVUAgBsg1VYXYmh0sa3UeTMoSJstLO4itXZ5GxJTPFlao1Dt92ScvK+wXjimAoI5HguJFNJ7nJLL1beF4EXkvotsO7+NlD19LIBWhNdX+9iPW1PyCVR7V3IHcy60DDQSMnRhXLEZXhvNeJh6G/vrfYEnc5va93thXvdKDpTZ20J4keNrhaXNWbnmwQUsn+jjVGXK6TPSvwNk7udidT/OySzfx//7vh3B/J0c/PhXlMyF/WU3BRHK8fvyTrYVv6b8/puyLsGSjGWeHS/uVfBCvM9LsO3wNMWTwA154EVKmkaCi82IRKXgSzGl9byv+wjnWxlUCFCgQnkXvyRfomQVWfJQHOWzNRJOcUuD470U9NejMsgvZn2HnwByG9l+fJrH5u+jtBeaCmHsUb4fF3+Szygyy8/qeB1oye/hjEiRMfZRT3lIneO5ap+sJ4cC1YK/nln5PVz+J795SGu9H6+AlRwsLCw6YlXYHJwmEFlwEFnSyG4XjpURDkt1DOX1LfQ9Tgq78A5S055HyTbm7tDQcR6rDJKK9pzgfflxB6VxkvL0abZpuLBTVygP83vsKDlAcYOw36EmAstCpana+dxp6q0eBp5mW/n1or+N8LOe1r4uX+3l9YYTpc45yG8We0eD908+SH147DsNTPy46MRKjAgixYSKGsUUzxmpJ3WJ5wrjDtyaXF7iufIWHqit13DnwivuK4sOWxlw4YjNpjzIz/4jbDO7WSEobSlZjkZCxhZtYeCQrMevzAIArhxZwwPf7se4SEKTb+W8imOiMy9quEE+h2BZuLWseebtZYx/ltdP38W56zFy5sixIHLbOC636MhY2F2tWyNxVa1Y60aC13Sf0tCK5xpJtl0ZZL/quZTP1fvExlTrljZ7NZyG2IqWOZbsTrEznHX9dyF2nGMvbOC54jjQf0ieyVqOYUje1/y6tiRspM/wgszT8TBwkGuW2cFnFT/Lz2ABuPJ2fh/9NtsuDbCBWp+L+ATbMO+XQ4EDkUXHXyNfYhEbnhquIvRoVNaI5y6jM1YFcTCiTultRTgiKrWqcQCAFrXCjTewvF2Gu8AFi4hhMZh14MixUJaXlN+VAwB0fzaByhDbyG6TJ+PyvjXfBTLbuMBhuS8kL4QXAJITPFalpOyLoG7e9YmCEV2TV6TpmkJhC4/Ve9hP/JoY13raRM8Ylhq0QeHqT7eAAp9y1wR/RLXb+NKUB0K+imNE6txGthPMt38Y5uVwl9hOK+Fhyx/OAABm30Iq2Arz/sJ6z1c/jKic3SxicROop2QOw/wRuUIo+x8PYeZ2Xj/0+2wzskXu08DM7WFZP87dGPcKG4Ceb/CZVrtFDVmWPlwPC3vl2Y5xYaIvU0au7isjckq+i7pWN1qaBvo+OAkAKH6KVursVo6l3qXQc0qIZDfbLg/x3MDhGupdhmCIOiE/Xi8IxKb5vTTKz8gs34neE1VM3UvOUtwk65IXY+BZhZIYbQ3zMepFPaV94hxaFgKX1P44S8Ncs/xOttn/DMe7vAMIyvtf+ngWANB8gg/bqSmUR7gQiSsrmV4wT4Mz5yeqlagxwYtRNEXlzu8XnfhP0RFWrbCwsOiIVWGQjIyM6bFf+g1yahlObis/jUEssqiR3cGTY4+QJE8+QPKY2JCD8x3KhUbsux6GgyfWUZqIfDkNAIguNVHpE9flu6gzJB4ld/NCCpWBlW65wruLAADncBKucOlqv0gOYiTN3NRE1yCvq55gP05duEYeSIibanGXGJMmeF8042F5K+djuIox5jktjeVtasV6pC6Lu3KDiyalRHRdbhs+AUoXRqw0ElVdJJXkpMbSLlkgZeZAXlEe0aj3tFa0ZUR6qDb3GzxITjd7IOifM9JLZTu5km7y2nCiht4vcW2XRdKoDrGP0EAZ9TlKB/GrIskttzmrcQ0GRB0xalAkA2RvoN6TOsXnaNzdhX01dD9L6WB5J+/rPdJ242b2cWFik2JsHOD/XZcc5G+hKBW4Qlbc6BK1puqg2cM5B+eDflsA0HMSWNwn7kZRyRopcctOy0UAes5yztU016DSr1DvFiOxuLmrIqH2HdcojK5UpYw0EpvTyOw2rmFRAWWddFAjKM9o/Fucy/I2viStkPJVKPNunPn3n7QGSQsLi+8fq0JyiA2M6c0/+UnUu9q6X3hZTgonCGc9P4jJuGB6j3PshXEH1QFjNJJrbuD/4XkX1UFxZS1IkIpQzNqGKoKT1PeMm9QE9ASL7euMS7H7qLicZj1ce7vYEy6Tgww9T07ZjLmYE05aHeJ9iQti2Lyx7AcVFcaNy459RJY0MndRR46dJscy7kNjRASAxP5FAMDSJZL/3pcd36XYSPK62JwEWDlAi8wTObGDmIAu5QHdZ9oBSgDQc1KCxT6cQeCzPf7aAm0XnGoB8Ske+//Y+89wy5KzPBi+a+2c98n5dM45TM6j0cwojgLCEtEGA/4AgyT8GTCfP2McXr9OGPOaaMAIkIQklMNoRpNjz3T3dM7h5Bx2znvV++N+au0+Ys9Iwga1L9d9XX3t03vXqlVVq9aTn6eKw/KdrJ1vQwEbe2hkuDjdx/lNhWU9FRLjwiGXOenF/QFvLOElcZXu4WfHWbGtbAFi08YtJ/PaLJbh6zFPJ3dlDYJZWSjdcu1VultSAQAUBzXik7KeM2TFC4fENb0rh94/5IWZTQGZZ2vtaiINBPJi86GQiORYE5UOY1sS24OsfWWwgeR57gHPmCrBVG5HHaFJNoyISzi3WX6LuIiN87riOq5Z8iL/H8xp1MRIbaRtM9/onPbsH+b9Ma7sylAd/c9wrkYKOfLpf2IlBwsLi+8eN4W3wg2SOrtBjfAGRgfpZxikUkuzTWa3i0CWFM9wi3JPyx5RWi/6a17Cmo2e5wDx65xmfIoUuZo214U9HdxYehtGT1xQWN3PPntfZl8rEoRT6vdBCQc2FvTrj5H69xxr6b2xMd43lBH303jEo+SGc5gQ8GJYoedbZH/59WyT28TPrlMusu8jt1ya54AHXuZ1S3sZ1gsApRFyl2onx9t5VnsWerebxoDIGd5D+4CVx9hnvSiW+wV+5qbSCOwVjmy8Dgui+6Y0fBWxgIun5v33vwaDz79OBuRP8n49b7SCw4pD5rmJjcI4j6qt0PaOs7J2H6Sb9KdGXsOf/uv3AACaIXER/j7XdWm38rhlIyLuY7FjdB31IbeNfztV44rgx7pvVjD+KB94LcW+Os/xx9nhCHLrRMJsss/wonhXMhr733kaAHB8gd6R2gt0ZS3td7znUJSw5PRpCVxa8Xtz7TrHMTXCbDv3MBCSEPz83fTUDPwVn9Hybj9Kw2zf+QafaalfFlorVMVW0XVaXNCbfeYnT/Juij0qkJdn5vcjt4736zr/1uHTNwVxUHWKVPWkQvRL3PyLd6wV5aNzPiwfFAKQERGN9i04VYX0KW64/Ci/qwyyraqqlsuuyE2SGqdRaXFvwHtZ41MmNkGiGtdrRKZ4n5rka8Sm+Vm8vQRdFgNYTgjAkoiU6VZshhEhDRFrxJvQiu16jq/9rZZqES0Tt2DyIMIrDaxcZWcR2YA1ifdPXWndJzLrX3N9PQYEhTCVsqLqiJFV+zScq1xAJX3ltnNdhh93sLLdqD1r8ygS1+C95O+/83UAwJkM4xyU0hj9Gu898QE+t+xGztdXa0XpGZewEre1rwYol9/1fYQ+4aksmcO/P/oIevyyfkIIlnfwxQ7mNHKbRQ05xzYrt/O5F0b8iF9bq0YW1ovaFIl4ka3JMf5Y/5EVruc3u+HU5UUSGmaMyDs+egb3i5U4IhbpJ2IkDrWuJgDO2S/uzcwhtolfCCJ9heOauc+4wIU5zAcRzMm4TnKCiwdFjSnDi0Z1RUUJc5ioJbnvAWD+Nn6mz8tz3AIomYOJMDYqp6/cUsFKPS1DaTtYtcLCwqItbgrJAQCggJ4TDSzu45CckjF68edafx3+iLit7qdRbu4cM/FiMwqrO437z+QCSGCOC6yKcTK7k301w2QJgSI8N2AzIC7Cq6Tws/cAgbyJqRdO5wr1vRxFwLgyxRDaDEu8+lATSqIf/cL5qxvJyn1zIc9YZXSP3DbOKTLt99xifjFyFdaJwXBXwOMERh1pilgam6a0wkWTMXUZcbOB3HquZ6CfImtjhtxJB7SX+dh1hu3n7hGuFlCo7KILbOSTvH5lO9csc08Zao6c+53pkxyvwzE9/x9uR+YA+wyLSuVFo9ZbnLieZvv1X+Tcw782i/NXKX3UVsXPdoSSgxpwvaAp4yatiru6llIILYvEcIhiSWiCbLE6XIOWXApydQCSu1KtBTxpcvZefjfyh1xEf8zFgpjmjGvwgbupSmyOLiDT5PpdyTNlU2/nw4qeiHtuyhVxSUdnyO7zG5twfeKi5dZFZEEM5/dUgItcpOi9iwCA8nOSDurCkypW90jWb1RUpZwfflFJTVat/4MLvMczvUhMSMCYSL1GQi1vqqHzCNdlZfdbOyOs5GBhYdEWN4UrMzzMIKhaRxORGXKcyIJJ4udH5u4KOp8mhV26TWwPEp9e73BbGZMTJmad/2/EtGcrCEtgzfxt/H9k3kFRQlB1hH2GZklVR79ZxNi7ySWMzlrrIaeLjgdaefmnyLFm7xAbhK8VGBMfNxlxIlXs0NA+cefNslFFXISx9Vmo58g1DbU3Ia/RaeXlI5gaESazLrsZXh6EcWl67t+V1ncmv8TcP3nZj/xmfufvopTQnBOl3m25/XK387f4Manr0K/R6OA6HNg+BgA4+zzTOetJ17MjmEAgE9iT3+B6HM6TluRZV/qa+Oy7fhsA8GvX3w8AuDxDqVDNh7x10KKnd7whNqeE8uwXva+LQfEuCXQKaS/3wKyBMdJFll1UZR2N4XPlaYaCl/tduHGuy6HtzDrYnpjnb80Ajq+MAAA+MHgCAPC75+8BALxjwzk0hdc+8cVbAdClyDVTnjHUPJvRr9Hnmtua8GxNZcmgrKV5XWhFeZKXCe8vjLCj/leBufdQIu36FhuZfqqdLRe5eQ8qIxS7ul4JeAGGxoB68TdsEJSFhcX3gJvD5iCVoMILfs/VM32f4ZBiiX0h7HHwxGUOu+MiyePSHr9ndZ6/g9d3H2sl4JQYj4OV28jl/YuUDqppDadbKkGV1lr69b9agT5GTtr0k5Kv+yJ/WzgE1OP8bnEf+zLJOpndDa+Sk7FnFG8IQEpcFy4hLqm0WNlrMx3oO04uvbCfF5oAm2akFZrb+7pwo16p3jTf8tqYACBz31qS0gPAYCkAqIjNo9Sn4Yg71n+a7o7O8VYlKOMZ0Kvi3hRPxi27rmFXgslcX53czftIkFn/0z6Ue3ifrCQRhVa5Pm7ERV48C8aab6ADLo5V1gMALk1J8JRIOInrjmdTSV3jeJf3yjxjTQRyvN+s2EtGvik2o7v9nhfHeKIyW80NHa8a2OpTlBiKko0bu+5HUeZaqFPkeHGBPuWppTR+as9LAID/cY3i56/v/QoAYLzWjS9O7QPQsj8tH+JYwrN+L4R/8Hn+NvWIcdXrli2ltzUGgDaaCIUWhDNiawpKiHsvoPVaT0RhnQR7bcoAT9AwkzksgXWXxRbTATT7JbTdERfIm+CmIA7+MtB1tolq0kGhX6IQJYLNpNYmxvyeeFgVsWvyB7j48VN+rOwSeS3FlS4OUdTSqlUAJjHOjWoKwUAB1RLfpPJ6XmdeSPXPOxG5XQjTNd7HGPfKw42W31kIjzG2Rab9XnGX6KzJwGQ/yavA8mFRX+YlU1CiHwN5ID/CCZoX0xhAw6uuJ6bHp/mwQ1lxs8YdZA6IDClux6Ened/pt7soi/sOYiTtOsZxV7uBoLhfTcZnIyKi9pTyclR8YhhOnuHnyOFVHFlZz/U4wQzBcMNEXbqocc+jVzILF27n2AafVlg4LH1JHkpGUrGTfQVM16hSxU5J1uPOmrcW5vmt7BQ1TYyJjl8hTs8nKt3s20ud768jMB9YM6/Iotx3fx2p07IXhMhv/WMS5vnbEnCEoG3czWjPN5YY06CbCn/53x4CADSlz7OjtJgv1hLoiTBu5H2PPQEA+MOvPMxx1gCnjy/k/K1C+MXd3XEOXik4V8q9xaQgT26DghK1f3m3ic3hdbUkkDgmkbRCOLSoX5n5BMKSSWzSv8PL/K3co9D1LL8zRvw3g1UrLCws2uKmkByaYWB1qw+1pPayHV0R5Y3xqdSnPLdhfExomohFhmsD8IqbGClDNYHVg2vLhHnFUUqt65w875O6xrbzt8Y8Y868xN0bTj78hMLsXaIqSB+u1EkYeGAK18ZoTKs0eL/Qkgko0ggurxWpO89K0M6wzysz5pcAndxWkViSdTgLnND8YXILY2DsekMhfpHrUJNsTjM2+DX8S1IardQyVnFMLQml3sG+yj0St58FBl7mIs3fYqI22ee26BxemKOY/cF3UcT+whWK07PDIUSvciym7FusmwtU6kvBHSJ3Th6krnMwSTZ4ZqUff36aRry4CdoTFdJfAlwJgjLBZWXJhHUayqtNYIr/muIyyVNBFNaLsVcMg7P38bPzmN+rgWFcynN30QpcHNRe3sT9KVZsffw1zi+04POK/q7eyo36iaN3AAB++Y5v4NwqdcXxMtn20HOtHBI9y+fWc6IVMco5tGo9GJd5LSFRrX6N1cOUnDuO8jmavIvYlAP3/gzbP0FxLSySavJ6S/Iz5fbyD1GqCR6NIyMGSRPh+mawkoOFhUVb3BSSg/ZJebXRMvznyc4SU/KbCYs/mEfwLKl7QXT6zpMSSLSxRX2bEu9iMgD9H1qA+h8MKilIQFVhnTRKNhCcEOkjQSq/uo3/L4646HtV2ruO9x0AlDsdBCUevuMS+zKhqPNPDqNXdEaj/5Yke1H7HYQXZD5C9ctzKW+efkk29IKuxC049Hk/5slYvToHKXrZMPOAiy4xvpqy+Cbzcv3vZTD7EKWY/J3k2umvUhJYPMw8EAAoFtYaYytdQHYT18GURK/spDjzh//hMbzzF58HAPzZ83fzOrFL/OQ7nsFnug8AAA50UipwJJLoI7/wFWwP0rr2qYxMRvDq0/vg6zJGW/kUvb+WgOf+M+5jLWX4dFN5UoRhc82g6cdBbFJcp+vketkcbkChtJ4cefOfyXPfTs6+/mtVXPsJNv9gjL7PX5b8ksq6JkJioHUkhP+huxkI9h/feBgdKT7AF6Y38j57OAen3qrpkd201l1Z7lFY/1VeF3snxZ+FPqYQ9H0+hJUa72eMsskrMpUQUDvDveNIQJyxlxUHHK/ehyP5IfnrtFoX1zVb5eWCNnzawsLib4CbJghq+B9/DMmr8KrbGFfk0oNkZ+ErYc/CWxqQbDnR5QubG+h9SZJdpNaDCb6JLrhelqQJEjEcNrvVRVgs9kbSWKF3DsGMalHiQf4Wl/qNxWGNRj91TnNgSu9RKQDa70NxaC0363/VVH/yeTn0JqjFcMPCXSXoGXKv8CIv7LxArra8y+9VcvKCwvbwt66jPuTIqDyJwQT7qCa8A31MspRxB2uH5f7ZXpKjpJZktUujIYFAXqDSgHCiikLnKVm/LTKmLeR8t4xMoCGi3pELHJRvRVy9WzKo19nX2zcweenrl5jm2pkqYukaleTolHiBJBtRBzTiVyW5baWVWATwEBczn9Ud/IxPyPrscr3aj1EJqEv9PUbDXRvrRfRa0FsHs1ZmfYJ30UtxoJftnz7OuPvkJT9y+7gfA2Gu/0gPF7tYC+KuPrphMnVKvycWGRKeO9flVRMzwUmuOePGadWxjE+KNNFrJM6m5242MNLkyJM1FIY5B3OkgNkbjZhG4qokEN7Khxq6SC9JfWcJ63+fDZd28btT/619ENRNoVbABfwFheIwMPwUF2jpRyliqSk6q8NLGqvisotd44YzFXX7n3ewInkTMVFHTIGOUr/yNowppLoiZcMic45XossYMM1DLPXDOy1KidhtqiAHMg66nueDMcTIpMtG71pCQ9Kqg2IANSXBmmHt5QKYQiDmnIXwV8NY2idqwR3ccPki3XvBHJDdLi5QMWj6JX09Nt/0qlWb+Ib0VYrMCwcCXsFes/nNSx7IOJ4qVpeTqAyhLA1q7xQlUyQkLmdpVLu0d/qV64n3vO7k53eifICLbNyIJgqv8/fjWNnB7zq2ihVXqHb2eDdC4g41Z0UYX3+5322VjPOKm0iMyd0NhCelmO4o71uucRF0yEWgILkxEnm4PE4VK34piJLsHUOIq92mmI1C6s/SAIBN//wEAODcBhKLn3nwBRzNb+DaVrkvByMMLvnS8QP4/OxBAMD/dfdfAQBeHCeBDGZUy5Ap+zQkp4YVt9SgivIsTREdSfmOX2ulehuVz1QDv/6DDvwSw2LO3DAG5thk628sS+FlYazBV6KYkQLBoTuk+u9/Q1tYtcLCwqItbgrJQftJuXVQY3mXuNOeJ/dVEv2XX6e9kt3mrEw3Ygx/Pi8SMLtNSmyFhDPM+eEvr83T6DkuORa3a6+sd90UDZHPam8TgRglh6HP8Lt6nBR+7g6N1R3i1nyapHz6Po6t67+ngcfIneujJPermyS6LR+AapgCMLx+4Rb27asoL7KuelIyEyU5rxnS6D0i2YfvFqvlBAe6tMePRpTXxSQS0BTNLQ83vJoURtWoSHZebBKoSrajv2IMmbx+4xdqGHtXSMbJ64xh0qkrr7Ta0l7JbRHpxA0Cg58lJ5+9ndclJjn36fv9uONeHibSLWmn/isSEFQEyvvI+fu+LoVOpFycDmi4kjFrCv6aU8rUBb+nUqVfMEFvsoY9DnJ75OSoOY6p61WRMkJARIrXmPwHr7RbSGNlJ+f1l3/yNgBA5zupXtwSHscTy1SFXj/HGw9+i20j2xyvhNx/ucrrelKc54o/fsNZKfw0Ul5oOuA9P3N0gnF3O7WW2mOiYMsi+aGpPLd/WdRsf97k8gD5bZQelRT4Nbk4tZRCWWqdpEVCejNYycHCwqItbgqDZHxrv977//w4Gn/ei/z7aWtoipvG6MpOHeg5SWo4fa/UKJB89sS4xoqUWTcVfmodrWzL/qfZfv52qXMgeQaruzUggU39R9g+P2LCojXcITk/U4qk1s1J137tVZ4yB96EJTS3NKA9ar/um7x+4aCUBQ+1ip0GCiaUuBX3byQN/zS5p6k5UN1QQVJCZU1ZO3PojL+kUJXchoic8mzO4dS+Vnkwr8qU4VxRoDQiWZkS9NP/qgRkDfg8qcJkmJqgpOic9gxmxhhoMmirHcozFickIzVzQG5Yd7wCCVs3MzdjNk/XdH4p5p0C3nsEMgfJfdjq92wORuoxOTW5dX5Pmug/IoViD7TWwLg5zenjPa/SvTr9aI83H5MhOvJN/r+S9nlVl0zpuuxO9j2waREHuihFvPxHtC+Y6lbdJ/O4+iHOp9HLOX9kPytlfW18F9RTlAaN7cFIseVu5UlnJTF8m1PMC9tr3rqMPCE5I3dK2HxvA6riyBgkSCwjLm0FpK9wgnNSr8IUpi33aG9fmdBxe1amhYXF94SbwubQKASw8lI/fAOAe1KCgnZRX/OdpFU4MeFi7jZy69RlcS3OkGzP3xZoBRcJ1U9cEb2t4YO/LBxSgnWW94vuel55pxBnN0p9BVNF6ZTGYpgc3HDITjltudKtUJUEI0esyPm7aIH3XY94nOrqD4i7LCJej5LPq+QTERdofl0roSYyH5b+5X4SWu1eDKIoZ3qakO+GZBwmr2s0xGVnpIrcIUosyaNhiHHdk3qMPuurAL2vrK0jMPU2cf9OKzSkrmRzPfvq/TLXojDkeNmfVQlBbsgp5umrDYTkOEFTwct4bPSWIv7R7hcBAL/9IpOXjIvSGXDR/YZwSwl/rorXA6iiOE/JKybnYs7eZUqra88bMvU2Ptu4BIf5K9pz7RpdfvkA3aWBrEZUXN2QGhG5Uck+3ewicV0yWGUvJC/wfqsz/TgqHo+CFBtuit0rszOOpjzb4BT72nQ7N2VvYh3mQpQcWnU0JRiqw0VVbEu6g/tEK671tt8t4+I/pLHBlPHvPiW2tGUX198vEt/L4q6WJMByv8biIZme97zZxg20gu2Mx+TNcFMQBx3UKK+rof8pv5eKXL3A3W9iDWpJhaoUWwlJdNrUdlMYBigPiNEvIr57OaNC+zUGns0AAJb2cDebwq2N2A2FXERMN5WGq2mFDimftnwHn2ilwIcezAChHC9ceEhSky/RQFjdUMHSgJxpEOXDrq/wpY/M+tAMy8uzhy+df4obITEGLN7GPs1LW01K2nmqJa43O+TEJUk7z22E5x41RqfAa7xfcqKB0oC8SOIqNO7cWlJ7Z1KYlz19VtKtt7qIb+SXxctpAMD6X2BswisXNyFynevQEALuXuSzym9onbBlCtwYQrn5wUl8aoySa2BF1kDyQ3bumcDENKlYYZMUuf0s57CyzQe/qDYmqtBAuUBpkxQxeZnrYQ5GridU68wGSf9OXZRYmH11dJ7n3z0vrC2Z5jRaqc+ROYkV2NQ6swOSBWrmtSyp9KruwIlz7HUpHBQQv+NPjTyPf37LY1yrE0LVtelHeXkW6/+cc7n2A5JHc3sKTpX39t9Ki3J5JQ0AmL3X752k5a/wvhmZQ3jB7+XzGBdo5g7ut/iJMAJFo5LiLWHVCgsLi7a4KSSHQE5h8Ak/Fg8oj4uZSDLjAvJVgdgEh5scJ2WupVrBP82Y1FCQNqV1ks32hh9z95oUNX6YiMlGuBUU5JOy9SYPQjVbZzWkTkrWo6gS2e1NIM3+w1IcdOQpctGrH4p6komSSCefBBn1nKhjXlSj7qfJHZYOS6bpTANLksVpalMYQ1O1t9Eqn3e5dUoUAJQGFELZtQE2GCZLyOwJout1WWPhErn1/Ow+pTH3TrKV9Fc5zpkHZSxX/KjmKQanJWrvpIQgbjxSRVECiEolckETQNZ1RqMwtLYUXEZOMf/hvlcQ6ONaf/yFn5L5iWv4L9ajskE4sJwxUZXMxPRVFzMPiBHXnFuSN5ISUBMj5cp+tvGVxUU5q1onTIuBsCGFcSMTAS/Ppuc41yC/XoqhdGqvzL2JdI2LmuE0gMVHKXqZk7ziphRdEGiWwzIGXv/bz38IAJD8oWm8feNFAMCLL1HeN0F69UTL7T79IPVC7bRUAFeK4uJZPg9TLtDprMJ/mdJqLcExdJzmfVf3tvZLSE4/a1w2J49p5MX9u/GzEhn1JrCSg4WFRVvcFJJDPQ7M3qOhg00kpKS5iRdPSRZa5u4KNv4B/x57N6lgo8fUh3e806XMCdXhaXLY8KqL/DrSQBMkVBwWN10AXoaasTlASSbkcw0UxWVnXGghiTaNzjlYvZfth14kJ7n0M1JUoupC91Gq6JWAnjUnKm9m+2KVRrb4dd5/9nafdyiJFzL7GP2j9W/0ILNfKlXlyOUT0+TCmR0KRQlfDkqgk09OrI5PuchtkMxVkXrqfbJmCCIwzvGVZH6+fEtPNfqoKWlu3LNj726VdVdSK6CVcejzjHjmQJkDH2Dg0xeXD2GxQkmjNChc3pzb2QTqXZQ0VMiU3pfw9G3Kk+rq3VzXRsTkWjgIzYlEJQbG8KqUC7xNe0cDRM9JAdbDlO4aCxF0nmaf4z8urj7JwPVVHC8QrucNcR9K0dr4hAMtrm89Qh2+kOA4U+d80OJaL/dLCoCUnis+O4zP/cynAQBve2g9ACD2Ldq/3EBL0jCuzKicDVo4XEa/7KGq7E9zGry7FMGev3cOAHD5dynVxebFWPmsg7IYi0vvonQQ/halkkqX8nJoxt8lm+Io2sJKDhYWFm1xUwRBRft4yvbA00uYeA/9eKauYT1BahjMON7JyMsHpHDnXCvgxSSkGD3THJzSccrxAnlMWXfjASmMtop7GpeUqQFQ7tOoi3ek7zl2YCo1VTs0GuIa9OXNmZy8fuTJJuZupxSRGDN1Gdgku1l51Xe8ku0HyIGGv+CHFqmlMGCs+eKhCGovwMUctGJCdftf0Zh+WDiGlNX3Aqu2AGGpO5HbS93ayZhqWMoLBDL2HXNmY7lPe7UZjcvNhNzGxn3e6domUctwov53TOL6HJ9f5CQlo96HmQkXC9RwdpKuBLfG+f3kYbo2//zCLWg2xPJ+gWy7EV8bEAT8dc+SclvtDEwIsb+kUBqViYm3wgQNhVZ8LZdwZM3liMxr78CgjsuUVFZ/ihtm6JfrWL6VfkcjXZg9VRp0W0lckszXQcaOwohCbQsX6dAGZgEePU7RePB51voEWs9P7Sa397+c9GqIaFMZTWp5FgeVt2ebUfGuTIvbsnaDdFYyXqrWczTSsjkw58Tvtw+CuinUimZUI3OoiuZDPoS/JtmUpkBKwIjaClJ9C/6CceG0+qhINJwpxOovSaTjIFCVaElT3KQgakXqEuATv3NuPRe9cS8tovpKEoEEX6jl3eZQTrlHQSE+aXIWJJJvh3mx/eg5wU1pTtEyGZuhVSAvhreOy1J49RlumisfSXs5AyUxhNV7OLjuvhwi/z3N+2zhfWMSYTn9SB3pk9xUWTk4dlUOnG0kXNTEeNjznByWKzENiUkX2R9kKGVphTvdVxZj6UmNhpS9y4v7MJCVEnZlIDRI0Tb+RnzNei791QjcW7lmhc2ysHm20VphfT/1sp4IX7bPj+0DAFRXIuh7wWx6XmbOOu096iCzSVy6+3lf30U+j8q6Kga/zvXIj4qxsmDcdBpOU3JxdnJMWty5oZWWsdH4+s0Lnd2u4YalBN8Zeek+S/F75hEHjqTAF9ZxnIYQ971eQ26dqACiWq1IlWynCgRCfN5XVriJdUwiOg8GEL+89jzM+JcZaZnZqj13qimDF5+Rw5LT/tZ5JbdyXdQUVYfSoOsdnBw9acoKci5O2fGymQ3jeDNYtcLCwqItbgrJAU0FlfejMptGQwpfdJ6SaEYJQAqttCiyOd1oXkrHx8cdDD5PSplbL+RXiGJuY0uaMBwuvCSuv92tPAijXoRfI5fwR4DAa+R6xsBkit7GZsmZAHjnNMQnhPNsAVblbIHUZTFomsK4405rLP1c+ul70nK9Qm6TcGk5GbkmBqqlmRTim+XEaFPhbkyi8bJ+LB0mN+mUE8mz91KkSr8QRkGMsebU8uAS+1nqUgi9wrkGRTowYnig6PfUNJPl2P8K7zf5A02ET5KzhXL8Lu8zZ4oCkUvknkYNaZzk/N793lfwxASLXyw8Tj9iSE4Ax8Emsps5zsghShfx52lRW9qjvcC25POUcKKLcrbJkIOFQ1IrQ9po4fqBZT/qnZJzcpVjMgVWgjmN3EEJUKtzv5iaCJFFjXkxNo+/m5JU/8u8rh4DSvvE2PysuJ0lp6fcHfZyTFTdRIzyN9UAikWu2T/9kU8BAH514YMAGPxW+rbo14V7OG5fzuepDia6d/o+mW9HDT6Za+oJ9r26W06W76whdJn6kpGojeSnHQ1tJFpzjsebwEoOFhYWbXHTGCQ3f+TjqKZbWYCmPoOcdYLIfCsbsNpDCtl5suUi9E6fjrUCeQCWCDNVl0wehcnyCxQ1shtFlxaqbWLfiyMuUpckGEYCZgzniU+2zt2syWlDDTFi1RMt/dCcuNQ8wC9CLyeA+8Xf+AwnVkvzv+ElhnMD8KoUuUnh9vOtszmNjhx6gBmGlWe7vWpNZTl3s0dcU8v7FFIXZSwyTmOEUm5rPl5dAHFbRhY0lg6xnQkY65Vgqkqn4+VwGE6ZE7dzdFZ5gWLGrWoMd26wtbbl7RLKK/qwr3JD3QI5JzR9Wewn23xo7KaNwhGbjzrDRsFsS08361PtlqCfqOsZcWtinDanY4WXFXqP0tYz9TZyWCM5jn55BfltnMT0u7n+/gXeJH2hVVLehHI3E6buXxNKTk3rfl2kScn9WdoT9LJjCzxqE2979A0AwEt/cRAGZr8YA7GvwhOqzN8AkJRTyapJhdX9UrpPpIrQsrwPm6teGbvAaW4qY8xtRFp2EiNJX/zX9qxMCwuL7wE3hc3B9ZFC+outmoWVOyUr8wzJaTWtPIkhIAVRjUsTkaZ37FfFxDLJZ6VTIfoIS6I3a6InzkqVqWgDuigh2ZLv3nVWCtquBhGfIjtpSnZmSWwP1Q6F1VvIFeLn1h6sU0/qv+Y5Gf1t9n31Q010fCMt42L7DZ+jjn3hZ9NIi5sqISHAWRrzEcwqFKS2YkAO36k/Qat3ZVB7Vmcj6cw/xHGH4zVP1zWISjhtcbB1wri5zi8JOc1wa+wd50SC8Bmd3EWMB1Nj9p61QVtdZ5tYOCBJWUZikHWpDDQQlpBeXTbZrbzQV24lSQUOUOQIP0gLvH5hGKlvkvuZ09WVSInpyy2OOPs2qbEp7m2d1p7YExqgWNj9DAe1ssOHqQcoMVQ3CkvOc6D5bSmsbpN6oEnuwchLnNPyQdc76du/yvsYd3p0tlU82Mx5cZ+U91/WXn0MU7vh2S9SYohlNSpd4mbcyOcWmuR1qtFKGuu8IMcidIn3ogT0P8e/5x7kjSum9kbFh3qV/+m+JnUdePYOdKSJutT3jE+/tdZwU6gVkYERveEffBy+SutsiGZaIuak+GZw1YfUFTHQHbphU4Evz41lsAASHGBtcRJDeIxfvjTsIiiEprJe4gDkjALHpxE8LZmWcsx7cht1ncxYGr2vSfzAhrXCV3lDzcsQNKnMRlws97noOSZ+cRmTiedohoH4mHGrsb05E2Npv4O6xDX0vso2C3eYSESfVxIsuCCEQ1y3qXM+ZHaLcUvcv64Y7lLnfd64TKm05DjXYOKREGISpWdUiOb+vDdH9xovNDEI5gV1A62clnVf5ncTD0uh2ngDna8JcRaiborkpq4Ay7dynANSdi0/4nh9GqOcySwNSMqxuiGDMiGnoGVNJOl8AF2n16p+JiYhvKQ9Vcgc1FyUateBnANnG4lCZTkiaycRtpMOcjs4zuFv8rrp+2VvJesY/opE9643aeuyML1Vr4xdYZRfmbyZSn/Di+799jgOHqQrBuwNphq4uOg31BG7Kud9SlSwu4Ebu+dLYazsbOWD8D5SsOY55bmGKzvZfuxHfs2qFRYWFt89bgrJITw8ood//mPf72FY3IBG0v1+DwEA4M/97fKv79c8/7bn9b3g6q/aMnEWFhbfAyxxsLCwaAtLHCwsLNrCEgcLC4u2+I7EQSk1opR6Ril1Xil1Vin1i/J9p1LqSaXUZfnsuOGaX1VKXVFKXVRKPfK3OQELC4u/HXw3kkMDwC9prXcAuB3AzymldgL4FQBPaa23AHhK/g/57cMAdgF4FMDvKKV8bXu2sLC4afEdiYPWelZrfVz+zgM4D2AIwGMA/lSa/SmA98nfjwH4tNa6qrW+DuAKgFv/F4/bwsLibxnfk81BKbUewAEARwD0aa1nARIQAL3SbAjA5A2XTcl3FhYW/xvhuyYOSqk4gL8C8FGt9VvVtG5XXuavRVoppX5aKXVUKXW0WSx+t8OwsLD4O8J3RRyUUgGQMPyF1vrz8vW8UmpAfh8AIAfSYQrAyA2XDwOY+fY+tdZ/oLU+rLU+7IvF/qbjt7Cw+FvCd+OtUAD+CMB5rfV/vuGnLwP4cfn7xwF86YbvP6yUCimlNgDYAuC1/3VDtrCw+LvAd5OyfReAHwVwWil1Qr77ZwD+HYDPKKV+EsAEgA8BgNb6rFLqMwDOgZ6On9NaN/9XD9zCwuJvF9+ROGitX0R7OwIAvO1Nrvk3AP7N/8S4LCwsvs+wEZIWFhZtYYmDhYVFW1jiYGFh0RaWOFhYWLSFJQ4WFhZtYYmDhYVFW1jiYGFh0RaWOFhYWLSFJQ4WFhZtYYmDhYVFW1jiYGFh0RaWOFhYWLSFJQ4WFhZtYYmDhYVFW1jiYGFh0RaWOFhYWLSFJQ4WFhZtYYmDhYVFW1jiYGFh0RaWOFhYWLSFJQ4WFhZtYYmDhYVFW1jiYGFh0RaWOFhYWLSFJQ4WFhZtYYmDhYVFW1jiYGFh0RaWOFhYWLSFJQ4WFhZtYYmDhYVFW1jiYGFh0RaWOFhYWLSFJQ4WFhZtYYmDhYVFW1jiYGFh0Rb+7/cAPDhAIKvQc6IOAFg4HAAANKIaABCbVigO8W9nUwEAUJ+OAQDckItYfxEAEHwyCQBITDYBANoHLO/kNGspXh9aVQAAXwUeeaxH+RleZpvCKJAY43fZ+8sAgMCFqFwP+Gpsp3gbLB/kH+u/rDH1AO/n1Phb+hI/F+5peNPtPMY2lU6OpbytCt9cEAAQWeR3+W1ci4GnfchsdtbMQfv56UZcJC+wr9xW9h9c9bHNxhICp7hGDrtCpVfG3QCcOu/j35Pld6+keP8FjWqnT8bH9oEi28amNRYeYGfB2QBuhL+g4MgUSwMuACA61xp3+gJ/W30Xn5XPx76r81EMP8m/52/lfUPLsi59GonrvC4+xzWeP8w2zbBGZJ79m+e2upOfvUeB+ASfWyPOcc7cxfUder6KqfuDMmZe33WK11U6HHRcrLL9PSEAQD3BuSSvOMgc4kNNnuL1lW7Zk02FQA7emAEgmOEcQqsaxWGs+c7HW8Cpa+Q3yG9ZmfNW+THvR2iJc/Xvz/C6p9MAgMiy6+2J8Aqbl+7he9H5pShWt8uzLfEzOif7RgHZbWzPb94cVnKwsLBoi5tCclBNILiqUE9qZDeRyhuu23uOVHtluw+NIVLU0LkEACA9K9xilwJeI9cr9/C67FbSve7jCjFp1yATRbmffQbyCi4ZANLnIX3xM7ja+i3+KiWG0gD7KUXItQBg6Fn2tRzk5+K+INxBcizftQgAILONbTuP+RFdZLv8kHAQmWfX80HUkvyu2sHv1n2RnwsHHIQyMpYpfi4dFInquh8NkXrSZ0VCovCE7k+GMHcr2xlpxA3y/76mQq2bnDj5JV7gr3JsS/sUYnKf9GUZw+0UCZphP1Se9/GVlbSRNdgD9B7j36FVrr9q8n7+osLKLnkOWXLkZpFcMTbnYPId7L/zDd4vlGU/xVEgv4H3qfTwvpEFtnEDyuPcxd3cG+HLYblvEyu7uDD59WzfcZ59XvugD11yn8KIzO9WfobngfF3cQ+GF/ldMMu5ZG6pInlKxh6WMcgecbVGpZ/rue33KRmNv5d7UjsK0VmsQeqaSMgHA4hN87vSoEghy7x/52mFQJF9zqzjXGKyh5eHFJJX2D63md+pS/yxGQCqvbzON841a8h4qx0KWmSGQF7hrXBTEAc3qFHc0IBTcZDdygfY/zIHPvMg22hfA73f4oNZuJMbSTU5/NCyD5UeXhfM8EGmLvL6wnBL1K0N8k30hflFaCKKBt9fZHZwwdZ9nZus1BfE6jbHGx/QUjMaUYV6XMS2Ih9y7/PcJeUeYPgzfLjT93NMqqG8e9QSfCFckchLG3h9oe5Ai5i97kuiMgRFzNxUQ22Vc9V9HJ9a5Fq4fiB9hfeZv419dp3idXO3+1CPs6/wEr8LGzF8RaMR4ViWD/D6jtNK1lChHof0z++cCq/TjkZs0pF1kDn08f+BPDzk1/Gz3iHEYskHNySEqcD7BuSli85pVA9wXtUObvCVg3xGkYkAap3swxA9M7bojPLm5Zvi7s9tYlvtU57KZl7ycpe0LSos38F1j1zlc0uJylPtABLXOS6jHvScbMjcQ9Aia4dW+Vt8WtSRTge5CH+8+AscS/KE3LdfewQg/0BR7sNJ1HaW4F7hJozMcnzdZ7hPx3/CRffXQzJp/qY4PfgLCtoRwisvuSMqrhvUSJ/ifjH7zA2wTTMMNNJsmLrK5/BmsGqFhYVFW9wUkoOvrJA+7QceXkGxTEpeS5CDdJ4QCWAEyI8KhSyRplW3Unx36z50vUQSuXyIVN5wxdgMoBoiRkXILYyRLr+pifCscPKRCgDg+mOk1EPPugiJFOLnbeC+Ryw/T3Wh9zj7qnTzvou3kKT3vaKwcIjL2oxwLB2neI9yn4PSsHC2TnKHwJQYvTqaiF3ndePv4XXdr8scLvnhM2OYJ5dpCkMJlIDFQ1yXAO1RHsdMXtVw388xl1a7AQDhJbapphXqCZFQQhzT6u6WdBCZM3MX7jRU4rpORlHtE8PnIsdbj/F+gy+Wcf3/w/bJFzjOXER5fRr4xbhZ7SMHywQcRI7EcCNCHXwejWW/J6n4qqKWRHl9blsDnSe4RrUUv+t/RVS/Hp/HZY2h1khw4UUf6s2At34APOmy3O/Cmeb9Os6zr4l38P/x6y1OjPcsAwAWr3QCAHpfdxGRtS32h9aMNz4OrOwXifgLFLd8dc59IRxBfILX5Tew/dIeXu84RVTTvHePSNLVTlmfFY34rOz1GAdVlL1Vjyskr7Kv5e38zhifO84qmNd+Zb+IGm8CKzlYWFi0hdL6Ozk0/vYRGhnRQx//KDrOKPgrQj33GReMcf1o9LxBfW3s3eQyYXF3lQY0UuIuLPfyO8MVfWWFAC9D92ly6+VdlE4aYereABAorv3MbPHBFbnK2AW88c76MfAyqfb0fWzUeUZcoCMOSkOkyPHr5GrVLv5WT7uepGJcoHVxTfpKCvWk6P5nxZW5XtyHMxrlbrE/bBCJY4HcoveYi/ywkUzETSmPNHUZyIkxzxh4fTKV/pdLuP5+0Y0vi8FPdPJmVHtuNSOxGPtCrUMjNiHS3HpxDctzSEy4KHeT3xRH+Fvfa2IPudVBI85Jq7pIKFH+37/s9/Rgn7gWDZdvRl30vC7uxp8YBwCMP7neW8NQRtyhaTG4yjNrJDTq0md4Tr4UScJXbbkSjc3JuNAzWwLIbZHr5rmuDbHbuAENLTa84afZ2eQPiv3qWhgVkahS53k/43qPjyuP41c288aJNygdlIY0wmIsNu5mMybtA2pp3qeZZt/Js2Lbuq2ARpX3CV/iczRSkBsAXDEn1JMibcneCvWWEPsm7R2rYiAe++g/Oaa1Poxvg5UcLCws2uKmsDlAkeJnt7as2xDXYE64mX/Fj0aEFC+YYRPjnuk9qrH6g1S4K7OUKmKTJJ3RWe25J8feK8FBUbLR8HgQq7vWekfqUSONuF6ATWCJy+QFlMxorOwg5w5R9UR2E39rRDUS13if5Dg50Ky4LdNnHAQK4n7dIdOUoJj0laYnTYSXyF2y28LSp0JQAmwCwjmKQxz3zP0AXP7dJfaZptgZanNdiE8aiUbsEsJdVnZGkLjWsmADLSlm3x2XceaZLVwrtdbl6i8o1OlJRtdJCVy6n1xt4OE5zI33AQB+655PAgC+dv8+AEBvJY57u+gX9Ykr7USefsREoIIXf/cWAEBuI/uOLPC+xWHlSYNX5uin9on7MLQMdJ2haDN/K0WbjstcxOn7gfg1PreBF4rShnujNKgRWhH7g0iOi/v5PMvDTeiA6O4JPn/D0dNXFIryLJd38bfRv+T1+SGN0LIJRlury2f2ak96USsB6Zu/uYMVONMRmau4MsX+kb4AZEOy/iN8AKFVXq9fi3uek8ImuWCK94hPac8OFJb9uXo3J+FeiKM0IPt4+n8DV6ZTByLzDkqba+h6WSbvE/Fb5l0YVZ54HsitfbD1qELoWfq5GndJpORpiU3oV2gM0rilGyKyiivNDWoYOXF5l4ilYugLZpVn9HNq8mKJq85XBxry/Mvim06MS5txRlcCQLmL90tc5W+rexuIjUmMAIeE4npOsLjVRfo4d/38bXxbjWjYiMALZ3O/bUxOXSE2yb+X7uQGiL/Qxes6gLC43HwVtWY9/RUgO8S/jToD2Suz/3UT/BsNsZM2oqbVupvoHMpwbZsc4B09cwCAVy9vBJritm3ywh0xOvjny9vw9NJ2rkuDz3h/B4MpfqLzJTw+wkADExdhYkpUQ3nqUupp9qlFbyoNKszfsjYGJTHJ8UbmfKjJmGfvJlEwLlDlAsV1DZm7UfOMW9BBfFzcvbJ2Tl1E87BCVdS62Fk+iFqc15feXoA6mfD6AIDYlKxhnMQfADKbJbpTDJux4xGUjPoh6lpuu6hbVYX6KO/X/3nOc+GWlipnDK6pcxLLIM+q0qVQEubhuY9nON7EGBAs8rdK+q2Jg1UrLCws2uKmkBwAABpIng563DqyvDawB9AILYlLaZq/VVP8f36D9sT76CuSSyDuy3Kfi8AkOzWGMxPVplygEZEgIRPL3y9x8dnW0IwrLDIvLqBdQNdpGYO4mgxXqsdbLkL/9bVGzvCs3xPha1vE0ldmn8GZALpP87vsNg4wPiYSzl1ZuMck2k6IffIqP91Aq/++5yQOv0LOs7rFh6YEUhkXnOE2pT6FgVckSu+AiNQSOVrs93nuW/M8at38LX3KjxUfx/LwvjMAgPkypba+vgwc4eqPL+8BAPxY38sAgFf9G7FY5iLlq+z088/x4X6xejtSE2JY7GwZNwEgt8GBXwzKZXpjkZgUsX9LBV1f4Frl9nDO2Q1B7/qGiOS5TWsjArtPuyj2cd0zByVX4jSvC2Y1isNioO02Eau8X3jRgX+W7Tz1UALlcDbhSYO1DjFqH+b69n/Lj8V9fDamjTGIFtY3PRadk5yHDdsobf3s25/Fn8zcxTUrUQVryt4KX1JY3cNx+crsIHmF13edKWHsvRQj+o7wu5m3SSCXz4/Zt3Gtul5969ffSg4WFhZtcVNIDq4fXvizccNVO01ATissulfcTblRofq7SQ3jV/0eZ6wK1a73se3IVxxk/j6NBY1jTFow7iFoePqluT51kZ/JH57CwjeZStdzRDi4ZEIGCgrJq7Tszd9NLhiU7DlfWSE6I8a0fjFu7iMbjp6OwC8GwcoI26ff4I1rKWBxHw1TQ0+Tsi/vZtvOv4gjI/HzCeGwy/tlLFkHuZRkQE6vpfWlkQZCkqHZd4zrMXM31y4xBszcI49f3Nkm8Kl8dwGOBC2lYiLNvNbLz+1NBJNkewERQ+IBYYNR4ORlcriFKKWJiPhOTz65HU0JQ2/Eed3oE5zn+AeAQpXrUB0mJ2+GyaG7zjaxsp1zMO7YRljsLZNh5NaJ/WlCskh7jI3F8ewQzehaQ1896rTcjBeNGMmP5cOuF2QXlXBm7RNblQvodVwP5yqfVU2kiuis4/WvJOAofo59F4Zb8/Ivcp7Dh2YAAN0AFnKUqHw+MSyHKSr99tiDmFmmlBb7h7RIv2NwDADwzdgOpF41Pk9+mD08d0cMqr5WsgnL3igOAdFrstbiXn0zWMnBwsKiLW6KIKhY94je8Z6PwfUDXafpksxtou3AcIlGTKEm7h+jtxv9uRnWXt2H0W+SGxnXVDPcclsZbmF0z9is9vL/e46yL2PHqHYA1W9L5jL6fjOkkbzGvwMlcefd2dL/TI0A40Yyrq30aAbqayTXxk5gXKf5jQyEAhj8ArRCjv0l5emxRnrRwoWjUz5PGuk5zj/G3x2RsSiPsxlvheHe4UXHC5Ax4cwmgesn/8UX8XKWosrTZ+hhiF8iF3RqrSAdv+jP+f38I3wl7I29LtJB6lJrTsaL4y+sdY8WttegyrwwMiPh0DK2ZkQjcU3WXwQdr+bGiEJFQrCdDkmquyaBXdeZgQgAvcc5vvnD/K2W0t44jefLcP3onPYCx4yEaRKxCutdbz6rd/B+G/+UY5m5szX3ZqT13ACuudrGfR0M8kb/bMfjAIBPz92KYoNr+1MjzwMAfuWVDwIAHt15Dg+nade5OzwPADgp2WdHipvxqauHAADxz/C7pf0yl/VFxJ/i+2Oe0eJhsZss+NB5gWtm3q3X/6x9ENRNQRzCQyN63c98HOmLrhcZabLtjGzjKwMDz68CAK59KA2gJfZlb6sgcoEP3hh6AnlJ5R0GAlLMJPQ1imjLh7k4A886mL+d7U1GYu+PMArv0rFRJMbE4Ll+bXx651mNxYO8zqQm12Mt0dUY8fQAn8ym3+Vv0/dGPWNaQ3IOSn3y8viB7pMSF3GXiKXjLcHORO2ZqMJAjOJ6+GgMdUlLqEoKtsl2hNMihOFFjmFFMjB1tOllJJZHuGEDKS5eLFpFZpmibnhMippskN8uhFDZS9Faz3OiG78kasZsDtc/zDiH1DUxbg5wLJUe7T1Tv+SABMWoVxxUXixBfhdfOifIuUTORBCfYl+ZrUJUhNB1nWug1MU3MpyRNlv4//iU6+WFLBwS16Koe7mNLnQnb5g6KmphTrIruxRyuzmGziNiqO0RA+Vg01MfjWpkXMqhZeUV0jGRoE6V9733jrOICSVcqdFQuFzhQ3vvwEn852MP8X5Pcw9Hl2TuH5vG9ZdJUQM7qVZ8cNMJAMAnXr/DU4VM4RmjVjhNwJH3wMRTmPmt3lpDWuZsShic+78/biMkLSwsvnvcFAZJ1QQCOaAw5LQyC6WIhw60xODrH6BB0biDMgdElJwPee7J8nb+mDhOKpy6rJFxKDEUJTMuJC7JUi+Qusj+TcGMyuPrAQDROhB7N4N7mk/0A2i5QHPrlBcQZT5raZEcKo6XERd8jRdc+1lxl72oURgmPQ5mxXBmyoUVgam3t/IsACC7U2TdkIvBb5BjrUheiHYkWMwBKoPC+TMimi+2ru97jV3UYxKQdVlqKZQcZLaJEVi4330b6QtztcKJr9EVWaYgAP+8uS/gv8q1NfUtZu+gnuEvRTw1Znm3cPe9rMxSO9aD0qBILcLxUhdNQZhWAE9HNxe0fJyBXAD3BQDPpWmM18U+n1dXY3WPZKJKjk016SCzVfaQiPumHkR0zoGSbNj8OlFfuihJJE8HAcnrMNJdbacU75kMY+j5qszZiIfS57yGYeXVEbZ5bNspAMBQaBV/cP5uAMBAmhJA4U8Zgfa7g6NIiNvYROtWpAzij/RcwkR1Hee6yAU61klJwgk10dvNvjKTfWvWNbzkDcXLNTFZq8GZoLeuJiL3zWAlBwsLi7a4KSQHpw7EZl3M36XhK0o+gxiK6kLtS9urSB4ntS7eTsNb8nVS0+KQ68WZ935TgmIkRh9aedw5fURsCCTGLPxpyrVJm9R1Uu/ZezWCnyVF9okBzlDa0qYmlIQJ11PS17IMQAO5TfyzGRI36SrH1H2iFZwSovkEHZfI9WsJB/mNxj4gMf2baStxnurA4gETrMXrTNYj0OL8zWEJE5eaD/Frfky9XQYtH2mpUVAcUDBs7+ABRlQtVzm2C09sgS+5ds6e7afWGntBgoVSV6WU327l2QNMVmfkv1Lac/uA3EbR/aV6UvY+sszguYgXxJZZFUVYJJDYtOMZQE3AminbFptrYHG/VAOTADmT1euraUSlvkLtx5hrsnqRxuCOM63QeDPOmjJuXcCX963pa/0fss31n6xg/rC4MCUPxRimI0sNlHspzY30cIECsnhL9QR8r3FBrw9zfqFNhrXDs6kYw3B9Mg0A+ONzD6G5nWvUnaLYdG2JEtVHDz6N3zl3LwCg2sX7JKSyk9IticEYJP1iOC+NuOh+nJLsgszlzWAlBwsLi7a4KSQH5ZJKJ676kD9AUpe+KIU8haPUVkOezhh/WarpiDU64ndQFktxflRqSAo3iyzUcf3viSvMJ+XuhQMt3NNA+pRYpPtM3QHew1d0kJiiHrqyk5zfhEiHF3y46z0nAQAvfZVZh8aVmroErOxbm+zkmgCbvVFPFzQW8Pw6UvuOC8DAC/xt/jYJVf5qWsYL9B8hd5h6m3Dri1IrogOA6NSpl2gLqEiYcTOsvfoRpr6gkbAqAw1A3KL5Gq+bWJEgsS4XoRU27DzPC3PrTMUljSqH5RV6Lfeybe+xJrIb1yYWGZdydE57EocppLv5P0utxHdHPNtG7KzMYT+lw3Ij7EkFzU4jFbCt09BeZSsTBt2MSHHYBQd58fD4zpLb+ky1pz6FkCTtVQbEwyPu6kqXRtdJjs9U94pK7YzoKR+SY2zvBjnP3EZxMQcC6Hsns76GYxkAwJHl9QCA39j4RXwuTpuDef6VIalh2VFG6j62vzIjWaeTXIPBw7N4uJ+Vj/sClCKzktD2n194BPBzfL1S76IgiXT6UA7VCW7W+IQJ75fEwFUHS3spMRTWSSzAm+CmIA6uHyh3OghmNWKnuTAmE84rkOm7QXzdZVKNWxlqgy+a+AYp0SYFNafeFkRIKv82Q8a4Ka4p7fey8cKjNISVslw45XOxsl384nfyt+BL9AvV0sDRT5AoVLZJYRfJg2hEWpF8pihKVtKzVw40EZVqwMalaVKiqx0KRYl5MFGhJlU5tKoR+gVOIvFVRiAaAsCXTjIDaTdFrVNi5487iC7yjZi7Xdps5+AiV0Iob+TfxbrkckSoW2Wqca+ojP8evn2N5wYBsIBMhqEPyG8U0V/KuBX7WgVyyuvE1TrFB1jpUuh5B7MwZ15k5OncXdTJ0pdcLB0w6p0YMr/OtV88CJTkHAffEscZ3U81ITfbifCKxCJIKrsxeoaygO+sxKyk1+ZruD+0BPcLXMDgsjGKynPoaSJ9maK8KQJrXjr39ixWYhxzdMYYlMU9uqWJ7HVSuPtuZWp6UyyEvz37EO55hMbJXXFWmv2vR1k5eWffHD429AQA4LkebpTwAa7dbC2FJ+b43XyWe6/ZMBVum5Ctg/y6tapxeSyOUNa42PmdIUq1ribqJhvzO0QxWLXCwsKiLW6OIKjBEb3+pz4O6FYx1/zOtbHo0VnlBW0YtaD7lAS+bPZ5J/qU+1rcFgBWbqmj5wX2YTieEdEzG/1e/H3vUQlgEffV0i0ufJKXP/zs2pJwyoVXckyJzuDVeqi1IjgLUqshcUXy7WNAZTPVJt8c1aZAoWWYMm7AsIjRVRGj42MKtbQ0E87slZlLaATFeNt9ivfLbGoJhMaga6IoTZBR39N+/Mq/+HMAwBslWmg/dZ4Rd3etv45XJ9dzzNfJPZ11NIhpDcRekOxKKRO36SDF6SszPYgflcIlo8Klw639lbgkKofYwUwUJNQNJ0GJAS26IGre3RqhRckPea0ubeS3W0JeX+Y6o975d+fQOJ1aM/fiRl7ffcSP5UMSwXlWTtiSczLyow5KUuvBcNbhJzm26QdbruDcHgmUep17qzSgvGdi9qfJ71FFH9571zEAwJEFcU0+SSlj+wcu4uoq1Z5d3XSdD4czAICJcicGwhLAJyGcT/9fzNLUPniBeMaAbfKKoIAOOYdlSQzZOszB9T3vw6IJd5KtN/aLtkychYXF94CbQnKI9o7orR/6GAqj8KoamTj6xBQpXn7Y57mfjDGpKKXP/EV4BklzHqMpheUvtdw4BkU55Sg2CWS3S/CS6I7dUvpsaZ9Co4eU3+izqWsSkx5RXhz7kEgVpj5AbmPLEOkZn6SoaMfLQTRNjYGtJsRWwraPtirzmJyAGwNZjFRhckeWdosu363hl5Onek5wLFPvlwKpV0Me9zM2hJjkSFT2ldCdZrDNvm7qwc9eZ2m4QKCJ7jglhcGYcC6x5u2JT+PLM3sBADs7yOnKYn10oFEXi+erYxvY16lWyfmgSFeJKXHfxlrZjvN3rg16C6xIWPSyQvGAhGsvS10OCVmv9DXQcVI4+RYJZkpy7sF5/5rixFxISFt4+TYmlNu4LWsphZq4p00wW07cjrFJ7ZWYN+tqxgtHw5+Vgq8SJl7YyjXveiWAhpRtM5KfMQxDAcXdFHs+tPc4AGC6nOb9AxU8+2WKByb8fXWviCdOq3CukVS0cWUvhpDckAEApH+HUl52A59RdptG+ry4b8UwfO4/2PBpCwuL7wE3hbeiGdVY3d9AuLOCUlOKyEpwyvR7yWX8sw5codKmWlDLNabh9pI7F4tU+E1QjdKtsxJNFme9i336ygGs+zqlg/wwKauRCOJjCuUKuWzvMVLk5d3sWzs8qAQAZu4V74h4QoIZxzu+LS6JW0rcstGlJpZ3kNMNf0s4nUSzFvt96HmDyvG1D0oSWb+cEl31o+MVNsxu4P0KW0SfrTmAw/vM/xDHKR5KBDMhZHeLy0wSqEwVrdhMBE0/FfZLs+LCZDIgDu28gsUKn8OZRbpAmk1xba4v4h+vfxoA8FlRXl9/fSvvMZrHvSMMqEpJTHAhQMmhMlKH/5y4jTuMd0W8Rr2uV4reuGhNWLO/pBG8xHGa55+SszkrvUA9weuUJG4Z70h4sRU2rR7jZlge5zwTl3xeXUr/XdxES9M0zqQu+LHxT2hDmX4vRczYlARtbWudbh6ZWVsLtLCh4Z0GbkLBE1LPwWlo77uglNI3iWbFEYWfP/QsAOB8cQAAcHqBn6ULaSiRCkKSOBXIid0mpL2zP+ffzb3vHwvL3BUaUwz4GvtBcZ1Vxa1b9HkVroz7/s1wUxAHuAqq6iD2eNxTJ1bv4qqrJVO+S3mFTubuNU57LlS9s4muF/kC5uTUICPC1uPKy0w0hWRMkZGrH1bIz8hJWRKbYDIhaym6fQCg/CsZAEDHf+aCX3+f0zoLQWA2Tf+ROsZ+UDbAFak0LGJqbtTnRayZeAWT3u0vAdUuznXwBTGOhvhSrOxU3jHt9W4pImtSqKvwsgEDx/lCNyRDNLuvhoAYdA3xmn+QhLH3mQCWD8pLNilqQYQv9OxvbMLY+yVSVcR7tYlqxko9hj+cugcAUBIX6D975IsAgC8t7MepZbo8f337lwEA/zXCjMNrr4+gKMV4jdvQIJh1kJCSeiaF3kQeZrdqL3V9+Bk5wyEoRDfahNOQMyHF8CmmRNRSCqXNfDF65MH7OuQc1MGIp3LUX+EzlaVGdm8NtRSJginGq8QlmbimvQOaDVEwOUDxa34URs2L/21z2Ax0XODYS/0S/SrvbGWkhuEgidfzy1Tr8jkOJlxoGTlNSQC/STGvK+8EuPhxtjeqSrlfIykHIPtO8hkZI7m/BFRlP+rQWxMHq1ZYWFi0xc0hOQA0vjmtgJXABCWBjgviouy+wbAkp1G760i+o2cjSF/h35mtvM7E/ddS2pMY9BDbTCRIRmNjyhP3Av3kjP4LFIO1X3nlvjLPUbQufaAh1/k9l1l8jNcX1pHOZjb50P2icPKicY9KTseWJgafYfuEGF5NKbl6jOc7AkDmAY5z8LPk6AOvaswf4t++CdFDZE71OBCbkq9EjDaRi6krAe8Uq7gYduc3cg7ZzUF0vSHl0CRQqrDdRF36Eeul6FVbpbhdy3JdX5lY75WQa1xiYM7v1Bjjf0f/OO6Rsyn+7ZV3AgAW36DLrhlzkZATwEydhZqoBP2v1jH2AY65/1m2yW7g2CJzCjEJOJq/xRRplRyUK62o2cHnxJC8p3XWhFGlVpck4ElU1fJAE9W0RI6KdGA+gZbR0BWR3hGVx6kpRDfQQBsWN6k5T6Tcc+NpW+TIiyLRxa/6vX1m1InAIwwu+/sjZ/Crr3Hy//gAN8fZV5kYFJ3XWNkr7m2JdDUFelKXeE+gla2avsbOl/YGsHynBLtdb2XTAsDw11dQHpEiD/fx+PEJtIeVHCwsLNri5pAcXOYyVDvUX8tDN7n89YSGUxf31rScZiVGoehiE1d/SDiBcGvDPQN55R0aYioX1VNSaWfBj+S4cM1RkRgc4egjTQRyvLcJijEH0jh1ILQqdQRuox478Ljo7U0Xy7s4loQYLU1Ya3jGh3k5lMQ7z1LqSZR7gYwc2tPMs6+ZuyWnIO4CQTmlSySH3uNVaRNCZrcE9Jwz8f5y35pCdFYCcsRtaPoOl1vuu5XtnFdaSuXNPlCHM0HukhYjW7koksvlANa9i1+uX0cp4bkppqFGfDW8IHrz9jQfwFyYhWn9BQeVTmOEFU4s3Hr6Xj8iIknN3SvHAJiCqJvrKO2QuhNy2pOXvbi3gNAx2llm7hNX9HHeY/HBGnxzXKuBl/m8S93mFDQHmbfT+KsuSZasGLC1L+iFRrt+cT+KCx0AamcksCpqQuONoc9BaCPFiIqcFB8KcS7OxQSW7yBXD4nBdHcn1+dYZhRuleP6recfBgCk5DCc+HQdOTFAxyWTNSP5PaUBhbqc4WkOe/JX2Lb/5RLyM5SOTeh3cTPvP/kbPoS/Lm73YwN4K1jJwcLCoi1uCsnBaVIf9JVbR4GZ05ZNeGxivOXe8ioySd2E7P1VpF4lB8juENdnRbII09RbASC/SxQ+CXmudGpM3y/eAgljTl1nm2pHwNMntSNnX0qth+Jgiwv6FzmYuXeRk/d9I+jpgBk5pMRYpp0GEJs2biTh2nuNy05RQgAA0enNUWaBVR82fEHOhLxd6lWMSo2IU01PUjHZkqa2ZmZ3AzUpmGskslCHmNmvBzy93iuMKhbt9OkAcpvXJn8ZG8t7PvgKLudF2RWs66A7sMPEKQN49lXW1U9eNdmO8KSYwtsYeZT+qpyWvtJy6Q68xOdv3NXLvgB8e6nnqzNk4ab8vLoWQ/qqZFXKfikOimfhZMgLJ567QyQGWfvsnjpip7hfjGsxs9MU4nWwcoB9mtOyTdCdcoGScOKsJNyt/zI/a0kgs0qpom/M1AQRafLuLPyXaRz5jQ/zDNEnV2V9AhX4JXv0wgLtM/EZrktmUwCueBRMMdiu0yJ9BVr734RKh/4Bk/PKv9+HrNTOqEoB3o7j5rzXpBcgGJD9/Wa4KYiDVsy+rN6bR9c3KDeZaEiTJh1daGJ1x9rNbDLiQqejXkHZ+FVOafFBvqyRiyHPRxy9wh1oKjknxoBVVkPzMjwnHhV//imN4iPcxL2fFsLzIzTSFZZiCEqpOXMeYn1Vskmbrnew6cBzJluRbSLLrpdSblxUxrjWfcLF4gf54kaOSO7C5qY3z+kHKOZ3XmDfKzskz0M76D0uqeU/zfFmx7lJk5f86LjE3yoSW1C/GPfWNT3B/k38h6/KPyYe9nkVt8ujvD64wPu9+G9vQ+pnGQfwwjT1lw9tfAMA8IWJfeiKkjKa4jeZXaImTPk9d2z3F7ieS/v5/w1fLCKzlS/E+Dv5QnWc47p0XHCREV9watq43qSITgAodZv78Dfzso79sAstTCByhZtp6HP0LWZ3jHol1bJSSi4qVa9TV5soDEksyW0kyKN/aoykAe88S6fO+17/oGSFXvJB3yJp1YrjDUnOg5pIYdujVMW+KAkRHx1kJubPn/shVOryGr5BAjJ/q7hCr2oEJRo0Ly76yFFDHBSqPRINKqUAS5+kmpBcqcJXk5KBooIbC3bmHUWEX+MeMOrIm8GqFRYWFm1xU+RWxLpH9M53fwz50VbQR0Wit7qPt1x9rtRoMHUETCm59CWNRSYUeiXETABL7xt1jD/Gv5PnW0VZAYqLRsQy0ojJQUBDITIhp1HJ+QVdp9i21K+80l5G3DNHnjeiQEhEVVNrwJRG91XVDVF+WDOW5VuaSF70e3O9EekrLsrCIc3ZG0bMD6040Lso0USepXRhgnEaqSbi18VoKyJ9hsGMiE/CO2vRlBfL7ZVj3qcCiDHdApVOo26Zoq4Ooo/yDIW5GUYcDg2xvkL5C31e1KMxwkYWxMV4ewPR67KeIrk1pDx8aCZww6nhvK518lnr3BGTG2HUn3pcIzove+AyH/jEByWvZDyEyLwY7CRXpSQl+BOX/ajcQgmn4xviY5T7dZzL4doHycGN69M8z1Kf8oq0KKmrYFyMnReamLkf0r/U9pDn6B7IY1svDZB9EcryGyN0ZX5u/ABq36KrNTHJsc+8V2phXAl7ORUmd8Tk4kTmFdLXOJ/8kEgJcr7s4kHHk0hNsJY5B7X/ldYpWGYPXf2VX7K5FRYWFt89bgqbQzMI5DbwdCbVQ5YTf42WyMVbWq4iJeQ9LicQmZoBCw/XEDtDdtJ1llR38SdoHMtkU/BLbYfcXvadPEkddGWfhit57unTXIpqBz87TyuvdJlXRUeCdqqd2uMcqeuSJXmXjHPVj4AUyZ292+QLiFQRa+WDGEOkKePV86oPDbF7JB4hZ156jQPIbnC8kvtGjw5OmdJ1GsN/yrnnpOis4aahPatwXmKtAHMaeGKMbSo9CtpnosP4YQ5xic80vYAjw8kX95uQZw08wXFF7iIXzD/OILFGJ5AYF914Wc773MX1jEwEpHw7UB4S+8DnjLTlIlCQ8ms+fjf1oIRFh1zAXWunSYp71fW3TgXLbGb7DjnXdPXWGsKLJptWwsRF+nIaQP9fcg9MvU0qRy2LTSaWREhqS5jKUXOSMRpa9qFfTjLPrRebzKBw+z4gaPqQGKNKP397YOQ6XplaDwAYD1DaOh/mmi1f7UTvUovjA4AumigsoLCOf8Zkn5gAsko3UJFALmNMrc2YA3ea3v6s7pIqWlI/JLNJedKxOVn8zfAdJQel1IhS6hml1Hml1Fml1C/K97+ulJpWSp2Qf++84ZpfVUpdUUpdVEo98p3uYWFhcfPhO9oclFIDAAa01seVUgkAxwC8D8APAihorf/jt7XfCeBTAG4FMAjgWwC2aq3f9AiNaO+I3vbBjyG/oVXPwSAkx5xVuhz0HqM0cPkfiKVeKGzqooPMXgk4kROS+1/l9a4Pnr7uVU+SkNvwokbu7dQ9w2GySP830wCA1VvqCMakBsKzojy+g7p19ZUuTw81yG0y3MXxDl3plCP28uuN7to6ps8kGKVFqlBuy35RvIPzdOcoEbhRF53HTYFXccOKOy+8oDx90mQ2Gs4XyAH5TWvLnhtdtBHVWPc1OSn8VpmfLH0j0goLNhWSluXQGO3Xa/ReAF41rWBOoSwFW9NnW2HMXB+gkRRb0arhgiYsueXaTcuhNBHhpplNPk/iWHybVF96gRxv+Y46Ol+jtBOUdTV1IQJZx0uc+3ZbTnROe0ciRidMlS7xAoSATh5PiaUHeL/hL8iBQtv9rbHcLvU4ZL/pzrp3tF5BJDgTfHXHr7+GgWAGQCu56tpn+dkMt6qWmezhJBNboX2tIwPn7hJ3qoT0hxcdL4nPuGxNNS3tB3ofZkz9whOs11kcNe5ZHzL7+VCGv86xv/z5/29bm8N3VCu01rMAZuXvvFLqPICht7jkMQCf1lpXAVxXSl0BCcUrb34TGhDdkQoKjlibhulGqlyQStNV4MpHuPjdL60VeJphDX9WfNnbMgCAmShlO9VQCMjir/s6X4brj7FPrZTnNjRVhacfFCPdmSCKI1we4+tPf4oiYWGvRuIexqUvXKPYHhsXo9CChitZg6YysamWHFlwsbrDvFiS/SmEquvtMxiflKqxOW5+ZQqfZHxe1GMjTSKYPsOx1RItl66JFTD+6/wm1zO0Gt+d7FFUejUKo1TdjEur7zWpcP2QA5+c87hwu0SXSqVjf9aHdd/krrzyYfHjVySSdKTpbV7jgzdGvdgMEJGqzibvpThiSsm53nwka9kTi5UGFt9O0birsyBTkdOwmsozNpp5dr2hvOnWViStWnJyisNioNxeReSinNIlL5iJqIXTGrshQtMPyMs3lEfok9w7Gz/DtZp8O2/c9UQAdVO+UAoGVZP87fl/fzsaPyxnZ2RJoZQQkJEnarj+EY558HFRE7YYt6qL3AZjYJc4DDHcdp9uoJaQzFmJ0SnL9tE+IPspvqJlUTl0QAzoER+Skjo/d4cwuM+jLb4ng6RSaj2AAwCOyFc/r5Q6pZT6Y6VUh3w3BGDyhsum8NbExMLC4ibEd22QVErFAfwVgI9qrXNKqd8F8K9Ac9a/AvCfAPwEPOF0Df6a7qKU+mkAPw0A/nQHslsA3/WwVzbeXSKF7jxHTjn5oQai5yhVmBLshlv4S63MRHeBNGr0ohSFfUChPkrOs7SPfXpFX1LaKxA7f6u4G6XArfaTowGAdkyZb+GCUwq5AnMGuqdaBiIAyG9QcORU79TnE3Idf1u4UyM8J0FWEsxUTXESS08NYsuLlGwqPSbzUgypowqNB9hn7NmUTJQfzQN5xJ+SoBZTcvwRcin/yQ4vB8SMofu0lF3vinjlxUybudvECFltBaF1H+V4V3fy/8mrwJUfldwRiSCMzcqRATt82PAlulUv/6LUshC/Yz3ZOo9UB8h1Y1LOv7DBRU2MeCYD00SXxqeA7qflJGzJC1kQt7U/628Vj2UZCSQkxbCaap3cndssUagSRRk6FkF+k5wSJWc+LO1nm9iUg2qPKQYkuTQi+alTCcyJJOU0+Iw2/w9KkJf+YQ8GJbpzdYsEUe3kvnNWAwge5QbR4hbveUOiRYeCSIr6maGmgUqvRKf2tSQ9EwxVWc/BTDym4EgkbtcJftblFK7uExp1yf0wcw5IseTihqZXsPc74buSHJRSAZAw/IXW+vMAoLWe11o3tdYugD8EVQeAksLIDZcPA5j59j611n+gtT6stT7sxGLf/rOFhcX3Gd+NQVIB+FMAK1rrj97w/YDYI6CU+hiA27TWH1ZK7QLwSbQMkk8B2PJWBsnw8Ige/vmP/c/OxeK7QMPkb9xEMNztf0d8v9bzxjX79jF8r+t59VfbB0F9N2rFXQB+FMBppdQJ+e6fAfiIUmo/qDKMAfgZANBan1VKfQbAObBq18+9FWGwsLC4OfHdeCteRHs7wtff4pp/A+Df/E+My8LC4vuM/33lOQsLi79VWOJgYWHRFpY4WFhYtIUlDhYWFm1hiYOFhUVbWOJgYWHRFpY4WFhYtIUlDhYWFm1hiYOFhUVbWOJgYWHRFpY4WFhYtIUlDhYWFm1hiYOFhUVbWOJgYWHRFpY4WFhYtIUlDhYWFm1hiYOFhUVbWOJgYWHRFpY4WFhYtIUlDhYWFm1hiYOFhUVbWOJgYWHRFpY4WFhYtIUlDhYWFm1hiYOFhUVbWOJgYWHRFpY4WFhYtIUlDhYWFm1hiYOFhUVbWOJgYWHRFpY4WFhYtIUlDhYWFm1hiYOFhUVbWOJgYWHRFpY4WFhYtIUlDhYWFm1hiYOFhUVbWOJgYWHRFpY4WFhYtIUlDhYWFm1hiYOFhUVbWOJgYWHRFpY4WFhYtIUlDhYWFm1hiYOFhUVbWOJgYWHRFv7v9wAAQDWAyKJCPQ70vVYDACzvDgIAXBlhcVMd4Y4KACD0fAIAULqrAABIfisG7WO7SqcCAHSfaQAAJh9yoEMu7xPld4HJEACgPlxF7GwYAFDYyvuGp3hfpwpUdpUBAJEzEQCAr8p71FKAqvPvYI6fuc28R8fGFZRe7+b9+BVS11wZm4PMQd4H8ltwNsC+KwqVbZyfbnIOXS9zLI2wQqCo18zPJ93U78uidiXJdbjG7xoRtoEGmvdlOYZPc83yw1yo2KyLhdvYLJAlj9AO7wEAgQL7KO/hGrgFjjN11o8mlwOVTrZ3Q/zUfg0da8qP7DOQkQfjANEZ9pndwefgKznevcxzNmsWWWCftVTrt1qH3M/Pz/QFhcy9smbLfKbJqzIXBcRnOJaZR/kZnOMc6mkX2sc+hr/JMa3s4DijsxrVDuWtEQBkN7LPao+LjjPy2zz7nHy4tdapC+yj0iNLMMB5Rqb82Pr2qwCAhd/ZAABYPMjrglkF90AeANCo8/qhP+M4J97hIDbB71LXeb+FWziW2LRCKMvxrW7jd+Fl9lnq14hP8e/oAttkNrNNaFXD4bCQ24i3xE1BHABAO0D3qQYqH10FAEQ+1QsAWN7Lhxi/HEAwx+HmN/C7jq/HAACFYYV63GxsfmY2sq0ONZC4LH/7+KlkcaIzYTS5p9BxVDYO3zPUEhrONAlH6iofzNI+LnDyGpDZJnfz8yGYl2llLoWIvB/uPj70xVQcAOBfl0fwEv8OL7K9G5SxzGmEMrxfdntzzTi1A4/41VKcX/I6/18/mkJjPSmVkrcov5nXDz0FlL/BCVUTWDO/xoMrSD5OItaImjnzc9u91zHz59zEgVdICapp/tZ8MIPSODvx9QnxPBmTfjSaZXnhs0LEhIj6i0B2myyMvE/xcbatxwElj6+WdtfMb+VQA5EpPpvoLC9MTMhL8UN5RF9JAQDKB0sAgFJZCHlJIRuWRXPZvtYl69pUQIh/Lx7gA4hNcwClfoX0FXneP8D5hV7jMwtdcpDdzHaFUfadvCxEtE97zyh5nW0aMXmxr7q4Wt3Eddwnz08IeTDvYm6UY45O8PkFM0WuWSGK0jDHHswJAZB9U08A9bgjfbCv/CaOO7ToQ3FI3oPdvD48x7bFQcCVPW+I9ZvBqhUWFhZtcVNIDtoPVDs1ph5WSDzeBwBQKflRWMroZyZx7p/zt8gYqf3qO6lWqCsxxESMKoyyfee7pwEA/s8NYuUg2Vf8inCgeeE8OxUCeV6XG+DtnAavr3c0EZ7l8uTWkQMol7+VexVSl/h3M2yuk7lM+pG5g6Ju5xPkOJkdbFsthJBcEtFvgN/5yzLuYYVKH8eVPreWZjt1oNQn4uwMr8tu5W9dJ11oP+eV2crfBp7jb5lNPpQGhfOI6hA+sMI5nexCXTi5IypA4jrvcenZjajt52+dJ4S7i8QReD4NfxfvkzpBkcNwUzcIDD3LhZi5h2uXPsHfVnYpcmwA4UmuZ2KS96imHFRT/K26rSprLBJATqEeE04sEk5xWKSumTh6RfSHjKUqqkdkUXvqlZL59R7h/+fvdrHxz3nZqqxj7VGqX+WpBBKTbJf6hpFM2Sa/tY7Yda51cVRUVJFmVRMI5kX16+L1jW7qfvN3+tDzuqzZdY7X7Klyt0LHSbY33P7yPxBp5pqCaI/QZkvcQ8lavdyBhuy9yhD3d/ICx5bfX0HsDH/0Fzk+I1mHMgqBWV4XnxNJ7k1gJQcLC4u2uDkkB4f6auKSD8VhUrhgTgwq06RfV39yGE5Or7mu97PkLsUBhWrn2j5XvjYEAKgOAt1HOM0iv0J+lH3W+mpoBklt3TApun+FFN1XdBCbEk7Qw7FoUdHK/S6KG4XripHzwLpJjvMvtyI4RqodKIq+t8j7lUI+T8IYFg67soP3Dy9pQIleKBwkPtmajzF8Fgf52YgJB1rvM2YWBIrCgQZE0rl7FXqcIpifaizcZ7lQIQWEZK7GjlFYL9ztMpCJsa/lWznOzte5hm6gpbv7znK8ymXbjnPAxN/jb4njIs1saRnsjA2l55TMfRv79JeBSrfc+yWu3epettnwV00sHKaS3BT7TGRepK11LhbukOsu8Leu02KIfsQHp5c2Ayzxeu3wuvRpP5pB0cl3cR27v8R16ppvYOpH2IczwbFEFthNPetHQ4yxwS5Kh+oqpUOnrrC8n33FRPL4wL7jAIB7kpfwsfBHAACBZeHkad4jOuFHNS17fY59x6ZEYt3iwhnkHJIvcPLLL3Zw7lvqSJ4ze4efRmr2zYU8Y2p4mc8ju4ltYnNNrHxYNsMzIg6+CazkYGFh0RY3heQAn4buqKOgAohNkF4VNojOe1L+v8mFqgsX2k82Op+keV0PlxAKUe/q+yQpeSMsXHQYcMWj4COxR0B0Qz0W9CznSrhoaTO1vN5nA3AD8ptwvMQEP1f2aGz6FMc38dOk0Kde2sL7bm/CEYt9qZ+f5X62iUz7UBY3V3TeSAn8rTSgkLzKca2KWzUXILfwFwE3IBzyihkDBz7yeBZj7yPXq/YIN5oRffZsGh3jbG8kq4pICb6aQllcbclLfhmD3H+HRsc59p95sC7j5GJUB1scy+jixrXoL2tP1y1LX4MHqeBOLaURPkm7QGFQxiceht4vh9GItDwXADxpaP6WkKenz94v9gUl9owLyltj035xP8c2+FwTyzt5v2YvkhfYCAAAi5VJREFUrzOSn/YBk2/nGEa/IS7C/fz/4u0K0bORNe1T1xqyPj404uwrcJ4DzW8Wt2zBB93B55YXL8KRxfUAgNsSVxGckzFfZp/50ZYk1gwZCVVsNyJVBGcCSH+dc5i5R6beZJuuo36s7OXYwwsi7VY54GZEI7xiXK38zY3XZQ5++I+J63tCNvabQGmt37LB3wXCQyN69B99HNWBOpyS+INkWOkLnHClUyEuYv7irSIyyaI0IhqDL3Kii/u4ORpigImPA4lp/jb+XvYZHTduS+2JgOmrXLylPfzNX0SLcMgDMf5vpYGyGA8D4mJqbOFGdxfDSJ8X45rEJJTEeJU+40cwZ0R3tp+7k0YvX7n1ApvNMvAyr5t+wI+AqFnKeANlfbTTMoaWe1rxBgAQyCvU1tHA55+maN1IyguWrKPrGX5XHBR33AYxfykg2UXRs/YGxVhjCM1tABqDbOcLcjB/b+cxAMBYqQuvTYwCAA6PUCf6sb6XAQBfXd2PZyc3AwDuGBoDAHzr7A4AfAn8ohJVu2RdC1zXZlB7LsyOy3xG84cD3pjqov4U7uB6xo7wZcrtaGDgWfYxdw/HHu4TcfqNpLeXjEvaXS/xHDUf4qdDMgaZs8QDuCHtPVvzHOoJMWjvqQE13q9/3TLvO9bFflZ9nusyt1mWeAPH4pyPIzrL+zjv4XWV5+hiLo42PdlemwcuezJ92u8ZaB0ZS26bEJVlHxpRtld95IjJF0jw/GV4xl//Q0sAgBPv/rfHtNaH8W2waoWFhUVb3BRqhVMD4pMaTj2AbjEoGTEzs8kEvmiUxTDYcUYi1tK8PlJQmL6PU2l0kavFLlMkz27RXnBQpIvRImWh8DreQKaTf2e3UgrpPCXBMAPK4+D1EfYZukaOEr1lCXiR1D11TaLwRuQeC45n2HMD5IKDT/Meq1uBzB7Ob1VE3ugM5xDMa+Q3ilRxifNcMlJQooEuUa9KfTL3ThNooxFcFS4r0Yn+HOeiFQCJbGxGhDuJ6Jl+PoTCqHDr7uaaNWtEAX2Of9ckCEc7wkc2FJGIcD0+tPENAMC5PP3Ar1zaiGCU3H1LnFa85SbF75dn16NU4Pp9sOsoAODaKDnrQkcchVVZjyu8b8clcXMmHNQlOGvmbq5xVFxxtYTyIiqdMXLGyDK/yEcayG4Qw3BG3HjnKE67QaAwIm5VMms0Sry/vwQkptiHicyMzjryf4XVXRxXdNq3pk3H6wGUH6Rr/dHB8wCAL9b2AgCKHUHUF7kO9R6u3Zb/yj6nHmxJbu5prsfgOa5hPR6AnwKR5zIP0RONYF4jdZ17KbtRolfPGUNm03tHmtc4r87zlCCmfraO2gK/6/3Lb7Pifxus5GBhYdEWN4Xk0IwAq7s13EQd+VtIfZ2vU9ctb5Sch5UAyodJRotzJKPpi7w+t6EVvuw0SEVLI6Tw3Ucdz1bQvEAWFF+RtnUfchIIFJRw3zyjhuErw9PvBr7OPmceJKVWR7qREB08s5kcJDwmbsEOF80OsTEcIxfMj7Cf8rqWMa/znBivRvn/xdtcOJ20D9TmSNnrCd4jNub38h5MjkVYOEhhxEG1Q9yweY6hGZXAp1kfnKYYq8QYa2w5qztdBAryVZRr4AbY1l+CZ4x1JS/FV5VQ3aMxvO/HyPmz4te7Nc1Y56Pz21FPcEt96mv3AmiFQbthhYgYG38WP8z1SVPv/sfbn0XSoc7/ry99RNaMY6l2aLjiduw8zc/oIjnr4v4gisMi9Uhg1dIBtun/ehBLEqpc7xZpVGw/4WgN6qhx8bKNCcle2enD4gHZL8NcNGeW+00Pl5F6SRR9Qf4u7smeTwaRF7fmyT76zN1nyJnr2xuIV0R6kdydhUP8DK20cjHMHmwGJQ8iA/hLEmwX4m/9r9AYP/lwEvl1Ji+H11eMgfdrYWQ3s70JRy8Nis1pPuCt1eJhEbv+DG1hJQcLC4u2uCkkB38J6D6mkNsQROM8qW2neBhCT3KIKzuB5qpk3l1fm0XYjGiPzBku0/uKCUlVqAkHDonEUJAElciUD6FFUtHu0/xuWbLzmhGNyALbLxxk38mLYldYdNEMKm/sAFAckczPhvIyLUsSzGQyMP0rfuS2k+uVe8XiLmHffS8prOyWJKeuv56cY7L44uPiWaDJA9EZ7XF1n8T8mISfekx7mXpG4ljdxc+erUtYuMxOwhPkYsYCX+3UaEY46OiUSVrjb+qeVXxlfDcA4FDfFADgW5M0+ccmFLLbzLoouZ94luYVIosS4JYnJ248xAEfy6/DmRXaLYy05NTEVftEGUv7RJKiYwezm2S8LhDIS0BbWqSsSf5/ebdGM8zv+p7n4B3JesyPRBCd52/FIQl1jvK30Co8iSongW4xeQ71xai3jibMvnqOY5v+SBn/9OBXAQD//vgjAICuZfF2aGaXAkBAHCahjJEIuGcAICCBbjP3SPj2Je0FSBnbyqUfpXSiQw3Er/LZFMSdGjvFsSwddBEfg6wj18Mve0PN+hCbkyzh7rdOvLopiEMzBGS20lVUE0NbcTcfTEoi83befxknrtFN5gZF5BRXYXS2ZZgym3L+bhEhZx1PRDYZmIYgOHVg06N88lPz1CcaCRPH70L75aUTD194SfIEdisvZdv8FtnEJ5v+RAKr29h/cb08tDGJioNGZJKDMS9ihTYoJMddRCWXoyaBa0XJ93B9QKNL/NQ1Xp8Qcb0RUwjLS7diMv42ZHiPFzq9yEOTbq4lXqL+pR7E46JKbTGJIfxQNQfJyyZ6UuYpomshH/bUrZde2ydjELex1t7LaQxoBo39BTiSq4CqPONzaQDAz+1/Br/lPgQAeCZNGTuY5f2vfSDsGeHKkn2aOhX01mnkSU5s+n7esMYuEV5SwLKJbJWXdD3/332mjvH387v0G99WGmBYo/M0/64vkViv3iK5C2eCyDK5EqkrJqpR9ssuF6OB5TVzNobG5EU/Sv2iEs23IkYBILsFCGb4t4lw9A+R4xQGFNRVvvAmt0KLkTu46ENhW23tHGSfa9VK1Z47zDYDX+WPC4ccuH4T2frWYQxWrbCwsGiLm0JygGJmZjOkEcwIVRMVwl8mdZv6482IiRjU/zJls6lfkgixxxOe0dEEGcXGOTVfFUhdNRyft6t3GLXCj6vfZIRL9xS5Z6XbZNk5iEtEpOuTPINHKZs542HPvVmXrLz4i2kAQG79DTUXzstvUnREOwplydgzOfWuSBCrW/2e9BOQ/PzqvfLH0yZsEOh7XWLlN7Dvcq9G7wwvHHpWVIcpqmZuAKj1inq2Ss4RucEFZ9aq61grQxAABh+axML1Ebkf5zn2A/xt18gcDnVwYT5RuYN9T0uW7K1VqLzUJFg1hlDhlJdjyEoAUHI/Oaw6SbHpfV/5RXzxPb/F/u+jb/eP/uJRtnGBjruZdOAe6QcAL/9GK2BpLyWGSr9kSaYp4sS+GMXCe/l34vMS7LVOpKg7CgiI1GIkNyP9JK8AvrpkdvZTv6hWA601lDySpVulNoSoPx/d+yxeLDDFMxHnPtElSh65wxX4ZjmGxISpIyEG3k053DrE9fz5vqcAAF/OHQAATJQ7kdrKvp4co+rmP9HKlQnmvk1iMNKFX2P5A7xu4+/xeYy9S9STBt81AIjM2XoOFhYWfwPcFJKDatKNE1pV6D4pnOrDxthI6pg+D3ReJHe4/j6JmReOWd2lPcPe0n5eN/Q8KXQl7UMtaYx5vF/eR64WzAHBLNsv7RF9X1w/vorywmYbKd535LMk0XO3AWpFwqxLJodD3IkF5cXf19K8z+wB4d7XQwgvsk9jXDNGs/V3TOLa6+TWfuG27gVKDLl7yoBwicpPM5+/cJksz1dWmH6X2AzE+NT3vJQG26YQkVBxE1ptDLalzQ2oogRLiStTSem0/B8PIV4TaUekJpMbMNufwGqc62+Cpgya80F0npX5xY3xUcJ4m8CiBOg6n+HYm1Lly5938LMXfohzrUjoshjuonetYP4Mq4J1jMmzEq7d/4KDQFEkqZ0yiDEubHiljliU9oiFg1IbQqTQzt+LY5qeVlQGW6HtAFDtADLb2a5DqmjVHiQXzmwFUlfE9XnFt2Zd/8uT78Ajd51gH0cpuY2K2zG7L4z4hITS90oOiXFTBxrwi8j4w5/4KABg94OXAAAXFvswkGIftw9x8x59noFVtXRLUqhJEFtoXqpTXfGh0MW/C0N8Rk3J4g12VuDMc1/5KtbmYGFh8TfATSE5BHJNDH8zi4l3plBLcUiJ06RbXtWmHQqLveKaek6y2LqM14JVeoBWfYUlcQuqJhCTxBa/UMqG1NMrDWivPoIJo41J/Yj4VBP1mIRWbxEJgCo2uk5rVJPOmj6D4v5a3aIQvSh+v7fTzF5ZSHhzDWdEMhFd1Vi0s38yDP9m431hnwUJngqfjHoSTvO4hG2b/DSfQvS42B+6OKb8OhPk1aoD0XuC67JwQLIre5sIigtTIqoRnZLM1H4gv5MSXHCG7XtumQcA7OiYx/ksdX/jcvXmtqCw9LDo21n2HRAbUq23geACx2kyBnuP8rfceoVYgPc72M2ErW/2UszIjXVAS1CX9osdo8DPzFaFqnDuhHhXzDrN3hFE9BtiZ4ma9RDpco+Dej/v52TXeoicRitxrRkWz4vYv4aPNLFwSFzdElrff4RjW/dPLqLaFLevrOfUQ+w0OKfxMz//JQDAX83SL74vlgEALFdj+NYbuzgv4e7nvknbRXm0jls20HUyVqK0ldvFfd5xvPXqugGO00hBcPzwXzGZpeKVOcv2lc64V0HKuEnfDDcFcah2+XD5xxIILQNVUQGMi9CkWwdzCikpFLt0UKIERUSHbhXRUEutVFgAcDRj8AF4i2LSglUDSIrRMfMA3Ud6no3CSw4SExzEyq6WXx0A8iOO585DRjbcHZIufd1BZqcUSJEoz5AUmRl5sgg3JJF8e3ifrnOi/qQcpK7wBouHZO4i7kdnAsjs4W/DT4joP8x59j+/gqmHKcbGRc3KbREVIqyRvMbxTTwqVYtFtYqeiHi+b5M7UOI7j2YQiJ+XdHFxYS6sksBtTC1hPi8FczeQIlYWpQLKgh8dz3JepkiMcsXwNu9HSNZq4ZDJkTB5LC6uzNKF+d6BkwCAZ1dlH1R9Xsp8VuaV2ErVKjOdRP8L7H/+TolJkGrNneeamH8fn1/0+Fq1IrOzifi5tc/UFHHpOl1HZT9fQN8pfhlcYp8z97heVG6pn+Pb/stnAAAvTW5ARXJH/KIqQorLbLvnukcURmMce1Eq11x8fgM6paiPiYUIL5vcnACq4mO9t4OqxsVj2wHwxTbu5QjpNqCEQGZa88mL+9a4u31V5cV4GCbyZrBqhYWFRVvcFJKDrwykzysUh4DMAySHTSmMGVwUiWBLHqUZqX0gRkDjMqx3N+CTTMQtf5IBAMy8jdw0suxidbuI63tItVVdRMmlKPKOnOPwuhQiGSUrKQwrZLaTusel5oMxIoYy2is5ZwrEhiW4qTTcgK9oxHOJdLtiovHCWDhMerzps5T3Temy2s64V6Cm4xz7XpYSaNldTaTOsc8VMg4vyGji3Z0ob+eaBXPkXMF1dIFWJ+MoDgjnlkAik+FYT7hIXTYSldz3gojvPmBpv4nSY/t/IdF/p0ojOHaEPuG+E+SwpjScv+L3Ig7Tp2U9+lvRqUbkD2bZp+Ha8XEHsa1cjyNZBqOZnILonMbCrZB1kaCm65TIgsPA/B1ra3uEVkWteE8didclU3NJzp+Qsxt6X/Vh9R20ePZ8gW1m7+Z1kz/UgCvGX+MeNyX5wkuOJ11VxcWYDlDivG14HKtV7qFThfWcw6wEkjV92Jhg7YRsnfe7uEQja22gjsYejqV8OQ0AKMoeDC358PgYa17M9tOFmV/H+7vrSxj6FMdZ7JXITxlnIyI65w3I3EnjbOpIGEsH5TnHbYFZCwuLvwFuCsnBaWjEFprovFjHwgopa43MwbMrZNNRhDJrcypM7sFSzEFCTjq69JNpAPBKxy8eArb+ESWG1fP8LSscueuk4xllvNObpCZCaWsd3c9LTYht/G3gRSkN9x4NiNFq8EXJIdhKTlkZ1mhKma/4hISs3kcOG5gPICh6d3YL9fblx8h5QkcVinspAfQ9zvtGrokbKqw9O4lXAv+GE7c6v8D7mPBg32ty6Ey6ZTDsOsOxz0q5sc7TjsfljTu1IUFXpUwE6z4nodVSp+J8mZbbr13f5VVrymyROgKvsc/SoPaCiXzGrXpUuJMGcuta7kKgVfrOqQHbO1j/odyUuWyS59Fbgy4ZQ6aS+xprrPZOzar08j71HeSQnU9HkHmQa1uWWg9dZ0xRXgcdUn6tKNJdSAzSlWAAAQngSo6Zslut6mSdlyWIKUTR7fOLPDbsA/cfwUvPUKKKy8EzpoRcthrGs4ssIzjwCUp32XeI4XvC59XO0JKf032Mv63sdvFPdz4DAPhPpxheHpajDUZ+ew5Lb6eUlWXX6Dzdej9M7o/ZN8HrHK/rB3xSqj8089cljBthJQcLC4u2uCkkBwCABq58JIDelyRJZodUAtogdfEidWCRCrOhniYzMTbhQ3yW7TMHhKvtIN1Ln3eQ2Ztm+x4JzU5RX8xuCnvW+K4zwsWk4tLiAb+n1wfFcj7xXtGDxwIY/Rqlkbl7yAZN4k78SgBBybgzxWT7n2pllhrOP3c/x5t+nqS90g0kjvOGcyJphEySDlr6+fCXGUpc3MbJN8MKs3ebXH/JLN0pWZndDa9s+dwdbGMkrOT1KlZEz/ZLaG/9orjeXGDqoVbwEgBsCFGES0Qq0CvUf02mpglnDi0rpA5Qt25e4PhMoM3sHQFv7qmrkhXYIR6Ud83hR3pYa/LxLIN8Xhe25Q824IxxfMt7+J3JXvT5lNenX2qPNuQ4vFK/Qug0pQNTfrEiNoTq7hLKQxK/npCkqte59vWk49UVrSU4iK6zbDNzjx+1BX5nsl3zo5RU7klcxFc2UXIIS/DaLT9I49GFTK+XxDX+QS5oh0yw0gX47sjwutfSAIDoPO9XfV8Bv/WJ9wEABh/kIU1zcfZ95Wc3eHVSwzIm8z6Uh5voe0m8fHJokzkUqREF4mOyr97aWXFzEIdGRGFpjx/+rPaKZoYWjCjHIXaeD3rRj8Y1aV7s8qESlETrxS+LeC9qQj2uEF6RRZTPplQX9heB/CZuVGOAMyJXdAZYPURXWPI0+x56Qh5Cl8bi4TTbSfbb/K2t4ZqzEvu/wQe69AADFkKrCoPvoi9x8dPMMM1uFePTioPCIRKtmBzca0rBRWcU8hvZbuYdEmMgonm108WQxH3M3Cv+fMl6DE8FvBfY5KzktwjRvasGLRu266SoYBLB2Ew3gDrbb9xMYjRb53yTwSr0+No1CwnxLO8uA8+TIhbllLHCaKu0myGghkibjM/RSAm/NfV2AMDpS1yruMSiVAdUqwq4EEjj5vZXWrkRxshp4kfSV5oo9ZgMUVFHpERfwHHhk9J6gUnJ5jQneuUcr9CvKy+WkviF2JTyigHVpFDv1mH6ET+5cDuqohIvP8SNeer3Sc3u+vnXMRah0fH0JNWzzG4+mB3/ehxTeYbiNuSZzko5vOF4EZvfPwYAeOoFZsB2jkuU6CHXO3ekRwzD4+/n9emTAeS5vbzTwsz6DD+RxcWfomW976W3VhysWmFhYdEWN4XkoMMuajvKUJNhrO4kpes5JgEz95AqZhoBuH5S686L/CxLcVjfy1FMvkPOYGBtT+9Y+EZMofFjtDbVv06u5gjnye5oIDIjtRaSImmIiypQiAANESFF4ij2mTJqGuVeEdcGxOV3WaI2OxXm7pKSYztIvo2E04gC04/TFyXHW3pndaauuqgnJQBr2YydbXI7Gt4JzIZTpi+LO+q6QkkK7/ZvJxdbOMEzRRtxDUcKiVSGJbKunzJ59ZUurBOOsyIqmCsnT3e9GvCi/CYz9NleuFfKk62mIXFqaEqNiZrmZHTTQfMwjZqxY1QBDdevdLbUkMh+iRxdlmAq1cSpKd7HL8Fshe18SP1fjniu6Gofn3FdgoXSZx3vtC0TqWrybyYe9nmFcUzwk8+oEE/FPGOscX0ad26g2Cp3b87xCGU4brgsrsPJiiRWo3pSaQS8c1X0KiVNk5vz5OduRfJePhufn2v8zttZzv/L/24vYmdMlyIZSybtb27+DB57/BcAAAmJ3PVXJUci43iZqJVOLmz8kkhbHa06E+YUraVDvO/Y+5MIeK7k/8ncCqVUWCn1mlLqpFLqrFLqX8r3nUqpJ5VSl+Wz44ZrflUpdUUpdVEp9ch3uoeFhcXNh+94qI1SSgGIaa0LSqkAgBcB/CKADwBY0Vr/O6XUrwDo0Fr/slJqJ4BPAbgVwCCAbwHYqrV+04iL0IZh3f8v/jF6ngt4NRdMaGjHRalRsF15Zds8/VDyGQrrXe/QF2cnOZcJZdVNhcC8nCkoRiQTCOTUWKYcAAZeJqda3Sq2i5km5m41hT6Fk+yQQWUD3mE25jAWYxy68QSjnqNrDT+ZLfAOaDEwRs9miIYkAIhKCLAJb86vdxGdWUvHy32iS2YUQsIJsodpHIue59yrndrL3Ete5eeqnJLU9YbP45Am1NbUmKh2NRGXgrleHH6PBAKN5OF/gcYCI+GYEN1GVMMRF2ur+pZxVyrvtK7eH6bdZalE0ShbiCAZ49quXKLCb8rzF0aBoedEwpGgtMSUOf/Rh6YphCvPMTYtHHPR9Www6tt3nmoF0hnJqlPC2MvdDkpSgctU21qR7NHUJYWsnGTuk+C3dbexVF5HqOQFQU28wqPAjATgy/sQWuKD2PIoH8Tp4zReROYcz8722HtplA3IgL94bS+Kc1yjxFXjBhbpoqeB6JgcO2DSAsQOolx4J8cZg210obUIi4ecNfM78Xu/9Dc71EYT8hoiIP80gMcA/Kl8/6cA3id/Pwbg01rrqtb6OoArIKGwsLD43wjflc1BKeUDcAzAZgD/TWt9RCnVp7WeBQCt9axSqleaDwF49YbLp+S7N4WvpJA+HkQjqlul0MVrUBiS/PdUE76yBKfcyYAZ/x9RAS+sU6h1iim7KPUfjkho6VCr5LcJCIlI4kl2q0bHWdFLf5k6YfYMh1oaaNHNspy1GImTM5cBhOQkbVOb0Wm2krtMcI85FXpZ9D1f2UFcinuaGoI+0SGn7/XDL9KI4XRG5+04r1CS1a1JGfrE9ZYrzMzLCIHmqL7IguMlLZn6hDGxXZR7gMiCZLm+ThFF/QbdkNdfG2mFpm8kR0+naG1PRyqYGKDoFhf9uzwg5f1XfF6x1B4p2FsRu9C7f/Z5fPoCM8oeSMmpNOKt+OqX7kClyElIbBFWxdsRng5g9k4p407vMWbuk/mGWscnbvpMWeZJMWjmHuXtpeCyuHalVkd4zofyEBdm3ddknlHJEN2kPanAJNMlz5pQ8NYJ2vltHN/ki/SuTO7I4551jMr7pQ8/AQD4p7/7k1yDHu1JUufnaA/atp8Zf9lqGHNLXIirBfoij52lsSJ23Y+QSBWm/qk5fT6yEEBYDvAxXqZNn+UazN8WQ2mQ8/IkarGf1R7NIvRyGkDL1f5m+K6Ig6gE+5VSaQBfUErtfovm7bynf013UUr9NICfBoBAogOun2cbmMKm5hh686KEF3xe9WLnE3KWoByiGp0GnLtl5zxL04cxqHWfbHqnZ5kTs0zVXdVoFakdf52iYKeIvvWEQmJSoiW7eX2xxJcillWeKGheYKMGhZcVqlLUtS5p3T3rOLbGV7qxusVEsbG9r2YOQW1FKjYjZpHgrYGJPGydvsTfKiN1KHlBfMtiGOwjxYqcCXtzHZVCrKtbuMsy2zW6zho3LCfjfIEG1PreOpwIX4zuJ/i2Rn+URsRYoOaNc1bOoFRxth3YvIDxa6RiMyKa/+7b/xgA8In5O/Hqnb8HAHjk1I/zfhKA0HHRxeo2mZcQ1sCKGGBHavAvG1+mGCLPs23quoulvSZC0pz7KbkkWaDaTVXRlbwbc15DacBFQArYzt0qeTeSITr8VAMTH5ZDnF8KSXvevtbVRFwK6DrCqOKTvG6lO4rGKH/71bP0KUbfRib2SP9Vb65bw7QQdvkpjH/85R9ENMFnM/7HEuooRng30KqKXkmx78wOWZ+C4xnFDQEYe48UQQq5nrpknpXHcMaTCAnT8gytb4LvyZWptc4AeBbAowDmlVIDACCfC9JsCsDIDZcNA5hp09cfaK0Pa60P+yKx72UYFhYWfwf4bgySPQDqWuuMUioC4AkA/zeA+wAs32CQ7NRa/1Ol1C4An0TLIPkUgC1vaZBcP6z7f+0X0PeCD0v7SPHcXlJTtWKOoW+VnzdFSxty/mMwpxCfltJsEiBlMiKhec4mAJQfo+WufppiXCOmMfKEFEGRE4i8iLsiUJCCpGGJlcedGY7t9bQndhvubo6hj021aj2YLM6KHHXvlB1PHTDBV4GsGD1XgcI6ftcnStnyHnYUXlae+9VkKxoDXKWztS4VMRqaE6obUQ1H1J66qF1hyRQsj9a9MvnG7RX4AOl7thgB3qCUVOlvqQwA8J9++I/xuaVbOBa58XCYktHuyBTq4q/8zDxl3fEsJbk/2fMJ/FWWasVnLx8AACSjFLdqX+lB6QFy0uTXqF4ExP04fyvQdYrrYHJHjKHWVwF8tZYLGWidX+HUW0fSe8cAiHo48mQTK9uk6I0Y+OK0K6IwDNRTbDf4PL/LbBLjbEx7RjxPzJeTuJf2y7MHkNvO590xKJZiAMUyLxjs5HcDUbqGX726AcjJKWgSnWvKGt44PnPminEHh1Y1ChLoZE5jN4V8glmgLkb7rjNS/EjKIEbnW2fOlndzIcd+5NfaGiS/G7ViAMCfit3BAfAZrfVXlVKvAPiMUuonAUwA+BAAaK3PKqU+A+AcgAaAn3srwmBhYXFz4jtKDn8XCI2M6KGPfxSJ6w5SV6lvLe2VE6ZFt29EtVe+y5T7MgFEoVWWaAeA/ldJKQtDpHvZTfCUJ8NFzf+bYe1R+/x6CSoK8tOJ11vWkyVSfWPccwPA+q9JQE7MVHbiZ6DQ4u4NKU+W3022f8vW6zjxIkuAmexMEyKtfdrjCjrNNeh8ieLByh7Xy9TLSD2H2LTolKFW9l90UsqUyfyGnitj4hHaDIzLtbROqlS94fM4qvte2hNKb9DAW+1toGeU0oAJHCuOiM1jWuHv/6OvAwCeXWK5dEckCL/j4sQkbTe/e+ufAwD+4ZM/KZNqneVp3J0GgZxC92kJt5bnVo+bcPbWqU/lfnk2PVLzIxvE4DOSCyMnghmXbSOsUOlZy3XLUluiEdHoFGnE6PCmAGv8uh/F3ey/65nQmvVsRFu2JvOdMepWOxVqSWNDEWlGckeaoVYgXW5Ta84AsP4d13H2ArVwc0SAcTFHFl3PoJvdZTYfP/pfcDB3nxinpUKakVST1zWWmaLiFbbNHuTm7+nNofbNnjXtL/zbj//NXJkWFhb/Z+KmkBzCQyN69B99HE4DSEhiSW6DuGBE11a6dbxc53FSWGOJT0y4nueisEss9VeF6vsBMQx751kaOHVW9wGAhCQTGS5VHGwFBUUkOzJwv2QcPtntJeeYBJzBFyUgaNjnSTHeISPyGSgo70CRlIQ/VzuMlR6opqWdlGUvyHxVQyEsZcfNqeMQa/SGz7qYv1VOUBaPhiO6tq8KxMTduLoLa9az2ttE70s+795Ay15TT7T0+ooUkTUJTaEVIHWN41rdysmYIBynwaP7gJaEEr8i4dBb6/BnzLmb7NMvthF/CSjK0W5KisfGx001LY3YzNow6IrozK4fSF7jd5kdsixih+o+3krYqieMl8p4kVzEr0uVJrFR9B43Z6X6Uesw68jre07wWTVDCqFVtlvdHvDGADDU3RzbZ5Li4rdwv1Se60ZBCiDHL0ngktiMql2uJ9VFxLZlAtzCuzJwX2RnnoQpeymYbe2loef5sK69n/vAV1Ko93D9IxIoVd3BNuGzEW9dTFHlU/9Pe8nhpsitgGIuhBvkUfRA60xAc+5CI6Lhz5mzAoQoiBtpaZ/yJhq9JCqA0LzicAO9L4tBSXzZxoBX3V6GM0O/oYmnMAel1obqiFwNyr3ZvnKGYnf/eAPzkmdhzp6ckbRpp67Rc1xcfY+IO/BFMRTlW25V8yKaXInIgkZVqge7UhrPqDWxKZ/30pnvjCtteXfAc6OaXAxjgF28rQlHSuKZjW6iSmspxytoatyxaSMO9wCBb9uwJpaicFsZ4UflJK6neuV+pviLg4ScLRHMri3skjod8MT60LwxvMm69roY/spa9azl6oWXAt+IrTXKBYrA4v0kKh2v8VmVe1vFck0Ep1EHjTqqmsqrru2Tk8AWDoq788ASSpNpAMDIS7xucX/r9LS5uznX3lfEyC170V+Elznr62eRmazsl4AfGP4Gx7UkQQBhISSBkuOpCqacnal6rV/oQEHOMe0ZoZq3fJF9pq5rzL6dv2WmRXWUZ9tYV4Eyp2GZaN1vSJGZ9+WhTjFE2GQEvxmsWmFhYdEWN4Xk4NTk+PbtTc/VFhDx0hjNyhGg0UNqH5skFSwMG06gURXx0Liv6gmJM68rZN5FOT14lOzPFDoN5CMeJzZc1xjewmNBL4DEBJQY8W9pjx+1rrUnJRUHW25VMy4T+rVyw8lO5oQsozqYPHutFAaekrnLKU6Zjew7t7/qqUlGnTBohoDiFqNqCHeKi4TUUJ7Y7N9N11nXf5dAmXAANckxMfMsvo8SQfKrSZQYyOcZ3JohMfi9GMbKqESHypqt7GxxT1OU1Zw3kr2V4rQeC3nrZwyofglESl1SWNwv7tfNFH97PkOOPvleF+X8WgnO1MtohAFlSshJduXqPq5d8oIf5Xdwzuo1uq7jIlEtDTSRkNJsJtIxssj71Z7rRljmtSynaJmzP3LbGh6Xz9CujFp/y7radYRjWQ5Ldq1EWsZmNeZul7J5Rzg+839frZXjUBgW1XiTlMbPt8q4VZ6mEVENcgCz97ve3jWFXIwb3jcdRvcbxlxgDN78X98fh7EgCkTnaQkARHtYycHCwqItbgrJoRnVyByoIzIW8PREc0aj4RbBjEJNKvsYHdlzMaVaYRQxOdna43QXFQqjZAXG1mBOkuo+1cTqNpJUwyFNAFFh1EVESrw3T5HzGC6fmGpiVXT5quQHmL47LjZRGGKfUbFZmFoR4RXlBbWUNouB6gLblAY1MsL1UmekmpUY0GIXQp6toB43wSzChde72Pxnwo1uY1/JMYm5/0AZzpyUoauwT89VmNBeuT2TSxCWsyELI8oLY5YiRZ4+m9/QOm/TSALdp4Rrb2sZyYw7LnlMdN3ddfQ/Z0KO1xpCV/ZqBAZ5Az3NZ7VwkM9h6BtNlL2TtSQYSsYU3JuBlnMf594h+SuLrfUMvc6H0yFFYc0BSYEVP9y7+TD9lzlnY9NppDR82/jcg8/wN+O+7D7iw9LdfG4JyZxtzgVknVzPmG2kSXPd6s5WkFx+pOXyBpjnY1zenvQrJ7crDc/OFiiYgDyxjV27oWKVGDeNkXvpXRUsBCi9mCA7I4FX31HF0F/xfsag/GawkoOFhUVb3ByuzOERPfzzH/t+D+P/eBhX6M0KI6l8v3Czrc//qvW4+qt/w3oOFhYW/2fCEgcLC4u2sMTBwsKiLSxxsLCwaAtLHCwsLNrCEgcLC4u2sMTBwsKiLSxxsLCwaAtLHCwsLNrCEgcLC4u2sMTBwsKiLSxxsLCwaAtLHCwsLNrCEgcLC4u2sMTBwsKiLSxxsLCwaAtLHCwsLNrCEgcLC4u2sMTBwsKiLSxxsLCwaAtLHCwsLNrCEgcLC4u2sMTBwsKiLSxxsLCwaAtLHCwsLNrCEgcLC4u2sMTBwsKiLSxxsLCwaAtLHCwsLNrCEgcLC4u2sMTBwsKiLSxxsLCwaAtLHCwsLNrCEgcLC4u2sMTBwsKiLSxxsLCwaAtLHCwsLNrCEgcLC4u2sMTBwsKiLSxxsLCwaAtLHCwsLNrCEgcLC4u2sMTBwsKiLSxxsLCwaAtLHCwsLNrCEgcLC4u2sMTBwsKiLfzf7wEAgFMFkleB5Tvr6H+KQ3L9CgCgbyBf3S8vAADGPtQHAGjENADAV1Hw1dhm+JtZAMClfxAHAARXfQhlsKYvza4RKGgESvy71M8vqweKAID481EECvytEeVn7r4yAGD0j3y49gM+9r/CT//2XGugR1OcQ5D/Da3yMzHVxNIetg8v8bvMngb7WfIhkOcYzFySY00AwPxtDmIT/K3rXAUAsLgvDABohoHwMtdh+S5e2PdUAABQ7HcQKPK3wsjaNRh9soaFQyG2f419Xv1Brn1k1o/ycB0AsO6LMvf1/K2wDug6yT6bIY5p6aALAIhN+FDYzPmoBn9zKvwMLTuIzvG61V38TJ/nb8URheY2rrv/XAwA0NjFxddjMWg/28dm2D67nesSWvShlua9U5f4m3mOjaiGw6EgMs/v6twSqG4vI3A1wnWXxxZa4T3qCeU97/ASvyv3yHOpKtQTvJ8zLBvnOsebvggs3cLfNn6Wazf5ENfXX1Rohs19+BnMa2+85t7ZbVra8CH1nGggN8J1z21m39FZ/hZ7YAHFp3s55iSvq/ZyXQCg63Xus5V7q/xC9nznCyFvPvEpjbeClRwsLCza4qaQHLRDip08E0Sxn9+FMqRqlW5SOdUAxj9IiaHnJFnC7F2kjtXeJoJCbS/+FCn55k+SYk4+HEGZBBb9r5KyNoPss9LhYHkPf0tf4P2cV3l9PQGP2ruySn2fJycoDDkIUEBBdJp9lbcIRf9WHLU0fytv4BicOq8rdTvwCSFvPJQBACReZONGFHDJ8FEaJJcorGPfwYzy+hx7D8WRjm0UPerf7EZ289r1rCWEW2c1igP8OzEmEsQoZH4+j2NNPSATDVLyCK0AzSAnPfEujmXTpyk11eNRzN8l3LOTEgdWhEOWgdA8r2vE2ab3KJtkNgGrO/l3M8rfaim2rQzWsfl3uH7j7xIp4Xmy+Uov4NtL9l4pUCLTjrSZ0tA+zs9IRo0Y++44pzzJJvCwiGnf6ub1VyKo9HMPhTIiEY2KpFMHmmGRxG5hm67X2EZpoJbkOEtuTNZYpJlNGr0v8+/VrVwPs2+cOpC6xnEtvJcboOOT/HHpNiCY5YOPi3QYzIpU0eNDjVOGG5Y1b/D+7l/2wEnzN1+Z1yUviJSxo45GlN91vsCxmPeo40IZ9RilprpISG8GKzlYWFi0hdL6rfWOvwuEhkf08C98DMGs8iSG7pPU6WbuJYWupjWS19neUMVGuNWH0dNT1ykdrPwwddbSchShOVLUJokohp+mTjh/OAjnEEUA33Mk0cVRUuhmzEWkm2MoL5DEJi+xH6cOVMiEPC7TeVY487ADxS4QEL0yuiCcMuF4Y/AJ082LdKBcoDxETuWUSbP9ReF8ReVxoWaIfSYm+P9yt0Ij3rK9AC1dudSv4Mi6BPNYMxZ/xUV+WDhiU3tjAICVgw2gKUqqT+Z3zC/9aGgKbFg8yM/woiPjbPVRGuB1tW7hvkf9WD7Mv3telnWU+y7vVWgKx0+fYV/aMfYIDbNDzTP2l8SGkNCenabzPPvOi44ezGokpsill/Zwo4RW2dPCXS3dfPNfcC9c/QE+mNRlB/k7+dwdRyZzTaTJrgb8WZGM0ryff0XWcF0JHd/gPjGSXFNsTm5Qw6mpNWOvDMj1WR++Hamrrb/rstfzmzhmY38JLflQ38BNpHO8Ud+LIsVsbkmolW5jx5D13FpD4izbV9P87cr/75eOaa0Pf/s4rORgYWHRFjeFzcFpAsGsQi2loVxSuLH3kgqH6aBA9ymNcjdpWX6/sN2C6GpXfah2kQrO9rONupwEAERzCqUN5A6xK2w/fR8/w0tA32+Sis7eyS5jk7y+2qmAKfbhS4k9gt2gloRn/TXW/+V3ckydj0dgWN3yPmPVJ3eo9GjUh8j+UkfJqeoJtglmFXwFtjPcoeuMcNbdQKAgdgThAJVOSJ8ulHD5xHW2X9krHC9VB3Kca+qacKBRDriwo46oSELJcb1mLqrhINRL7tm4St2/sK71W3yC7Y0XIH2FXG1xv4NaHzli7Brv6wZa0lbyAr8LFtneeG46zmkUhvm38RR0neE6ZXY7SFwRyU84sXkO9aSLRoRjWIiwzcDLvG78XT4s3C97ocT7BXppNwkpjeoK9e6lvfxMXjV9a0TeiMrYW2sMAPErAW8MG77EQczdzka+iRiaAXmWGY7JeKsCeYVqB/82e8Of43z7jrrIruffyQneZ3mX7MGBOnxx3id0iWNKX2SbehxYljULLbO9r87fOs83Mf0o5+zk5RnL3qj0+OEryxjE9vZmuCmIg2rQHRefAjLbxaXUwU22/st82AuHY55Irla46olrXJRmhC8XAG/xy31cqEYEGP0Sf5t4Dxf6rj2XAQCvXNuA64PsNDLP63wVdtCIaaRFdE9flpf0A3S3BV+Pw38b/ZONS2kAwMBnOKbVzcpzHxqXlNlk0RmFkuwYs1m6Tst4e4AGpVfErvO65d3sJ1BUHhGJzrFNxbijJh0UDvFpZ8WCOvAC2+TWhxGf4jqs7GL77lP8f+S+HFbnqRtlxJgaENXDV9RoXiFR8IuqYlx+ldsLiL7B++Q2cXOubvXJWDTc2YBcZzYjP5cOavgqvE8wK6qDz7hZFaKzQnCWOb78MPtR0SqC8iIZQ6tRLzpPOqjHhWiKyjB/C9c3vAjU6vxt5Em+KAsHxJepgJDspeiiK33yc/Z2nzeu9V+jbH71R4SgblaIjnNc44+SqBjxvRkEHCHSSrQW81vtwSzqV8ho3KCogGJELPb6UNhal7Xi6+gXL2n8UgDaz/tFFmQd94vquKyQPM25GqPljOzv4f5V+Jf5ZeoIr6/IflNNvi8A4B6UB/4msGqFhYVFW9wUkkMzAmR2aCSvOEhdEs49xaHN3ikqwLL2gn2cmlDydWLEuqAQzgiH3EYuE5ljm9s/dBKvLe3jjVxKI2c/SZ+ae6CGyCrbGcONMTSGFxwUh/j3Shf7Dp8i59F3ZtF8haTYEWNgqYv3jc26HpdOXZEgml4R6Te63pwjC+KCG+ZndFajKOpSdQs5QPRZcpvc7WX0fYWsbvY+9pG4yvXRDhB7g6yg6yyvK8tYovPaCyYzKsCCGBH1hS4oYQ3BAxKl9ZzMqQYkxPi7fCfZ9LrfIxu8tDGK2TtEIhIu2BzluIOvRpBfJ89IApACOUfmqz3VJJQ162AkCWBVpKR8fq20pRuOpzrkd3EsRlR2agqJa7yuOKjW3LeW0khd5Hfzt4joL9poM9gynM48IhcYrl910fM6x5BbxzUfepyNZ+5rBQ4Zo2P3aV6/utXvSauReVELxODX+bkEFt8jKs15MW5fZ9vFRypQ4goOrXIMzUPk6LePXsPvDVMMvOPXfg4A0OjhM24Wgihs5XoEYvxuw3/nuswfHkBUpJbl29nm4T1nAQAvf+6A9x7VziXwVrCSg4WFRVvcFK7M8OCIXvczH0elvwF/XrieBBeVhsTN1gRCy/yuJhTZcLd6vOUCXbyflHLw66SiMw+5XmCOcQcaY2K9uwH/Mr/slpBgN2D0Wo25e/hd/JpII4ti8Hu0jGZNjIcVMSyNMi4293Iv/GLwCRSMe5P/d5oKrhitBp8ntV84SL3RX27ZIQxiM63wXaOHGs5Yj93wf5mPMZYlxGCYvlDAzL2JNdel3jELAMg+PuC5G417LZiR+865qKZkrZMSVt5hXMwalY5WkBUArOyUAThAICcBOePGdiBu2VIrVLzzxFr3nVMDVsR4G15wvPUAaHgLLfNvc314ls/MV6O9CmgFrBlXb8dFjeU9op8vtsYHAL5yS08362LsA7k9NQQWxJgqw0yMCUcP0YYFAJEls3Yytm1AbYDPNDDP6+v9IgFeDnpGTRPMZAyS4W1ZVKucz8ZeTrQzxE6HIhmczQ4AAJZKfOA/tP51AMDj87sw9QRFsdo+2sLUVbHmasARe4tZR2PXCCz5EZ0VaVXc2kc+9U+sK9PCwuK7x01hc3Dq1Ll7TgJzt/I7Y1ENiBeiPNKAr8Lh3v0QTfxHvrAXAPVHX02kiTPiIoyKRORqOFX20fMGzcgZsbJH5gOeNKF0K3AIAMr7Khj9c/6YHyENzW4Sivt6K+7U2CrmQd9iBNR3AQD3ZAAAjetkU+u+VsP8LRzf9N8Xf5xYvwPFllQREU6X2SFc8HRLBw8UZX3EDVXcU/GkF+MKrYkd5fIPx4AOKtpfufe/AQA+dPSnAADv+bGX8eUrjB2vznI+lXVko5WeoBfopOpiA5KEo9m3NxG7TBGlGeYY+o7yusX9fiiZ+uwDXOvkOY6l3AdEJ7meqzvZl2kbyCn0bGWI83wyzbksc8LROYVASVy7x4Tb/gBdNrPne5G+IAFxwikDh7nmtZm0F0TmiPfAEZsD0PK+FIfZd/IK/x+aCnj7pbyZ4kTRi2YCtNkv4nIvHRBp6LgPmOCYSxKyHhFvQm5nHRH5TYuhxzUSTrSMUJJ9XJ5mnP+hDXSTpfxl1EV82ZimVPG1OT6zq1M9SJnEwEuUKnpOcKJztzk3eLVkwrLYW/5gBov3DnJc699aNrgp1IrQyIge/sWPITqtkN/KCZpor0onJ5DZ0/DUg1oH26hOqhDaVUiIe81XNi40IQgnGp6BbnUH72dUiNVtDjrPiyFTRGOTsRleUt4LYtQYkwUaH3c8MdaI2yY6Ebolapo2JmsuOqs8VcOIw8aQFp3XyD3ENz/yKg2fxRExhC44iE/z7/m7+Rlc5uAC+VbGn+mzvIHr8i/v+iLWB/nS/cXSHVxrkaPPZ/uxPk5V6Fq+CwAwFMsAAG5JjuN3PvMuTse3Nuai3KmwepiErbOPb5jzBV6/dEcDgUU+o85zYqwUNW3pnjoCETGmzZPy+/NG9FVe7EJiYq3rtTrQgE/yH/qP8LdqgnsiNVbF9D2cfFByXYzr1FdWHlEwLtrK/fxDn0sgIbEdZk+MPskBzN4e9ORpvxDi4qjEDNSUR4xis/xuWmIpQssOSlu47kFx57obKNP7rkQw+Dx/W95F5vC+n3gOAPDZywfg83FeyQipV9gvuUPPDePB9x4DAPRIiOuB6BgA4BNzd+L4NSbK+GfYZzMie6uuPAZTlvUwz9FfVp4b1WQdn/nNj1u1wsLC4rvHTaFWQAFuQKNyZxGqTo64cCupoQlKCi76vFz6YEaCdibIgcoDLRehEcW3fIpkf/FADNWHyOECb1C891dErG22AoBMfQVf1eQZaJT6hLMVTWaoiLARICWuqJUIr+9/lZwhenEB5/45xcPgHDlI1yljuKN4DTAg6kaY/H4A6LhILlZPSERhTnuut/RpqbmwJFw0DS/oauHtHEMiRY41W+/ALWGKqM9c3cqxL3DNAv0l5Ktc45WcRASK2HRkYj1qEvsfu25yFcSAejjgjTN3nhJDRAyU0WsBlAfJUQtDfI5GbQssBNCISe2LPo6vFuT9u591PCNndIFzKAxTIojMBxBdYJ+zjwlnvsI5rO4MeVKBMUjX7iY7DD8dR+Y2tu9/XDj5SzTO1mOtNfOM02ZspZZaaCJPDdd1ag5W9oth0S+BX+OiXtxZROezkluxVYzbYrTuOaeRH6GKYYzAyzVKh7FwDVs6FgEAr7+4HQCQvMYxVXa6+MaLB/gfGe5n1/P/P771VSwNsI/ZSRot+7cznLj0lX4kpvj8Oi7yuqU9fBDlTVWkjnPdy/1vrTVYycHCwqItbgqbQ7RnRG/74MdYCaefRiC3LnHxK6T6vUeBRti41cTNJhQ+vKRQGBFd04SsdpHbhOf8XrCOkQCMizA/qpC+g3HTvj9i9JMJoqontcfdjTRhuEx+PdCMkIOERPc3eQluUCPF6GxU02vDfUv3FJD6Rkz6MFLIDe5EzyVp5iLz7XSRvGKCtdjG2Eryo47nXvutH/1DAMDFKg1Ode3DH37uURkn25hqSNvecRnjf8FInsY7MgCAeJiLtz29gGdf3Q2gFUxmwpObQYXC7eIfmzXSXUsKMm7RaifHN7KPrtPpo4OI7WSwlf4WjbelQQkSOqs9Ca7aI5mpEui27+BVLPzWRn4nWZyL+yXMuNiSTPxibCwOud6YTD7K8MPjAIClT1FHLw4qNHdQwghIYJuBGwSiM2vfCdN3fkR59qOYuNrDK7xfboODWqp1bwBIjPG65EQdtY/TvvNvt34eAPAfJ/lcHuk5i//y5XdzLGKDMbaDYFa18knEntTz0DQA4NH+c5iUBJsnrm4DAERf4lwGXshg6SCl5GWp0uWX4DLVAIaf4Yacvo/P7/L/39ocLCwsvgfcFJJDZGBEb/zxj6Pcoz0ub6zCmR1C+YoOsF4yBSVoJHqelE/7gPoe2hgCp8mZjYchfb7lBiyLDcG4DJshuspuRH693K+kvOCZjssk29P3mQQlx/NcGO+GqW9YS95Qy0+WNjYhGYdheiX4t7gixSMRXHWgZZzG8m64RvNwHsEXqS97gUs3cGtT1WjrlhkAwEqZum/1mW5vfMaDkt/U8oCUd1MC+On9LwIA/vI6Y6vzhQiU2HPWfY1sc/4W/r8RhSfhmPqNBr6K8pKOausphegin1Vowe8F5BjpriC1M+ITjrdWRrIxiUbNkPLmXO9k576CsRM5nuRQ28kJuktixzjueKHpxutkpJnInIOqhMTHx6WvVf4/s8Xx3MW1Q5QuOr/E9Zy72wWCbBdY5MPyKk+ddTyJtrxLQqUvcM10AHj7uxm89Or8egDAz296BgDwH84/jGaTYxj+TU5m+t6Wq7w8xDl3viGc//0UATd1LOH1awyCOrB+EgBw5TO0K+W2NNH7GsdivHANkWo6TzjIURDzgghP/H77eg43hUFSNYDQskalsyXGGvE5MmdSW2tY3MsXP3YPRbS6kpiGuEbiGTEGbRNxf4n9rO7UXsy6IQqm8Ea1t4GcRMr5M6bwK69LXnORE9G/MCBRmxJZ5isDZRM9JwscLIjR7P7WvEa/zrFMPszreo+0incYmMzSRgxwDTGZkdTd3eIvfyGBcp+8LCJy9pwQV9wHi3hwhIPI1mnEm/sqxef8tgb6XjAxGpLWW2mlE2uhHN+Y3QUA6E/Qurc6nwSS4jP/uKh5r0fkulbhEmOwM+qaG9SeWzX1Kv9IX6UIO/GIi5q4fbWoZD0viZGsFyhsF91L/PHxSYkR2NRS7wKFtdu1HtOeizUnNc+KG0gom0GgPMy/oxOShxKXCMvlALrOcQzlLvad22ByOrQXQzL8ByQAs7fzt3VfaWJpnxgWd5Boxs5wnlppz/WZeE3KsN1PKt+o+TES5p79enYXvh2dn+a+nnrQxN9obyk6xaC4IHbJjk9TlTj6zih+8/a/BAD8+vn3AACSk6KSNXxYeDufW/czfEeWJZvTqbfybEz5xDfDd1QrlFJhpdRrSqmTSqmzSql/Kd//ulJqWil1Qv6984ZrflUpdUUpdVEp9ch3uoeFhcXNh+9GcqgCeFBrXVBKBQC8qJT6hvz2m1rr/3hjY6XUTgAfBrALwCCAbymltmqtm3gTNBMuVu+vIPZGxMucMxGPBpNvC+D/be+9gyQ7zjvBX77yrrvad093j/ceGAyAAWaAAQGQgAiRohFFaWVPJ3Kl1YlLSSGd4iIupLhQhHZ1q9CetKsQtVp5Q29FEiAcAQIgMIPBeG96unvad1dVl3cv74/vl1nTZAGEVjyxNy5/ERPVU/Vevsx873355Wd+X3OQtFizomIrSu/Br4Wtq685xpX4NvJMo+qaqLbGblEXg2MJpMRWZYNw4j8qBrTZyKAlxTDqYnE7pfELYZtVaVynftgEUfnwqtKH6fu5/ZmU32YeqaH/eVmN8n3mPI63rCzJS4BRiZ1X5bzcZm0Nl4OvyPGZbXKNtd0ZXFsWNSsUkIEu72zF0c/dx4lp+PY6AODtzSNIstINHaKqnpoTQ+bgswEbPRc6I6pVRbyWiCy2ApYMv4bZRqmaspqQ4WWY3y8DjM+0aP2SEyvXpMKOqo2INMQlGc5rZEkhdYsBR49Km4Yu/9bRoJ0HE/EYnmfJgIJG5zkS2FILNcFUy5uBZoRGTbrKd77zMgDg3NNbLV3f9Q9L20n5CZXugDX+dhyXwZgtT3lAw6dG1HmBbtzTMneN/ib6gqKV/fr+bwAA/uN5WTNLt5Ko3rkyn8RQ9ysfyA7IMxc5Iw90nkS40dNxXNorLszcVUnKKe/lXOSB1Anp38JBmbvO89Kn7FZtaQGN8fjN8D01By1gLBVC/PdWhor3AvhHrXVVa30DwFUAd3+v6zg4OKwuvC2bg1IqAOB1AJsB/Bet9atKqccB/LJS6qcBHAfwa1rrDIBhAN++7fRJfvfm7Vc8RC7FMPhqCbN3i4TM7iMx6THjtlIIX6S0Ju3b9t+VPd3EewcszbrRGIz7MLqoWkEtFGnxl8Xq1YgB2UOifaRel7ajvyMcCrWfayD9BuP7GXAE2jiW3lFGz9M8foHS95D0pVSIIDpJQxTtEiZPZM2TQczR7GOyTrsZ8LS8Lmjp1ee4kpg9fTjfIh01dOImaOvBviv48U4Jsf3I1R8HAESYtZiYBLJHZCK6vin9XTrClSjg4y8O/CUA4OOXPiR9r5DN6pEGVIhBVnPynXHPVrpb4drGgGqyVkMFjVCRGsMdK0N0q90+NI3No9+QOb91VPbawfmwpb8z98q49SJZDZ/088YQOn1Yxpccb4W4G83GaHSZbaplWGS4fYQFiOoJbTW2PIPSLnxVjHk6pq3R0RCxGu0A9y3Dr5DpalJuxJqX5Dmduj9otZ4gc0F8tg0fmKyJrWBDRAKeyuOi/a55USM/SpvU66I6TDwqEzz4agP5EXl4lhjQNfxVGcPM+6t4Iye2pfcffRUA8Pln75HreR7AR7brNG1OW2iIXvBQHqQRPPcv1BwAQGvd1FrvBzAC4G6l1G4AfwJgE4D9AKYB/Cce3u6K36VpKKU+opQ6rpQ63iwV3043HBwc/hXxz/JWaK2zSqnnATx2u61BKfVnAL7C/04CGL3ttBEAU23a+gSATwBAvH9Ux+Y15vfFrZTuPGuogLiSFFraAMbkt9x+STkrbGigsEl+2vFHsoLnt6YBAPP7Axj+pqzOt44aRiAjqxQi10VKB5mwZVazka82USZpbZXchSXaHLyAxvwD0qaiBb3zG3K9aGfLDbe8Ra5iOAZmDt1GUivbRBTWGPLUFo28sZ+YBKr48RaxaWaPcUXKCvLy0kZ8fnwfANgw3BubZQXa8NAkogEm8QyJRkTnDLZ1zuGvF++Xa/u0E0Skc/1/F8PiTrmg4WWYp7V886fyGH9MVr31X5Z9dH6jaGJzdyk0Orgqce9f2SJagsqEEWLY+/ydMscmcKzzCuD/sNg9As+ICmCCfrLbNNAv855+Jcb54T6/r8WxYbIlTdGe7rOteaz0cHz0AtSPlFHymLDFPhl+hkZMIUANqnavjC98SsbbOJlGjG7mxQPSwRAzYf2wRp3ESossSqTjorGEp0M4nBDDxUeP/6Rcj165hX3aaoE3/q18mrdl6v6gTWALkmOixCzLZjmI6ZLc09cn5XUzJfrqEwlEyX2ydIf0oe8V6WdhBIjf+m67XDt8T+GglOoDUKdgiAF4BMB/UEoNaa2nedj7AJzl318C8PdKqT+AGCS3AHjtra4RqGkkbzVRHAqgm61U0/JZN2Qj3drWZzBuyizdc50XPBTWym9j7+9Z0XZiSmPsg1RmAvLw5/Vt7NPHqNZ/VCxaHSG56XPxfsuk3PsKffXXZVsRujOD2hWyYzPyMDkjN2GxO2DjKUaeZS7Akrwg0/d3tHz2Wxl4wHiC0v0FNKekzdGn6UYMt/IYqt00eMaYCk026UvJ9Th69DQA4FZJXv1YjBFwhU78+iYxgP3uxcdlnHm5Rme4jP9zVOT5187+kozlBuNG1gJRbtMKa/gGs4bDrQdTNk7h1lF5OE3GaHReocG4BpMHEwgZ5mbP1ns0BDKNPbLnaLyUROGKSMsQhaYRlPEZzxLazh2m8KILNL9O263Yuq9xq/KAzOf8oYZ9IUyFLVOdKnImboVPYppExIw7SU41MfUgf3tN3nYzF8VhZQ2SxthZ5TYovjmLwi0ynjOupcqMYmwp4revvUf6wpiG2tZWBqdJG08cl8ZNjYrUjQCWf0QEVP8nGb+TNPdD44eHhLrgb8sH5byLsnWJLiqbLRxekL6Y+pi1dMu1b0iB3gxvR3MYAvBXtDt4AD6ltf6KUupvlFL7IVuGMQAfBQCt9Tml1KcAnAfQAPDv3spT4eDgsDqxKiIkE72jeucTH0etQyFHPoeOq0b6yjHRO5bgPy//MRRpxpLRcdNHYVgk6vYnRH07PSk20MCFhOXsX94oEtOsasGSQpy6T5FUbobmLDGlES7IcQt7mbFHV1N5wLckosVR6e/okybzMmgrXRnKORMJGFn0rIHOe1RUDv2UaDrNCJDfIatJ8qKotcZwlBz3LDntyDPyOfljsorqpbDN76hTpQ+OkjbsXAp1alvGoDj0E2MAgI+PPoVfOflhmePrpE2n2ywwE0GX8JFaI26wzLnY79moS7MCGQ6FUE5Z8tIySXNMXctmFGgyb6bztagdMwCkrzawuFPWqa7LrCx+0BTRABodTfs3ACSvy7HFdU0ESWDbGKY6w7qTgbKHXgaKxWdkrsae4G81Zauemcgzw7NQHtI2sK1GqjzjXl/e2rqeMZia+aknNBLcDmTuZDASKQ/9rjp+4YBEoX5mbD8AoPqK3PeBY9UV15ZJ40cAaCRl7L3HV+bwLN3hQ3XI8/Lq0T8GANz30i8CAIb/MmJJdTuvyX0rDraIlOtDcl7HSbnu2T90uRUODg7/DKyK8OlGQmPuUBOxW0H0nKSEo+nAZMjV890YekK4CW4ck42mpVt/tIbYJVmNpv9IMg0DO9jO2hoUrXl6l+zfIidlL1nt8bFMQ6YJCDH7y+VNyq7IlVEaBq/RrXdZoSjxQkhdIzUbV+3CxgZCdJk1Rsjsc8lQNQEF5m50csnJ7GHA0mLQtl8abmkagBiOuk9L//LkSYieN3Uub8sjOc+w2AUZX6gIdF7jykiy1Z/quwAA+PzSXUjFZLVNiskC86ze5Advq169xmhS0pdmVFtjWoGhygmGJ3dfaCCzjRWWrrXyVwBgaY9G5NpKtq74E5IRm9WDKDOgrcIAoOh5Y3wEvPXyW8/n4xwLj5kKGDMSYnPStmHDqieBCrNiq50yr8bOkLoBLOwO2/ZlzPIZWlb22TN1UCv7RWXs+mYMS/sY8KVXEiFXerV9hjr6RD0s5sQGtG/jJC4Wxa+pVEuTAoDpwxH7jLd4Q2D721xm4NYGhs0flDlL10L4/V2fAQD86MWfAADocVGpp44AIQaFmSAxEwgYXVAIVCIr5uPN4DQHBweHtlgVmoNqKoSyAUQyrfqSMWZLZndy/x5tQi/L3tiuDolWG2YvliEvgHEZJrrLKNGCnjhGlxQtzoGKssE3hT2yigYvyeqZutHaT4JhxiaXvzTatKuloTjPb5Hfuk4GkN0ty1H8fNQeDwBeWSHOFbg6LzG9gb3MNK15aFJDMdl8JlCn1qntWG2Fafal40qrIrXhOzD9Vg0JBgJgXVt/cUW4JB8cuYrMCfGLhYYZkHXC7EuVDTn+TlakDV+qIP+b1MA+J+eX6V7Lbg62XK5SNwj9x6kFZQKW70AZ1+kn5ER/KxC9LHPVfUHmauod8pm8HgCuyOBnn6Amdk7mJzGlLZmrCc02Np1al0ZqQvo++xATsK6GbX9NUpbZ3yvyR4QKyrqejebW8SQ5L6saHkv6dTPha+YBehauBu15lXNpaZQaz/mXmQYJYP8RsYld2U+P23wSwZL0q/+IGC1mXxK1tLyjgmhcnonYC/Ls558cBCAU+hN1UXHGZ8QWF2dAXmFjA/Fb8nyWRlidm27VeiaIFFMMTCX7N8OqEA4GlT6g7zgJLzjRHcx9mDkCBJ9JAwAaNFKaOIDE+ahVD7+z9Hj8qQ7U1oNtyqcx3DUTPsKMt9fMh1jcq2w7RnB0nzCUZ4xYvBDA0i4Tf7Ey5itzsG4JRo07LsKYhFqnb33hDaqsYVZAasQ1YtdjK9oKcIuT3elbf3rvWXmoJx41AsGz2Y5+hP3LyPnZvQ0ojsvbLm/Nhk5x1H/10m40e+WBqfXzoe5l1a4JZVOnTQSomc9aKoT8i5LOl2hQvWc//aCy9yFEFXnug/KChE4nEByWhzHC4rzL68y8aCQmuBhsYchjosrfWsqtNykSwLi0y32tPA9jzDVGy65jIVt4GSQOqrAiVccVDxW6m7WpI/Ii81qqPpa2M/aEz8v8IZmn9OkgohJKgsx29inFfnYErfFcb5Rxxo+T2GdzAyou963UEEGQnZaXfeh5DzOPyxw1XhahYJ6b4FQEDZ/uZbZdWSfX+5Mjf4tf/sLPAQA6r3E7SSHd/UYA2e1mv8RIVcZJxKcUsjsbPJ6BDqyt+p1w2woHB4e2WBWag2pITHx+UxOLW7n0T8sqkZjkypeswg9KdwcOi/qV+Sfy729rIEJKdJNjUdtsshAD6GDNSiPZu87RuLc+iBrVc48EDZ2sX1CPK4SX2RYNW/k7RMIPfyEE7ZnVSFYV4+JKnvUswejaQ5MAgKmnxYAaLCnrBjT0+Fmq/Y2UjwCJR8y2wLjSdFDD2ysrfnlMVhxDrJrd3nKrlTcygrMWYZ8C1g0XXS9LbMH8FmogPMHtC7W06rAcE7gWQpNkurZOBoOEspuDNgLUVIQyNUW9KlBeK20EqZEFGCxWGfDR/wX526y6VW6j0ifCKKxfuZ0ceEpWuoW9rRwV0C3acbmVrZom30HzOquXk2gluyNoIykNzH0PVLU1Nlc3GjIbBrhtyqO8IP2MUTWP36TrdESjnpb7HZvi+MblOa2urSJ+WdpIfo1jpmEzmAvApwZ3cL+owssb5LxD99zA556SrV6KLnezVQqUlaUDnGIdkCPbxSf9i0/9DJCkm3KEwV4R5sPUPAw/zwzdddJPEw3p1TUiDIyq9b51+JHTHBwcHNpiVWgOfkRjeVsD3W8EsHSvSLXhl7mv5F438UYMvWdkZZwYkDx2FhcGvFZF6/m7GPRzmfnr+2soc/+la4Z+XFal2ByQ3UPqMQaUFJdExB547DxePiHEnSbAqZ4QaV/s13blX9rdIu4EZBU18fr5/y5L6sg12e/f/KGEpVabvU/OTzHbsrqhjswdXCK5T0xdZR3PYwHUE2L5NPn5iTGuFtGWm6r7FVk9TfBWM6xshe/svOzzU6PULk6nUB6VTsdYiSr9upxfT7YMvIX7ZP9saOP8skKRBswKNQ2P++nIpRjCC9TuXpNO5Yfl/6FlZV2QMWFQty7mQFm3UvNMgFWWBsnxEMIsBGQK5JgxwdNYuJO5FUk+DDQ+I9FAuEPGGn6V/B+0kSzs1/BpoFMZujQ5h9XJJAJ9zFy9S/yB/nPdtnNR1lbNs/akKbSUOhOxwWBz60zQFjWd8yHUk9Lnv3zlMAcox6xJ5myQ18JBamusWOY1FebfT/tFSI45NSvPVDgTsIFtpjCP0bJ7ztcwyTwiY5Mx2mFxYwOJ6/Jbbb15gdrDaQ4ODg5tsSrCp6NrRvX6X/hV1FPaMhWZ+osT72JgyaUAiqSfX/c1kezjH6GbZjxuA6KMN6D7kBQLzLw4aANlSsyWC5hgletAcWRl4El5k7QdvxKxnIAmrFWvlf1s9GS81Rb7a6z6yUnfBtQYNqU0Q1gT01XceILFWsg7UNjJQKQLERS54iRuilZgpH5xjbal3cwKZ3gVswdqCLF4jllJgqTSX9ir0LNXzOu9cRnM+VPrAIiXxXhjjA0h+4Qc05yIW55Iw1Vp6oXiSAbNVyU7yniGzJzrgEb6EoO7dhh7ifwWqLRchP46mcdmVlbtvtcClvTUaCzGUxAqKDRY99Tc4/xOmvO1QmQqxOPpfsxy9V0GUu8lLf55CUCKLJEpKdcKQjJu7eoGalQLIRvkpe8jR8ct0brCSwFLna/IR4k85/5cwLJKLd4h93vgFXps7gZ63mC/GAA29S7jj/fQtUauEwuzcNAp6W961yIWF+Xav3LXswCAL05JBu742SGrKeQ3sU9G4xxZRvMVuUfGjR7m2JWvUO3i88jzz//H9uHTq2JbobQ89MmbCrmtrAOwli5GTtjy9lZGYn6UT86YfCQnlFVHDRFr/inxB9f7NeI0MoL+dZNFmNvoIbK0si+R49J2cVi3jDh8Fhs0khaHfesD775I1uLNdCMuNzH1AFVpGg2NEXLs3be5XE1RV8YPFNc2bZyBMZzmN1D19RUim0XFrV6S7cUyyTuSF8MobJSHI7NrJYlso7uB0jPidtz1ky8DAKY3ypucvdaNMF+k+buZk3FDXG8dY0D2kLwshXkZc5nuzvX/KYrJh6R/JaqlinEggYKHBPMYikMyj6V9rBd5JYYg3b7RF1mpjJmCC0eriF1ukQUDQuQijcMWhS3JbhJDT8v8zhxqGUwNG7iJOwlUFG6dk5csTGOxrYO65EMrpuG/X17Mzojc5PmlXstW7r0ijfUwnTtU9lHndm7xQd4b1hbJ7mkgMsf0/Yus2vZDXEzOxWz2aHntypiL0toG/KfFcjm1X/rwsSeEhfGPTx2FohH2/3nuXXaOAcDvraMZljZMrkmVW/DKmTRSNMyXmDNkBh+f1tZN/ZZ8bnDbCgcHhzfBqtAc/BBQHmwiUAnYehXhPCPrGFA0/EIVN36GqwONO6a+Q3FE27h082nUfh0Udx8A+HT1lAdEJkYyQDgnbeaEJQxDB0UVrT23BvEZGiJ5vd5nZTWdfEfEilUTv1/iijCVDFr6eFPxyNCeN1INDL5AFyj5GVRVVpnukx6WHiCBba+oHKEvS5hix0QDN3+Yqi210d7Xed1BCc65HcaFV+/w0KRqPFESNTNBTayY8+w2Jj7JXBCq7/WUsoY6E3k48BpdakdiiDDIqpGQ6xrNyg8DU0dEA+g5K21XeqWB1M1WNGN8nvehj5mGlYDdLhnCX6NBqrqy89h7mnU4B0nhPtO6z+u/LONa2EuNZY1G70nOEenaDOX//GNVaG5pPGaklpkxGsl4ViPqOE8VghrHzP3azntghsS53Oos72rY2iBmfFVGupbWNSz/g8/n2WzJeo4HLJmyIXT5qz8RIveuorbGV0O6U+ozmb4N1DrJS8Lo1zUvyMOxtD1kg6CiQ1StZmSc1bSyJRYaK2PuvgtOc3BwcGiLVaE5eDUgMRFAsNSSlGZ/afbt1z/soecF6e7i3SLZR75OVp2Eh7mHWGGapKBe3WQRtgrBmKxFY/wqbaohstRiWwKApSclsKo6qJEg14PhUqinZBWsdTetu8kEB5kK3MGSQoPBKSZ81wTxBIqerUDUc452D8MHkFLoeklWvewG+ezjvnFxRwhrnqVtYxMzDUURQCN2G/VcWj5j3KNH5wOolWXFuBCUVbd8UgwaoQKgt8ue2DAWxV8VI0tpSCM2Ld8NHDPaEue1BlQZ3GNYu5ZYp2X4+QZym2Q+5+7i/AzKfWmMhVFi9SbV4MpPireOsQBymxmKTcq0AeFMRbm7tcoaSnXDptWMaWtHmDosczbwmkxG43AJSyPSl4HPUZvg/fCmotYuY8oBxF6VZbSaBqI0cpqsRVMNCwookafC0LcZQ2jvSyEsHJaxevUw507+P5EO2rqgJkzcFKCZfNhD6hqNhetklc8OyHO+7jMeamkGg/2IqCXBM6JBNstBMJrfsj4Vhk2wlg/NfsWeSnEsNDo/UkYjRwrAS2/9+jvNwcHBoS1WheYAJXaHWqdC+E7Z0NaPydJoqjsHT4VQk0UQiWuUpnvk/6EC0HGGnA0Ud4a1CaplvS9z72lsFV2vBWzJObMqmSrN/ceB2Bw5J9eRqYeJO4GiZ7Mie0+bgCOyMaVaFOiDXDlK/z4LAKh8vd+66IpM/EGyyusHLHOUCZ4xHJleA1jasZIk1Yyv0qOsh8Z4Qh7+iFQG+MxrB7HuywwOy9IFcoeoYo3zKfg3xTsRIreiodCHB6QfEvUjtySTZrSfYLnlZlzcz+QxalE6qKyHx9goZg/SKr8GWPu17ygPSGbRcq9nE5rqvMcm0KqRbGkH1WnzuBrXpmc5IJfXyy+zd8uz0fHFDuSoZWVI9KsOyo2Pv9hpbSlmzs3+OzWuMX8fg8MY4JTbwOC5ZY0quTOM3SXEAK1Kj2cLFmWYSTx7d5jHCFMU0Apjzv+C9EUvJaG0TGjqeRau4VgmjwKKoQZ6gqS8VHR7XgmhnqAmvFees3JerufHmwC5O/2QSW6T88Jn4lB7RUOpJ9769V8dcQ4jo3rklz/+g+7GqkaD1HY/KJgS7g5vjh/0PWqHt3Pfrv1W+0K67o47ODi0hRMODg4ObeGEg4ODQ1s44eDg4NAWTjg4ODi0hRMODg4ObeGEg4ODQ1s44eDg4NAWTjg4ODi0hRMODg4ObeGEg4ODQ1s44eDg4NAWTjg4ODi0hRMODg4ObeGEg4ODQ1s44eDg4NAWTjg4ODi0hRMODg4ObeGEg4ODQ1s44eDg4NAWTjg4ODi0hRMODg4ObeGEg4ODQ1s44eDg4NAWTjg4ODi0hRMODg4ObeGEg4ODQ1s44eDg4NAWTjg4ODi0hRMODg4ObeGEg4ODQ1s44eDg4NAWTjg4ODi0hRMODg4ObeGEg4ODQ1s44eDg4NAWTjg4ODi0hRMODg4ObeGEg4ODQ1s44eDg4NAWTjg4ODi0hRMODg4ObeGEg4ODQ1s44eDg4NAWTjg4ODi0hRMODg4ObeGEg4ODQ1s44eDg4NAWwR90BwBAhzUaI1XomodAJgQA8GM+ACAyUAIA1MeS8CPyXfxWAABQGm4CAIafB3Ib5DufI6qnNAAgOQ4EK/JdzyuzAIALv94jXzQVEuNyQmiZX8Xk0w/d9l1UPvNbGgCA9NkgCuukfR2Qz/C6gvT3+Q7bh56zVQDA9OEIAKCxtQRMyAUCFSVjmZHz8+uB+LR815TDEajJZ2LKR2FYxpe+Jn2Y3x/ksRpegxO5XfoQfD0FACgP+khflDYLa8H+ymdyDPBD5jfpQ7Ak/w+WAa/OMW8zf3icH43Rp2XeS73Sh0oP29nURHiRx4U5j1Fpe+hFDT8ox1W65dNco9KjkJiS40qDCrejkQCSN+W3xf3yGeE14AGJSfmumpbz/BBPVEC1W56X3jfkq4X9ckxkSaG6R54rvyYT0vF6hGPXqHUo+zcAZPbIeGPTQdQ6fU5kqy0A8MMAPOmX4v1QTfmtOlRHbFw6VhmUtqIzct3UTY1wUdqsprwVY2kk2C4AtVsextQX5d4u7QbiM+r2riC/VS4cnQqiMiqT2+BUBQryh1cDus/LuIpr3lo3cJqDg4NDW6wKzUFVFCJXo6inNPwoJTOlbuNGEgCQvgRkt3NV4krefVr+nx9RiM2JNAyV5fxaUn6r9CjU0nJ8aG8fACA+JlK7kdR2lS8PUtO4Kf/PbW2tpPFZaTOcl/NCRR91tp+8JefNBaWfjRGN6BxX0hER+x41gPircduXeoe0aVaX7rNAfp381ndKVoDioFwvv9ZDLS3XKWe49HO10AHAK3MV+6asKsubpO3kmIfcVvnb75GVJDgtfSqOAI24/KZD0raXDdj5LY3KCjf0jHyX3SLXiC4qjD9OrSkk/QxlZBK3/kURk490SF+WpH/VLjkvnKti/k5qUHHOx7S0U+nz0f+69C+7VfpnNKtQHgiVzfWkv9EFmftIzsfsA+znc9LPcjefiY0+4lPy99zdcp7RBINlIPB6nP2UtrPb5XP0GzV4Dfn71lF50OITMr7UhI/8Ws+2AQD1hHxqD5CzgAa1i84L0qfkRBDRrPRTX5JjioPymV+r0NwrjaW/Io1V+tmSD3h8Pro+Kb8t7aLW1dAIFeS42IJcr9wv1wtWgL6XpM/zd8t1FV8rr67Q4PujVypp3wWnOTg4OLTFqtAcAlWg85qP/DoPjYTIq/qI7Ncj5ym9FxvIck9XGZEVq94hknLoWz5q3K/NHZBPoxH40SY6L8pxUw/Kd7EZ+UzdAJY3r9QYFu+RtiMzQeR2yN/lBTm/3iXiNzYVQJir0NJu7vvCIqG7Xg8gsiznNbmn7xiTz5kjPrpPSluZbjm/kZDfVFMjkpXvFvZI5/tfF5UjkgtiKSbjym2T/tZ75beOs2GkJuXa9bgck7gln8vbGwjmaKv4tqzI4Tw1nUfrCE7Jd7GbcnxsUcY3d4/GmuekL7mN8ls4x//vbLTGsFP6HqDmsrgnhdIOMfCMfF7GcOsu0QiWJ6MI0PZT6ZE+NCO0sSSb8H9jUf4+LUtqJEvNMQ4UB6UPAy9zzmjHUD7Q/bpcpyaKG5apKan+CvJJGV/vq9LfcEF+K/UFUBqSNnrOyb1KUQPMbKPBB0DvaZnXpe1mRdao9EsbRrMJFqnhFAHlU4OjBlYcpcbjAcuaz3Va2kzcgD1PX5DOm1U+tMz7uG8JuetdAICpR/jsTUjbkbyyWks1TY02zzaXNQqj1NioaSYmzag08utpCxkwxqr2WBXCwQ8DhREPOtB6qWNJEQ76bvmsTnWgd8c8ACD3aj+AljFv6ogIGEAMdACQGjPqpUJuh0w6kpwMGoOW9vt2++IH5fj4DTEche9dQvWWqMiafer/thyb2aaRmJLvjCqp4tJ2aSCI3CY5ISbdRXa3/NZ5LogS1clwdqVOt7TPt3/HpmlcDVEgbPDsNiKYN+dJP7UHLG0LrGirMsAH8GYQ1a7WiwQAjRi3B1ci1uCZ3yrHF/mS977uoZKW3xJTcmJuEwXIZBDFNRxzk21zWnNbAW9eXsj5fXzAT8hcBKoandfkJnk1OaZs5qKrgskFuWDHVelDcloanXyHh0CZ93JUPtPXaKxeqiO3SV5mc49iNOoWU0Ekr8uXwaocv0yjdef1JpRPAbpWjslvANtsGVODJRoIOYfVDs9uGc2ND+dbc6E9CgyxdcLctNS4b+dPeysN50PvnMD4ggiAyHkREua5Tr7SidpuPpcc18JBuVd9r3lW2BljfHmtCOKOmwGEitJ+aUDOW97M7WVII0jjZHhx5XPznXDbCgcHh7ZYHZpDRKO4qQ5V9xBaEnkVfVJW7aWDNHqlFfKXewEA9GgiSxV74DVg7gDb6pTj/aCI/02fLGP6sOhfw0+Lqy+zhyvXGlijVWmYxh1K6PSfpBDpld8W7tBs07jgFAojcj1llpCstNlItFbp3lNiaMrullV+eWsTo0/J8UvbZeqLIzy4o4GeXlmGCgsyzolHjez2rftu9BtyzNWfSPB6ClVud8x2wmgeuDuH9JdlHhW7OX+vrDzJ60EUd4ueH7skW7dKH425Hcoaq7IPyTGJ1+jj9YAtj12WMfyOWFCnHpCxqyagaeza9Ygck6mK4e/GuTWAluMyB2WFC8/IvIROJFFbTyPssJxfoZs0VNCI0mgYzUj/Zn5UNJCBz0VQ5hYzeY3biw6uupfCKGw0ajPnmkZWrxpAfqO01XOSBuFz1IKaGqV+mcfFh6i9luT8xaMNvG/XSQDA5964EwDwm4e/AAD4x+m7cX1eXOShF8QwbOZwaYdC93n2/UFjHJU2574yihpd5MEEXbVZasSHAzAqSk1uI5I3jJEaGH5RVL+lbXJvel+VNqudCtW0HN97lm1zDLkdTTSSfF7Gnebg4ODwPwCltf7eR/1/jETvqN75xMcxd38DoUW6jcbktzL3TJVNVcQuyf6y2rPSDRhaVjZYJDku48ltkf+nbgI6wDZkQbZ7wlqnRhcDQmYPU6LTmKR8wKvKebU0f2MgUPrgHAoviN0jNU4j3t3SpvZaezpjtDJ7+9L2KiLXZQxRsb9Zt2wz3NpLW1fae2gcTdTQvCb7UcXAodS4fDZiyroujYsvNWYCq5R1o7b2oPJboKoQKnA+6Co02lAz2tI0jA0nSaNlM9Jy384/TEMPV8jYhSjK20XTuHezWNwOpa8DAP7bf3s3CgdEkwrekEFH9mTlvM+msfBOOU/Nyfx0XuI961G2/dJ6GbxXDNgxeBtlEMmnaJEkah0K2sRKHc7I+RfTAIDhFxu4+R75LXFdtBfvkBxzz9A4XpuWiLF4hEbfsIxzR3oGx+blt6kxeZhUrWWnCVZlXqYfkPuRojbjh2BtFOmrtO8MGCOpRrXTGDXpji/JZ+OnFqE+Kdcx2muEmrUf0mhQ0zDPqXG9x+Y0lu6VuRr5ilxn7sdk7qMvJ+HRBGfm58x//tXXtdZ34TvgNAcHB4e2WBU2h0YMWNynEV4IIjEh3ylKt8YuWRk2/mkQ5V75slDiXsmsDNVW2PPyo7JERk7KShIqaBvibIKZ8qNy/vovFjB3UPaHqt5aUQGgEddWCzEX2vg3twAAxW/3I0dNIcLgltQ1E7cLFEe4fzUW7Rz3jdNhVOlJqNP12n+cq0RMYXk93V1JutleEHUovy6CRrecZ7SSpT0MtLkYsCHSxgK+9DhXiZNx+JyqIFej8LJxncIG0ZT76CFgKHJs0cetd1BjuN7a+wNAaY22lv6e52WVX9rHYKadZXz9gT8GAFyvi6/21eImAEDhQBmb/ov0+eqHOdfHujhPPhInuW8+I6v19H10s85qVHpp6ymt3CMPvdTEZFRsL4v3y3mhWTkvsgjrOq29LtdprJdjFneGoRq0+j8i93TsumiCZyODSEXpRg+K5nZlYsD+v/pZ+RuHZGWO3aStKaaR3Sb97Dkh/Vx+RJ5ddS2ByKL8NnuP0cCo7c15iNBNbALGFg5L26F8DB3UiBMT1Bj4/2YY6Ly8MhzdBMoBCskLcmB2o3zT/UWx/czf6VvbVD2Ot8TbFg5KqQCA4wBuaa2fUEp1A/gkgPUAxgB8SGud4bG/BeDnATQB/IrW+sm3bNzTaCZ8jDyt0aCv3sQkRC7KSz7+Tm1j8Y3gSF+RCS4Oeigzqix8muo37XxLe4BGii/wVRmucb1Ve6PovC6NBio05jD+YPCVOsbeJ30Zel4+L39UfHhdF4CB+8WXeXNIHqqOC3JeLS1RhEDLOGbch1ppG1NvVLoy8xJStxpYuItq4RT95HQZVvsa6LwYXDGuuKSJYPZe3z444WW5Xt+XRG0v9QGavvCFI3x5ZuShCW7Jo/EGrVyEyb8o9wegmlR/18tnlUIptOzZB9QIhdg0jboqgk/lxDL8ubF90peECOvY2RhuvpsPb4DbNG7vFu4AAhReNz7A+WAkYXaHj1BO2u/bugAAyJyQSNel7UF0nza5FSKoui/K/Zw7ELJ9D/fJhcyzVNjQwMjTcp35Iflu33bZp/lQOHtKDK3dp3hdbs0ubd+ArveJfzpyXPpgXKhQrZgHcKve+RQF134fUcbKmG2aOVYHW67TwlqZl+A884u8kM010fvEEB1mVKr3Ypc9r9rLiNibxvUKpK/J2CffScHdK9ePzSgkGRfTDL91iOQ/Z1vxMQAXbvv//w7gGa31FgDP8P9QSu0E8GEAuwA8BuC/UrA4ODj8T4S3pTkopUYAvBvA7wL4VX79XgBH+fdfAXgewG/y+3/UWlcB3FBKXQVwN4BX3rQTRYXeVwPIblHw3yGGofApqpzUEuLTCqU1XKkY4TjzsEjRjjNhDOyTpXT+uKh94RxV5TGF3DaRgQ1uPYzLrusysLSztR0AWgEot46GkGBk4+y9zEHgPqH4njyKz8myHqX0jmZoTCoCpSGq7ltFraxcbxnLTFakyT1Y3sKAnmUPgy/Kd1kaU+Oz0mZiKoCFw7J8DTzDmPkPyNZBz8RtpFyegS7G3VUc9dFxmZmCy1yNwuzntzqwvENW2dRl+a3aS+0rq5A+L+cVaaQceknmOrchZDNSu87JOEv9HFvEx6HEFQBAZlh01kt5uR/VdCt7NMJVFK24L3SI3RJLG+Qg48rsO+5hcS+NxuNU64aoQiKEwgZ5QAZeMkZgE8EIdHHlz3CLMkjj88z9Cou7GFX4QqfM52Myn/f2jeFCSSKiqlTXjZYXnQPqXxCNoblZvjNZoX5QoZfRliawKreXEa5TISzvla1K/3MMEjso/awPNpH6Jl261CbqjJotD2pEF7nyn5Ltb95kdcaAZtrcL5NHJH1qRjRmD7LTVJObdF/W6p6NTK3Hvz+awx8C+A2suJ0Y0FpPAwA/+YhgGMDEbcdN8rsVUEp9RCl1XCl1vFEpvs1uODg4/Gvhe2oOSqknAMxprV9XSh19G222E0ff5S/VWn8CwCcAIDoyqnNbAa+u0bycBgAMnRCJN3NIJHx0AQhRG6iKUoEojUHhRxZQ/ZSsUFG6hYwLL7/RR3J8peZgDIUzh4IIZ+XvWif7pRgEdUljaTezMYdFeIVfFuldW+xEkBpNk8bO7Fb5f2KypRVEviRL+vIGwzWgERbFyBot/ZSMsxEJIbuP/fNp/FMt11QgKsdV0zTUfVu0keXdNaROMJQ6IHNV6WPbnXWU1phYYPlu6HnOyyjQz8y9uftlJY7eogbRo1FcR23Jk/Mm30H3YVlb41qJtrkGDahexcM/LtwLABiJykB/eeRZAMDHGv8LQgWGNq+TscRmGUoc1li4h7kDF+UmldfIMYXhIBITtOF0MSOStgrtAd42WfFDJdFU5pj5GcprVB+TBJjoG9QO7qQmdtOzvA9Gm5k5L2vbzsdfxlfpgi4NrcyjCEQUfLtPp62D2kUzAlT6TKi4HBFcMvcF6Dgt/YrkqO1eNq9eAE3miphQZ7NkN1JNZNjnIF38RusKFVvGSZNTEai0DMpTDzON4BLvaZdJKwDmJX4LnVfwlng724r7AbxHKfVDAKIAOpRSfwtgVik1pLWeVkoNAZjj8ZMARm87fwTA1Nu4joODwyrC9xQOWuvfAvBbAEDN4de11j+plPp9AD8D4Pf4+UWe8iUAf6+U+gMAawBsAfDaW17ElwCOQAUIcodx60Emqhj2p3kgy8COrpdFCvsMbtKf70FmD0Ns57nKDLSCd4wbr8yNz+g3ZNmfPRiy0tcEJeX2icrRHAvDD9Oq/opoDCVyPoTyCoXNDDL5OpOj1jN3f6aB2bvIgUB3Y9xoLp3asjyZvb/HMNpKr8Lo03LtzDbpVG4XORiKIajx2Io+mD06Gh4K72WK6GlZIeudMr7+Z8PWHpFhJqRhkGrEtZ3r6DRZirhaD3zLgx+QPs8flT75JmltPmwDtwwzV3SelvBphau7JGin2JAx/HyX3PraYB1qSq7T87ocb7xPXk0hTW6O5YfFThM7J5pRsAwU7xFVIXRJtAMTFp24EYQ6KfcmQ82txhVSBxS8b8l8FBmevO7LJiw5aG1ZpS3SCcXAqicXd1ntzjBl2QzfsEaBgViBJfly9Kty8LUPd6HvDZmPxT2832m5bv+3gihwY13qoz2I4foKwJoXGPIfkvlJTknnFncFUdtO21LOeDuknXpcoU5vmAn9Lg3Qzb0zgAi9WRXakeoD0u+ef6qhPGBsYG8dAPkviXP4PQCfUkr9PIBxAD8KAFrrc0qpTwE4D6AB4N9prZtv1ZAOALVOH7GKh+JWeRjv2CI51Fe+LNY5P+Bj6CsyeUYtMpGAS0dqGPk8qePoszdZj34YyOyiajYsD9nNfmkneqNFANLgSxSgKhif1pbQpUBjUjDCiMUXE4hk5bisuPHtC5PdFMTaJ+WGzhySF7q4Tobfc9zDwiFpQ1WoIvOzntKY/QVxzOuz3C7Ny+0prG+i7xhfno0mclCu13UqgOw9cvH+q3xZqYIuvKuM4Jj8tvarcvz8XvnsvgFkt8nfMaPz0S9XXNPaloUnV5Kv1DtbcSNrn5TrjX9IHrz6bARboiJxFisyof954Yicnw0ixBgL44Iz+7vBl7V9oVLPyXlmy1Lp0YicEaFgCVaSzK684ds4jq6PiCuy9PvyFmY3h5CaIJ0d6dAWmeEYm4clPAEF1v/1ob8HAHxq9iB2fUicchcXZTXZ3C0u1IbvYSwrRtHCjHze+KDscetdTdRSdFObPr1h3M8aJT4DtS4SDfXKnPV+K4Txx2k0bjDL0mxRJ4AY4z+Mm9pQ2HVfbFgyoECdkbX1VpyDvafcmpa0jPPKL/kY+WyDbX4fU7a11s9DvBLQWi8CePhNjvtdiGfDwcHhf1KsityKyMioHvnYx5Ecb9ky66ItIrog/WtGlFWpTMThEl1cvScUlt5Fqq1naNCicacZ1eg7SZIPZlkalbI8oGygUvc5+c4YLatdygaSZDdTFVxLibsUQEpSB7B0JyMVz5ucDG3V0fKA6Tvj4hc91EhsCm4TGv2ygiQvhhGh28pskcxKG5tTqDwoVqfgqzIxVW4TtNcygJlcExPkVR5uQMUZWUl6uHoXB6+0TRuMkQZt4LioCwv7wggzM3CZmpFPKrlwzkMkY+ZIPusp5hJc9/CxX/kMAOCRuOx7Hjv+UQDAxp5F3Po7cRGavJfYXMvgavMSHmbA2gXyauQ0Fg+sDGIzK3OtQ4MLoiWcMca9xC2Fsngd0XmNEZk1bi97PGT3yCTt2CYsKJenRFX5qd2v4syyuKnHl2WAuYLcq9pyBD9+16sAgKzhuiPO/c5eZLZK/zrGV1KzTR1WiM947AujfNfwmTpUQuCKtFUnvZzJFC6ONm2GrWmr65KcP3PIs9rxd+ZYVPua2PrX8pyN/bBsIYzm14wAnVe5NR0XTfXZ5/8Pl1vh4ODw9rEqciuCFQlJzq+DdS02D8hKmSPBbGwWyO+Qla3vW7QZMAuxuAYI3BDpXueeLDZPI82gQnajYVaSto0hLH3Vx/ydDKO9S47f8CX5ceznfESXZGNq8vLNyuWHgQzdnMlhMVpkI7JX7j4WQpR0aykGyCzubO2RZ0kQCq5wXeRwqKkeSxlnDIUmy67SDVQy0hemElgjGZRGWqgTvouePXg1iOjSypUnu5WU6GNAjoE8BuEcg3YWQyi8i1T7QYaoz8qFB59sYOZe5i9Qg+i4Ltede0cVN6tikHzk9LsAAO/ceBEA8I0vH0SAMUzG9Wb5JJ6LoPhhSTDo/WJaxsxVv5FQCPfIcdVF6UOAmZDBskJ5VNSmOKnly7RV1FKtMc89QCMiyx40Uw0EmX1bbsh3nR0y6S9+7F5MPBxZcZ06w5MT0x6+eOMwAKC0gQ8Rr3HgN69jaUFsFFO3RLvruMTnLtZAeV+VY5f7aGxUwYtxu0Sb8GdjD+k64yGcp9ZLqryp98s96nwpiv4PiV1u7EUJ927GGLZ9IYDrH5D3pvckteydbPOixvI6UirezwflebSF0xwcHBzaYnXYHEZH9fDH/z2CRe82lhpa5xmC2nUsZEN5jbvSuLjix+OWONWEhBbWM4BlsIzIG2a5lQ+TLBPOaZu5Z1aZ5XfLitn3D3FMPi5fpk+JhA1wX1wcbnEFdNygZN7TSkKyBWHu5Mp4UZaC1Lhvrc3GTmAy8UIFbftgiGkrdL0m71pA+K9l2S0OGC8HrdcpZc8zMNmn5V4PdXqtTOCQYUBKjnsrtCsAKI6QPj2o0XHJuDzlvN4zskefvTtg97q2vw/Iqh94sRP3/8QJAMBMmS5GMkFlvjRsNRrjGTKh6ssbgUZCGus+w4nlYxnJ+VjYS3awxVZGKQDkdjSQHGM/udqarNe+Vz3rjakxozV2i4FEB5eQG0sDADqveCvG6VVb82hWd4NaWqNOG1EgSp7O14zG2uIZ+eCD3wYAfP2v7wMg7mfzzBo71tydJqBLIUGKfqMlZKjdVXeXET1NFzYLOHVckd9yd9QQZtBakvHI5v0IlBVKtI9Fp2XMaXqylnYpBBmMVmIw2vhHf6OtzWFVbCugxIc88HwDU0eY7rpf7lIsRXWsGLIGSfNyJ18xLi6NzE6Z4IHXmAlJFutKr2eFgnEHmQpNuS3aqo7GYKO1Id5oAv7KNFnzYkfngcJ6Ptg/JMJk4NN8CbZKpSkASJ2Up2t5G6m6ysFW5h3j6KPiJUNmp7YRhPURESq9z8qAF7q60UnqMkMXVrqPlcByEQy+IL+Ve6guMp18wxcruPkYIwZ5PRMdmt/SQLVb5rrWTSo+w1R9AZg/1KqeBADj7yPBTsGzxj8j4Do+KYJg5oEmTszLE/pT68VwFyU7ze/3vQ+1TkbpXZd+Gpq/6Lxno1gX75LrBlgjJD4VQPqyHJfbwsjPrbIFSL+QbBmuGZXaxyzNzGaFKN15cQoFEwVbPd6NTsa1GGFp7n9sTn1XXY3CWv42qxDJ0LB7l+yNlslQHp4PIM5U6FvlNADgT/+3PwIA/PjT/xaaLvLpw9xW8pnc+Okl3PiACP4EDZHGwBi6FLOV20yVN0MUm7gcRseY/F0YZjbmIGNvvuYhSvLY7h8TyXFtQIyswZyyQq/nmBxD3qDvgttWODg4tMWq0RwQAKaOBDBwjIEgG2XFGzgmknNht7IumAoNW/EZOXb2iI/I3MogKOPSGv2HIIp0KVb4m5XGUx6SpF6fP0C3I42QN3+6jsSZ8IrrmGpDlQGN6CyDl5qit08fpRtqAggWV7pMPUOtvqkJdIp0j2+Slb/2qgym56QCPiDL2cKULHGRZa4SkwFrFK1sIelpQfrWeT6ImSPM8aeWpRdl7ubujNoVNbedKzKvHxqPWg3AVnQab/FjGI3BbJsqQ8x6LCtEmIFq+TWYC5IaXkaT2laJ6tY3mXTSdVFj5gjzJ6gt5akJFDfX0fMl6UtmH1fpGcNNoC357/BzfDbmZc4Ts01kWLOhdEQ0OJ85J41Eq9aG0RhMEFVi2kc4LzcnR1r/xC1qjgHAv0uMzN4XyA3S4HOTbNG/r/mM7I2meN8bCY1av8zx+b/ZAQD4XzvECvjz/+ZZfGVkt7T/ouwVKwNy3uWfTaPjqvTLGLl7TlDr7QPqu5nZOyHX67xssi2BOW4ETATwmm8waG8ogGiW0ZqfFk0u2ttye9bozg5e/v7xOTg4OPz/CKtDc4CG9jT8sLYUbiZ+vpP0a+UBbYNfuvfJZnLhHP1dPqB2iIQtK5H2xniV2RJCYQMLetCIFKJ7LzXpY3GXcf8xj4IVgiLXg5ZJyD8oe1x9RTa4I8/48JnlOPlD5GOg4aeW1ug09RDXSNubPiVL1sQ7E2iSpNZ/QzSUgNmChoHyK+IG7GHdz+xmGgrXN9H/svzdMSSqgPeHcuziHiDC3IYIjYjV+xgwdTFp4/QjHHN2B9mtXmpi6gGG+7IPzWjLlWq4JObu4/lzbHuojryiS3eN2EaC4zIW/2Ia4d2i/Xx9ZhcAYGxG4rwH/FYbhgbPVOSOLgaQYWCUYdSq3i9jqJdDiF6R9sP5Ks9nNuldHjquMxDu/5bPaz/K/7+hsUAtxDwLZq+9sE9h9GkTuCXPV4La4fJ6D+VFMQI2t620C5UPFQC6lE0VLpMp3OjwbSVtU7io1iNzdz4/hF3dQkJyYrmffZHzR56r4NqPyXg8htIvWHtPCAHmjiQYDJinrasZ1dZuZQL3piVSHcGCRo61P7fsYpDX9SEAQOxmyGbDmtqsbwanOTg4OLTFqtAcggWFwRc8LO1Ulp2o5zVmK6blmEayVYG78nX6+Mg5EJ0JIk5C2QzLynVeY1bhA3UkLpugHZG0S3eIRJ9+sOXeSh8na9AO9qkMJG6yRuIFcRGU97ES9MaQtWiH5rn60VUVyWnEFlkab0hWkqu/KO2s+7saZg5JX0zCkClgUk8oy4RhNA5bVi1ZRz3BCtWflmUpCV5jsFXwxtSgrM9K52r31BAk85AJg17zgsxPdlMQQVLTG5emcetG8k1UyJ1gSqaZ39BQ6GTQVW1OlizLlLSgUWB5ut/b8VkAwCv9ohL8eekIPI4n9Rq1kZyxE3lQD0hEVeQLaRkf9/SZrZ7VGMcfJaU97SilngbyTfMI09ZAvtDl9UGAXBTGxmQSvlI3gUrPSuJcw/MZyWikvskM21sy6LmPkzb/bBqgXaFGL4npm9dVRfJYbMV1QgX58ZXERhzdIZNmQs6Np2f6vig6qS0Z5qj0G+ae+eh9g6H4/cblLcfG5pTVELv/ScYeZvDc4i6FID06c58VYtAIg8OAlvcseeOtX/9VIRwaMWBhv0IorxBJy42op0iLRRqw5FgAUEzj5ktkDEXhPJBfL991n6RQYGWn+NWWyyezlU8xH5rUxZB9IQ2Dr4ney6/30eSDFszJNMWpghZHNJoJ+s5puDOp3oFcED5VRlWj+88SpoTQdYHuu4ycnx9u3QJTWyLI+gWmaO7IYAbTG0UoNOh2bDL2ouNqKyqw65J5CYywiFjfpakDkh9txf+bArHJaaYI75Dflh8rY+0fy2+ZXfKymzyI+GQQxdvZOtB6WZUG/MvyoGb3ycv9pYk98qOvEGDEpyGOMXUxQ0WN0mvy1iSL0pcM+xbOtuIpDNHKwLfJMJ6J2/yaxftoHa0z83JOo8EYD7ONSd3gvHQDgUrLsAe0qNlqnWLoBIDJo4xh4H3xNpUQOyfjMgLHxGMEr8ThMTuy3Cdtm5gG/94KTs2JK9HGG3B7l77io9rBLYohlqVwSV9S9jmZOyIX6jnGRfPxZfiXZdEymbqqSbKfkTrW/5kI0lsP0XjMLNv4DFBhXEvn9bdMlnbbCgcHh/ZYFZoDghqNdAO9pwKY7xRxXyWHwpqvSheXdsGScBhVsIvqWGK6huxmU+OBJBqset2MBFHupsaxW9SC0b+TFSE/AuTvF1039bKo4p2sOlUsedAeA4/upYFviqQvnQ1s+ntm3jVlxbo2KNdvdjVaq0kPxfUNWW1CywqL+0k9l2WGIVeuQKW1xTAVsqLcsjT/fAChLUazkdXFBP+EcxpdFzkfp2QJv/IzYgT0ahoFEp2YSmKmXkLndR9hqvULu+U3kwFbno1h4hHS85E0xASgqWaLOq73lDHiybFLu4DNBySk5tPz4mczQWVeso5mkqv7gjRmgsqKG+u2ilXOX0ndX08C5aGV0YFXflbm2itrRDhHkavMreCOM7Pbt7kpnVfkM7uTBumsZ9tf+9PiR7w0LyeWlqNQHKypdBZjlGJlX8mSE8enDaGttJPf2MTSLnJS3OQz+FExnJemulFj9fEQtQLjBo5mmpg7RL6RbmnMm5TnpbhGoTQg92bka9L3hb0ylsRXO1CnQbH7gvRp+lEa8U+GsbBPflOsnlbrpsayvo6Rz0qb0/dxT/RptIXTHBwcHNpidWgOnoaXaCC7NWRdUxnmHBjXZmWgYSsFRycZ/ET67VoqbANd6szSU4sioms9TaSY7VajoW7yAyK2O44HoSa4KpBZKS+UA2jGG9b11mAASsTE+0+HxOAFoDAq/Uxd41h0AB03Rco3YnK95fU0MFZaNSCNhuNtkA2tOpFEisVGqp3S3wSZN5fXeghwpTFGyuwBGWfymRACZACafkjcmx55C6ILCvBIesrwYEPZf/P9DWsVSx+TlTJzp6w8Pa8GbXGf8h1ywdgbMpbSGh/rvirHzd8R5nfS73sPXMY7e4QY40RBlrVwjxx7Tg+iUqPWUyZz1RuiIk2mQ5a01tC8Gcq16cPKFpcxuQNm3714oImNj0pm4uxfrwfQCq3vuObZeTAVszouUzvZV0N1t4zr7JS4+OpZ6VNsIogyiWUNNZ43wWC41+No3ieRVdHTst83to7wVAj+JnFZDxwSjeHxfpmLP7j+Lgx+m0FJpMQ3mbfT9wVtCYTeL8n8zByS36oDDSRoNMwwWCsqNXWweH8NiYusOLZDzl/3OYa1f3wS16+KjSqQItXgTfpx8xHM38FnIYu3hNMcHBwc2mJVZGWm0iP6jsO/gsmHgmgmub9k+bcaXT+qDvSdEkme3SS/mbBa7Wn4hgFo3FDZk31p2Udmy0puP8vMlFHWpZS+JhJ25h5WXa61AoFqpLsv382En6/HsUDKcFP1uPc0LdxPNC1ZaXKMbD93yIoy+o9BlFisxQSuZPbTYuwDKsEknpuyIhgC0XDWQ5OEtCYk2+Tur3nRt/T9eq1cp++LskpktnnwOdb6CEOrq3Js77eDaDDoycxjB7Wf4pCyXA2WH2E998MLYVtExXBKVAek33/6yF/go8/+LABg/QZZPceuSQPJq0E075Ww5MqtlRWxcVtl8nqaHqJO0Sp6vxK1NUSNLcdkc+oAEM1QwzhCbwqTlwobG/YZii1wv71Tjuk+r20Q029/5G8BAK8VNwIAji+uxc05UZua1FR3bBIVbrkaxXxO+v6eLWcAADeKonJWmiFcJJuUT5uKpkcrejNsw9CL75M5aJ4SVbe2pYzRf5DrzB6gZkVGKK+qbKBTeZPMx5Y/l7le2BuD9mhL2SvP7uDzcm8X9yqkL3JujWePz012Kyz/hyEfPvUnv7aKszK1uKuaqSbSZw3JKTPwmOarmkB2M38bYXrvWRMhpqB8TtR2k/VIJt69Cmm6D01svSH/qI1opM7KjRz7ET5coqUiktFY5hbDpnV7rfTsjquM/X9AjJULEjqPnq8nUXxMnNF6XKyGyddFEiyv1TYz1JCIhhdMPQgFHZDxmZyAgWNyR28dbRVNNX2JLRk1NYg6WY4Tp0X1X2ANy/AyAD5A/d9gvYt5U7kKNv6+eEQmpliWp0X5QH6T/LbuqzJXS/OM6KxoS7/mU0CZtOCxeh8Uo1AnTorrziO9XKVfo/+T0n6Zar4hp6n2+rIFArDmBW6RDsuczd6jEZ+W63XcoAF0A7ddtzRm76ZBknEmXZdZ72KztjR9xWGmSzNyFRoobJLjfvvPfhIAkHwHK6ad68PDD5yS8xrybEwW0gCA946cwqcaBwAAT/+56P5m++WHNDq4DTQ1QBdpPEzd1FiSgFEknxGh0DlHwtmJKAJlefHNYoQZCoQ+ZRfE5Zzcv2sfMNtEoIP0d15ZnqHZx6WdxJko8CHZf+jPyFbT5BVFF4G5+xjncO2tX3+3rXBwcGiLVaE5NOIK8/tCgKpbchIj5RdYUzA+GbARYd5VOSjHqL9gSdktgzHYVdPyGVpWNsDJqOlRGphSN1vaQZIUcEHjmtoANHpoBSRxq4mrj5SUJSyp0e3UyiIEYs/QHWuMekMtAtbMe2iJqkmH61npS3T3MuosSW/GUicnhd9TR25LaMV1lkjlvuHLZUzESKpLHglj9Mxt9REg6ejCfg6FdPvxm8oa+NJPisaRZTy+V1XWxbqwR1ZPUxkqWNZ2PLGZldGTf/9r70bgAZlHU03JVE0vDWnM3iONDr3Malqsr5ycVJh7SFa9YFHGWeujcfRY0OYqFEkxb4hPiuu1zR7tPyH3an4/iXkSZcTPMGhteOX2MFzQVtM028/Zy1SHeuv41ufvkL7wWRr+hiRX/MMD70SR9SfiXOQ7aHwsPd2P4ii1Xb5VIbNVSgLJSY6Blc6Ko9R4FhTCBZY54PybGqSNpLZ1Loz7PpJhINdgE5U+s32lu5g5KNkDNZTP93I4K7Xm4oiUMwCAzN30mb8JnObg4ODQFqtCc9Ce5JmraquGYZ7uP+PGWtrto+tpWZGXyBaUPikH57b5lpvAhNqaUORCr7ZFZiaPygppjHSJ6ToKo7JyG8mavUsMd5GJMIIs6FKhwS02yUCivQVUJqWtGHkdzPnaA5bJU2CyM03ocaUXWPtfRWrffJdIeWMcKsx2tViatpOKrMKMwXMRq1EZGNfk+C81EXpDvqutY17KB2U1KxwbQWzahAVz1eTKE1vQyDM/xFDrmVyLxJRGdruxcazkbvBDytLEVXdx0DOiXVS7gkjRZpPdaTgzaMuZVjDWsQVGVBtKe6+hkDrHuaZL2dTtrHUAUeZ+GAYvY5gcet6zla5mD8rxljJvOmJtDp3kS8htNSu7h8EXmWPyQeYnfF3sQ4t3BVDaQhvAVenTzff2cg41hu6Q7MrF58UF6j8lwVP57Q10nmf4+ZYmx2XYu5SlZDOh+8kr0t9GEij3GC4JU5ymxS1RfLf0T58R12nilgnbDlijtuHXyBySfocnw0iS3smEZjcezQIABv4qidkfF1Uv0Hxr3cBpDg4ODm2xKlyZ0TWjet1HflVCc7nfTpLWfWk3Je2EZ10vZi/YuE/cQuXlqLULGF4FEzrbjAKVYRaO6RV1IvSUWIzz6wCPLp4GLe+Gx7HrokYtxSzOAyL1+16mpfjH5pB7WdxWhmfQhOMqH2gyQShBi7QpWxafbWL6QTkuMbay9mFlSxWRMflPZZDBNww3jp+KIUHGqgp5Io1NxQ9pJG7J38aGYNxflf5WxqbxgPSfkMm79UDc2mCSXO2NRbsZbnFnNNfKKhO5JMtUYlJbUtzOG0wsIiFuboNntR9D1GrCvCuDTWhyYPR9m3ve7fLb8IsN5DaE7BwBQH4tOTcmfFR4PWMHKaxn210NgKX5FC32JpO2nmrVPzWaRscEPTXrgigPfsd943PQcQ024CxDOnejKQFSY/R2mNVbB7XVIks7ZM6sSzqlbSi8CQXvOmuIbZUtQhRb/A5OyB6NDQeFA7Lwp3JzTSmF2EyL5LjSb9zq8lthUx2hzMrw/GaMLFMnPZukaLKUz/7Br65eV2Y472PkuTKufyCCCGtRLIvbGckJUqxtaCLJG9/9mPiMbl4VlS51NWjZp03qrVHtek54iJDcM7eX+RMhswXRlqIrT1+6edmLH8qhcjENABh6lg/epKjRN873oTksd1SrlYbMZlheBADIUrD1kupu+gGg7zVmjT4sJ3S/wDj+rhASFIiJCSM4pO3sngZU0+RGyHVMHMLyXVUUQvJlD8lVDUkM0FJDTb3Qwlp5mr2GRnOUZC1kxzaxFPHpFsmqn5fjTTxIdhswcFzGZ7I6zQvjh3XLkHw32ZkZ4ReZD9jiwUYodDKuYvxdHkLLxu1II16XzG/jUAnViZVVmww56+a/qWP2bumoEWYlurm7zrViWAwztck6TV9t2nqb5Y2iiseuyxaikQDo0UWStVhNmnWl34dmvszQV8KcAwrfvBIaQACxS3JBk7kZznqWNDZQNhmzfMlntXW1mrRsQ3EYyivMfV5SroP2fBO40KK/qw+w+HNEnvOhZwO2nqzd0jJTdOmhis1tCeVbz0k7uG2Fg4NDW6yObcXIqB755Y//oLvh8DZhohj/pQix6tS/Fr5f/f5+4F977G+Fa7/VPkLSaQ4ODg5t4YSDg4NDWzjh4ODg0BZOODg4OLSFEw4ODg5t4YSDg4NDWzjh4ODg0BZOODg4OLSFEw4ODg5t4YSDg4NDWzjh4ODg0BZOODg4OLSFEw4ODg5t4YSDg4NDWzjh4ODg0BZOODg4OLSFEw4ODg5t4YSDg4NDWzjh4ODg0BZOODg4OLSFEw4ODg5t4YSDg4NDWzjh4ODg0BZOODg4OLSFEw4ODg5t4YSDg4NDWzjh4ODg0BZOODg4OLSFEw4ODg5t4YSDg4NDWzjh4ODg0BZOODg4OLSFEw4ODg5t4YSDg4NDWzjh4ODg0BZOODg4OLSFEw4ODg5t4YSDg4NDWzjh4ODg0BZOODg4OLSFEw4ODg5t4YSDg4NDWzjh4ODg0BZOODg4OLSFEw4ODg5t4YSDg4NDWzjh4ODg0BZOODg4OLSF0lr/oPsApdQ8gCKAhR90X4heuL60g+tLe6yWvvyP9mOd1rrvO79cFcIBAJRSx7XWd/2g+wG4vrwZXF/aY7X05fvdD7etcHBwaAsnHBwcHNpiNQmHT/ygO3AbXF/aw/WlPVZLX76v/Vg1NgcHB4fVhdWkOTg4OKwiOOHg4ODQFk44ODg4tIUTDg4ODm3hhIODg0Nb/L8fIijgnAw0CAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "model.eval()\n", + "with torch.no_grad():\n", + " for data in test_loader:\n", + " data, targets = get_batch(data, seq_len, digits_per_batch)\n", + " data = data.to(device)\n", + " targets = targets.to(device)\n", + " output = model.visualize(data)\n", + " \n", + " fig=plt.figure(figsize=(12,8), dpi= 100, facecolor='w', edgecolor='k')\n", + " matrix = torch.empty((2*data.shape[0], data.shape[1], data.shape[2]), device=data.device)\n", + " matrix[0::2] = data\n", + " matrix[1::2] = output.view(output.shape[0], 1, output.shape[1]).repeat_interleave(data.shape[1], dim=1) * 10\n", + " \n", + " plt.matshow((matrix[:matrix.shape[0]//8,:,:].view(-1, data.shape[-1]) * mnist_std + mnist_mean).cpu().numpy())\n", + " plt.show()\n", + " break" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 1.636068, + "end_time": "2021-01-22T23:26:18.776641", + "exception": false, + "start_time": "2021-01-22T23:26:17.140573", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "### Average pooling - one layer" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-22T23:26:22.076489Z", + "iopub.status.busy": "2021-01-22T23:26:22.075952Z", + "iopub.status.idle": "2021-01-23T04:38:00.616365Z", + "shell.execute_reply": "2021-01-23T04:38:00.616794Z" + }, + "papermill": { + "duration": 18700.201623, + "end_time": "2021-01-23T04:38:00.616950", + "exception": false, + "start_time": "2021-01-22T23:26:20.415327", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 125.20 | loss 54.70 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 124.82 | loss 22.90 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 125.54 | loss 19.74 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 124.98 | loss 16.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 122.49s | valid loss 24.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 125.96 | loss 13.94 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 127.50 | loss 13.62 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 125.84 | loss 11.53 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 128.52 | loss 10.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 123.82s | valid loss 18.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 127.65 | loss 9.61 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 126.12 | loss 9.12 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 128.74 | loss 8.81 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 126.68 | loss 8.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 124.33s | valid loss 14.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 126.98 | loss 7.56 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 126.64 | loss 7.81 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 126.87 | loss 7.28 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 127.11 | loss 7.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 124.05s | valid loss 10.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 126.78 | loss 7.18 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 126.84 | loss 6.81 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 127.00 | loss 6.48 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 126.60 | loss 6.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 123.97s | valid loss 9.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 127.04 | loss 5.97 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 127.23 | loss 5.70 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 127.39 | loss 6.40 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 126.62 | loss 5.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 124.18s | valid loss 11.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 127.34 | loss 5.49 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 127.61 | loss 5.27 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 126.95 | loss 6.06 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 127.31 | loss 5.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 124.41s | valid loss 7.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 127.42 | loss 4.78 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 127.49 | loss 4.86 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 126.87 | loss 5.46 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 126.85 | loss 5.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 124.28s | valid loss 6.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 127.37 | loss 4.62 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 127.08 | loss 5.12 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 127.15 | loss 4.39 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 127.15 | loss 4.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 124.34s | valid loss 8.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 127.42 | loss 4.71 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 126.94 | loss 4.82 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 127.44 | loss 4.78 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 127.31 | loss 4.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 124.35s | valid loss 9.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 127.13 | loss 4.50 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 127.43 | loss 5.15 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 127.44 | loss 4.03 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 127.31 | loss 3.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 124.40s | valid loss 7.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 127.28 | loss 3.78 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 127.06 | loss 3.80 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 127.22 | loss 4.45 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 127.36 | loss 4.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 124.24s | valid loss 8.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 127.62 | loss 3.71 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 126.79 | loss 3.53 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 127.21 | loss 5.13 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 126.83 | loss 3.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 124.22s | valid loss 8.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 127.21 | loss 3.47 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 127.06 | loss 4.17 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 127.14 | loss 3.75 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 127.07 | loss 3.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 124.19s | valid loss 8.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 127.40 | loss 3.56 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 126.99 | loss 3.38 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 126.72 | loss 3.99 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 127.34 | loss 3.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 124.30s | valid loss 7.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 126.77 | loss 3.40 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 126.98 | loss 3.80 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 126.95 | loss 3.19 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 126.57 | loss 3.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 123.92s | valid loss 7.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 127.09 | loss 3.06 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 126.88 | loss 3.01 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 126.87 | loss 2.85 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 127.65 | loss 3.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 124.24s | valid loss 7.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 127.10 | loss 3.20 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 127.07 | loss 2.97 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 126.81 | loss 3.13 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 127.39 | loss 3.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 124.22s | valid loss 8.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 127.90 | loss 2.59 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 126.55 | loss 2.71 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 127.07 | loss 3.49 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 126.93 | loss 3.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 124.25s | valid loss 7.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 127.01 | loss 2.90 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 126.93 | loss 2.81 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 127.33 | loss 2.46 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 126.49 | loss 2.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 124.08s | valid loss 7.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 127.14 | loss 3.25 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 126.98 | loss 2.75 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 126.84 | loss 2.94 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 127.12 | loss 2.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 124.10s | valid loss 6.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 127.19 | loss 2.39 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 127.19 | loss 2.47 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 126.61 | loss 2.30 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 127.41 | loss 3.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 124.18s | valid loss 7.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 127.08 | loss 2.15 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 127.29 | loss 2.16 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 126.76 | loss 2.64 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 126.92 | loss 2.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 124.13s | valid loss 7.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 127.10 | loss 2.43 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 126.78 | loss 2.38 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 126.94 | loss 1.87 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 127.10 | loss 2.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 124.17s | valid loss 7.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 127.68 | loss 2.79 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 127.32 | loss 2.57 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 126.90 | loss 2.19 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 126.56 | loss 1.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 124.56s | valid loss 8.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 128.00 | loss 2.32 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 128.01 | loss 2.47 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 126.96 | loss 2.01 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 128.47 | loss 2.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 124.81s | valid loss 8.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 126.93 | loss 1.80 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 126.81 | loss 1.95 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 126.51 | loss 2.22 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 126.83 | loss 2.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 124.28s | valid loss 6.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 127.09 | loss 1.54 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 126.73 | loss 2.14 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 126.58 | loss 1.88 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 126.76 | loss 2.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 123.91s | valid loss 7.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 126.67 | loss 1.74 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 127.12 | loss 1.89 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 126.59 | loss 2.35 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 127.44 | loss 1.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 124.09s | valid loss 8.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 126.69 | loss 2.18 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 126.68 | loss 1.86 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 127.14 | loss 2.19 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 126.90 | loss 2.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 123.96s | valid loss 7.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.59 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 126.95 | loss 61.46 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 127.69 | loss 26.95 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 127.32 | loss 19.40 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 127.32 | loss 17.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 124.43s | valid loss 32.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 127.79 | loss 14.23 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 127.10 | loss 13.14 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 127.43 | loss 11.57 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 127.48 | loss 11.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 124.65s | valid loss 25.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 127.42 | loss 10.84 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 127.45 | loss 9.72 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 127.61 | loss 8.51 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 127.56 | loss 9.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 124.58s | valid loss 20.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 128.33 | loss 7.67 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 127.77 | loss 7.48 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 128.09 | loss 7.65 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 127.63 | loss 7.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 125.06s | valid loss 10.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 127.82 | loss 6.72 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 127.50 | loss 6.89 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 128.32 | loss 7.08 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 127.78 | loss 6.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 125.02s | valid loss 8.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 128.00 | loss 5.85 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 128.05 | loss 5.67 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 127.65 | loss 5.89 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 127.66 | loss 5.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 124.94s | valid loss 7.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 128.14 | loss 5.99 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 127.77 | loss 5.61 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 127.51 | loss 5.13 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 127.99 | loss 5.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 125.01s | valid loss 7.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 127.70 | loss 5.23 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 127.53 | loss 5.32 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 127.65 | loss 5.08 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 127.77 | loss 4.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 124.77s | valid loss 9.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 127.91 | loss 5.03 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 127.38 | loss 4.32 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 127.60 | loss 5.25 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 128.02 | loss 4.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 124.80s | valid loss 7.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 128.13 | loss 4.59 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 127.46 | loss 5.34 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 127.26 | loss 3.60 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 127.43 | loss 4.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 124.69s | valid loss 7.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 127.86 | loss 4.19 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 127.35 | loss 4.23 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 127.00 | loss 4.29 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 127.07 | loss 4.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 124.37s | valid loss 7.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 127.22 | loss 4.36 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 127.36 | loss 4.63 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 127.01 | loss 4.10 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 127.17 | loss 4.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 124.29s | valid loss 6.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 127.08 | loss 3.82 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 126.88 | loss 3.58 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 127.52 | loss 3.93 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 126.72 | loss 3.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 124.12s | valid loss 6.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 127.59 | loss 3.04 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 127.13 | loss 4.49 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 127.01 | loss 3.70 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 127.19 | loss 4.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 124.29s | valid loss 9.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 127.18 | loss 2.80 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 127.17 | loss 4.44 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 127.11 | loss 3.31 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 127.27 | loss 3.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 124.29s | valid loss 6.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 127.34 | loss 2.79 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 127.07 | loss 2.87 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 127.48 | loss 3.35 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 127.65 | loss 3.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 124.47s | valid loss 8.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 129.04 | loss 3.57 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 127.48 | loss 3.06 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 129.00 | loss 3.43 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 127.26 | loss 2.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 125.43s | valid loss 8.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 127.77 | loss 2.59 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 128.87 | loss 3.50 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 127.17 | loss 2.79 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 127.48 | loss 3.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 124.73s | valid loss 8.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 127.42 | loss 2.80 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 127.19 | loss 2.52 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 126.99 | loss 3.26 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 127.17 | loss 3.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 124.30s | valid loss 7.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 127.44 | loss 3.06 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 126.87 | loss 2.07 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 127.01 | loss 2.58 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 126.72 | loss 3.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 124.16s | valid loss 8.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 126.86 | loss 2.46 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 127.53 | loss 2.89 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 126.55 | loss 2.56 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 126.99 | loss 2.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 124.14s | valid loss 8.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 126.69 | loss 2.58 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 127.02 | loss 2.26 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 127.11 | loss 3.25 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 127.39 | loss 1.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 124.07s | valid loss 7.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 127.43 | loss 2.44 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 126.54 | loss 2.00 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 127.19 | loss 2.76 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 126.84 | loss 3.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 124.14s | valid loss 7.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 127.31 | loss 2.23 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 126.69 | loss 1.80 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 127.02 | loss 2.49 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 126.71 | loss 2.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 124.07s | valid loss 7.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 127.43 | loss 2.47 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 126.69 | loss 2.07 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 126.87 | loss 2.62 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 126.88 | loss 2.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 124.21s | valid loss 8.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 126.93 | loss 2.53 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 126.91 | loss 1.89 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 127.23 | loss 2.06 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 126.77 | loss 1.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 124.12s | valid loss 7.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 127.15 | loss 1.95 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 126.84 | loss 2.03 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 127.14 | loss 1.99 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 127.04 | loss 2.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 124.48s | valid loss 7.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 127.43 | loss 1.71 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 127.24 | loss 2.38 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 126.92 | loss 1.85 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 127.36 | loss 1.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 124.34s | valid loss 8.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 127.05 | loss 1.97 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 127.23 | loss 1.72 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 129.33 | loss 1.67 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 126.87 | loss 2.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 124.63s | valid loss 7.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 127.10 | loss 1.52 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 127.61 | loss 1.47 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 126.95 | loss 1.80 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 127.00 | loss 1.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 124.25s | valid loss 8.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.25 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 127.56 | loss 58.10 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 127.63 | loss 24.76 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 127.63 | loss 20.33 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 128.23 | loss 16.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 124.94s | valid loss 25.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 127.40 | loss 14.17 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 127.85 | loss 12.32 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 127.75 | loss 11.77 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 127.12 | loss 10.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 124.58s | valid loss 13.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 128.40 | loss 9.94 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 128.36 | loss 9.52 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 128.11 | loss 8.40 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 128.16 | loss 8.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 125.31s | valid loss 18.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 128.23 | loss 8.21 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 127.86 | loss 7.57 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 128.46 | loss 7.60 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 127.15 | loss 7.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 125.03s | valid loss 9.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 128.32 | loss 6.38 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 127.47 | loss 7.14 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 127.67 | loss 6.27 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 127.97 | loss 6.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 125.11s | valid loss 7.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 127.84 | loss 5.68 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 127.60 | loss 6.64 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 127.65 | loss 5.88 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 128.02 | loss 5.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 124.87s | valid loss 9.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 128.04 | loss 5.64 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 127.53 | loss 5.63 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 127.66 | loss 5.04 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 127.75 | loss 5.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 124.85s | valid loss 10.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 128.10 | loss 4.85 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 127.67 | loss 5.44 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 127.93 | loss 5.51 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 127.62 | loss 4.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 124.90s | valid loss 8.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 128.47 | loss 4.92 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 127.60 | loss 4.85 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 127.98 | loss 4.82 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 128.07 | loss 4.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 125.00s | valid loss 10.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 128.21 | loss 5.33 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 127.47 | loss 4.60 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 127.93 | loss 4.87 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 127.86 | loss 4.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 124.99s | valid loss 7.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 128.44 | loss 4.26 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 127.54 | loss 4.33 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 128.13 | loss 4.55 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 127.46 | loss 4.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 124.96s | valid loss 7.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 127.67 | loss 3.88 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 127.60 | loss 4.65 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 127.88 | loss 4.81 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 128.22 | loss 4.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 124.91s | valid loss 8.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 128.14 | loss 3.91 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 127.56 | loss 4.26 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 127.38 | loss 4.11 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 127.68 | loss 3.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 124.75s | valid loss 8.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 127.74 | loss 3.86 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 127.29 | loss 4.48 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 127.95 | loss 3.87 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 127.52 | loss 3.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 124.79s | valid loss 9.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 127.80 | loss 3.76 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 127.72 | loss 4.19 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 127.69 | loss 3.21 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 127.58 | loss 2.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 124.81s | valid loss 8.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 127.48 | loss 3.63 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 127.34 | loss 3.25 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 127.81 | loss 3.57 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 127.67 | loss 3.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 124.64s | valid loss 8.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 127.52 | loss 2.94 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 127.67 | loss 3.09 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 127.26 | loss 3.17 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 127.46 | loss 3.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 124.64s | valid loss 8.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 127.44 | loss 2.67 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 127.94 | loss 3.02 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 127.01 | loss 3.21 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 127.42 | loss 2.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 124.57s | valid loss 8.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 127.81 | loss 2.66 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 127.32 | loss 3.47 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 127.56 | loss 3.46 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 127.84 | loss 2.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 124.66s | valid loss 7.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 127.84 | loss 2.79 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 127.24 | loss 2.88 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 127.93 | loss 2.46 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 127.75 | loss 3.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 124.81s | valid loss 7.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 127.64 | loss 2.57 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 127.23 | loss 2.74 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 127.86 | loss 2.52 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 127.65 | loss 3.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 124.67s | valid loss 8.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 127.60 | loss 2.38 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 128.32 | loss 2.60 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 127.37 | loss 2.67 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 128.22 | loss 2.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 124.86s | valid loss 8.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 128.22 | loss 2.33 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 127.19 | loss 2.48 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 127.54 | loss 3.13 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 127.28 | loss 2.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 124.68s | valid loss 8.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 127.27 | loss 1.90 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 127.44 | loss 2.37 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 127.98 | loss 2.09 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 127.21 | loss 2.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 124.56s | valid loss 7.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 127.39 | loss 1.66 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 127.46 | loss 2.71 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 127.96 | loss 2.32 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 126.86 | loss 2.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 124.65s | valid loss 7.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 127.24 | loss 1.71 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 128.82 | loss 2.27 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 127.11 | loss 1.66 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 128.98 | loss 2.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 124.96s | valid loss 8.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 129.08 | loss 2.21 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 126.82 | loss 1.70 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 126.65 | loss 2.43 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 127.72 | loss 2.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 124.46s | valid loss 7.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 126.92 | loss 1.77 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 126.90 | loss 1.79 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 126.69 | loss 1.76 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 127.02 | loss 2.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 123.97s | valid loss 8.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 127.49 | loss 1.86 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 126.97 | loss 1.65 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 126.77 | loss 1.37 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 126.32 | loss 1.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 124.06s | valid loss 8.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 127.43 | loss 1.89 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 126.79 | loss 1.86 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 126.75 | loss 1.55 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 127.01 | loss 1.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 124.12s | valid loss 9.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 7.16 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 127.32 | loss 56.21 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 127.25 | loss 23.87 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 129.53 | loss 19.09 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 127.47 | loss 16.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 125.36s | valid loss 28.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 127.94 | loss 13.89 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 129.83 | loss 13.02 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 128.35 | loss 11.59 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 127.35 | loss 11.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 125.30s | valid loss 13.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 127.70 | loss 9.40 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 127.28 | loss 9.71 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 127.36 | loss 9.60 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 127.43 | loss 8.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 124.51s | valid loss 12.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 127.55 | loss 7.76 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 127.58 | loss 8.06 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 127.37 | loss 7.72 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 127.66 | loss 7.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 124.72s | valid loss 10.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 128.22 | loss 6.82 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 126.68 | loss 7.20 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 127.51 | loss 7.00 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 127.60 | loss 6.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 124.55s | valid loss 11.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 127.99 | loss 5.77 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 127.54 | loss 6.33 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 127.42 | loss 6.53 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 127.99 | loss 5.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 124.82s | valid loss 8.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 127.07 | loss 6.08 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 127.54 | loss 5.27 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 129.25 | loss 5.35 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 127.12 | loss 6.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 125.19s | valid loss 11.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 127.46 | loss 5.50 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 128.89 | loss 5.22 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 127.48 | loss 5.35 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 127.25 | loss 5.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 124.75s | valid loss 7.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 127.55 | loss 4.97 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 127.17 | loss 5.21 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 127.16 | loss 4.80 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 127.44 | loss 5.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 124.37s | valid loss 9.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 127.53 | loss 4.36 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 127.30 | loss 4.86 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 127.39 | loss 5.21 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 126.96 | loss 4.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 124.43s | valid loss 9.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 127.39 | loss 4.65 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 126.90 | loss 4.44 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 127.15 | loss 3.95 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 126.76 | loss 4.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 124.12s | valid loss 7.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 127.31 | loss 3.77 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 126.68 | loss 4.34 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 126.93 | loss 4.11 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 129.28 | loss 3.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 124.60s | valid loss 6.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 128.88 | loss 3.92 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 126.89 | loss 4.02 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 126.97 | loss 4.81 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 126.64 | loss 4.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 124.34s | valid loss 7.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 127.28 | loss 4.12 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 126.99 | loss 3.14 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 126.52 | loss 4.02 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 127.26 | loss 3.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 124.06s | valid loss 7.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 126.79 | loss 3.07 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 126.94 | loss 3.53 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 127.38 | loss 3.89 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 127.62 | loss 3.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 124.68s | valid loss 7.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 127.89 | loss 2.70 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 129.18 | loss 3.74 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 127.57 | loss 3.62 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 128.78 | loss 3.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 125.22s | valid loss 6.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 127.65 | loss 2.88 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 127.00 | loss 3.05 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 128.93 | loss 3.99 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 126.98 | loss 3.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 124.74s | valid loss 5.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 127.50 | loss 2.61 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 128.86 | loss 3.93 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 127.06 | loss 2.96 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 129.24 | loss 3.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 125.09s | valid loss 7.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 129.11 | loss 2.61 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 127.40 | loss 2.86 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 128.01 | loss 3.19 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 126.46 | loss 2.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 124.63s | valid loss 7.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 126.65 | loss 2.78 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 126.60 | loss 2.57 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 126.36 | loss 1.76 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 127.06 | loss 3.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 123.78s | valid loss 7.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 126.91 | loss 2.63 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 126.59 | loss 2.58 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 128.75 | loss 3.84 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 126.75 | loss 2.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 124.68s | valid loss 8.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 126.58 | loss 2.41 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 126.41 | loss 2.78 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 126.52 | loss 2.52 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 126.67 | loss 2.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 123.73s | valid loss 8.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 127.12 | loss 2.45 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 126.83 | loss 2.48 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 126.87 | loss 2.51 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 126.86 | loss 2.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 124.08s | valid loss 6.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 127.12 | loss 2.57 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 126.71 | loss 2.12 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 127.21 | loss 2.04 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 126.31 | loss 2.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 123.97s | valid loss 7.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 127.17 | loss 1.94 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 126.45 | loss 1.86 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 126.75 | loss 2.22 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 127.24 | loss 2.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 124.01s | valid loss 9.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 127.05 | loss 2.08 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 126.47 | loss 2.08 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 126.95 | loss 2.03 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 126.36 | loss 2.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 123.93s | valid loss 7.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 126.58 | loss 1.98 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 126.87 | loss 1.89 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 126.70 | loss 1.46 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 126.54 | loss 2.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 123.87s | valid loss 6.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 127.18 | loss 2.13 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 126.59 | loss 1.76 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 126.98 | loss 2.60 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 128.73 | loss 1.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 124.37s | valid loss 7.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 127.10 | loss 1.78 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 126.93 | loss 2.34 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 126.53 | loss 2.03 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 126.86 | loss 2.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 123.94s | valid loss 7.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 127.28 | loss 1.96 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 126.52 | loss 1.87 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 126.66 | loss 1.55 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 126.71 | loss 1.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 123.99s | valid loss 6.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.78 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 127.82 | loss 58.27 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 127.53 | loss 25.02 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 128.22 | loss 18.99 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 127.38 | loss 16.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 125.00s | valid loss 28.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 127.38 | loss 13.48 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 127.84 | loss 12.66 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 127.49 | loss 11.98 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 128.19 | loss 10.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 124.74s | valid loss 18.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 127.63 | loss 10.03 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 127.86 | loss 9.61 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 127.82 | loss 8.97 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 127.71 | loss 8.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 124.78s | valid loss 12.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 128.40 | loss 7.96 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 127.31 | loss 7.46 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 128.30 | loss 7.34 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 127.51 | loss 7.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 124.95s | valid loss 12.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 128.43 | loss 7.05 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 127.59 | loss 6.93 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 127.28 | loss 6.53 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 128.12 | loss 5.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 124.82s | valid loss 7.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 127.52 | loss 5.65 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 127.83 | loss 5.99 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 127.42 | loss 5.72 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 127.37 | loss 6.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 124.76s | valid loss 9.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 127.59 | loss 5.65 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 127.39 | loss 5.73 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 127.63 | loss 5.88 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 127.69 | loss 5.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 124.71s | valid loss 9.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 127.95 | loss 4.67 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 127.61 | loss 4.65 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 127.33 | loss 5.39 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 127.33 | loss 5.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 124.68s | valid loss 8.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 127.90 | loss 5.44 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 127.40 | loss 4.71 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 128.05 | loss 5.01 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 127.16 | loss 4.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 124.74s | valid loss 7.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 128.08 | loss 4.81 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 127.47 | loss 4.34 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 127.64 | loss 4.30 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 127.43 | loss 4.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 124.72s | valid loss 6.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 127.99 | loss 3.92 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 127.21 | loss 4.23 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 127.60 | loss 4.47 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 128.33 | loss 4.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 124.82s | valid loss 7.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 127.38 | loss 3.95 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 127.95 | loss 4.00 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 127.48 | loss 4.24 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 127.41 | loss 4.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 124.65s | valid loss 8.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 127.77 | loss 3.90 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 127.32 | loss 3.69 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 127.28 | loss 3.99 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 127.12 | loss 4.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 124.52s | valid loss 6.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 127.45 | loss 3.38 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 127.41 | loss 4.05 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 127.57 | loss 3.48 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 129.39 | loss 3.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 124.89s | valid loss 7.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 128.83 | loss 3.84 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 127.43 | loss 3.10 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 127.38 | loss 3.53 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 127.05 | loss 3.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 125.15s | valid loss 6.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 127.93 | loss 3.51 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 126.90 | loss 2.93 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 127.30 | loss 3.67 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 126.96 | loss 3.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 124.41s | valid loss 7.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 127.42 | loss 3.28 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 127.02 | loss 3.48 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 126.79 | loss 2.51 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 126.79 | loss 3.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 124.16s | valid loss 7.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 127.12 | loss 3.11 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 126.91 | loss 3.63 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 127.37 | loss 2.78 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 128.92 | loss 3.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 124.65s | valid loss 6.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 126.76 | loss 2.67 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 126.58 | loss 3.33 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 126.91 | loss 2.63 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 127.26 | loss 3.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 124.06s | valid loss 8.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 127.03 | loss 2.75 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 126.32 | loss 2.84 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 127.10 | loss 2.90 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 126.95 | loss 2.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 124.02s | valid loss 6.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 127.15 | loss 2.63 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 126.57 | loss 2.47 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 126.59 | loss 2.55 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 126.86 | loss 2.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 123.99s | valid loss 7.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 127.01 | loss 2.94 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 127.03 | loss 1.99 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 127.02 | loss 2.79 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 127.00 | loss 2.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 124.21s | valid loss 7.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 126.55 | loss 1.97 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 126.73 | loss 2.74 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 126.95 | loss 2.07 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 126.74 | loss 2.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 123.93s | valid loss 8.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 126.98 | loss 2.02 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 126.88 | loss 2.47 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 127.07 | loss 2.40 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 129.36 | loss 2.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 124.52s | valid loss 7.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 129.10 | loss 2.03 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 126.66 | loss 2.50 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 129.33 | loss 1.92 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 126.34 | loss 2.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 124.82s | valid loss 7.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 127.21 | loss 1.92 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 127.47 | loss 2.44 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 126.62 | loss 2.01 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 127.21 | loss 2.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 124.18s | valid loss 8.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 126.46 | loss 1.83 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 128.04 | loss 2.12 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 126.33 | loss 2.35 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 126.57 | loss 2.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 123.95s | valid loss 7.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 126.89 | loss 1.96 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 126.98 | loss 2.27 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 126.79 | loss 2.50 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 126.76 | loss 1.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 123.95s | valid loss 6.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 126.88 | loss 1.47 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 127.01 | loss 1.63 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 126.76 | loss 1.98 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 127.04 | loss 1.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 124.04s | valid loss 7.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 127.16 | loss 1.72 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 126.42 | loss 1.73 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 127.50 | loss 1.84 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 126.32 | loss 1.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 123.96s | valid loss 7.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.34 |\n", + "=========================================================================================\n", + "Result over 5 runs: 6.206099987030029\n" + ] + } + ], + "source": [ + "num_runs = 5\n", + "best_model = None\n", + "best_result = 1000000\n", + "results = []\n", + "\n", + "for i in range(num_runs):\n", + " features = nn.Sequential(nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(64),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(128),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(256),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=(0,1)),\n", + " nn.BatchNorm2d(512),\n", + " nn.ReLU()\n", + " )\n", + "\n", + " classifier = nn.Sequential(nn.Linear(512, 1028),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=1028, bias=True),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=10, bias=True))\n", + "\n", + " conv = nn.Sequential(nn.Conv1d(512, 512, kernel_size=64, stride=1, padding=32),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.AvgPool1d(kernel_size=64, stride=64))\n", + "\n", + " model = ConvMNIST(features, conv, classifier)\n", + " model = model.to(device)\n", + "\n", + " optimizer = torch.optim.Adam(model.parameters())\n", + " scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "\n", + " epochs = 30\n", + " model, result = train_model(model, epochs, train_loader, test_loader, seq_len, digits_per_batch, optimizer, scheduler)\n", + " \n", + " if result < best_result:\n", + " best_result = result\n", + " best_model = model\n", + " results.append(result)\n", + " \n", + "model = best_model\n", + "print(\"Result over {} runs: {}\".format(num_runs, torch.tensor(results).mean().item()))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 1.852108, + "end_time": "2021-01-23T04:38:04.326499", + "exception": false, + "start_time": "2021-01-23T04:38:02.474391", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "### Max pooling - one layer" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-23T04:38:08.057108Z", + "iopub.status.busy": "2021-01-23T04:38:08.056403Z", + "iopub.status.idle": "2021-01-23T09:50:54.065842Z", + "shell.execute_reply": "2021-01-23T09:50:54.066448Z" + }, + "papermill": { + "duration": 18767.88477, + "end_time": "2021-01-23T09:50:54.066651", + "exception": false, + "start_time": "2021-01-23T04:38:06.181881", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 126.44 | loss 62.70 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 127.29 | loss 22.79 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 127.44 | loss 17.07 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 127.81 | loss 16.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 124.56s | valid loss 22.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 128.19 | loss 12.38 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 127.82 | loss 10.84 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 127.91 | loss 11.79 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 127.91 | loss 10.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 125.07s | valid loss 14.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 128.38 | loss 10.17 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 127.43 | loss 9.53 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 128.49 | loss 8.89 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 127.82 | loss 8.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 125.07s | valid loss 12.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 128.29 | loss 8.21 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 127.72 | loss 7.97 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 127.98 | loss 8.15 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 127.64 | loss 7.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 124.94s | valid loss 10.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 127.77 | loss 6.84 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 128.02 | loss 7.77 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 128.06 | loss 6.86 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 127.64 | loss 6.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 124.96s | valid loss 10.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 128.39 | loss 7.20 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 127.78 | loss 6.12 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 127.52 | loss 6.72 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 127.46 | loss 5.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 124.83s | valid loss 7.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 127.47 | loss 6.30 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 127.37 | loss 6.40 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 127.47 | loss 6.15 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 127.32 | loss 5.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 124.49s | valid loss 11.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 127.67 | loss 5.63 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 127.44 | loss 4.79 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 127.50 | loss 6.39 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 127.22 | loss 4.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 124.53s | valid loss 6.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 127.89 | loss 6.07 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 127.36 | loss 5.55 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 127.49 | loss 5.63 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 127.58 | loss 5.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 124.83s | valid loss 6.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 127.44 | loss 4.86 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 128.26 | loss 5.31 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 127.53 | loss 5.66 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 129.32 | loss 5.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 125.22s | valid loss 8.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 130.37 | loss 4.81 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 127.61 | loss 5.29 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 127.27 | loss 5.13 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 127.75 | loss 5.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 125.22s | valid loss 6.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 127.91 | loss 4.43 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 127.10 | loss 4.76 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 127.84 | loss 4.57 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 129.69 | loss 4.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 125.21s | valid loss 6.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 127.48 | loss 4.49 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 127.82 | loss 4.13 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 127.52 | loss 4.53 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 127.47 | loss 4.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 124.67s | valid loss 6.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 127.62 | loss 4.00 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 127.59 | loss 4.27 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 127.52 | loss 4.22 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 129.67 | loss 4.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 125.06s | valid loss 7.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 129.96 | loss 4.12 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 127.21 | loss 3.73 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 127.63 | loss 4.76 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 127.80 | loss 4.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 125.09s | valid loss 7.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 127.53 | loss 3.06 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 127.36 | loss 4.22 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 127.51 | loss 3.94 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 127.19 | loss 3.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 124.55s | valid loss 6.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 127.75 | loss 3.55 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 127.14 | loss 3.89 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 127.16 | loss 3.98 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 127.64 | loss 4.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 124.59s | valid loss 5.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 127.45 | loss 3.16 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 127.11 | loss 2.98 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 127.39 | loss 3.82 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 127.14 | loss 3.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 124.43s | valid loss 7.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 127.55 | loss 2.72 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 127.52 | loss 3.26 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 126.93 | loss 3.22 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 127.32 | loss 3.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 124.53s | valid loss 7.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 127.62 | loss 3.27 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 127.45 | loss 3.38 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 127.34 | loss 3.15 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 127.76 | loss 3.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 124.61s | valid loss 7.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 127.69 | loss 3.97 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 127.33 | loss 2.34 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 127.60 | loss 3.37 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 127.43 | loss 2.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 124.62s | valid loss 5.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 127.64 | loss 3.51 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 127.34 | loss 2.65 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 127.31 | loss 2.49 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 127.09 | loss 2.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 124.48s | valid loss 6.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 127.36 | loss 2.99 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 127.65 | loss 2.79 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 127.33 | loss 2.12 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 127.32 | loss 3.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 124.53s | valid loss 7.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 127.65 | loss 2.33 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 127.47 | loss 3.14 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 127.48 | loss 3.01 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 127.58 | loss 3.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 124.64s | valid loss 7.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 127.65 | loss 2.15 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 127.32 | loss 2.38 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 127.48 | loss 3.40 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 127.27 | loss 1.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 124.54s | valid loss 7.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 127.85 | loss 3.01 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 127.22 | loss 2.96 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 127.29 | loss 1.86 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 127.18 | loss 2.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 124.54s | valid loss 6.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 127.79 | loss 2.09 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 127.22 | loss 2.54 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 127.10 | loss 2.50 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 127.36 | loss 2.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 124.49s | valid loss 6.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 127.67 | loss 1.89 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 127.21 | loss 2.27 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 127.16 | loss 2.94 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 127.48 | loss 2.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 124.45s | valid loss 6.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 127.43 | loss 2.03 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 126.76 | loss 1.99 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 127.26 | loss 2.35 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 127.09 | loss 2.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 124.26s | valid loss 6.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 128.01 | loss 1.92 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 127.08 | loss 1.60 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 126.97 | loss 2.02 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 127.31 | loss 2.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 124.45s | valid loss 7.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.38 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 128.04 | loss 64.85 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 127.67 | loss 22.50 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 128.05 | loss 17.65 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 128.11 | loss 16.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 125.03s | valid loss 16.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 127.82 | loss 13.09 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 127.85 | loss 11.24 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 128.33 | loss 10.66 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 129.26 | loss 10.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 125.42s | valid loss 13.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 130.08 | loss 10.00 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 127.90 | loss 9.01 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 129.90 | loss 9.39 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 128.04 | loss 8.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 126.31s | valid loss 12.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 128.41 | loss 7.75 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 129.92 | loss 8.41 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 127.28 | loss 8.09 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 130.76 | loss 7.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 125.91s | valid loss 8.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 130.61 | loss 7.43 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 127.40 | loss 7.03 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 128.51 | loss 7.19 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 127.91 | loss 7.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 125.61s | valid loss 7.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 128.60 | loss 6.42 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 127.89 | loss 6.96 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 128.15 | loss 5.74 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 127.75 | loss 6.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 125.22s | valid loss 7.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 127.84 | loss 6.09 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 128.01 | loss 5.97 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 128.08 | loss 5.94 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 127.66 | loss 6.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 125.02s | valid loss 9.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 127.54 | loss 4.95 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 127.53 | loss 5.39 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 128.08 | loss 6.55 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 127.63 | loss 5.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 124.75s | valid loss 8.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 127.79 | loss 5.38 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 127.70 | loss 5.38 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 127.31 | loss 5.96 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 127.16 | loss 5.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 124.55s | valid loss 7.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 127.72 | loss 4.72 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 127.26 | loss 5.90 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 127.50 | loss 4.97 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 127.77 | loss 4.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 124.57s | valid loss 5.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 127.64 | loss 4.39 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 127.46 | loss 6.07 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 127.26 | loss 4.97 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 127.42 | loss 4.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 124.48s | valid loss 8.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 127.70 | loss 4.61 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 127.41 | loss 5.57 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 127.33 | loss 4.30 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 128.00 | loss 4.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 124.79s | valid loss 6.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 128.15 | loss 4.41 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 127.77 | loss 4.96 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 128.12 | loss 4.98 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 127.48 | loss 4.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 125.10s | valid loss 7.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 128.22 | loss 4.28 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 128.07 | loss 4.56 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 128.15 | loss 4.28 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 127.47 | loss 4.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 125.07s | valid loss 6.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 128.60 | loss 3.44 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 127.60 | loss 3.75 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 128.04 | loss 3.88 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 127.79 | loss 4.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 125.12s | valid loss 5.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 128.21 | loss 3.20 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 128.34 | loss 3.84 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 127.78 | loss 4.15 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 127.95 | loss 4.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 125.12s | valid loss 7.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 128.34 | loss 3.15 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 128.04 | loss 4.43 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 128.57 | loss 4.72 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 128.07 | loss 3.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 125.26s | valid loss 7.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 127.94 | loss 3.77 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 127.70 | loss 3.36 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 128.14 | loss 3.84 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 128.10 | loss 2.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 125.09s | valid loss 6.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 128.59 | loss 3.75 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 127.43 | loss 2.87 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 128.02 | loss 4.02 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 128.25 | loss 3.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 125.17s | valid loss 5.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 127.81 | loss 2.72 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 128.17 | loss 3.96 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 127.65 | loss 2.86 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 128.47 | loss 3.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 125.14s | valid loss 8.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 127.39 | loss 2.98 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 127.38 | loss 2.64 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 128.10 | loss 3.40 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 127.54 | loss 2.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 124.84s | valid loss 6.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 127.95 | loss 2.89 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 127.47 | loss 2.26 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 127.81 | loss 2.99 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 127.69 | loss 2.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 124.87s | valid loss 6.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 127.98 | loss 2.89 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 127.63 | loss 2.44 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 127.30 | loss 3.11 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 127.92 | loss 2.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 124.84s | valid loss 5.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 127.76 | loss 2.45 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 127.46 | loss 2.74 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 127.86 | loss 2.40 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 127.55 | loss 2.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 124.81s | valid loss 6.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 128.09 | loss 2.61 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 127.19 | loss 2.92 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 127.91 | loss 2.90 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 127.41 | loss 2.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 124.64s | valid loss 6.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 127.00 | loss 2.41 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 127.33 | loss 2.85 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 127.48 | loss 2.74 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 128.13 | loss 2.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 124.45s | valid loss 5.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 127.56 | loss 2.33 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 126.84 | loss 1.97 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 128.18 | loss 1.99 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 127.28 | loss 1.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 124.46s | valid loss 6.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 127.25 | loss 2.05 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 126.64 | loss 1.92 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 126.88 | loss 2.16 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 129.20 | loss 2.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 124.46s | valid loss 7.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 129.45 | loss 2.40 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 126.94 | loss 2.83 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 129.18 | loss 1.95 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 126.77 | loss 2.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 125.38s | valid loss 6.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 127.29 | loss 1.68 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 128.87 | loss 1.70 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 127.25 | loss 2.66 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 129.20 | loss 2.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 125.02s | valid loss 7.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.27 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 127.86 | loss 63.62 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 128.15 | loss 23.22 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 129.63 | loss 17.38 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 127.79 | loss 15.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 125.37s | valid loss 16.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 127.96 | loss 12.83 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 128.24 | loss 12.19 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 128.41 | loss 12.36 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 129.96 | loss 11.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 125.55s | valid loss 15.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 128.46 | loss 10.12 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 127.03 | loss 9.52 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 128.37 | loss 8.68 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 127.30 | loss 9.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 125.02s | valid loss 8.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 128.10 | loss 7.90 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 127.72 | loss 7.40 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 127.75 | loss 7.59 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 127.61 | loss 7.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 124.78s | valid loss 8.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 129.99 | loss 7.06 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 127.73 | loss 7.19 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 129.34 | loss 6.89 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 127.89 | loss 7.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 125.74s | valid loss 8.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 127.54 | loss 6.60 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 127.74 | loss 6.49 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 127.49 | loss 6.51 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 128.11 | loss 7.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 124.80s | valid loss 9.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 127.99 | loss 6.68 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 127.77 | loss 5.70 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 127.76 | loss 6.37 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 127.74 | loss 6.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 124.90s | valid loss 9.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 127.96 | loss 5.89 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 129.81 | loss 6.70 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 127.41 | loss 5.18 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 128.27 | loss 5.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 125.24s | valid loss 7.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 127.72 | loss 5.40 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 127.67 | loss 5.64 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 128.05 | loss 6.59 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 127.71 | loss 5.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 124.83s | valid loss 10.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 127.71 | loss 4.99 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 127.79 | loss 5.50 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 127.11 | loss 5.57 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 127.44 | loss 5.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 124.61s | valid loss 6.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 127.87 | loss 4.75 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 127.64 | loss 4.67 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 127.35 | loss 4.65 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 127.47 | loss 4.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 124.66s | valid loss 9.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 127.77 | loss 5.04 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 129.49 | loss 4.62 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 128.14 | loss 4.84 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 126.90 | loss 4.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 125.06s | valid loss 7.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 130.07 | loss 4.31 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 127.19 | loss 4.21 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 129.62 | loss 4.64 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 127.65 | loss 4.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 125.89s | valid loss 8.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 127.88 | loss 4.10 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 129.59 | loss 3.94 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 127.34 | loss 4.03 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 127.28 | loss 3.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 125.01s | valid loss 7.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 127.70 | loss 3.67 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 127.12 | loss 3.64 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 128.25 | loss 4.33 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 127.41 | loss 4.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 124.70s | valid loss 6.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 128.24 | loss 3.71 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 127.08 | loss 3.02 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 127.22 | loss 3.42 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 127.51 | loss 4.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 124.86s | valid loss 5.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 128.20 | loss 4.02 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 127.41 | loss 3.02 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 128.47 | loss 3.92 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 128.14 | loss 3.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 125.09s | valid loss 8.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 128.45 | loss 4.62 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 127.70 | loss 3.80 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 128.07 | loss 3.42 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 128.11 | loss 3.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 125.21s | valid loss 6.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 128.35 | loss 4.03 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 127.57 | loss 3.19 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 128.20 | loss 3.71 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 127.87 | loss 2.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 125.12s | valid loss 7.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 128.16 | loss 3.62 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 127.74 | loss 3.57 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 127.87 | loss 3.66 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 127.60 | loss 3.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 125.10s | valid loss 7.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 127.97 | loss 3.04 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 128.01 | loss 3.38 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 127.73 | loss 3.01 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 127.71 | loss 3.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 124.98s | valid loss 7.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 128.02 | loss 2.74 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 128.01 | loss 3.05 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 127.71 | loss 3.09 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 127.72 | loss 2.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 125.02s | valid loss 6.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 128.20 | loss 2.52 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 127.72 | loss 2.93 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 128.13 | loss 2.80 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 127.60 | loss 1.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 125.02s | valid loss 6.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 128.03 | loss 2.22 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 127.80 | loss 2.85 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 127.69 | loss 2.96 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 128.04 | loss 3.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 124.95s | valid loss 6.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 128.06 | loss 2.46 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 127.34 | loss 2.88 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 127.58 | loss 2.89 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 127.83 | loss 1.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 125.06s | valid loss 6.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 127.62 | loss 2.15 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 127.81 | loss 2.90 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 127.82 | loss 2.27 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 128.05 | loss 3.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 124.89s | valid loss 7.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 128.21 | loss 2.64 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 127.07 | loss 2.57 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 127.69 | loss 2.36 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 127.51 | loss 2.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 124.74s | valid loss 6.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 128.36 | loss 2.32 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 127.52 | loss 2.02 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 127.48 | loss 2.39 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 127.36 | loss 1.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 124.93s | valid loss 7.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 128.04 | loss 2.31 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 127.71 | loss 1.88 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 127.43 | loss 1.51 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 127.81 | loss 2.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 124.94s | valid loss 7.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 128.06 | loss 1.83 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 127.51 | loss 1.74 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 127.44 | loss 1.82 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 127.73 | loss 2.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 124.91s | valid loss 6.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.19 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 128.50 | loss 64.36 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 128.66 | loss 22.33 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 128.26 | loss 17.91 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 128.91 | loss 15.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 125.69s | valid loss 17.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 128.93 | loss 14.42 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 128.31 | loss 12.68 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 128.62 | loss 11.36 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 127.63 | loss 11.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 125.41s | valid loss 11.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 130.18 | loss 9.85 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 128.03 | loss 8.87 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 129.59 | loss 10.16 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 127.75 | loss 8.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 125.70s | valid loss 11.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 128.22 | loss 8.61 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 128.21 | loss 8.48 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 127.03 | loss 7.46 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 128.65 | loss 7.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 125.21s | valid loss 9.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 128.49 | loss 7.08 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 128.59 | loss 7.71 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 128.36 | loss 6.91 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 128.26 | loss 7.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 125.46s | valid loss 9.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 129.15 | loss 6.67 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 128.19 | loss 6.40 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 128.21 | loss 6.15 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 128.22 | loss 7.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 125.54s | valid loss 8.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 128.69 | loss 5.60 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 128.32 | loss 6.82 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 127.92 | loss 5.75 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 128.44 | loss 6.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 125.33s | valid loss 9.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 128.62 | loss 6.79 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 128.01 | loss 5.70 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 128.12 | loss 4.84 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 127.95 | loss 6.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 125.29s | valid loss 8.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 128.73 | loss 5.97 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 128.15 | loss 5.44 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 127.89 | loss 5.22 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 128.17 | loss 5.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 125.32s | valid loss 7.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 128.90 | loss 5.06 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 128.06 | loss 5.70 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 128.13 | loss 5.32 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 128.27 | loss 4.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 125.47s | valid loss 6.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 128.11 | loss 5.31 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 128.41 | loss 4.62 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 127.86 | loss 5.29 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 128.20 | loss 5.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 125.30s | valid loss 8.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 128.53 | loss 3.63 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 127.95 | loss 5.05 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 128.07 | loss 5.19 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 128.12 | loss 4.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 125.25s | valid loss 7.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 128.18 | loss 4.78 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 129.07 | loss 5.03 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 127.66 | loss 3.96 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 127.73 | loss 4.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 125.29s | valid loss 6.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 128.17 | loss 4.44 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 128.14 | loss 4.42 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 127.77 | loss 4.03 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 128.43 | loss 4.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 125.15s | valid loss 6.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 128.33 | loss 3.85 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 127.99 | loss 3.44 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 127.82 | loss 4.08 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 128.10 | loss 3.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 125.24s | valid loss 5.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 128.10 | loss 3.16 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 128.29 | loss 4.03 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 128.09 | loss 4.43 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 127.58 | loss 3.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 125.16s | valid loss 7.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 127.94 | loss 3.60 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 127.84 | loss 4.07 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 127.76 | loss 3.35 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 128.02 | loss 3.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 124.98s | valid loss 6.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 128.17 | loss 3.66 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 127.80 | loss 4.00 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 127.90 | loss 3.36 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 127.48 | loss 2.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 124.97s | valid loss 7.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 128.06 | loss 3.62 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 127.83 | loss 3.25 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 128.15 | loss 4.50 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 127.74 | loss 2.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 125.16s | valid loss 7.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 127.87 | loss 3.29 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 128.42 | loss 2.98 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 127.47 | loss 3.50 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 127.83 | loss 2.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 124.99s | valid loss 6.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 128.49 | loss 3.23 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 127.41 | loss 3.26 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 127.51 | loss 2.39 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 127.21 | loss 2.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 124.62s | valid loss 6.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 127.51 | loss 2.54 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 126.98 | loss 3.22 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 126.98 | loss 3.41 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 126.78 | loss 3.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 124.18s | valid loss 6.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 126.88 | loss 2.90 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 126.73 | loss 2.46 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 127.03 | loss 3.30 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 126.62 | loss 2.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 123.91s | valid loss 5.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 127.33 | loss 1.74 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 126.74 | loss 3.04 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 127.01 | loss 2.93 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 126.60 | loss 3.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 124.04s | valid loss 7.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 126.83 | loss 2.23 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 126.93 | loss 2.16 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 127.74 | loss 2.95 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 126.81 | loss 2.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 124.27s | valid loss 7.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 127.54 | loss 2.33 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 126.86 | loss 3.11 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 126.98 | loss 2.52 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 127.19 | loss 2.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 124.31s | valid loss 6.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 127.35 | loss 2.46 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 127.25 | loss 2.18 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 127.14 | loss 1.41 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 127.04 | loss 2.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 124.41s | valid loss 9.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 127.64 | loss 2.63 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 127.13 | loss 2.76 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 127.49 | loss 2.25 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 127.24 | loss 1.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 124.55s | valid loss 5.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 127.64 | loss 1.49 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 127.20 | loss 2.08 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 127.35 | loss 2.44 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 127.82 | loss 1.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 124.65s | valid loss 7.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 127.16 | loss 1.92 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 127.13 | loss 1.97 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 127.12 | loss 2.37 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 127.32 | loss 1.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 124.31s | valid loss 7.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.30 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 127.86 | loss 63.81 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 128.29 | loss 22.89 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 128.40 | loss 17.83 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 127.99 | loss 15.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 125.20s | valid loss 19.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 128.57 | loss 13.39 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 128.51 | loss 12.16 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 128.09 | loss 11.26 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 127.77 | loss 10.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 125.27s | valid loss 13.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 127.83 | loss 9.40 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 128.49 | loss 9.40 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 127.64 | loss 9.24 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 127.95 | loss 9.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 125.13s | valid loss 9.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 128.31 | loss 7.82 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 127.75 | loss 7.84 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 128.07 | loss 8.66 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 128.30 | loss 7.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 125.08s | valid loss 14.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 128.19 | loss 6.83 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 127.98 | loss 6.71 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 127.61 | loss 6.65 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 128.09 | loss 6.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 125.06s | valid loss 9.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 127.92 | loss 6.85 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 127.64 | loss 5.99 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 127.78 | loss 6.24 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 128.12 | loss 5.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 124.92s | valid loss 8.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 127.98 | loss 6.52 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 128.07 | loss 6.21 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 127.87 | loss 6.43 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 127.84 | loss 6.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 125.07s | valid loss 8.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 127.93 | loss 6.47 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 128.03 | loss 6.30 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 128.01 | loss 5.41 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 127.95 | loss 5.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 125.03s | valid loss 9.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 127.72 | loss 5.31 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 127.59 | loss 5.38 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 128.35 | loss 5.56 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 127.55 | loss 5.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 125.05s | valid loss 6.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 128.00 | loss 5.35 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 127.98 | loss 5.16 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 127.63 | loss 5.37 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 128.09 | loss 4.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 125.12s | valid loss 7.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 127.67 | loss 4.96 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 127.79 | loss 4.79 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 127.86 | loss 5.95 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 127.68 | loss 4.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 124.90s | valid loss 6.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 127.83 | loss 4.10 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 128.48 | loss 4.18 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 127.75 | loss 4.90 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 128.39 | loss 4.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 125.21s | valid loss 5.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 128.08 | loss 4.59 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 128.18 | loss 4.52 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 127.86 | loss 3.51 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 127.70 | loss 4.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 125.03s | valid loss 9.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 127.92 | loss 4.35 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 127.58 | loss 3.99 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 127.64 | loss 4.31 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 127.59 | loss 4.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 124.78s | valid loss 8.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 127.92 | loss 4.72 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 127.38 | loss 4.02 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 128.04 | loss 4.04 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 127.29 | loss 4.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 124.89s | valid loss 6.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 127.32 | loss 3.47 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 127.79 | loss 3.99 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 127.44 | loss 4.57 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 128.11 | loss 3.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 124.82s | valid loss 7.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 130.00 | loss 2.71 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 127.02 | loss 3.99 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 128.29 | loss 3.57 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 127.24 | loss 4.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 125.19s | valid loss 6.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 127.66 | loss 3.59 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 127.62 | loss 3.04 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 127.42 | loss 2.96 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 127.42 | loss 3.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 124.60s | valid loss 6.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 127.76 | loss 3.30 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 127.09 | loss 3.25 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 127.14 | loss 3.00 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 127.31 | loss 3.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 124.52s | valid loss 7.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 127.43 | loss 3.23 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 127.35 | loss 3.12 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 127.26 | loss 3.23 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 127.71 | loss 3.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 124.49s | valid loss 7.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 127.77 | loss 2.64 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 127.22 | loss 3.23 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 127.22 | loss 3.06 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 127.17 | loss 3.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 124.47s | valid loss 6.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 128.31 | loss 2.19 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 127.47 | loss 3.33 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 127.21 | loss 2.91 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 128.05 | loss 2.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 124.76s | valid loss 6.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 127.75 | loss 2.98 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 127.68 | loss 3.36 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 127.46 | loss 3.03 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 128.10 | loss 2.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 124.81s | valid loss 6.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 128.04 | loss 2.84 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 127.70 | loss 2.11 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 127.72 | loss 2.68 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 127.38 | loss 3.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 124.87s | valid loss 6.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 127.69 | loss 2.68 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 127.42 | loss 2.89 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 127.53 | loss 2.66 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 127.52 | loss 2.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 124.64s | valid loss 7.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 127.83 | loss 2.58 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 127.66 | loss 2.56 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 127.04 | loss 2.09 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 127.60 | loss 2.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 124.64s | valid loss 6.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 127.35 | loss 2.35 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 127.82 | loss 2.74 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 127.67 | loss 2.47 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 127.26 | loss 2.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 124.63s | valid loss 6.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 127.42 | loss 1.57 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 127.15 | loss 2.57 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 126.99 | loss 2.62 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 127.37 | loss 2.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 124.37s | valid loss 7.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 127.77 | loss 1.64 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 126.79 | loss 1.83 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 127.31 | loss 2.44 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 126.99 | loss 1.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 124.39s | valid loss 5.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 127.55 | loss 1.90 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 126.98 | loss 1.82 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 127.25 | loss 2.10 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 127.15 | loss 2.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 124.36s | valid loss 6.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.82 |\n", + "=========================================================================================\n", + "Result over 5 runs: 5.279451847076416\n" + ] + } + ], + "source": [ + "num_runs = 5\n", + "best_model = None\n", + "best_result = 1000000\n", + "results = []\n", + "\n", + "for i in range(num_runs):\n", + " features = nn.Sequential(nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(64),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(128),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(256),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=(0,1)),\n", + " nn.BatchNorm2d(512),\n", + " nn.ReLU()\n", + " )\n", + "\n", + " classifier = nn.Sequential(nn.Linear(512, 1028),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=1028, bias=True),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=10, bias=True))\n", + "\n", + " conv = nn.Sequential(nn.Conv1d(512, 512, kernel_size=64, stride=1, padding=32),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.MaxPool1d(kernel_size=64, stride=64))\n", + "\n", + " model = ConvMNIST(features, conv, classifier)\n", + " model = model.to(device)\n", + "\n", + " optimizer = torch.optim.Adam(model.parameters())\n", + " scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "\n", + " epochs = 30\n", + " model, result = train_model(model, epochs, train_loader, test_loader, seq_len, digits_per_batch, optimizer, scheduler)\n", + " \n", + " if result < best_result:\n", + " best_result = result\n", + " best_model = model\n", + " results.append(result)\n", + " \n", + "model = best_model\n", + "print(\"Result over {} runs: {}\".format(num_runs, torch.tensor(results).mean().item()))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 2.053413, + "end_time": "2021-01-23T09:50:58.194440", + "exception": false, + "start_time": "2021-01-23T09:50:56.141027", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## Gaussian noise patches" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-23T09:51:02.308917Z", + "iopub.status.busy": "2021-01-23T09:51:02.308220Z", + "iopub.status.idle": "2021-01-23T09:51:02.310701Z", + "shell.execute_reply": "2021-01-23T09:51:02.310296Z" + }, + "papermill": { + "duration": 2.06299, + "end_time": "2021-01-23T09:51:02.310818", + "exception": false, + "start_time": "2021-01-23T09:51:00.247828", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def get_batch(batch, seq_len, digits_per_batch):\n", + " batch_size = batch[0].shape[0] // digits_per_batch\n", + " width = batch[0].shape[-1]\n", + " data = (torch.zeros(batch_size, batch[0].shape[2], seq_len) - mnist_mean) / mnist_std\n", + " choices = torch.multinomial(torch.ones(batch_size, seq_len - (width - 1) * digits_per_batch), digits_per_batch)\n", + " choices = choices.sort()[0] + torch.arange(digits_per_batch) * (width - 1)\n", + "\n", + " a = batch[0][torch.arange(batch[0].shape[0]),:,:].view(-1)\n", + " b = torch.arange(batch_size).repeat_interleave(digits_per_batch * width * width)\n", + " c = torch.arange(width).repeat_interleave(width).repeat(digits_per_batch * batch_size)\n", + " d = (torch.arange(width).repeat(digits_per_batch * batch_size * width).view(digits_per_batch * batch_size, width, width) + choices.view(digits_per_batch * batch_size, 1, 1)).view(-1)\n", + " data[b,c,d] = a\n", + " \n", + " # Adding N gaussian noise patches of size 20x20 each\n", + " noise_patches = 3\n", + " noise_width = 20\n", + " choices = torch.multinomial(torch.ones(batch_size, seq_len - (noise_width - 1) * noise_patches), noise_patches)\n", + " choices = choices.sort()[0] + torch.arange(noise_patches) * (noise_width - 1)\n", + "\n", + " a = torch.randn(batch_size * noise_patches * noise_width * noise_width)\n", + " b = torch.arange(batch_size).repeat_interleave(noise_patches * noise_width * noise_width)\n", + " c = torch.arange((width - noise_width) // 2, (width - noise_width) // 2 + noise_width).repeat_interleave(noise_width).repeat(noise_patches * batch_size)\n", + " d = (torch.arange(noise_width).repeat(noise_patches * batch_size * noise_width).view(noise_patches * batch_size, noise_width, noise_width) + choices.view(noise_patches * batch_size, 1, 1)).view(-1)\n", + " data[b,c,d] += a\n", + " \n", + " return data, batch[1]" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-23T09:51:06.471403Z", + "iopub.status.busy": "2021-01-23T09:51:06.470291Z", + "iopub.status.idle": "2021-01-23T09:51:06.738531Z", + "shell.execute_reply": "2021-01-23T09:51:06.738937Z" + }, + "papermill": { + "duration": 2.363034, + "end_time": "2021-01-23T09:51:06.739088", + "exception": false, + "start_time": "2021-01-23T09:51:04.376054", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAKoAAAOfCAYAAABYKVgBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAEAAElEQVR4nOz9d5RlV3beCf7O9c+b8D7SWyATCV9AobyndypxKEpDSmxppFa32BoNudSzxoha00aL3a3RSD0aUTPVElkURc9iFcuiqmCq4IH0NryPeN5cf8/8cQKZQCFRSBOZyCDjW+utiHjx3rn3nrvvPvts820hpWQHO7jXob3fJ7CDHdwIdgR1B9sCO4K6g22BHUHdwbbAjqDuYFtgR1B3sC3wvguqEOLTQogLQojLQohfu4XvjwkhnhZCnBNCnBFC/Feb75eFEF8XQlza/Fm6yXF1IcRrQogv3e54QoiiEOL3hRDnN8/z8dsc7x9tXutpIcQXhRDOzYwnhPj3Qog1IcTpt7z3rt8XQvz65v25IIT41A2O9z9uXu9JIcQfCSGKNzredSGlfN9egA5cAXYDFvAGcPgmxxgCTmz+ngMuAoeB/wH4tc33fw34729y3F8Ffgf40ubftzwe8AXgb2/+bgHFWx0PGAGmgdTm378H/K2bGQ94CjgBnH7Le9f9/uZcvgHYwK7N+6XfwHifBIzN3//7mxnvuuf8Pgvq48BX3/L3rwO/fptj/gnwCeACMPQWYb5wE2OMAt8EPvoWQb2l8YD8pmCJH3j/VscbAeaBMmAAX9oUipsaD5j8AcG67vd/8J4AXwUef6/xfuB/Pwn89s2M94Ov93vpf3PS38TC5nu3BCHEJPAA8AIwIKVcBtj82X8TQ/3PwD8Bkre8d6vj7QbWgf/vpinx74QQmVsdT0q5CPwLYA5YBhpSyq/dxvm9iXf7/lbco18CvnI7473fgiqu894txXSFEFngD4D/WkrZvOUTEuJHgDUp5Su3OsYPwEAti/9GSvkA0EEtrbd6fiXgx1HL5jCQEUL8wlac6Lsd8jrv3fA9EkL8UyACfvt2xnu/BXUBGHvL36PA0s0OIoQwUUL621LKP9x8e1UIMbT5/yFg7QaHewL4MSHEDPC7wEeFEP/xNsZbABaklC9s/v37KMG91fE+DkxLKdellCHwh8AHbmO8N/Fu37/leySE+JvAjwD/O7m5zt/qeO+3oL4E7BNC7BJCWMDngT+9mQGEEAL4LeCclPI33/KvPwX+5ubvfxNlu74npJS/LqUclVJObp7Pt6SUv3Ab460A80KIA5tvfQw4e6vjoZb8x4QQ6c1r/xhw7jbGexPv9v0/BT4vhLCFELuAfcCL7zWYEOLTwP8J+DEpZfcHjnPT472vm6nNh+yzqJ36FeCf3sL3n0QtHSeB1zdfnwV6UBuiS5s/y7cw9oe5tpm65fGA48DLm+f4x0DpNsf7vwHngdPAf0DtoG94POCLKPs2RGm4X/5h3wf+6eb9uQB85gbHu4yyRd+8J//rjY53vZe4ppF3sIN7F+/30r+DHdwQdgR1B9sCO4K6g22BHUHdwbbAjqDuYFvgjgnqzWZFCSF+ZYuPvzPePTDWlo13h3yjN50VBby8xeewM949MNZWjXenNOojwGUp5ZSUMkCFIn/8Dh1rB38FYNyhca+XIfPoWz+wuRz8CoAwrQeNfAlneGzLog87490bY93IeP7ywoaUsu+HjrFVJ/MDeM8MGSnlvwX+LYAzPCYnf/lX79Cp7OBex4Xf+NXZ9/rMnVr6tyQr6rYgwDvgsfeTU3j7PaR+V4++gy3GnRLU286Kei+kNiRCQnpVooegBxBlpRLQwZjk/hb/4MS3GXBapLI+3mCEU5GkVyTuiPrdHYsAsOtbeWbvDsOF2JGEefWK0xItAqcqKV+I6T0doQdgNUGL1ctsg92AsCA58bmzjH10DncyxB0PAUhsiVNV1yINSK1LpAlmF7ILEi1Wxy1eTnDHQ7ILEm8wRiQQ5iSxI7FrEBx27+mH+Y4IqpQyAv4BqszgHPB7UsozW3mMKCXILEkqTwR0hxIMF3RPENsSJBwfWeT52m5eXhmju5HGbGzeBQ2KZw20GPLnDPKzCYm1lWf27vB7JMVLgISoEJNZALMFflEQZDRWf96jMxHTuC8kzEnkpgHVHZIceGSGDS/DpcV+iAXFwRYAqVWBVxZYGzoDL4XUD0oiRxI5ENtgdMFqSRq7NdIzJu6AoP/7AgQULkPQF1OYDUFIvIng7kzELeCO+VGllF+WUu6XUu6RUv7zrR5fC2HjRELxJZv0kkaUhmCfS2pNYPR5vDo3xiunduO/WGZscgORQP1xn8gRNB728AuCKA2JIdDdu5NBFmUkQU4QZxIyMwa1YzGdMUligTsgSGYypOd1zIqB0RE4FUnzYMShJ6e4uNLHWKbOI3tm+PDxcxRSHu5ECFJp6fwULD1poAWC3AzYdUl7HMIM5OZCgqIkN5egBVC5X2A1oLkX0rMGc5/SyD+Twpm7S0/sLWDbRqZELJGGpHEkRmpKe2RfSRE7MNzTYLDcRO/oIGD9+SHMpkBftklMkJ5O76mAKCsRMYT56+39th6F8wK7kWAPqjzigWc0cjOoZTgrkRpIHXRfEB1r0/l4m//2w3+KG5k8Mj7Ht567j3U3yz8a/Dp520M4MZ2n2qTWBM09kFoXCAmN/RKjC2ExITFh9RGL/pcTWhMaUQq0yQ6NgzF2VeAOJhTPaWghpNbu3ZTPbSuoflmQu2ygdzQQypZzNiQf+/yL/MTI6yydHsBwwWqAsw5BSVK8oDSxtWYw9xmDMJ+gxZLsQvLeB9wCuIOC0ukm9jM5EgMq9wmqD8RkFiVRStmSWgR+OUY7myUMDE51R1lrZ7kvt8gvfuy7TGar/GnzOGcXB9WgUxlEDNk56A5KwsEAaUpakwKzptFzJsbowtrDGt0JZaMar2cRkSDISaSd0B6DyuMhufn4rszDrWDbCqrVgCgFuSnQXWiPJbj9gi9fOsy3Ng5SPCfQAkGYhfaEJLUiqN6vNmBaLEicBL2rUTsoSPS7o1F1DxY/WsTrlURZSeJIRCqiPSqICxFhXjL2RytkZ3X83pifOfIaL6+Po2kJ1SjDl+aP8p0r++jGFvuH1tB0SViOEVLSmlAmgOgaZGd0snOSzBLEtqC9O8boCKwNg+6gIMwpzZmbBb2lI3Uw10zWH7hT3srbx7YV1PoDAVYTvD6BU5UUzwv8suTvHH2OqpsmtgXuSER+JiFOJUQpcNbU5SaGxFkyGH06wu+Pqdx/d87Z6ELpYoRdEWTmBT2vCWSkzqn4uonRFqx9eACpg0zH7HHWkECSaPzF7CGCb/Qy/gWdL37vMS4sDnBwZIXsZQOrpcyG3IyGtaETW9AeF3QHBZWjAq0YYNUhKMUkJpgtQXpRHdeuCqJiRPE85KfuzspyK9i2glp8zcLtVzZm5X5l2wXDAX+8cIyNZoYwCw8enWLlkyH2hk5QkNh1pUnilCSzKFk/ZmKt61jNu6NRgxzU9xkkFnRGJN0hQeF1C6sF9aMR+SsQWwK3P2Hv5Cp/sHSCgu2xr2edf3bkTxAJLHzERPM0fuE+VQ/X3hvR7Ve3sXkoxKlAYoNdUxo2Tkv0aYegBEZTx2yDkOCXJY29kF6RlF436IwIVj+4I6hbDqeWkJsFpFpS22Nw3+5FNhpZkkQQnmjz6tQ4Zipk4sOzOAcaOFVJlIbSaYHVkdg1dSPdybvjlvH7EhId3H6J7gm0ALxe0HxAQGYtwi+DPtblyrlhFhsF8pbHfzH0bf5fcx8FoHAJklzM9zd2ASBCQVBQWtJZMmnuSfB3eWRWEuJsgrOqUbgMqRWJXRcgQQqUXTsv0GJoT4BTkVjr964jddsKamwJElM5+sWBNmF/SI/d4Q8f/X8TVVKUcl2GBur8Tw/9Hg+V57ivf5m1h6EzkuDUEzaOCdoTkLuiQXJ3NKoIBWFO+T4TSxJlILEkfg+kFg1mPydwhyI+vOsSew4tcXxwkUGnyd/9g1+h8jtjxJbyuWIkNHwHgI88fAYtBD2EcH+X/BWN9DmH6mENva2R2OCVBV6fwK5JgjwUryREuzz8Iqw9GaH5AhFBWNrRqFuO7qCgPa5+j+Yy7J5cY096nb/26t/mQw+epfpGHyvn+/m//vP/PV+ZP0Q7shHDHlogqB3QSSyJ0RbUj4ekp8y7cs6pVUFiqmiUFqpNTWZRqOU4hv0HlkgtGTTCFIkUtEObqXYvzoagNam0Xmt/BFLw2MAMADPtMnoAiQFxx6S5N1aBAglGRyBiMDuSxJRUH1LRqzAtSJ9MgQBnwcTsQOXJgMzsjkbdcuTmEoyOoHglIr2s8bH+C3xj5SD/6tgX+d7XjpKbASRsPBxTmy5R91LEVRujK9A9yMxpWA3Qunfv5rgDkuJFJZQIiMohXnkzguTB9EtjeEMxBdNlpZFjX26N0ycnyM4nBMMhhRkfEQmEJnFj9XBNzfVDAlKDwad1pJMQ5iW6r1xxQVF5Q6SA/FkToeRcLf+J+kxnLKbneYuguONH3XLUDgqitCTIarQOB4RSZ1e+wi8997dIL0NuPsIea7N7/wq7Di+zcLEfEQmkLjFbKiegcThGCwRadHfO2WoKWuOCzEqMiGD0yzphTuL2Jxgf3SC1LNh1YJln53fjLmf5g+cfoXRKQ+ow/sca0z9mMfisIHvK5uVVlfPzXz7yLaQOUVqy/oDAXjbJzimXXfFyjFXT8AZiEOqBcCdD0hsxdk3S3aXyBaQpCTOCzMLdmYdbwbYV1Ny02tWuPpGgV03+w5lHWHVz9H/NQvdg9scESSKYf3mEkt0ltaSTv6RhtgX1j3rENqQXdOJ0gl+6O5rEbIF1okaQ1XB3BSw9JbBrgv6XIfWFEt0RiSYkbstBpmOy0zqdj7dZ/5RPY5dB72sCw0voPODSbKUB+Ddf/hTisTpmSyBNSbyvS/cjbRDg9mjkZxJGv7F5fRKcRZPlx3WCokBYKhFG8zTsuopm3avYtoIapQXFCwLN1xAxfHLfeS4t97P6REJmJcIqeTw+PkPxvg1evTKBN6DCiboHxuUUIgavR91Ao3N3NlNhDpqrWVLVGK1hKO3eVjbj4qdjrD1Nlup5/vaJZxFtHXdAEi5mEJqK3Vfulyx+WCPxdSLP4PziICTgXSjgDsVYVQ3n1TR+wyG2of5YgJ/XWPioQG6OYbZQQYIIxv6zQXPMUGHkrCBK3ZVpuCVsW0FtTUoVDq1qiIkuj+SmSGKBWdPp/6dT9BXa7EmvU61n6fuWhYghvZaABH84BAH5K6C5GqPf6tyVc5Y67P69hPpeg+yshl3RkQLMrqT/uyZuy2Ff7wbfXDvAxx49Td+rEquuMfYfDCrHJOkljSQV86uPfZ2+gQZJLHA2BOUzUDqjkVlWaX2lVw3CnCR70ia7HGM2NESibNLsQoK9apBal6w9aFA/EaB7gvRqjF+6Ow/sreCe4J66lQz/xJakjtWwf69Id1Bj+HOz3Fdc4lxzkN/c9fv82At/l3g6i9lSqX89ZyQrT0rIh+RfcogdlY8pIpXyJu6CnWp2oTsgyc2C26+EwhuM0HIh1qUUHG3x1Uf/Df9w+qd5vDzFRpjlmZU9dAOTyVKNmVoJKQW7ylVOXR7FXDMpXFQPQGsXpJcEfkltkLrjMdJKSM2YeAc89CX7qncgsyDoDkmsliA3p1x15v4mwaU8ZuvuC+uF3/jVV6SUD/2wz2xbjZoY0D5XovqjXTojCf/Hib+gHqa5vNbLf3Hx5/nlw88jdUlsS8JyQntYI7WkY007hFm1S84sqNS7KHN3Hlb/8RZWQ9DYJzevQeKsGMhYkJ+W2N/N8XcufZ7ZeokHUjOcrI3Qn2nzmYlz+JFBay3Lob5VzswPIVwdoy3ojAjCnEB3BYkBZgfiFOTP6+h1g9ycpKfcJjunlv+wHOP1bSaZAxvHBWFfSHQ+jxbelWm4JWxbQdUilVQh5zJIW/K3v/VL7E2vEaylqf/JCP/+9z5F6awgGA1Iz+l0R5Tbxq6r74pYaaHsnCC1epeSUl7LkZtPcNY1zBZkjlXxd3sMfNWidhgahyIuXhrm+MAi//j0z5AguLDSz5BVZzJXoTzUYLFd4Nj4Anpb3TpnQxIUIDsvCfMQOeD3xLR2JRhdQXdIUJkt0dolyc4DRkLvqZj8FQ2jA7or0OsGvW8oDXuvYtsKajDh4w4IolyMva5DAr/1pY+TG23SOBhTvJTQHhM4MzbpFUlqWSNKJwQ5aB8ISEwY/m5EZ1QSZu7OOXfHIzqDGu5gQpiHxuUS+ZcdpIDyKWWGmAWfZ6b2MJBrM7dR4jN7z/JqU0U2jvctsfbGAOe+sQ9pKo3oDgq8oZDOiCB2JH5vQnpRx2poBKUYuyrpfVEjO6cqAbSWweJPhLj9Eq8PjA70nBSsnxBXN5f3IratjbqDvzz4S22j7uCvFnYEdQfbAjuCuoNtgR1B3cG2wI6g7mBbYEdQd7AtsCOoO9gW2FaCWphOKEwnZBclVlOFPxPrGt+St8/HHQ9JbUj0QJWpaBGUz8eIEw2yixKRgDRVgsZ2RX5W8UgZnsp5cEci9BCsFvi9CdJU1+zuDohyEsNT85BY4O4OyM8kSB3cvT6p9bf70e264vWSBpQuxhQvJ0Q5xZUlpCp5yc8mirtrImTy4zOk1iXFy+q46TVVCm52uT6n4y1iWwlqc0Jj5YMJ3UGVfMGuLj2PrmB4kF6WFF62EZFGYqqUNb+kMt2DX6oSnctTP6iEN72kbt69AmlA6rENdn9iGr/3vZ+g7oC6bZ2RRJWd6BK3Tz2Eui/wSwmLn04QXZ3clAqz+iXFLAOw8VkfEYNoG+/ImOoOSmJLULicsPRJlRqZ6Kqs2u9JMFsSt08j+5MrmBsGy38wSXsC6vs0/N6Y9qgg7Ilw+5WwbxXuXcaB68CuSaRh0NkdIjyNvq+kEV2HtY/HoEmcBZPxLyWsPairFMCWILGgdrqX7JKgPa7Iw0Qs8EsSzX//Y9vSBOt4jaFci+VWDqP93ueUWpO4/QZICHpiel8wFMNKUSW66J7AXjIJc5LWLkjGPUZ/2yDM6qwZJlFJ4Dxawatn6OYi7BkbEUKYl0S5mHAiIjzvADHVowKGPYLIQfMF7THoez2h+edDZENVLBjmE/pfBvPFmDADSzmdKJsQ9sekpraGz2pbadTOqEphIxHkplWt0/pxDc3VyJ8xyc5LFn4hwq5D7EBuLsY/0SbKxzT3K00VFCWtPfFVprz3G+Jok7FiHTcyab/eg+6994mtfTSgcBmcqsCq6dQOSxo/0qYzIolzMYYr6I7GishCA/tMivVjJs2/3iTOJAyOVTnat8zP3/8SvX1NCo+sIU40KB6pYPV4jA7USEwggdFvhaReT2E2VS6B2RZUD2m0difovqpdy8zphBnB9F+X6H5CZlFDxGLLhBS2mUYF6OwLKL5mkZjg9grCcY/C9x06Y5L2LolzJkVqI0H3BcsfFFjnsjgtCLOg+9B7KmTpKUMVwN2AUNxpZFM+AOvtzA2fT+93LdwBQWxDlJKkJlt01jKUpwAMakdUGUtQlERpSWIJ/sHP/RkbYY5LA/0stItU/QxzrTI/MXaS56u76U1fSx6fn+qDoQi9pVPbr8pUuoOKIimzKAnygr43EjbuM9BCQeeAT3ufRvacTeWISsFM7AS/D+z1rdGF20pQtUiQO2sROYCmKikH/9xi+SMRzoqB7gm8vgSpqYI4q67q1bsjCfaGpogqhtUlDz4P9b1379zdXQET4xusfH/oqskRlJOrAtK9UuBG9U93UBDc1yX1Shq7puF28xgm1I4mijQOcAclzpqg/5WY9V/o8JuvfJwn9l4hpYccKK7y9bOHGR+u8GxlDzOVMuGVHEZH8Q2IYoJV1fAHInRPpQq6kwFokjoWxfvWqT3dR2xJvP4Ec9kisyDojCliYC0UlE7qBAVBskVFvttq6SeB+AMN2gdCZYM2JRvH1E23GmA+UMNZV2UXhcsJZlPtQjVfbQQQ0NwNu/9Tk+auu3TpAuTxFkf2LpK1fPyxTVYWDTJ7GmhCcnZuCKt64+cTpSVRx6R1OEALpSrs02DkW4rQIjujNlHd0YT5T8GB/jUemJznyeJlvnVpP9/61nEyp23mzwxy/uwYvJbHbCoOgMQAqUv8/pj0jEmqkiAiMDZMSi9a5KZhfa6kKiSKCaVTGmFfhN1MyMwLRKJYW4KCYPiZrasW3FaCmlmWWE8XSE+r+nS/KNBCgVXRGXixS2s9i1+ShDlJc7eG1ZJYLbXEp5c1/A+oDPtLv2r9QOuLOwd5rMWevg0A3MjEnrMBCDOSsWIdAG395mw5/VALIoGVDUCCO5Qot5sOY9+IidLg9SrBO3J4nqzpc25tgP/l7EfQ5x0SRxJloXhOMPDs280NZ0OQu3RtoV34TIKQUD6tmKtFDPa6jl0Ds64RFAWjX9HYOA6t3QlGW8OdCBARLD+RvvWJ+wFsq6W/sWfTz7euCL6Gn4lZ/Ih61tYfSJO9CLGl/ICJqT6vhYLUOsQfqWN/p0iYBWPOIXigjXEqe0fP190VcGRTSJdbOVpnejA2FWo8qH65sDSAVb85fZGcyTN0NmH5ExZmQTEDSh0SXeD2GpvsKPBTP/Mc1TDDU/kL5E2P09UhatUCfhnCXILhKeLft6K9P8RaN7AqOk5FUpjWccuw9lSEUTMonlekw7onkAa4x1zCTIr8FUmYE3TGY8xsQFA0iW1lfm0FtpVGTa0rmyfMCrKzgrUTigrc3tDoDkliR9UCdYYFrckEs6Xq5v0ydBopOiOqPLl0VpL63p0VUq8/4dAe1QjGjUwaF8sYrWu26a7RdfXBuRTcZPAhsSS1QxrFVy3kIw10T5FLrDyh+hf4ZYlfhm8sHWDFzTHl91MN0qzWcxhPVhk8sUJqVSPMwMqn314olTtvglS+0+6goDmhERSh+JqJ0RVEjsBsq03V5Jdcsi+k0D1Vvu5sSMyaRuFrGXRf2btbhW0lqH5RMvrtLokBjQMJTlWVPHf3BGihmrzBF2LsOmQWNbRINWpIdImomZgdQZBXDvP02p0NTYkeH01Iwlhn9vXht9mgSV9Aygipeym0ENzxkPIHV4hyN3ZjnXWB0YGgCIX/nEPEiltK9zS8HkHPKfD7Yjrf6sfQEv4/LzzFG187SLia4jeO/DFrjSztXRGdUdBX3m52ZJYTtEDZmql19fArQYTEVCsVgN1KuPQ3LDrDKmqlhWA3ErRQ0B5VfFqZxa0Tr20lqEZXMP/xNO39IWZT0P5Al/oBsBctwmxCZ0Sw9oChOJ4uRQgJZkdgNQRmUyPRVXivM56w8qE7J6hxWjLWXwMgSHSMriCx5dXX0YlrLbfCCZ/9e5cpOi5R5sbOqbU3ojOpehdUfrJLY7/aUEU9IYkBtcOw6w8j3AHJ5f+4H+FrxClJz+sa/+S3fgnLijh2eJbMfVWi3rdr1I3jgtSaJLMkkLrAGw2JbWUSmJsBFKlBbb+uGl1klCuwuT9m/YSiqfcmAsIs2NWt06jbykaNMhKjK3DmTQwXwktpnApogcRwBfUDm32TKhrzPxOTPm/gbIBTTTA7CfOfASMfkH8ujYgFXs+dOU+9K+iGJlnLJ2MG7H1q5m3/TzajDUXHpTjuAtAObJyVG/PlWBWd1LogtsB4MYueBrMtsVZM3BNd+v/UYeOojRarzihmQ8NsCcIMuMMxxa8WOb27yMGHZyilXZbmR6/yGlh1Qf1ITPGMRu14TOGUiTsgMeoGXo/KoYgdSZySFM4aSB28PknhrE70iTrulTwEGkhojykO2K3AttKoWqCWFMNTbhSrCbon8XoFTkM1StBCgR6CPWUjhSof9koayx/QEb5G7pkUQR7i1J119q/Pl6i4773rbQc2s7USC6cGb3jscNwn+WgNpyox25LipYT64YSgP8I+laaxS9tcsgVhMcZsCtyjLmZH2bDZpZjyGWj6DokUeGPXpCkxFSdXexKMzd5cpXOSuD8gtS7oOSlJL2rIfh9pqPmPJz1aT3axv1wgtaZhtHRys5Ls/F9RjSoSaD7oIWoWVl0jMZUNGhci1kwTU/UIQwvAH4vJX9LpDknEuiCzIJCGImnwyxJrRsDWeU/egdS8QW2tn8p+5dC37YiJUu3q/+frRZq1NNaihe6KG3b2A+RedbAbNq1xQeliQm2/Rs9rkva4ahzhjkSIdIRmJlhTKWIb7PMpGnsl+cs6zXG1yoh/1Qf/cJXJiXVWp0cAZbaIRJCZE7QnJd1hiRQaw18yCFOStYcERhcG/tymPQLdj7bJfyuL1ATVo8p3nZ0T2K2I+Y8kpK78FYz1p1ckxrJNZk4jMy+VX7WhkTtvkhxok1lSjHRRGrJjTWIbUiuKmbo7orRJ+zGX4kUl4Hcami8wTmUxTmXxzxfohOqmnZkeJni1ROqyje7evGaP0lA7pFL8KvcJvF0+rclN5pVHKtjrOua8jTabQu7pkp2XJPe31JJtQWojYfWJhIWPaXxs4AIZM8DrV/axVVfBg+6wpHyaTS8LhCkNr1dg1QVaCO1hjeKVCHkpS2cMlUuhS6KcpD0qWb/fUCbAVs3llo10F9Dcrey/1Iak8kRI5ZEIrz8mtSaRcxlauwR2Q7VmDN4oYTUlXq/qGDLydKhIwr6fwusRW5oreSOIMgkZMyCMdYx1Rdp2q7CaamMZ5hPCYkIq5zP4vQgtEtRni6TWFJt14SKYpzKquUQ1RZKL8fpUrL54xuBzT77Cil9grZO92jEmO58QZdScgYpShXmpElyE6kOQWpO0jgSs329gVwS7/riNvaGRndbVdWkQFhLKr24dSfK2EtQooybM8CTFVy2yF00ohKx/3FdsdRG0RwQ952KCXtWYtu8NpSlmfkJD9yRuv8RsgnmoedfOW+pQnKwDcOX1UczG7T0lPWc9opREdzUy0zrON3Ns/J0u4f4uhfM6XhlSKxpRRtB7KqI7IHBWDFKzJk5F0Ngv6f2peSKp85ULh9mYKl8dWySKezYsx6T/xtJVf2jtEHTGElIrGrVjCf3fMZUm3xdx4ZdTGO5mAruAvlel8tPmtk4bbCtBlXmlFet/rUVnVOL1SYxFG23donRWaRqpQ+WQjtHQCDOCtRMaUoPel3TaY6p1TVCAwhfvrMP/rfB3+wznm8zWSjcdhboeFj/okJ0TmE1l1rj9Au25ApmX0jQfcwkPuLgjMUZXUjls4JclpXMJ3l6f9sGAX/rE09h6xN/qfQZZtXDWrmk+r6zhDghEIKj92cimay0hvSKU1pUw/DRUj0L2jI3e1eh/Tsd9sMvGA6oKoD2ikVqTlM9uHevathJUEuXjC0Od7CyklwV2RdD3CjT2QuNwRGJKShdjnMN1wgyEPRHpFUn1qCToi8lfUUM19twd7n6pQ6HUoR3YeBcLNx2Fuh70QD1swSGXnnORSpZ2Vf5t5vUUsadDLqR2RJL90BrpFcGhf3SagYE6Hz18Hl8abHQz/LWn/x7pJf1t59QZVT24srM6YUatYpk5g+xiQnBflzgFi59MiJ0ELVY+Va9XMPSfLLRQkDiqqqI1Idi4f+uaeOxwT91hJLZk71MznJ0Zxrlk35Vj+j0Jmd0NiimPvP3Ompsr672EM9kti8PfLm6Ee2pbuae2IzRfMPX1XTh38Zh2RSNslFjY4yG0tysisWJjdDSsu9MDbsuwI6h/SSEicC7czcfjzmJ72ag7+CuLHUHdwbbAjqDuYFtgR1B3sC2wI6g72BbYFoKaXlVcU/mZhDCneKairGq7owfgjkaIZJNvaTLE3R2QXZDEaYk3FGN2oO9kxOTvr+GOXWso5fcmJLbErineJiFVCNGpSEaeVXmiqQ3Fo+QNxbgjEU5VUphO8A+6eP3K6a1F9yaXlTsWETsSdzzEaqnMqCin5i21IXHHVLdpd69P+XyMHqiaNFDzu/sT04x9dA5vn0+Uk/h9ybXXIZf4/jaxc3f88NvCPVV5KCZ/zkCLwXAF2TlVF6UHKpPIquhEGYmf13DmdYqXE8KMahDW0TSiNHT6dRaf6kPrXptYEatiwMaBBAyJFBIRC8y2TmvURiSQXk9Ye0DDWVVd9qI0xJag76s2rTGN1Kqker8kN6UR5N/HSboOBp7VqB4Fs2JgNSSNgzEiE4GAWs7E2tDxJgJE16B6WCcoqDIU3ROEgwFX1nv56X2vc39pkW/37mMo17ya9P0mLsQa+pk7mC+5iW2hUa0Nndb+mI1jAr8/JnZUFk9+NiExJb2biScbj8aKCExC5Aj8ospa0gJUo96c6iYtTcX5FBcjdv2Zi+jxSS3qFM4bmHWNxtEQr0fD8KA5pqP7At0FpwK5eaVFV59MsGuSKC2wqxrp9XtPpTYnNcqnVcVokBdkZwxomGirNpk5naA3ZuxPNTKzOiICs61huCrzyli3+BcP/Ge+ePohfqL4CpVKloqbvvq6st7LxecnMS7ceSGFbaJRdVdQvKjR3CVIrekgJdZIh/qeHOllMLsJsa1hlTz8xMHtM9A/sYH19V6E1HnkZ06y6uVYbef4Zx//E/7VwkcBVcI89RM99HwDOsPg9YAWC4SnE6XAqUrqByVGRxBloZNO6IxoiFhSel0nzAvae0JSCyaV+wT6vdadWSqWPS1SOQeaD+NfTVj4sAYfqFP8epG1B2DsWy5LH0hRuJzglTSCgiTqiXi5sxtdT/ilL/59yCXULl6rQhDcXeHZHhq1Bd1+gb/bgwTaH+oi3sjhVCXtCcnqgybaWIfQNbnvyByf+FvfQxOgfbxCd09AK7LxY4Nf3v0c/+T0T10dt1bLYjUF649HGK6qBeo5FTP8jMRqStw+QdIX4I+G6F2B3utjV1UNfXdIEGYlPS8ZqnVj/t7TqFYTEBJ3KKIzmtDaG+PndcyOQDxbRA8kUpNM/biNU5FsHFeVreklDTMd4CcGcj6NFCCzt5FAuwXYHhrV31xipxwyqwnubJr8dELtkGIDCQsJSWCwf3yFqUoPdS/F/b1LvLg8Dppk5t/tp/Jxj9984ceIRnymY41dPVXGB6tU8ymyz5bojCZk9jTwzxeRBpvUNMA5hzAnGXwxRL5ssvKoKtGOMop0zS8LcrMSu6YT5N7vmXo7gqJKdh75lqA1oiEkNHcBCbh9ktaBCGfBJH9FUD+8WSYdqU3lRF+NmW4PVk1VnuqzW8fMdyvYFhq1NanYObyBiJUnJGEhpjuokVoTRLkEs6mhrdpMf28ct2Ox+uIgT790BCkF+oZFYw/0fN1BapB9w6G7rnpK/p/3/Bm2ERPbkJ3VCF4r4ZcEtQ95ZBdj/LKq2swsCmZ+QrB+zCR2JGFeVXR6ZVW2UblfET7ca9ACVS6+dkIjyoDVkGiRSnBObIlRNXAqioonvaQhBSS2KogMY53zlf5N1kOV7udOhviHXPW6AcLhrcS20Kjs6hC0LdJTFu5QTPaKQWc0ITetYVV0wlxC7+vQ2KOReAbFS7D+KGT+KI/Wt1kDNAiZRfB6ITvQBuANd4JKNYucCBEzpmKnLkqsiyk27gME9L6i0R4BdFVdIMa7hM0MmSUJQrmmYltj8IWQ1Qe3Lv9yKyA16E5EWOs67mCM4er4ZUW+a7QFIhF0B6Vim6kqW7w9Ick/WKEbqmv54EdO8dLKGHpgcGhg/equPxkRXFzuR8yl7krr9G2hUaP1FKkZC3cwZvjbqo7c6CitFpRiRr6dsP4QZBYlCEn1MOgdjSCvOERBcTEFBVUD9GY16P/zux/HtCPS0yZRRtLZE6L5gjglcYdiEkOycUIR1up1A7MDxtkM3nCM1ycwPEljjyC9LKkeureEFBTDYXrWwGoKdFcj2ax7Sq8IzI4gLCRYTUF2Vq1afknVYtUvl4kTQb2e4ZnpPfzjg1+nlOvSCS3OLw1wbmGQhu9wcHiV9MH6llKgvxu2hUbteVWjuVfxbupBgrMh6I4kRFnJyLdh9WGD4lmJ2ZXkz1j4JcUc0njUw1iwcTYE9WMh5rrB8EPLAPixgeZpiPNZgpIS/MIpRexlzwukrqMHkjCjSq6lKQlySohTCyorvjmpYXZUnXzlyL03lUERzLbaVJpfKWN0JYkpFDO1LRGRWkGMriAxJSKR2BWBFmjsebjCad9ib98Gv/Ha5wBonM5cLeuuGinMJ5aYKNU4NZwlNXdnH9RtoVErDyUYbYFV1Vj8kIZdlZT3VtVO3E1UqcmJmOaERvNQSGpVUJiOERsWYU+E/2ibnzrxCv/iZ79AJ7CoeylmXh5l158F2MdqSAHFSwndYdWUoTukqjBjSy1pZkuoYrm0JOwPQUBnV4xfUje78ksdOvvuvUxku6aaUDi/U8LsQHO3IDzWJrMoMLqC9JJGbgo6I4q7SvNVDVbsSF55cR8DhRZnXpvEeiND5tm394oXEcwu9AKwe88qiX1nI1TbQlAzs6p5BAL6X4YwJ6if7kEPBMtPGiSmoHjaQA9UFCa3GLH4mZgknfBTD73CsZFFmlGKC/4Qv7b/L1h9YwCzKVh+zCF6oQRCsvZZH6Ml0APFANg8ogrTpK4iWHZVqgK3eQsthOJp7Woph/h+gf7v3HtLf2IoO7Q1phHkhCp+TDRIYOi5iPYxj9YkmG0VjdIiRf7r98cYHUHjD4cZekbi39+lue+dm6fUtNKvKSPEH9m6Qr7r4d5br64DqYE3oEp1w7SgeTgkPW0SFFXUJTFVaDNKSwqXYOVRnfv2zvGJ3nOEUuePTh/n+K55vrD4GPKlAubm4+lUJO0xZZeJiymiox2iDQdnTcdeMWhPqMYJhemY2n6d7q6Q9JRJe19Itq9DeKFAlFM8+u4AWNV7owbpTTSOB+RPW3QHJc66QIshapsIKfFLOn3fMFj7UEjpaYPOkEZQlJhNgTVnUJiOCXKKvTv1clq18flBaXmr7Go7GhWA4nlB+0BAbiGk/xmD7liEVRdkTmzg9Som5CifKCbpyS6WFvGvzz7FV1cPY1gxp57fizvzdkdnZ1T99IdDwnyCnE8jQkH9aHSVZz+7mLBxn4480YRYkF6VFF83ca/kMbqCeNSjcEm7SuBwLyE1beH1SvpfkYQ56A5IspdNwqxg7SFY+0CM3jCQmkD3YeDlGKTAcCVLHxRsHJc0x3XC3HWEFPAGrgUB9I0762e992b3OnA2JF5Z0PM9k5VHLRp7BWjQezqi81oPpXPKN0gk2HgiJKzbTNfLcDpHzUuxf2gNpyqQtsQdvja5qRWBsyEovqb4n0pnoHRGYDR13LEILRRUDyqt4rZsNFfDLykXTuGSwB+I0RYd0qsxzhZSLG4VpKHs68pRAUKSXlWuujAnkZakeMbAWdNojwqaexP8nGI60UKQlmTfFztqpcpIzM47x0+PKjdfxU1j3kB/rNvBthDU6v0JegCt3eBtpuYZTZ3afgO7Klj/uE/tiGT8qzFD3zAQsaDddfhHn/9j0mbI7uwGnSMeUpNMHFwh8/gGmcc3yP34Mu2HXcZ+bgotFPhFgdcnyB6qUTxpkEy4BMUEqYGxbmF0BGZbbTw6I5C7rJObho1j2nU1zvuN/BWJU1HLueYrV11hKqFwGYy2yllAKJ4AmY5VWLiQUHksxOjxuPLTWVIbkvSyoHn02mZRGhAc6bK7pwLAynz5Km3lncK2EFTN1+gOqgmTdoLREaRWBLk5xQbS93Ub0efjlXT8vKD8hkZQcfj3v/FjfKBviueWdzE8UMfucXm8d5qBbIuBbItDpRV+8b4XmKmViA910D9W4eiPnudQ7yrtcYl1Nk1uWrmg7IqgdF61X0wvaaRWBF6vxHDB6Ajqj/nv9zS9E0LZ90FB5fOaTfCKgvyUS2pVKL9wV7WqNDcMgrzk8EMzfPL+M/QU20S9Ic3ditgiNWMRZSTeAY++DyxzcHiVRApmayWM6p1/Su9BPfBOFM8rPtPWZIIIBUEhQfc0YlstN2tPRWRfT9HYpyZeaiCkoLlb4xv/8xM07pN4FY3Dn7mEnxgsNgpoWkLFTVPLpfF9k//hoT/gf5n5GC/PjqPNpa5qmjAL7miMs6Sz8VmP4jdTRBnVizU7q87PL0to33u7/soxiVVVpktQUK0pg6Jg5b8J6CzaJKZOdzCh93VB/XMdQs/A0iL+fv/T/HfhZ1id6sVoCwb/6yvsM32WOgVMXZlOTd9hqVLAvJDGvMPaFLYJU8qbTNNGF9q7YspvaDT3QjLu0vclFcNffTIhM2Ng1VXLHq9HQwsltUcCsmdtvB41RnZB0hlSHT28ITXDmWkDdyBB9gbsGVmn8vujeL2KJz/KgF0Fd0BiNVQPgKCYIHp8si+lCDf/Lp0RdIburV1/lJHkZlSmV7DXxbqcAsAvJ5RPCcLMta4obp+kfE6y8omQI7sUdXuUaFx+ZZz/w2e+yh8sHKcnpaJTMys9WJdSW7bc/6VhStE9gX/Axe2Y5C4aWK0Eq6ZBLU1rTLEk622l2VJrUDssiDIJ2WmN9EUbv0didlTX5aCprB0RqQz4zpCG1ZQgNILAZuXUGElR9YGS97fJ/1mW+uc6xAtp1XbdAKuuERgWYQ50F0xNo3IswVm7twQ1Ow+JpRpHpE+q7iXNAzHOqk57XDFCtz7ewXk+CwLqezVSV2zOb0yS219jpNDgpz/+PX539kGaHYelmV5Siwb2+5DRuC006g7+cuNGNOq22EztYAc7grqDbYEdQd3BtsCOoO5gW2BHUHewLbAjqDvYFtgR1B1sC2wbQfX2+URZSZiTiAT0UGVMRUc7ildpkwPJ3euTWKroLspI9BCinCQoqoa9PNB8b54oDcIjXfZ+cgr9obr63l2EXVMJ22ZHlZLk5iTZRdV/1OwAAtzJEG8oxm6oJBEE9JyL8QZjtEhlSAkJ2g+W42vg7gquckrJt/Tc6H8tREjFtyUSGPlOF28wJsoqHq+gqM4hdlQyS2KpHgXpFYnfmyiur30+cVpueTbZthHU7EkbJDhVlTuZ6BClBJlnMjQORaqH54okNWWT7O/glzdJ0sqS0lnITQuitES8kqcw9cMl1R0POTC8ihuZdBZyKs/1LqK5L2Hiy01yCzFoEOQEjb3qwSufC1Tb8VTE4LOC5mMufW9EuBMhy08Iel/RVFpeSxA719qWgxIs48EaR/YuYuoxhyeX8PuuSfLCx3T8cnL1s1f+jkZqSScsxCSGKr2O0hKrqbrT+GMBYVbS3Ae5KQ1nQ0DbUEk6h7Z2zraNoLb3xAy+GGO0lbZxh2KiDHSeauMsGzhVQe3DHkjIPJNBapC/pJGbgda4oH4ioOcNQWLztpt3PQhL3bzFWgFnRcdZujutft5EzxuCi7+YITYF7bGE7oik96Qkvawx+zldtTtvm6w9DLkXU6w8opM/Y6K7QhUg2hCnJFZd4L2l/t4fiBgv1t92rIG9G1cToLVAYNU1vLIKu1ozDmYb7A0dw1W5D3ZNkc5VH4wgFKRXNGJHEhSABCYOrCihbm2taG0bQc1f0AkyGn4ZClMxpTNqgvLfyOANR4QZ0OcddA8aR2LiXEyUBr+k2FTsRYv6AVVS7ZVv7LK99t1pt/ODaOyF8imNlY9FWHWN/GVojWlk5xN6X9EQHZ3+53QSQ7ERWk1VWdp7UhI7ULikaCNb9/lk59S1Bke6HDk4D6hE57NzQ1x4cZKeVJd9RxaJHUn5jKTwxCrucIyzIfEHIoqXVQl5sMclzEmVxD4ckj9r0vt9gyAn0bsCuwrNIyEzlwbQfQhHtzbtcdsIql+G6o92idKS1phOogsKl6CxH7RsiBao9L72/hCyIb0v6Fdr+qWmbNbUqqAwFdAZkWQ/sI47cv30n4lhlRBM4/1J3TM6gvpBCYnKGXUHVN/7tYehtUtgNjSCgiA3pRHbqvZr/KsBrVGVYFM/pBgNtZpJbCn+2IPDq2hC4kYmK7M9OBcczIbgynovph6TPV6helSwOtVLz2sa3SHInzeY/RlJYSohdSZF4TLkZyLGv6R6zlYeStADge4LuiMSIxtib+hEabCvbG1Hlm0jqCRgnszCri4ICHOqw3JmQVD6toM3EBOVI8yKgXPZoTO82X5xKCZOS7QAENDYZRGVI/ozbfLDrXccJnYklhaz1MxjV96f6fF7E1K7m2gdneKVCC1Q3bH7Dm6QGJLyWUnfKx2EVEu81yOY/nmVslc7ojikel8T5Kc0unsCdu9bAeD0yQkWvj9Cav5a0py3rlL/hnItxr4W0POyEn6jKwiKUH7BZPUxCPKSjeOS+Z+PSAxB5ED/9wTpJVXxkBgw/DsWqTWB2bqWQrlV2DaCanQhtiHzXIbWZII7mJCYKrE5dgQiEox8Vbu6oy9MJwS9qlgttpRd6hcVjeQHj14AoLnyTlazsJxg6jFeYN5WB+jbggTvUoGeNwQbRw1aRwPsqmB1rowWCWqHBFd+NoUUipIztsFcMRERTPx5jFUXVO6D1mSCmQ1IGSEVN42zor+j7bqzarDWUX1hZ37MZOORmNpDIVZdYlcgyAvsmkZYVhu79KkUrVGd2FHnUX0gUYWRuqQ1quMXVA4vztZO3rYRVL8sifZ14RNVjj9wBc0XlM8AUvXv1EJBY49Oz0nVy3PjfoHe1rBrGmiqStWuC5L+AFuLSaRA677z8qX1/tNHWnWNqCekO7jJG+DqKu+1rmN0lQkjLUlQUlUI3lhAal3RYbZGTYSE3KwgtaZRzLlU3DTVV/qveywRQRSreRj4PuQuG2i2qp8KiorZz++J6X9eJ7ETjC609iR0h5Rtqnc0hp4VDLwIjSc8DBcKlxPyJ7fWvt82ghplE8pfSxF+r8zCb+1FiyHMCFIbyh6z6oIgJ1l/UBClVA27sy7IzEt0VxBlVTdqfclm1Knhx8Z1l/Z8n6qsfL82UqD8qOkrFoWpRCU9z+lIHRJT0h1KMLoSEvD71KbHXFNURK3dMbX7JGZT+TDbuyNefOA/U2ul36FJr4flj8Y4FUn++ykQqqV5ZyQhf0nHLwpylww6I5L8JY2oGJGfVeXpsQWNXRpJoEqr3X6NaIubBr6noAoh/r0QYk0Icfot75WFEF8XQlza/Fl6y/9+XQhxWQhxQQjxqa060dy0TvUIuIMJ7TFlfwYFRfqVWVDulDAvKZ9SzNFiX5vcfEJjHyDAWRPkr4C2q0OCYGp64B3HiB1JKe0SJRra+vvHB1o/GpFakyw/oVha/JIksSC9qKF7go2HY4yOBpmIKC3ILKnNlnQSyicFiaXMnUx/h8PP/wKhf2OFHEbdQA8kiQ7p41VEDLlpDcNVtJperxLe9oQke8kk/GtVJh9eIMwIooyEUMNwFX9AevXuO/z/f8Cnf+C9XwO+KaXcB3xz82+EEIeBzwNHNr/zr4UQW+KE7A5KMkuKAyo45BIVI7zeRO1wheLWz01rSA38/hhxNsfyhxMyiwIkmB2JXxIYRsyKn8dafefNC0ZCspbPQr34vnZezp83qB1WK0FiQlSI8cpqRRC7O/S8puNsCMrPWTQOxlgNqTwcQrnj2uMJmZWElBXi1lJXa6XeC2ZL0fp4/RLLiPEPurQnE2JLENsSPRBk5wV9r0iiNITf7mX61VEVeFkEe+0aL0B3YGvn7z0FVUr5XaD6A2//OPCFzd+/APzEW97/XSmlL6WcBi4Dj2zFiUa9oWrukAK5apPrb4MGPa+rzURzQiNKQWdYYDQ1MguSzIxB40ikuqOMCrqDCQf61phwqopR5QdwaLOorbOWeec/7yKCAmTnNXpPSsLDXTIzBoklyc5C7hsZzI4kN58oGkldEbr1nopIzZhK6+mw9hBkrIDMFfOHFuGFeUl/Vpk74ngDtyxIDGg920/puw4MeTT3JBQugTcU4pehPaKRn5FoIcTlkMSC+iFJelkFY8JyQm5ua239Wy3uG5BSLgNIKZeFEG9a6iPA99/yuYXN924fAvyCoiHvDGt4XpGkJ6I7ZBKlwB8NyJ6zEIlq8aNFkjgF1rquuJbKCUZX49N9Z/h/PPM5RDlBlj1SqWvECtpmSWZ2oE0nVh4B4Yu77qYy2xDkQcQaYja12f1FQ4tUX4HYFng9kJ2XGC2dMCPweg3MrnLbIUHaks+PvsS/NH78ug/lm0gsiaElhLGON5vDMgRaKOnuCklMk+z30xSmIxY/pFE4Y+L2q5h/ZxScDUH6soXbn2A2FYFHlJFkr+i0R7d2Tra6CvV6+v660ySE+BXgVwCMfOl6H3kbJv5AMP/xhCQb48yb6L7AuWjibEjqB6H/OyaJLglziuDXHYDiBVUaXX9URUniUsLvzD/C3j0r2Pr11Uw7sCmkPAp7PBYXy9hrd99WNTuqiUaUFvS9Kll/QKAHks6oIDcjaexFeTn2Q9+rktoBFdcPB0JEVye1pPMzn/8Ov7vwsIok9bz7MmyOKK6ei1ND9JwVtHapPAEtMDE7yiU491lFAGc1JPnZmIWPCbSeANe0cNZ04myC7usUphNq+zU6YwnZGY0ku3VzcquqYlUIMQSw+XNt8/0FYOwtnxsFlq43gJTy30opH5JSPqRn3nupXXzKwGwJhKeprn1pSWcsof2jLcJCTJBXN04LVZbU8LMR64/FhAVJ6XmLXLHL333guxRtl781+jw1L8WZC6Oce32CMxevPf4LpwapPqNeqSnrjlPVXBdSlYinVwTLH05ILwvC3R5+T4wWKVdcbCtTYP0BgVNRrjdnzkILFP/BpXY/x3sWcPvfXUiDUsJIuUHTd7CXTFq7NpkTeySZJWVSFKZiiqcNtFDQmhA0xw3sDR25ZpOZ1/H2qfyK9KKgvlcjTku1jyhu7ZTcqqD+KfA3N3//m8CfvOX9zwshbCHELmAf8OLtnaJClFcO5PJJDW0zjGzVNTJ/nqPvRZ3m3gSzJeiMQNAXMftjyu+o72kTpQQnBhd4ODVN3nL5b5/9STZeGiA1Z2Kvaxh1tbCsdbJY9fffY9c4IMksKHrN1JJBmFN1+cPfgcp9Kq5uuILuoCAqxIQ55YryBiPifExiwkvPHeTLF48QFq5vK4Z5yZ4HFkgZIWuNLFoAAy9uOuk1pYXbeyKWfyJAxJIgn2C1VGMKNLXx0l0oPm+TmTVo7k/wBmLKp5TPWttiXuP3XPqFEF8EPgz0CiEWgP8L8N8BvyeE+GVgDvhZACnlGSHE7wFngQj4+1LKLQlRpOcMes5FVA8Y6D6kV6DxiEds2xgdgb0u8HsSjLYGhsRaMglHA4gFZkfyzNQeXlwcx9ATjA0T3bumaaJeRUK7caWM8/77+zHaGl6fpHDKVK6hoiDMQm2fjphoE9azir3lQy3scznCrETzNEXfE2/mp0p4aHyO73X2EIUmxpsNITTwSwl7jy1gGxEr7Rzyglqj1x8wGHo+Yu1Bg+xCgtkyqN+nWhnZFQ2zLel5Q7D+SASGJLVm0p5UJBy5KXXs9phyGb5pZ2/ZnLzXB6SUf/1d/vWxd/n8Pwf++e2c1PXgDsUsF3XSKypKpQeCwgsOhemQ2Z+CoW/qrBUFhcsSLpusPySRnk7UNEhM6P2yw+oTFkZLJ3YUwx0AAsp9TQCc9bubzvduGH4uoHLIIsiB4aokaC0U2DVJLt8l9ZrN6sMmfsMhX4UoLTA6Ar8scSqbVERpOLk6jHB1Bh9YYa2RJZrNwojLoeFVAKpumsqFHuzNh9bvjQkzasXyepTgOSvKt6pFgIQwK+h5VVddAQ8nFC6qVp5BAWJbEvRHlF41kPrWuqe2BaUPgIgFTkXglRVFefNwiN7Q6Q4b5M8IVp6IkWZC4YrP7OfSOOsaiaXoFtvjkvwVNcF2Q6ooyptXLiDv+MzXiz90d3w3MfdJgzgfkr1oktpIkEIjMaF2f8zIb/XQ7ddUEvm8SbK513uzCsFZlxSnfCpHHJzfz/Gp/+ZFLrQGyPd70L9x9RjTlTLR2fxVIQXV1GPlyYT8RcW1FY17JL5OYpnEltrcRZmE3JRGmIfSaXB/tEnyWgERQWYdnHWTzpikfDrBL26dGbVtBNWuarj9iRJAA4qvm3g9ULiSULkfUss67nDM4ocyaL5yOoeHXfymhVnVQajdtFcSuEMJ9vrmJCawuFFESoF5jwiq0dGIswlBSRJkNRoHJKkVgdHSaexWHK2FKwmNvRphVrnmUjMWiSmpfjCgesxE2iHpWZOv/OFj+D0J2d0N2i0H1m2MruLs13/geqv3SdILOmZb0Wl6rRR2VdKahF0nFph9cZQ4H6P7gnYZOhMxYiGLKCWIWDDwSkh9j4mzJsgs+zR2b10cdYd7agfvO3a4p3bwlwY7grqDbYEdQd3BtsCOoO5gW2BHUHewLbAjqDvYFtg2ftS/anAnQiZ3rZExVdD8/CsTWNW/unplW1651KF8PsbsQOlSghapTKDEAruh+JDckQj/kEvsKOc1QlH+CMm1MuOhmDB/+37k9KqkeCXBbKtxE0udj+6r4rvYkfiHXLQYDE/V4ZttKF5J8A+5ZBfVd6KM5OM/+RKlD65wZP/CVSEFsCba+H3qWp2K4oAKC+olElViHack3rCqFo1TigcqyklGvtMlzCt+KL9P1WHZ9du+7LuKbalRo7Rk45iGswFGN0EPNOyKhhZDkIMwK7GqOuasQWIo8glvv0eYs9E9kJpk4KWY1phBd+D2BbUzKpDLEJQgf7wCf9CD3ZLU96oeolFWYkw7dMdizJpGZ0SSWhW4vRrOmZRiefFg5IlFFrpFSo77jmPs7q1wtm2TbNh0xkBEgigXgQSjYzD2jZhOv0FsK/IIuybITVtoESx+OE1iJyRWQv6SgV2VNHeLG85wcsciRiY3KKe6AExXy7QrafbuWmVqqZfE18n3dmhuqHTN1Kz19oa+W4DtqVHHXfy+mO6wZO7HJWEGElvlUnbHIwoXBcluV7GM7ApoHI7o+batCgD3uMS7PRY+ppqaOZWtSZ4QETgb4H+rlygtWPy4xB1IKFxRtVypNYG9rhOWVE8qswXtcaUN3QFV9zX38gjjmdq7HmNiZAOjC8FwgFMBq+yRv2QQpySrD5ogoPZoQFBMSK8mqhVkQRBmVVHe+JehtUfl7prv5N54VwyMVyk6LokUJFIwUVJEa7YecWhshSN7F9Xnhusc2bvI0IcWMB6u4fdtnbRuS42aVG0KF3X8EqTObCZmBJCfSQgzOv0/N4f73QnSawmddZPsHFROJPS9oNEIHcymoHO/hx8rcoXbPh9TdcRT9DkxRkMje9lAGlA/AGYLIgeCfMLgM4LaAaXNomJEPBhDzWLwjGT1R32m2r3vepwg1nGqEufbFp1h6PnDNMsfihj/MqwfM2jshcx5m8yypDUmVItJAVFPhHA12iM6uUuKiytOccNab6OWo+XahIEBM+nrfkb3BL4uWR0o0TdaZ7xYJ7yvxZXXR7fEtt6WGtVe0/HLkFlS2lRE0N4b4ucFRldwcW6Q9JKkvlcjOwt+UZBaVOXWyf4OUofSczbZGQ3du/2lP7UqyM/GxA+0yE6pGvzg4TZ+UV6txc/NSczRDlYjJr0s6e4N6H/WoO8btuIdcATGjMPpNya4+PwkF5+fpO69vXp0/fUBvB6VMC0kbBwT5C8YLD2lBNhsKRKKjRMS3Vcmj9WE3HmTgX0bxJaqavV6VQ7pjcI6m0K+WsA4ncFoi+u+RASaL0jNmTRe7aUd2Jh6jD15E6r7h2BbCmpiKzLZMCtwJ0PCPOgtnTCnboS2YV7Nn2zugfxsgtVUfEriSga7LqkfUFSJ70VBeSOILQjTGn1fTNMdTiieB+1clsySajteuCiIUpBMZakcMXH7BflTlirt2C3oOSWpH4LMAjhrOkZH5Ze+9RGa2ughvSjoPOBi1yRBXhJlE5yqJHYSuoOCoChxB2OMtiDMQWc8Jk6B0ZW0nx5ASLXxC3NSdSu8A5CmqkK19Ygo0ehWrq+BbxbbUlCDvojsDDQPRCAkmg/5S4q4y6kobs7WJHTHIgxXtU7sjEmyj2wQ9EZXiRW80ZD2h67TiP4mEWWU1l59VMNsaUQZQWJKmnsSEktSOyJp7VIkGO29EfJYi8xKQvlsTGxJ3H4NvSuIHYE7rqoNgnJC2gyvHsP3lImTeDpCqix+LRAEeVVo51SUZk8v6YqwOIFdfxTRPuTjFwXdoYT60YjWhGDXlzzcvi3mLRDgjkSkH9zg8JE5TD1msVEgNbs1jIjbUlAHn9bRfTBrOsNf1RXpREngHvJo7U4wXEUeVjqlK0aVUoKIoPt8L8LT6X66hd+jxur/zzdGzvDDYNdULZHuCRJdkl2MCXMJw89I7IrG0LOS3tcl7YmE8S9BHGmsfECycb+OUxG09sbs+ei0IsPdTBKNc/E7KmVbB0NyZy1qRyTpRYFVU+3h9RmHMAut/erz0pRYdVh9yMZYsxASklSCvWoQ5iXzH00R5W5fo4Y55QLz9vn4B12OHJ5nKKeW+rNzQwSnC7d9jDexLTdTKx9KEL6GzEbU91oYrrLH8l+xcPuU+yc7L0gMgdmE9JIgdpQLSMQa+lSeKAM9LxosfioiNXt7JShhFsyGRu8bMYsfgepBHYhZ+2mPsGZjV3XcAUF2DpY/IMg8myHMqnKR1t4Iq6Kz8tuTGAWwF1TKvp4N33aMpG2SvWLg9UlSy4r1Ocyp60yvJyw/IclMGxSmYrRAV3aor4oARQI9L+tUj8eM/YUktdLlwt9zMFo3VwrujkQcOrTwtvfe5EJ4K86cHSO1ZGxpxcT21KjfUUKaP22pp7pX0twb0xnS8HqVU3/gew0AsouS7rCivOkObdb+CIhSkvpHvKsVqLeDzKJE92DpgwK5WSZg1XT08xnsDZ3ag5Ha+afAaijefd1TGzurqiuvxEMR7okumRMbuLsDDm7WNQF0QovUvIHRVbvrMCcJSopLv35fxMYxAcVg87o0jK7E2VA+XH+Xh1WXtMdB8wStMZ3FD+chufml3+lz0YR82+u6ELyticVWYFtqVP+v1+j7wzLdARQB2rqgawmiNCAFZgOmfjZP2BdQfMVCC1Q0p1GWdAfBbKvdcfb7KbXc3iZak4oAwugK9JqBOxrT84pGlBLKXq0YxDYMvhiwftzCnHZwapLUnyuO08Y+OLBvibV2ls+MneP13NtpRmam+kmhqjr9/hgiAUIS5jSKJw3KF3wWPmrj9SU4GwKvV2A1lb0eO/bVDjH2htLEXq/EWbx52zG5mOXClXdnlYgGArJFl0MHF6hOpFm70oOzsjUSuy01avNCmfaYEjazKdBCSK1oRGnFjWp0IbUiKLxu0dyX0P9aRHdQcXi+edN6TitNG25Ba56+1yPC/pBgJCAoJGgln8YnukQZaB8IsOqC7r6AxadMojSk1gTtEUF7VCP9i0ukj1eZ+v44e8sbfHnuMHBtSa15KVJzSqi6ExGpBdX4YeRpCPtC6kcjFp+yyV9WJctunyKg8Euge2p16YwIsjMqcme2FWeUN3HzHPt6V2C23v2VumwTv1zk3NQwRcelNFlDbhG7/LYU1Ny0Es4wJ7HrEt2X2DVJZlHQmoT8XEzzSIj+yQ2MrsAt65htWHtQEA37mLtbrHw8IrEkudnbP5+lD2oUX7MwnQhnXSPzYpqwZmM2JcN/oeP1SPqfVpGj1KqkNSEZeMkn/cF1qp0044U6//Ln/j1BbPDk8BQAiRRv+wnKBRelJc6aYPlnAuxFCxELrIbS6omhcgt0D8XRNSrIzENQTHD7VXQqzArMtsBcvXNURakpi1ZgM5Rr4Ze2Jjq1LQW1OyTpjqneR50Pd3D71XLnDkj6TqxSPahjLxtE3+glKKmQoeGCNCTlZ23MZ/Jkz1lYNY32xO2fT2pVw+uB4f/NQgvBrkuMto7XI2ju0ol7QrRIYrQFfknZsTM/auI+3ceRvhU+3HuB//L3f4l9uTU+Wzz5trHzlo87GhFllGkRZSU9ZwKMyym0SJWRx7biUU0saO2L8PpURMzf7ZGYgv6XlA85syRxH+zS2a06ndwpvNW1pvd5WzLmthRUoyvQWxrZOUH6O1nSK2oJ10JB988G6Y5HWA2B3yPBlDSPBhgd9X/vM030QDnMg4IkMW5/6Q/ykvhQG6+sU5iOSVViUiuC5FiLRIfJ3xO0xrVNVj7V/6r/JeiOJlyu9/KJzDnMvS3qYZrfWX/0bWObesyRQ/Ps/8AMD33iLPZYG/7xOoOPL+EfcBn4nurOF+xzae+JEJHiotJCyJ50aE8mNHZrDLzkUz8I6RfSOIsmvSdvjcDG609w9wT4vdfXlEEpYe8D81dda8ktbNquh20pqJkn14kHA9qTCe6HWxiexK4JnA1J+kdXcFYM/JLa6efPmmhNgyAviNMJhd/NEWZVCFIPYOCF2xfU8lmJcTrLxglY+IRk7UGD9v0+yeUsVhOqB03MFsSmsh/DLLi9GiKC3zjwx/z867+ElIK/0fccz7xxEHin20cTkudfPUAh4xInGkvVPIYZ05xUzCa5F1M4Kwa5y6ptUeESGB1J/rKgOx6xfsImLCZ0HnHJLEmWP3hrAqT1+uhOBH3+de1PaVw798VGAfv87fupYZsKavwnvThZX/VZms6S6ILy+ZDYEmy8MoCzjrI9pXLRCAloqqlaY7eG1FUaXFhIqBy9/V2p26eRXpYMvKCW4kSX6FZM8SKIWOIOSIpTIe6AIh7rjsdXe2D95twn+YcHnqYn1+Fvv/iLfObBk0xXykxVegjja+eWSMH/+MkvMpGvMZxtkHJCkpkMXl9CYSbCHVRdTNoTydXrttqKPCJ/TnkdzKpG+WsOUueGOP3fMe8pSdw2sM6ksc+lEG939RLmJaLfuyqoiRRb5kvdlu6pMCtIfS1H7WhM+aTa6QZ5k+KVCL+isfGAokzseV3xImUWFMFXbKvYe25O0c0UzutIcftkXn5RkYNlFjScNWUvyjNppJDEKUWMO/vXY7Q1A7OmeJvMNiSmxly1xBfCx/nY0AVesCb56oVD2OdSSODiWIbBCdWcTROSf/zcz4Kvk5k2sOuSchfWH5Zs3G9g1VXegl1RD2LtMMS9If3fUplW+emEKKVs5NaemPwlneCd3YveFVFOUrxvg/X5d3LZSgP8csL4oRWy1jVvgutaWyZg21NQc9A6FJGaNUkMSXtfiLVh4Pbr6L5AdyXpZYFfBrdPYtcF7TGVL5peljT2agR7XQrPOjT3y9tmlDY7EJQk7UlJdlYjyKuNW3dQLbv2moE5b6P7itcUuUnfKKE37fFk/xX+48lH0I0E3bhmO6bmDRrz15piGDmpmvgWJLovaOyVGP0u2mKW7lBCdk4JqUgATdL7XQs9lEhN0B3QiC1lY9oVnfAm2d8THfozbRiDTt/bPQamnrBns8dqJ7RwQ5PVmTLO2taJ17YU1Cgl6f+uQXMPdEYgtWBiV0H3peISXVVarjMWk53RFedUj8r+74yAkBKxatOa3Jrz6YwkSG1zORUquwsJYUGSWjDw9vkUX7CoPRCipSL0BYfeQxusrRWonOrjK1/qJ52BnrMhqw+9u+MxP606asdPNYgbBUSschWqh1QijkjAqkOqkiA1jcSUrB8URPmIvhd0gpygO6Y0a1xOcFZv3OwxXJivFxkr1uE6Qu7HBjOrPTCfwmwKtsYyvYYd7qkd3DCkCd5oQLany0SpxtmZYWSkbF1z3cRo39oG7Ua4p7alRt3B+wMRQmraIp62mKLIFvc8+6HYlrv+HfzVw46g7mBbYEdQd7AtsCOoO9gW2BHUbYb4/jZ7PzmFt+/m0/S2M3YEdRvBG4yZ7K2y1sliT71/bdrfD9xzguru9QmKktKlhPysYvuIshLvgIfhocqjCxJpQP+rIYan+KcSW/E36b6KvhSmkus3vLzHEaclVktVJCS24pjKzypeADSYq5ZYm+5BDyC1Luk9rVIA3ZEIu6HKob2BmL6TEe5k+N4HvAnYDcguqHsjTfCGY7ILqnTd3RUQFiQ952KCUoK3z0eL1fV4/QnuXpWu6PckZJYl2k0mb91zftTUlE1iwPoJ1bbwzZ5Q6WWH9kRC33Oqb1RnGGZ/EkqDFVaulMjMC9oTCVqgkb+isfZogjRjnKV7o3fUjULqKsIWOQKpSTILUD0syBypUdISymmX5p9lWf+YR7duUT0BRhNKp3Tqh2KcdR0tFKwf17GXtzaY4xdACkF7XFC4lCCnNGqHFVNM5pKF0YXGpK4KKi9aRGkVSctNCcKsTWpDYnYk3X5BZzIiNX/j4nfPadT0sioniR1FECEi6O4J8Hpg7OsxtYNQ36+y1oe+qRM+24PUwO2XOBsaQVnlfEoB9j3S4OymIMHtVQ9d7xsJ3WEwDzSpr2cZyzfoc9p0hgS9T9uYTUHPKzpxOiG2BQPfE6RW5ebfEmdj65aUxFZsgF6fIrBY+0SoWlymJPtPzNE97NF5vEv500vIoy3aY9B+yCU7J3EHlRYOs4LGXo30WkL2ys3pyHtOo3YHBWYHsjOK9STMgp6O0D2LmZ+TpC+rvNPaMcnaw4LhZyKilEF+SqJFCd1+jdb+GBEojqXthtiW2BUBGtQO6ITZhOODy8ymS5w8OYld0YkLEi0SgGJ8cQY7BOt5mgcTyq/pGE3FtuIOSPTurQtrYktVTAiUR+rsKVV4fX6U3f0VxrM1TvUPcaRQJaMHxKMazcCm5VsEXYv8uiBqp6jenyCdmI1jBlZNlcl4ZQ2/JDE6N35u95ygegMxckOHoy2SC1mkIck9n0YLJdYrFrGtlsfUgmqAtvy4Tn5aooeKa8obiLHXdIyO4iL1t44D4a4gycVkl6A4JTH+3iI/Nfw639w4yOrFPgxXEBQS8pc0DFfS2hsjQoF4NY/dAG+ITX6phMRSPALuu3Ou/VBEWUnqUJ29m1lRV9Z7uVTpwzidIX5e55UDY7R2g/5yLyuPQemcoL5f1bPlLFQP15KgcEmjfiSheAHsZoTuJqw8bhGWY4zOjYvfPSeohXM6cQpKX0hRPSyIfUHjkGofaVQN4lyCVVHcqCzpZJYlGw8m2INd/JU0zqpOekWqG7YdN8ZGgttrkhgwaoT8u0sfoLmaJbWh2kqWzqJKvjvgLBuIWOW2+kUoDTfQXi5jtgWJIandn9ySjS4NMPa1mCjVSKRgtlZCns0RN0DmYfZTNoYriO2EjfsFPW8opprcNOiBxB0QiFiZBUFOkJ4zWH80pvy6ju4rG9boGMQ3UV94z9mohZmQ9GrC0pOqelNqMPhdgVE3KF4EJPjDIUZHklqTuH2C/CWdaDrLyLcgOy+pPuXj9YPuvv+ZYTeL8vctMssx3eGEhu/QPV+keEqlMRamEjZOSPySpD2uKnEnvlSjcTjC8KB9pozXKwhKCcFwSG6keUvn4O/y2d1bIZGCjW6G+OUiWqCKBwuPrKHFiiw5s6BhNYWiu1xJEFJitSVSk2ihxOwIrBYgwKpomB2JFknaxzzs6s3dm3tOoy5/wCAc8xn+U5P6Pp3YguoRZcvUDkvsdR2p6VRPxEhd0vOSTv2wJC6H1PfZ+EWJOWdj1SDMbz//lNsn8Es6g0dXmMxXWSr2ImZ0pCaoHhZII0HqArsGVlNy5a8VSS0I2rtiRDHAM23MpgYNDe3VEhRv7vjeYMzRXYskUtD0Heov9V/VZl6PxPhKP/FjLvlnHFq7lLClVwRBRp17c6/EqgoyazFen05rQnHRp5c1Nu5XvFjS0zE8yc34D+85jWpXBc4lh86AjtWQ6IFirYuKEZlFlQyseyB8QeGsclUltiR32lbMyjoEIwHNQxHuwBbzc98FuJMBfR9bZDJf5ZXFMcWIkhdEGUXno5d9nnrqFOFHGxg/v8ov/sjTIKAw1kBbciheEIh9bcIxn/b4za8opcnaVS6BhQv9b6NPzywqDi/7TIrWrk32wGVBmIbCdEB6VUIicCqS9eOqjbqIwapv/j7ZoXBBkL1kEqVuToncc4KKUKW+diOhfighKEhV01436Iwk2FVBmJcYrsDrlXTGIHdJxy8rXvzCZeWL1Vztjtau3ymYGya7cxXOrA8SzmXQ0hHd+10+9HOvsP8TV/jcgdOEUuNw/woD6RZ/NHs/vU8uE3y/jNEWdAcE+ms5UucdsnO3WGkqJFfWe99RAZCY0P+qj3vQIzcDfknSvC/AcGHxQzYihqHvKeWw96kZ7KrE8BRBhtsv6f2j9GajC0nl8ZsLRtxzS7/uQeuBgCht4ayDOxyht3UQEt0V+CX1M7Ok7NPYgaCgNHFzb6JIGlIJhYuKOzS5567wh8Pc1yRj+HRcC7OpYez2OdS3Sp/VohnZ/NkzDyFCsUk+AR/4+GnKZoc/6R3AWdcIihKzIfD6Ewz35vVQq+OQ5Frs6dvgzB6b1JVrO54gL5n6aZ2+bypyDasBVtMiyqi9gV8SxI6irD/7xgROj/LrNg5I8pc11h9QnAyJLTEqN8f1c89p1MbhmOKrFkFvjF2F/AWD8mlwNjTMjsBwBdKQyB+vkNiKHz+zrCh9nHWN2FE+Rrdf3czthDgl2dVTVTz+Mxk42uJX9j+HF5v8bycf5dU/OYruCaymIE5JUmuC52Z2MelUSExF3itC1XSjdFrQHr9508c4lWW+XgTgvj0L5J9YI3qTn0vAyDcFrV1KU3d2xZgtiTsUYbVVnVj9cITUFQ9WakMihYAeXxVcHqxjdCEa9yhcvLnzuucEtXhKQxqQmVFlxd1HuiSmatmT6GDXJNl5MP9TmaCYEBQlUUpQPSrxexLickh6SeCNBbddBn23kZjgRiZ9Tpvdj8zxkclLPF3Zz6lz46RPpvB6JLkpsCvQ85qqsg2rDr/5/U9QPKfhlSVmW+BUFcN1dv7Wbq93usjZM+OsdbL0pjvsemQebyCmfFbS+htN1QGmKrHXdFUgaUhaoxp+jyR32aB8NqE7ImnuUeFg52yKysMxxpeLxCnIvJ7C69nmNmqUFqQ2Eroj6mJTr6SpfCAkdsA74BFmhKL67tewahoIRaNo1zR0V5CaspE6iju1sM02UwIyZsCThUsUbZe/OHeYmd/ZCyj69fSKoLFfIhLJ+uMx9338Anavi9YyMNsQ9YV4vRKvR6CFArfv1txzuitwlnTqL/dxbn4Q24g4fP8ca49A51IREas9hDegliy9paNFatPbfahLZtGDBLKzShjdkZjMtEGUUi61N92ON4N7TlDtmqQ9rJFe0EiMzRp2OybKxwx8xcIvqy4juqeY8UBRTPaeirCaKrxavBzR2h3T/8L7fDE3Cwkr7Ry/u/QwFyt9ZN9w8HqUd8PZgNbhgLgnJPp0nf/pY7/DB0uXGSo10QJBZ1gw9HWD1Jqy3/3eGN27vc2k5gvs86lrzIK5CN0DowvNCR0hBeNfC8hd0QiKikFQm02xcV8KIRX/AhqklnS6h3yaByJEBPGYhztyc/Hte2+rIcDsSNx+QWpVqI4n0w5OW2D4MVqoUbisuu+JBHIz0NwtCfMGUVrSGUvIrAqkJVn5oCS1eM89i+8OAY/0z3Gp2QfAj/zCs7xUneBEeZ7pTg/NwKEV2Dw5MMU/O/9ZOq6NvJjFaQh0H2r7NcwOtD/Txj6dI70it6SphCYkiRQITyfoj0ivqMhZZkZn+Qkdv5SQ3tWkYRUwm4L64x6Flxwah2JEouMdchHrNql1jaAkkTWL8klNETHf6Dnc9lVsMVoTgtYuQIBflvh9scqkykvWj6mOIJUTCY29ilqxejwhHAg3d/+S0imN6gGdwlmDwrl77zn8YTDagq9cOEwzsPknB77Gt5b3U7K7PJG9yFo3x6WlfpZXSnzjf32c5tke9NdySAO6QwmZFdUlsH2/h/Fijigl6YzcvpBGWXlVo0pNkpozMVxJej0hzEmcdeW7dv60QJRNGH7WJXPGoTMqyczq6B5kXkthNTW64xHpJYG0k5tOGLrn7mTsSKyaILy/g7icRqYj0lM2WqhYlL2RkOxlkyAnyU+B16fjbGiblD0afgly8wmtMU2F77YRRAz6goP/nMO/CD9PawLChT5+vecA3dGI7LSB2y8JCoKoEGF0DUQIRixY+mjM2F9A+qyjdtb7I4rPG3SGb15YpaHoI62ahrGvhSYkl9b6SC0YWE2VZgkCp6oCEYULyhMjEsHMjzo4GxDmY8yGQXcyQVoSYsUi4/VBz0vG9rdRdU/g9SWIy2nCYoJmJrgDii1Z9yBzxcSqS6SpuO+9Xkl9P3jjKoTSmYyo3C/wy5Joq3ll7gJ6X5d0RhWFe+ygOvVNhqQXDKIUjH0jJDbVBiYoKr9xelGQvWJQOWSoRg8aTPwRuAO3plG9gYjxI8tvi/lHU1nc0c3Nk6/chImOckndF+FUE4y2oHRmsy/sqqG6abuC/u8Jhp6V9H1fZ/KParh9ivbzZrBD6bODd8AdiThyeP6qbTpbKxGeLN5wN+qbxQ6lzw5uCc6KwZW+XgaLTWam+0nNmu/70rsjqDt4B0QMvJ5jldyWs/LdKt7vB2UHO7gh7AjqDrYFdgR1B9sCO4K6g22BHUHdwbbAPSOo3nCMNBQlj1ORRFmJOxESlBOKVxLitKTvjQinIlWxWEvRy5htyM8kGJ7izA9KCamNa75hpypxKlK1Sn+wQZRT6YBCqgiM1eRagwZUn1S/J8GugTcUIxLVodqucQ/N1g+Hvpk87x908YZjcnPq+sOCogiSpromsw3unoDUhqqiGHglxOxA8Yqi7IlykszS++9nh3to6q2KRmxL/LKkdjxWpcGvGWRnNCpHBSISVI4amB3V+GD4J2eI0qoyMrZVm8WoELH/wTmi9LWIjPx0jfp9mwwfZ/P0vbrZ4W88wtlQ6XMiUb2ohNxs3y2g8YiPvaZKXNYfi0lVE4zOD78GqYO3z1dse/s9otz7c5NjSwlr+mQKvaNRO6TyJuJBHxELvIGIIC/w+iTlF0xqhyXO4TqVwybptYREV0QYEqjetyOob4OzLmB3hyibMPpV1Tna7RMYnmTsm74q6Itg/SGJsyG59OIEmVWlBsOsEjxhJ1x4fZzu4LXJLf/LDM6KTmpFEJRj1h8QWHVBatEgvRZTuBKjRZBZkmgRZOcEqVWNwos2cUqReTlLBt0+ldD9w+D3qgpOgKOTS+x+ZA5v4O7TtVgNiG31QFs1gdVQ85k656D7oHc1mg95SE0RVqRWNHK/UyCxoHZQ0NytEQ37WI2bD3XeKdwzDn8tkoRVh+y8Tn0vFM9LmrtVE7OlJxx6zsTU9+lYQx3qn9awTmZYe1BD76oiM4wEbcPEqim2FLOpCtPqeyxKFxLSSx59JzWu/LzE6Jpk5yQbxzWC4QBnRqdyv2TouYSVx1TFZGIorZw6WEf/iyJRWmB0IC6++zUYnbc/95Yec/C+ec5lhklN3bluzj+I7pBays22ag2fmGDXBak1SWMvZGc1ojWHwnRCkIHmgQSp6ei+ykHVA5ChRnciIjNtqLzgcgKDipM1bpnouesX5+0fXsXQEk6dH7vavn0rcM8IqtcrMJo6QUFlf3dHJFZVoz2qHulun2IKsb6fo7U/Ir8u8aQq7otTEnRJakVTtVNVDa+sxu0OCdoTkO5PI2JJ5ooSuMSCvtcSVg0L43id6GKBhU8m6G829+1TCR/di0UKoUoCLl6J8IvvPmVGV3DmtUnGjyxjaTG2EWFoCQd2LXNBDJGatuAuFB1Eoz72lENzf8T4n0Ntv0F7IkEKjSgf00XD6AhyU21mP5un/JpG8xNt5LkscUrS/0zEB372DOf/7/eBiFi/3yC9p8H4Jr3PeyGRgrHJDTbmh7asxeQ9s/SnViW9byhNoAcC3VVVjkZXYNcEucWIKKVsrcIZg8rDEX5REjmS9KKGtWThneiiRVC6cC17wmpAeknQeFiVsRQvxURpaI8KpIDcNMSvFhn5dkRm1kBEqoyj91VBbgZiJ6G5T2UkNSfe47lOwF7XWP32CNMvjNH0VYMbS4+5b88C3rt0ZN5qFF50iA+2Kb2h0xw36I4k9L4mCHOKwMNwBe5oxOV/ZKL7EOYE0Wqa2JHYG4LZzwm++Z3jrDyq0+1VK5OUN5eJ5UVbqwPvGY0apQW1YwnCFzjr0DiUINd1rBbk5mKqB0zSq6oJbfvxLoXvp2kcC9BrBt1RpX1Hf0tn/Ziger+JVVXjOlVJ9TCY8zaxDSuPCyDBWdfoDOl0hyVmAxp7TN68F95QhEgMwpwktazj96rOfLn5hMrhG+By0hSBrWNcWx79yLgp9rrbQgJCQHsSMgtQPiVo7ONqZz808GJB5qUUUQpau1Vv1OFvrLP2gV7EFdWdO7ME1WMSe10Qv1zkzEgWNBDpiP5eRRfU6KTwKiojwCx67B9cB2BjuoyzhfbtPSOo/gdajHwxTWOXarBLopbbKCXpDmmYTWjvSXBWdPRX0zSOB9jzFlJXCdRhBqY+r6E3E3pfg+aEGjfRIc4kWEs6dk2CpkwIv6S6PuenNLofbCNPZnF3B1grJmZVx+uPyczphFnIXxJYbcnSTwfY5949TeNNO07TEo4Mr77tf5fOjuDcJUFNLNDOZtF9aO1SCdC5aUljP/hPtNBfy2FVdBJTPXzpVQ2pS2Z+pg+7BnZV0jnu0TIcyicFnSE1bmrxTXExaF5Qq4WAq4krok9tHFfbuS3npr1nlv5wIUO3T6M7nFA5JklvstAVLkPhIrjjIUZLIzjoYrYlfd8xCXpjjK6gNSHof7WL8DSMtkbls97VcWtHJNJSflYt2uwVui6xGoKek4IwA4WvZMjPJvR/27y60y2f1AgeaiM1Se3BiOaEhjHz7r3qpAG7ji9yeHSZg28R0qbvcPrUxE31Hb1dxI4iOjObKhHd70kQCRgdgf09Vb4S5hK6wwmrH4pp7OVqbVV7VNKeEPR8w8HZUJr4PSHAnQiZKNcAqFSziC3mVLhnBNXZ0JA/UqV4TlA6IzC6inLb7VNkEsLT0AIY/W2DxgFJc7dAc9VOVvdg9tMpcld0gt6Y3DPXtJ5IBL0vGLR2xUgdGkcijK7yefpFQVCU+EVB7aDi9wxyiqk5TAuM17MkJmQvmiD4oUu33xeTMt65E144NYizrG/ZpuJG4PYn5F63sVoSzVeExt0BgdSl4jpIFB9U3ytgrhtE2UR1na6C4QmcDbA6CYb7w6/5TSSW5Mj+BTQhqbhpzLmt5/u8ZwQ1zEuc3y7R2qUc+NmlmDAnsR6rbmoDjdJ5ydIHDZw1DWlI0ksaflmQm0vILEFnNMGsaiTmtcktnoPMaoQ0Jc3dIALB+id8/N4E3ZMYXYHuKd4qoyvILEJ6JaF5MEIPwNkQyoebqI3cu8FoatS9d5oFE8eW7sh8/TAMPyMJs1A5Jhl4OcRsa2gxFC9K1Zn7tRBvPMAvaBQuQfmURu6KIgcWIaQ2ElYf0vDLqmDyvRDuuraCrSyVVJftLcY9I6giBrdHQ4sE9eMhrRG1VLY7Dr2nQnRX4PZoOBtK2zprgjAL7cmI2mFB+ZyPXdGUu+oj9avjdgcFc5/VyMwYFC5D6YxG8Xs2zppGY7/E649BQGdU0vdGSGIKqkcEZl1RXmaXVM2W0VUeiHeD0RGsvjTIqakRouTatKaMEP8u7fbfxMJnlTa0qxprJ0wy80rY/KIGGlQPmZRfNDcrdwVhRhXeiVixdNf3aox9KyTMSUpnf/ix3JGIQ2MrACw18zhzd8ZffM8Iam4a2hOKv4hYcdhn5jXSL6WpHDYpXk5o7UrojCb4PZL8fIweQM9rOmZTUNtvIw3IzQiK/yF3dVzDBb2tLtMrC+yGpDukNgzZeQ2rqlOYCpEazH0+Jk6B2RKKGW88ojWmnOatvfF7Rml0V5C6bDP13UnOL/cDqiY+PXlrhLq3iuLrpqLs9NRKVT8kidLQfMxFRGp1aOxTodbsYow7IGk94pJYagPr7feoHLIwuoLKiXd/yIKiZPd+JaRhrNM+U95y2/RN3DOCGmYFUS4mSkvyFwxEpHavQQG6I2oz4GxoaKHAbAo6/RpRCsK0cs4HBUHpQqIm+y2x/jAL8aCP1NVN6wxqiFjglwRaoCpbm5MmiZNAzUJqigPKG4rJXzSwmoqENrWkv2es/01oAYTN94+XPchDZ1QQ5iAzL9B9AQkkLVPlOQxK8lcUg/XK48pfnHlNNYYwOqCtWTQPh2g+pBffZROoQW5/jZQREiUal06PYrTvnFfjnhHU1u4Yo6XCeO6gpHE0JDHAqkF2RmP5UyEDL/oYHcWLWj8kCXMJnUe79L4G7X0h1YMazcMh6yeujWu2wJq1VRx/FrqDEquulrgwqzKKwqwgtaRjtgVBQWVPpZZ0yudC/JJ6T3u4jneDXE7SgOHxCgBRotFezt6BGXt3ZBclugveYESYU8QWUUZiVXSMrkq+aU+qzzrrGt0BtSk1OhphTkUG82dNRW35aP26xyg9scJIoQFAO1Cm1J3EPSOoSIFdVVlSUTpBBJpqH1NWwqXVTSpHbRJTkfhiSKQpMS6lqd4nyJ9V5sHQN3WVAbUJISHa45Fak9TuS4iKMXZdkl1McAcS7A1B+WyoElPmIRn3cNY13IGEhY8YmC1FTpb+wwK56Ru4DB3MB2r0pLoAdEPrLf7HuwORSLxeibNq0B2P6O4KsauCYCTAqaq2OdGwT2wJorRKxsnPeZhNQZTazALrl6RXJMlLxXeM7w3G5CwV9++EFqtv3AQ3zy3innH4v5kPqm16eIaeEdT2C8wWBCWBsyDwehVhr9QgfVKjejwmtaZyWN1+WH8Qkp6Q9HlbkSOgMq56/sKhfhDMpkZmAdYfjclfMMjOQulSyMrjBtk5RV8Zt9XN7XlJpz0B7oDEagq6gxD0RT800UIaYByvM1lS/sROaLHwxhB3Lx1FoTWmqdY9/VI9JFJtlHq/a1F5MEEv+9jn09T3SwZfUHnA0z9mkxgJ0kwQoUaUkVfZ997avcQbjNl3eBFDS+iEFjNXBkg173wg457RqPkrikDLqUiQgiCrnvbgyRbhPpfm7kTZflll8HeHBEZLVzZpRpCbTdBCAW1DjfEmhPqsXRXonqCxT9L3go5dV5GppacMRr/l0xlWfKN6NkIEauJje3MZ/3aL7DxgXH/pj3ISb7/H4JOL7Cqr2G0iBXOvjmDV7z49e3ciUrZ5WuKOh6qDyUpCmBMYTQ25mMKuqyTx+j6DjfsMlQa4qiGkIDOvkZ3V0CLeNpeJBb27q5i6ikDNXB64qTaRt4N7RlAb+xKyM1B9IFZx+BGVjOI1bPLPqiYKsSPxJgJFd2NJomxMkAO7IWn+ZJvsDAx9R23M3kR7Qu1+pQb6psvG7ROsfzAkN6soK+c/ZmM1wO+JybyUYt/vuNQPKh6s3DRUj2RxewXFV96pG4OiZO+jsxydXLq6HAKcnRvCbLw/PQSMfIDXnzD+1Yjhr2k4VUGQE+TnIsyWoHheUSEZI120QCUElU6rpJXUoo7XL1V+xVGJ3VBLXWJLMic26M+0ATg3P4hVuXvRtntm6e95QzVKsDZ0oqwkyiaMfBNEbCEFmF3VgzN/0sLwJGFGEOVU7mR7TBCfz+H1CmonEtDjq/mfsa3cUbqn6MTNpiC3kKD7Ft0BsOsqTdDrh95XNer7JSuPZ8ksQfhEi3Y7R5SRpJYF3WH5Dmd2XFSpfG/CjUxmXhnFuQvL4bshWXWQlmTtAQvDVWx/3OcTvaYCEvWPuZgX04RLKvXRcLnKye8OJJgtwcpjGqk1gdTVtYlDbYZy11jnzGnnrjbz2OGeuk1IE4afXCBjqtTC02/c3bj+3UBQlBx8eAZQuQsL51Vji60KC+9wT90FiBCWnx69+ve7p61sUwhw9qiARc1LsTTTS+oOu6Kuh3vGRt3BvYswVCvE8mrxrm2efhA7GnUHPxwS9JNZpsi+r6vFjkbdwbbAjqDuYFvgPQVVCDEmhHhaCHFOCHFGCPFfbb5fFkJ8XQhxafNn6S3f+XUhxGUhxAUhxKfu5AXs4K8GbkSjRsB/I6U8BDwG/H0hxGHg14BvSin3Ad/c/JvN/30eOAJ8GvjXQoi/XP6aHdx1vKegSimXpZSvbv7eAs4BI8CPA1/Y/NgXgJ/Y/P3Hgd+VUvpSymngMvDIFp/3O+COqURI8+Ea2Q+sYz5cwx1RfTnNruKnEokKKxamEtAgvSKJ72+DUBlHUUbiToZXmyrcLgZeCdFDcEcjws1MeT1QIVfDVecSZSXpNdXuPcxJ3JEIs6MoeWJHqmjcYEycvkP+bqGOk12U/3/2/jvIkiw77wR/17U/rUJniNS6MkvLVtUKqtEAAXCgCCzJJYYCO1zuzM6SO7azNmNGM87SlsalkbQhZyiAJbgQZJMAGt1o3V3Vpasys1LLiMzQ+unn2u/+caOyqrqrqzIyo6ozk/GZhUXEixfP/fk7fu69557v+7Ab4K5KooKy5ClfSamdi/EmopvnqAwklP+sNxJTuJHe1OuqXPrwVGE2teoXQkwADwKvAANSygVQwSyE6N942gjw8jv+bXbjsR98rd8CfgvAKJR/8M+bQni4x3itQTCuM16oczC3iC5Slnfk6bfa/JvTT9Jes8lPKg5VWIDUUBe99OUs9UOKm2Wvg9k28Ya35oIvPaK4VtUTOr0BQelaSntMo3RR8cHa+xL0roZf0ZAa2HVBXEtpP+iTueCAgGg0QHYNEkeivw/D4HbhDyT0vaoRFgVaqJRpkoGA4vMWzV0a3iCYK6o5xW5ImrsV4bI7IrDWdBp7BUMvxiw/ZLD8kLzlnt3N4pYXU0KIHPAfgf+zlPL9Wtbf62r+UDqQUv5LKeUjUspH9Gz2Vk/jPTFQaWHqCUtLJU48v5/f/fbHebU+wXO/+yj/5vSTfOHgGaSdbqj3yZudWr0BxUIdelHpT6WmYnBu1c6StyPeMHNTl6SxW9lmrjye4FeV5U7tBPR2JPTGY6QGxTctpKdcCI2ugIZF4ZKBvfrhrHtz19VN5NcUEzUswM7fBcNXJnTFS4IkoyzO1x5M0ANBnBEkliQqpwRjAe1RHS2C4pUP5RSBW8yoQggTFaS/J6X80sbDS0KIoY1sOgQsbzw+C4y+4993AB8qw21xtQirNu6Shj+QMvgiXKnvIt+UaLMOr9XGyNww0APFEZICKudTGns1SldDlh+x6Y0kWHXFIJAam3aWey+YDR0tEogURaUZVduOekd9sH2nAmY/aZGZVXpXVkvSGQejqSsFF0fi7Y9oHYdcybvZmeXFJjeWK8TL7h03LAcl1cYYllLKZwWtXTD/cRt3STL+NZ+ZZ11lUJcDa12Z84oU8jegaWtYcza9QShdTlk/LH58GVUIIYB/BVyQUv6jd/zpT4Df3Pj5N4E/fsfjvyyEsIUQO4G9wKtbd8o/DPuCi7ug0dsZIUsR3QENLYL1z/nYdUHLc262voVFSW88ZulxMDsw/XkLv6oUUXRfXWh/eGvmqKmhOGCdsZTFj6lm5s6emPyUwFmTLP+2R9iX4K5I4hw0DimKiDbRpTMq2PWrV8gUfD53+Dyd1bdHHdeIODC8xL6jM4SHe3d0jmEtoTuaUD4r6Hyuw+i3QsWcGIGpL9j0n4gJiylWUzVUJ7bErynSZJJVegkISf2gQN/bucMr9qNxK7fj08BfAp4VQpza+Pop4B8AnxVCXAE+u/E7UspzwB8C54E/B/6WlPJD117s7UgonTKxp2w6Y0ouMgk1jn/xPJ8eu4zIxBhd1UVVfU2/SX9OHSWOKhKVRcMilN/cmqG/elo1fo9/Nab/RcWrR5NoiWTl4xHetQJ6WycoC0QMpfNKTCNazBAd6LE7t0oU6Vxu9qO1dS4u9HPm/Bhnru7g4kI/ndBm98DqHemwilhgdDXWj0qc7+dZfsjGbKopiruoMf9xjdyMRueJntKsvaiulV2HgRcEXr9ExIpmXv2DzJZct/fCBw79Usrv897zToBP/4j/+fvA37+D89o0pCk3mn8lQX+K1y9wciEvvXwAe01DK0nau1LoC/AHTQoXDXojkvJZjfVHYmrfSVh8zEQcalN6wWaxcud9+a0vdCj+cZbGbpOgJMjdgLCg0d4p6f+uSWLB2hMRgW+i+8qsuHBNMviJG/zi4Bt8u36AsVqdqaUqhUmNYC33jg/MYsXNIh9ZIiokGO3b2w13F9Q0pP+NlOVf7OC8msPoKS5ZUFNMYC0G7YaLvS6oH4TCVUn9iKRRihFGStQzcJd11g7pWzJlei/cNztTxfMG7cc9fuP/9FWeevASRx6d4vDgApmdLaQOfSck+SkN94xL/4s6rYMx7pKg/mQIkWDhCYPypRS/7jD/9NaQR4KlDL0BjfqjEUJCbiEhLKfErsTsprQnwFpUQarFytJ97bGYT9Qu873Gfq40+lj4xihJ28RdfTdtObVAJIL5hfIdfYq5GclTv3SS+l4dprL4/UrDILWUsl9vSN6skugBlM8rMqRIBMVTFtqyTfaGgTQEwa7ggw94m7hvAjXKQto1+IPph2lFDvOdAm+c34l4vkRuRtKa0PD6VI1w5bFUyZ6XJNa0RfGiweCrCb0BDaNhMPjKe4vUbhaapyF10OsGRg8VDEDthKC5UyccUscZec4jzihmgjtj0owzXGn00f7uAH5VUXOC4jvk3k2oPr5I/uFVjBULs3H7U5XGAXjhPz6I2YWoEmM2Bb0Rya4vRQRVidUQ1E7HZOeU/Hx2MaK9OyV3QxCWlI6C2YbYger3Pjx22H3TPZWfTilf0Vg7MkArAP3xOmYhpLNLw+9oFK5IorwgtcBd0ukNSvS9Hfw1F6Nr4Jd0tFAJss19wsDcAs0IbUcPsZileEng9avV8tifR0z9oo4IBcaayqbLD6vO+95QSlKOWY+y9GfadNuDpJbAmjZYfyTCva6IhcEu/ybLdbVauCM16zgjcZYFzQcDyq9Y9IYlqQ6dEQuzJShcT5l/RieuhRjrJjOfsShcVZJLSJ04C509EeWTxrsoQFuN+yZQuyMaqQ7ZWcnaozEs5/hrTzzHq/UJTr85wdqjEmtVx2oImodiChcNtLk8Rha6ownFKUGrKtBDQW4GguKdn5N5JovUN5i1Uu08Tf0aaGYEqzbJYECSMcm/oilB3axGUrf4qjiC7sSUfYlsKpEzc9VAmmrBZ96wWa7l6M92EJ07+wjL5wWNAynOlM3QV2eY/M1RypOSpY8loMdEeRO7IbCalppuxGoaUN+nK9XDCzGJpeSBojsrh78v7puh316TSAO6w4L+7xtgpnzp//MsF76/i+F9K4zvWsZZFXT2xNhL6sNtHInxj/fIzOmsHRHs+I5H4kBmaWtWBHF2o5ogoXo+Ic6mODdsnHMuxYsC55JDZlYV3Ls7FLvW26UoLc7pDH5VkJ9NaO1JiTOSYCIgyitF7mZXZWGZu7NSWq9fICJBZkly5W/swGoq6aO+lwwy1yzQwFlRW71GT007ElttBQdlydzHdewGdB7xbm5jfxi4bwK1tVvJ86S2pDMqwFdU6mTCZ365xPyJIaJnWmi+htmF1r6YiT+WFPI9wqIqsawfcogdycqDW3NZ7DW19ej1C+r7ddAgyirBh8bRFG8kRovUVm7sSqwmCCMld0HVdq2WpFfTyCxoyEyCOWspZmsKybUcZ8+O3zFvyWqBuyzojihFlfY+tUvnVwTeSIKzLOiOqCFdD8BZBQSK/bsmcFcE2aWEzGmXzI0Pb4C+bwI1KifErrp4/kEPs6FTuB5T+6pN5Xmb6rFlHhqeBRTTMjfUYfaTBuJPqkhN6ZuGeUHxCpQubc05hUUwPIkWquHSnVP128RSvlqYksxiqnyhAkFnPCV/wiGx1RAflATNT/hqUbNsoIVvzwHjTIrmC6zFO3MeSU1oH1DKfSJVgnLNQzG94ZTSWY04C+FeDy0SJBbU3vSQGqCBN5ji9Unqe3Uq55Xu7IeF+yZQkaobKqhIsqdc3GVB4mp4fRpSwOqb/fx6/0ukhZjqm4LwbBFnVYBU3lLWQI84q8pDfnVrFgV2Q0mzaxFULij1QaMniHOS2umUkT/XWH5cKsl2ASPfk7QOqiwb90W4y5J0zSI/Be6SeFeN0h7qceiR6zcrB7cLkYLW1clNa7hLqlvLWtVBCnpDAm80wjnvEhZTvKGUyZ+3lYjcWMjI91LyN5RQyMwvJtjND4/RfN8EaumswcivTWF2BO19Ee5yyuLjGn5Vsv5oTFxMSBAQaqwdBbMrCMtKYKHxQER8I4ezosTBzM7WXPDYhcaRlKAC9f06saO2IN0lQXOXztLjGjKbkJtPKF6G2S8oX6e+UyHWvInfJ8jd0HHWf1j6MU3VzVSo3tnmelgAZ1mjvTth7amI6imlf1A9Ddk5SeGC6gATqfJCcFY17Dq41ywWntbp7FBzZhloZBe356jvi8SVlH5+jmtf20VmUXlOLT8hiYtKpHfomzqYKf/XU79I4ZJB6qT4tZTMvKA1oWMvKuW61t6U3rAkzm5NRpU61F7X8PsTpAb9JyOVsVoSZ0X1AZRfN1l6TEcKGPimSZyB1WMWZkfgDagAXfhCRFB592vnMkrlubV2Z0ttkSoJoOJFncFvGYQloWw9j0usjiR21KIpO6tRPG9gNZRQcnZBEmdSnHVB9WxC/pLJzGc/vP74+yJQw5GQ0Vyd+FiHxgEQgY40lblp2BeT/61Z3JJPGBp4A5Kxr6Ts+E5CUFJaVrUzyntVDwSJmyonwC1A+VJKZ0SQmdXpPxGh+ynV06pDqT2hrB8TS5CasPZMyNKnYhCSKKcWXElOzbvNGZuhF8MPPuBtQKRQfUM5pPQGNGIH9FBJTy4/tmGI7KnnNo+FBGVFn/argtJ5jbAoWT+g5ELdhQ8vnO6LQHWu27zw8iF++8j3eOIT59DKAUcPTlMeavHFh08CEEzncE9kyMwL1g+ZLD1qUrmUkOzt0R3QsdeVkJqzohOOb81W4NphjZHve3T3hSw/bDL7KYvukKAwqeavRg/izIbkuq8jejrhWIiIBclTTUa/IvAHEsKBiObE1tk1vhNBWdWd/ZrE65eYXeiMQumiILVTDF/1UHj9EqcQKAfwk3mKUwlSV/Pmwo2U1FAGyB8W7ouCv4hV4fofOZ8nP9Tmc3sv8ubaMPXFAl9uH0W/7uB0BWZH0twvkbpECzRWjmsw4xI+20SeKuIuqUw29KJg4Yk7P6/amYSFJ12ylyEsS+JsSjiQ0tuXYs1ZqsxTTSieM4gzOokj6R9o0Ds1QGs5S/qggRamOLMWrT3KuA1UfbaW2Zol9ui3YlaPKin1oCzojqgpiV8VFC8YzH0xoPiqDRo0+m0yoRKcW3haMPhigtR0pFCLwNxVg+RDEtre1p66B+H3pxw5fh2A86/u/LFIW24lbkV76r4Y+v9Lg8y+vbp+a/54v2M7UO9BiN6GxbvvItJ7O5veKu6LOep/aXCWdK5+fRfwozva7zdsZ9Rt3BPYDtRt3BPYDtRt3BPYDtRt3BO4rwI1PyMRUvH1c3OqbS2spGSWJblZid+f4k1EaA838QcS/P4Us6e0qbyJCG2jyTkqSMytbFnTQIuVFn6SkdhNZQukxRDUUjKLkuyCsi4Ky0ojCwH6Iw3inKpzh5WU7Lx8tzXRhwRvIiKzJPF2hsRHuuRmJXqkelfthrL0iXOS0tUUPQR/OKF0NcXfGzD4SkB4uEdU3NCzqm/NOd1XgfqWw0fiqMZfLVGuz90hgdlLKV0QEAvSk0UoRdh1jcQG/4kO/c8ZeINqX9vaYtud/A2J35eSWdhoerYgyqcICVooSGxB/bAktQSj30zojcVEBUlnNYtdV+9D6tDcD0FlC89NoCLgrf7S0Rh/b0B5oMWv/52v8uC+G1SKXR777RM88QtvEuVUR1h3FKyGYPVBQWdXzMQfJ+iRJHPRpr7fJmlZWHXBytMx3R1bc2PdV+UpfYObFJYl/ceXWHt5EGdFWaTPfcLCaiphL38gwZiz0QJFqHNP5TCClOy0RntPgt7R6O2Mb5Lp7hSJqczG/JokqiRokYGzohFloTAJnR2SHd9KiLKS9qjB2FcSpn8uYnhknQW/D6OjIQ3VwdQZVRZEWwFvLOLwvtkferzP6dBntClaPlfDGl958UEee+Sy6k3IgtlSe/zZGUEv1pl9VsdeFfSdilh61GTv7wbc+Ekd4WvILYqw+yqjtiY0vEHJwKsp81f7kJokqMLM55SltzS4mdUQqrt9/KsJ3g7VhhdUJGNfVu1/1Ze37h5uHFDNJ+ULoLdVsT7OgLsC9aMpqQ1Lf9nHXYkwPMnC0zqlWof56Srugk7UF2Gt6SqjVbeGyp24ksHR9ff823SnzP/j5S8CEEUGzqrG+S8duFm0lZpKBiJRjdbuoiCoSpo7DfyBhNlPZdEPtjFbWxde91WgdscSstOCzqDO8HeUK1+qqwsZliWpCdGnm2RndWX+68P6fhO9rbF2RGD0BEuPmSQu+H1bN8Q6q0qEbOVRZTYW5ySFSUnzYEJuSsdsC8TpPI3dNs1dGrnr4FgRzpxJagFCdVrpPpjZrQlUuad7k3L9TnixyVCmhWZIXvr6EeSFHHFW0t4T03usR5RTLAp3UU1ZzPEuZlvS/3pKdxgyMzqpLREnClh1QVzbmvO9r4Z+o6PRfNxn7A902qMGrSc90lgjzpnkrmsYnqRzvkBhReId92ExQ3dUeagaXaHEaWsCbyjFXdJgi/qAvYFUWYtf0WntUXPT5l6BWVcaT8F+tWGfOA6pKWnsDrA7GfzhCK2nk7tkgYT0Ew2Kf1qkO3SHJyRA1989d+yENjfmqtgzFpMj/VReNfH6BP4+n9q3bECnO5yhdC0l1WHpmRS9o+G8lqe1WxLUdfSDLaILeVITzBZ0DoZkL1mkWxBl91VGtdcEpZdtbvwF1apmX3bJXbBJS6pDvnFIbvDrBfqkS+KA1dCULXhO0h5XQ1hmTqMzsXW0isyChrOq0RlPqZ1Q5sG5GfU3d1lS/aaDfdElrCS4+xpoZspYpY7eMqi+KShMbSj+nSxhtX+YlrJZRHnJ7r7Vm7/PtwosvDKEe9VGCwTVF03WH0yIM5LBr1jUD6pFlLMmaY9qxK6iWJcvQPewT/4GdI/46C8XcJcFZlvR0nOVHrp/x6cL3GeBKnVIDYE9b9LcLxW9d0VCoBMMRdgrGlFBBavuCyWhuCfA70+UBPlIhNTV62zlJrpfk8QZSd8JaOyHOJtSfyrAWRGsHZesfipAJND/ok57Ic/ugVXmmkVGvpcQFAWNvTpaLMlPS9YP31malyb0H1u6+XucarQul9H9t99way+MfEOJcTR3KgnP5tM+xcmQ9uGQ3oBA8wWrD6WUX7Rp7pMUX3WQumLe5mYlVtmn17EJqluz6r+vAjU1lDhCblr9XLie0toFdsXDXDWICpLcDY3Bl9s3V6P5N22kneIsa5jrBs6qIM5C9vrWzYrMliAqJxieVOJjazoy0gjLUDsp6O9rqbpln8CdNxBCUv53ORae0umOpdhPrOGXNdY/6+Msf/Dx3g+xKyk7b/cGNnwXkQiigvIwiHMSsylojesMvBYRliWZJYE55bDwtI25rCohVkvgLir6t7ugpjBCKkJgakCaaKRdE3dxa+74+2qOGmckUV4Vpp0VjV4fhH0xxlSO2hnJ0k+GBKHN1M/mFE14f4fgfJ7iWZOwCPFwQFTRyV01iZ2tUZ2GDeW9WLD0uEb+uqD5mI+xYGN0UXPi8zX6PLXYa+9LmPnGOP4zqpheuCpoOCUyGbDPuXiD8o5cneWOd4/FtUyX2uOKydqNLAwtZb3nEsYG08ccBv5cZ+VhidnWiN23DSd0H7w+NToZnhqFgpLEr6ra9fC/t1g/aBB/tgGvl+7g6incVxm1fAEyi4KwINAD6IxLnEVDlZ5+pY6+aBONbfChhCT/5RxaBL0RFSSkAmfWJHYhLN35XPAtmG3IX9Ox1wS9QYl7wUHqktxcSlSQ2Gsaa0cFzX2S8V3LBEc8pCkx2or8Vzqj4S5LzPbWntcPImuG2HrM00NTdNYzONmQxU8luItKut3w1KLTXpcklsqgYUG8a5pkNQXWQ3Wmf0YtIt+plH0nuK8CNTXUfLD6uXn8qiQ7I4iyErMjkH9eRYtAJhqprZRRgrIgyqusIA2JPWXjDyplalndOtan91APoyvJz6QYPaXpn5/UWPxsBAeUnHhuGtJqhGNEpGsWWiCQJlifXaVxUBK7gvZjSrHkjjD/ox1NvdgkSAwuNAcZHlnnbx36Hpob4w2ljP9pHX8opu/NiLCgFkuJA1ZbzUG9AakWiBLiV8qIRLFrM5Nbs2lyXw39iQPRjoCZc4PklhRRzWoI5R81IMjMCzLzFrELYtGgdTiCRJCbNLCaEikkIjEQEoyFrWOp5V7OoAcSPZDkn1km+Q99yi5o2mbHdyXX/mKMs6bzv33s3/I3f/+3sAOBXVeF9frFCrKY4Pdr2BddguqdZVSrJbhwcpzUTRGBhjRT9K5aoGkxSmwiArm3y+ulCX764Fm+XH+IqV8oYS/DyjFBWJQULhl4j3WJ5rJEeZBCaWzFrpKlr5zSaO5Rsj93aogB91lG7e6AwusOzrKGFsDQywlmT7mLiAS8frUb5Q2ouWztJQN3xsA75tHcK6kfV4rQnZ3JlhLmmodi6gehM6KzfKWmvALKSpOqscdG72rs/c1LnPFHIYXiZEprb4K7lmI1NKqvGBgd1WzjrNzhR5aCvarhzhg4yxrunIHVEFgNgdERGG2BSAVRw+Fas0YqNY4fnyQ/pUah7q4Id1lg9CTG+SxaAju+1QVNqVTXTksyC4LEFAy+snXTlPsqUMNKQvPBkMQBbwBWfs2jNyTJzqkSSzgeoPuSylkYeC3C+otLSB3ErEOSTam9qmM1NIa/A72RrROj73tZJ8kq7nt+SlPmEgkYvjIWG/1GzKm5EZpxhqggae7RGHxB0OvXiPKSsKiymNmTW7Z3/n6w18EsBuhaymyvxFS9SntC+WJlriuJn96gIDMv8StQ/x88nBVVP107ouyR7FaKX9LQ72Dh907cV4FavGBQeNNSYrcdsL6fx11SpmfOukQmgsZTAesPSG78BUn0+wMgIC4mDD6nsfpURFiQhPmtvSxrxyRaNcBZUyIPVlPJXIZFVbtt7jL5wp6zPLeyB5mLSWzJ2gMCb1BieIKgLKlcUFJAQeXDW0yB6gFoHQ15bPwGf3Dg31OwPNJvV7DXVX+CSFW3mZrXg/lQHf+7NbojKUZPLbC6OyTN3RpRVpBa23XUH4LUlERPasmbdb04o3Z/OqNK58mctrEaGsQaa8cksSvJXjfw+jTcGxapm9LcA7U3trChIhDkXsrQnlBZx//ZJrqvyjjZayaNwyn/+eIxjpdnKb9monuC4mXVShfUEqpnlAfB8mNql+v2ToIPtPlJHEnh+BqfOXqBN+ZG+YsXfo2Xb0zQ3q1GF2dVreTzX8/irEFuLkF8p4zZlWQWNPRAEpYTjK7So2oeikkzWzMy3VeBilDzu6ikDLz8KhSvpkR5oQr5rkAPlDlC5aRO7rpG6kishqQ9kZJZkGSvK7e97vDWzVHzN1QtNTnSwRtOsL9cxPDArqtj56Z0bCfiT7/+OF6/IOhPWH1IyVE6SzqNPRpWE6pvitvWxfJ2BYw/Okt46IeFAPz+FH+/z44n5xjItTmen+ZvHnqOVArSVKN6SiO1ICgrfS6vX8l1rh8waO9KaX/Mw2xLGgfVSOGswuJTAnRJdnvV/8MwepLUFNReE6wfkST5hKWqYOi5lNUHBFok0EKong2Y+gs6/S/rhEWN1t6UzKKG1CRo4A/FWGv6lpnkegPKETCZzWCNdmkcyJGdhfVHY5xZUxX+r+dJywlRTVI8bZLYKjDSo22iuSz+oESUQ3Kvu5vXyhdQqnSx9ZgDI4t4AyYLjQJDJeWoYesxmnj7BvhHJz5DpdTF+36N9ECA1U5ZfSLFmTVxVpQ7SlCBKCdxVjSsqxkMX9V9tUijdC2k8YCG8DXFVtiC5p77KqP6VUF7TLD2oMRdERQuGYhYLUoSV4nlmh2Y+SsJVl3H8JQdeu66Rm8wpf5AirssKZ02iHJbR/mQAvw+SZJJCdZdhFQZNjNlYtdVZtUGfY4enMZcNdBD1YII4D6XRyRQOa2hzzp4g7dxXhL8ExUW2nn1mkbErtoarhHhGtG7gvTyYh/6rEPjXJXsnKT/GyZ+RcOdVtKciaNkKrNzktRW17S1N6E3qLZhi9dSlh6x6H9RZ+yrKZ1dW9Pcs6099V8QpAHhHo+Do4vverwVOMwuVJCejjtn3LEvwGZxK9pT99XQv433h4jBvugyeXHnD/3tR+9X3R24r4b+bdy/2A7UbdwT2A7UbdwT2A7UbdwT2A7UbdwT2A7UbdwT2A7ULYDdUPvkb+k1+f2p0mmqg7siCWopUVHijUUYHng7YqQB+WlV2PfGI/y9Ad7OkKC2+aYTbzRWdBBT0XD8QdWu6A8m+IMJhesp7qqkcF1RtTNLSvrIaqsmE7uhzILt+rv7AQxP9UvYDeWKmFmS+Pt9pX+1IQEU5SXR4R5RQYlRBFWl52V46v3pgarf3qlm1nagbgH8PmXB09oF9rzawTE7SqVl/XiCu6RhdAT5SybOmsRs6qSGpLNDqE6qUMOcs8hesdC929i2TSGzlBIWUuIMOIs6Qy+oPgG9p7H2gCAsCta/0FO27DFULkX4NYm7Kul+rIPo6pg9+a7j+32S3liM3UxBQG4+JnPeQRoCfzChclJHD8G8kEEP1K6VFgs6Yyl8vI7VSbGbEmcN6sfurDllO1C3AKVLSmAizkrsdYEWCsUS6IKINMKSJKgldB4IcBoJuRvq/wwfwqJg+LtQuKpaEZ3VzQeq2dJZP6qOmdgqS85+XhIVVHN2dlaQn05J5zL4gwlrxyXLx020QLD2UIJ2KYeIBZmVWO3Nb8BdErhzBp0dqoe2vs9EatAbklTf0PArSl3GrkPhmvJ8FQmULgikFKwd0TdIgIrceCfYDtQtQPMXOqTWBvnNkxQmYf2oYhHkr2vKulwHOgbNnQbGzyjxh/auhPxMSrdfpzsiWHsyui2fptopxWDoO7RCsNsnsyRxZwxyNwTlixJvQLL0OCSZFHdWdYgEVTUdcGcNrCYUrwjW95t4429zxcSGimB2TlK6ltDdoaK49qZk7eMhUldZtPFAxNpnfKJKwgPPXKH+QEpyokRQTWnsh/x1iVW/s1Db3kLdCpwqkJ+TdEcEjQMbBL5YULqSkpiCcFxQuKSjRZLuiKQ1U0bLSooXdRq7VVOKN5JQOGMRlCWxC2kmpTTaoJrtsdTOEb9W/pGH7w5ohP0RizeqZK8btD7fofC1LFZHsvAJSf6qhjEDvUEDZ11ieEpus7k3xV4TaDEMvFDn+s9VGP6aTn2fet0oB0EtoaFrlC8KsrMag1+YZuahEo8MLpDuFmSMkFU/x28Mv8QfLj3Cmp9leM8K9blBrLqGPxGwVjCgGOFcvv2N2u2MugWIM5LKhS5xRqKFArMlGH4+oTuo4VcF2TlJZ1QSlAThQEz/CzqVs4LOqMR4vE5rj6Qw1OYzv/4y4XjAocemOHLkBsMF1YY3kO+87/H9Pone1Ol7ScdqSKLFjKJ/S0luUqe9M6G1C7QQoqygdUBZrYtYYDVVe+TMT1ZAg/VD7+7Js9Z0CpOw8qgaAXYXVjGMhBPTo/xM32kqVo//YfzL/MPLn6VseRyrzPEb4y/j7Q8oTEkqL1tIO0Vbse7oGm9n1C1A5bxk+aEcZlutcP3+lJmDCYU3DLo7JEIK5JDH0J8JZh2H1JBY7ZS4JPmdY/+W33Z+hfVOhjm/xP/4+Jf5B6c/T1B3MBrGLWmhxjmJiJRY8foDgvykhl8F7Qt1glNVZD7GmbWxGpL1hxJGvimIMpLE2eDkS0hcpaKiB2/PJ1MLzK5a+ZtNjcSWnK8PogtJEuj8v85+DvFGgT/tfwSjI3gprPHsz77B6c4omYKP4Rm0duo4cyZ6COkd9KVuB+oWoLVTw2yrgMnMCYIySF/HakvClsCvScSCw/JDgqiY0vmpLoU/yZGdEvzqv/o7RHlJMhRw5ssHuNQ8gFYGdxPHdxfVwNjYo9P3esryoylaKKj9ixLxuCCxlRqg05BYKzp+CRJHNUD3hiVGR1C6lNL6xTbWt4oEG7MMkaoFUphX05j1Q4Ll7w0jUqgtSbx+myinZJJiV+nLFgyPq90+ALpDOtk5SXMPxDnUXP02sR2oW4CgougumTmB3weZeY3OoZjE0sksS/yKwEqhszuhNrFO40yNtWMbNUZNkQvdyw7+QIpdF2xWoc3sgB5IcrMxQUlHGpLiOUF7REcKpR7TORrQHTOQespaH2SmDYKBGLPk489nCKqQfa5Ib0jeFExzViRev6omvKUXa3bBrypGbMcROGuC0rWY+Wd0tB09zreGWPGydFcymEOSqBJjrRhExRSzefspdXuOugXof03JhOcWUqQuSRzIn7NIbMVvd1YlwUNd9HLA6loesbOLvbNNtCMgGfUZ37WM35eS2ind0c0fv/FgyPrTIQtPG9R/vkv+mk5vUChadqpW7jLQGPkO2Ks6pbNv5ydxLUuaSUmKiarBrr99k4RFQVRI8UYj7LpS9/OrEmlCY7fSTohyMPNZQe6GoJj36HM6rLWyZK6bRH0bYsS7POy1O+OjbAfqFqD9i216T3fwS0pZWve5yXnqjEusjkQ3Ug6OLPLE7imOjczhdWzcXMCzey/T9ByOP3gNbDVkbxZa22DkTwzCWoJxIo/dUDQRqauFUliQlM6YzD2LCjhfKhZsU6dyTmI0dYSnKVXud8ST2YX9/2wBe9GguyOleDXFWRXYa0p9xmqpZmxnRSdxoXm6iqklhL6Jd8BHbxiIBJJVm3jfndnMbA/9WwD9+SI5XxLlBJk5ne54gtnQboqKrR0Fx4yJU42Xzu6hcMGkGEF31ObFN47Rm4jwCy2O7Z7hzXgM98bmmJvOssbaIdD8lN4hn7Bo03ciZf2wTnMf7PhOxPoBi9rrGr0ByHSVqIVIwS9rlM9LUkvx8L1BRdIDNS1Zf3JIldouwtpREFL93fAE7mqK1DTMz6+yNlkGKZhsV8kXPHQtpXfdIclItEBDvo/m1a1gO6NuAUQC9UeVVv2ObzYpn1X04rCkBCYSG9IXy0z/+QSFi0risnbaQwLeYIJVDHi0coMrqzWqr24+d7jLkuJkiu4L8iccdE+w+KQg2OnjLgrmn7ZoHguJXSUilxoqSEUMzYMJQUUQZQW9YamMODYQ5mH9sFJG7A4r7a7+11OivCSzqIh9IoXG+Sr5SR1npMMztWvsLK+TfLuK4Sn/AqkrxcI7wXZG3QK0d6Xkz1nYdcnUzxWwG4K4ElHqb+NYEYvTFfzUQAshiQWJI5n6OZskkzC8c5Wq2+NMcxjj+SJRfvPHFykklnrdoAK10ymtMZ2w7eA92cF9OUcwAs39aoeoPQHukiQTCnqaWrH7hzxkw8J4h5NJWJJIQxLWYrJTJp2JFL+mIYD1ZwLEmoU0EpwVHWdN0lpzebU+gRebOOuStWOSHd9OSCxVnooyt3+NtzPqFsBd0PD6JI2DULqyIX1jJzTWs6ysF3jyyFWqR1f47E+cwDjeYOdjM2jDHlbF5xd2nOLczBBnv78H/dk1Wns3Ty/uDguKUz7OkobVEDR36SS2aj5JZ7KKVRppuAsaVkvx8TujavsTIChLDDNBqwRk596eI5tdQfmswFpR+cwY7pGbhfwU2FMOpUvi5iKpfhjGd65w9s1xjpdn6YwIkkxKfZ+JSNm8FsEPYDtQtwC6r/b5Sxdh+WMxIob8Gw7OlE3tqzYvnd6La0ZcaA4QRTpX5/uw7Bh5Lcvv/8PPY19yyV8H848qlM7d3iB39dcMCjdSzK6kOJmQmhCUU9xl5a+Fpsphuqf0rKy2oDuSogWC7Jwgl/HJvpKhceTtG0X3lZSk2VYKLdbrOdYfTBAxZOYliSVwHlpXUpgTXWZXyvwfP/ldvnTuON4Bn8y0QWKD1ZJKV+EOsB2oW4DUhmCXz9rxFKNukJ1XK240WH0QctcMOoHN5EwfxS/nyL3h4t3Ib8zfICwoe8nYVeWezcLf61M+YeCX1MeZmII4I7FaGrnZlM4xn+FvKtWS+rGE7JxEi5REkLOuivr271WIcjD8nbdDQguhfDkhcaB6WmI3JMJJsDqSzpjKyIX/vYC1rmG+meMTu6/wR1MPos07OFccorxEbij/xXcYqNtz1K1ACpXnbeyWZOlxpQ4dFjcMbpc1/H5onq1SWBKsH5ZYDTDbgigPcVagRZLGkRhrRSc3o0wnNgNjzibKCZW5bMHSE8qwzOxClBE4lx1Wj0mycxJ7RUcLoXU4RHMSCi+7tHemtPYAMmV+JMWdU8EaVCSLwwJnBYKyRuN4iDVrsfiUJDsr6I4lpIZx0+ugZPbo9mxSSyINQW5ayX8G/QnmHXZPbWfULUD4aIfuiKAzrKGFqtBudqA7LGntSYkzkjibQgruslpFS11VC/xaitUU5C8bJLs9mns3f/zKOYnfp7Jca39CflLDriuRNb8mlD2RK6lcDKicl3RGBTu+oqPNOpAqK6PCNQ1nRUNk3x76E1ditjQyi5L2uMRcMRl+fB53UaM3kiISgd+v2Auf/cVX+fr0AR7YMUd2VkP3ob1TidRlp3RKl+/sGm8H6hYgrDtkFiSJrWqP3kiCvS4pXQRnVSN3XeAsqkWH0ZV4gxKrKfBGYmonBd2dMVoMg1+yQdv8ENn4Yhezo7Kw0dGUw0tWIDWonovJX9NJ+0Ku/YZg7aiaFqwe1YmLCYmr5C+lUPSR2rffbogtXhJIQ2L2Usrn1e7b0nMjaBE4Sxr5a5pSrV4UfG3yIMOFFm+c24WIQd9w6TYCibsiKUzdmTPatvbUNu4cGuz5zCQAU+sVkk3a9dyK9tR2Rt3GlsCLTVIptsyu5wexvZjaxp0jhauXh8gNdnCvb41w7w9iO1C3sSVwZw2S2dKH9vrbQ/827glsB+o27glsB+o27glsB+o27gnclYGaWVK+8X5/evPLG4kJ+lLk8TZBNb3pYKeHkNqS/hMR4eEeYUli18GbiG7qL93W8fNS6SUJpSGVZNT+fZyTysrmoMfA6yHenuCmMYQeAEIpoHijMbWzMf7A1jkA/rghdbUtrEcQFSRCqu9xTuKsq8/rLZt0qas2wagob2pQGZ5i6aYW+Ps3twFwVwbq+oMpcS5l8CUYfFH1W4pcjL2qUfnDDMXLAt2H8pUUkSiRBLMd4Z7KMP7VkObjPuaKgTRuT0qmfkgS51ISR+DvUxc0rCYIqbYbtRjyr7rc+Fkda9YiNdU5ml31oUVZyE4ZJJagcvquvMS3BT2AgVfbdA/7SEMFZmZR4C4KGgdTdh6ex3hmne5EjLOqni/GurR2qg4ur18lEXcZxPrmeP53ZXmqeEEns5Sw+oDi5qSliNwZh7CkdtHqxxLceYOFn4rQ1kzcBYPJX9DQPUl32MSYU55SmUlB+XLE0sObCxaroVGcTFk9niKTDWbptEFuVtIbVGZgUVaJkakbJqY5YdAZgzibUjspWPpCgNRclfnf0TWfWhAMxIzvWiZnBVyYHcS+sBly9I8PUoO5T+XRDI+0GmFnQ9oVExnoFPo6PFmb4iu9Q0zsXmL16giZeYFcytEZT0k9gdVUohv2usBa39xnclcGql+D9rgy2s3OathnLFafiMhdNVn82RD3sovflyB7OkkxhobF8HOS1WMa7THVEOLtC7AbNvNPG+ib3GbWQ1h6JiU3aVB8yWDxyRQ9ELRHhRraYkXpQCoVv4UndbJzqiNKC3WkljLxbzSy/+MkZ67tAAmFaheAjB1SdRXRLU410ujuz7hBNcUZb/Mb+14lo4UMm3Uu+UP8weRD7O1f4enKNf5s/gjPLe1hV3mNE6d2w0TC8Hdg/lkY+3LK8kOmclM87NP3HYPFx82bXVe3grsyUJ1VAEF2RtIdhfbBGPe6Se8BD+e8S5SXlC4oSkVm0VCZdFAjsZRuU+KmmHMWnbGUtBDjTm5umOmOx5jFAK9PuU33vaG4+V6/0g+NsylGRyMqJYS7PMS1HGO/MEnOCJj6Z/sp/9Y0F0+P0d/NYWVD9g6s/NAx4lTj8tkduEtbYGv3IcIfSth9cB5bj/lX557kC3vPMmA2eL0xhmUkBLEKod3FVd5Y3EGYKF2B4Z2rrC8M4s7CjS9KMtOgxZLMaZf5ZxSDdTPKKXft7eysqg4gEYO5ZuDtiCm85JJaap7YOJgSFxLMtsRdl/hVxTtPsil9r2vkbiiz3NvpRhKpYOCPHLJzgs6oUkJpHIsoX0pxlwT2qk5qSYa/oxEuZvjcZ06w2MmzHmR45r97BUNL+e1Pfx2AX9h3iulGiYb/9vAeJTqXz+3AucuDNLWgOlHH1lXr32itwZfeeJh/9D//CifP7qSa6bK7sMobzTGWvDy9roP1LysMf0ej+2eDhCWlcWCu6xieGom6hwIGXk9xVjf3udyVgRoWIChB+PEWZls9ZrR0gqrqOtcDsOo6hUsGzT3Q3qERVFKsOpAIvJqgvRP2/H4T0dr83rPe1lg/oGN4EqOn5snV1wxa4xp9p3xSS5K7rtEZ0She1HllaZyVxSLt0OZL33uc87ND/NNvf5bO9/r54z96hvi1Mmvtt5s1rpwbudn2dzcjcSUl922T3wG3zdC3dFoTGlqg8TdHv8N3b+yhaPpcuDZM35dtmjsNOsMaQVXJUxoeDLyeokWw9nCKMW8x92kIC5tb5N6VgWq1VBlEvFEAAX1vpIgUvJFYiXalSrOzdTDCaqoVZfGywGpLzJagN5ISVWKmfr50W8cXqFJKe6c6VpyVNA6oEszCEw7hYKQWRSXILSZ0Xq+Ru2jR/vogY38eY15yKV3Q8IZSUgPCQx67+1aJEp2ry7W7PpO+BaMrWOu+TR19/XsHSGwBx1sMHVzm5c4eavku37m6j0KtS2oKWodDekMSs63m+b0dCcsPKdE2e0mneAWIBd0Hf9jl+n3PZYvf25aguS+hckbD3yPJTjRZaLmUSl26pys0D8ToxQjzeQc9F5NZMsjPRvhlAy2WRCUoXNEJKhpWHcy2tmlxXBGpIEwsSeViQrCgWJ2tnZDaKYUzFq0jIc6sxfzHBWZLEmcE/kSA3bDILig6ythXIxaftKiV1bAwuVTDPHcHnOGPGKkhqa/lbspfjjw6z+xEmV/de5KMFvKfZo7R9mySQCc5U6Y1AbUXlTT8ymMJ+csGcRbyNyRrRxVPqzMuQE8xrM3Vl+/KjFq8pONXBZXTAv3rZTLnHcIXq2ihwFrXKX/LQUjIveLS3AcLT1kYfsr6AR1nSaczKileSUlt8AY3b96gRYIon1I+J1g/oKYXSLAbApEIWocjslcsYket/AvXoP9EDIHKlK1dkJ9NCIsG5vE6tUyX6+sVzAv3TpCCWgvYMxbzrQIAE/l19g8vcbIxyputHdRP9GF+r8iu35O4SxK7AfXDkrXP+ww+L/AGlHRQYz84a0psbeCVCKOtMfZPNzeq3JWB2jga0zvkb3CKFLcocVCqI64ivwVlQXdYEhUSEkvSGjfQEqWXlOQSVh9Uq8rilduQOhRKkW/9eIqzLhn9hqpvWQ2J1CSaGxNnJYYvsNc1tFiy9KiOCAXdEUVqWzmmM/+plIlynW5kEVwpvKueeq9ACwSdcxUALqwP0AocZholXn15P6kB7cc8Jn8dTE9JsJcuCDJvuCw9Cf1vSOIMDL6U4Fcl1dOS2U/rVM5KJn9+c8PcXTn0l0/pgE5YUKJj/nBK8ZwBmOz4jsfqEZfOuFroaLFO5bxUukgJeAMSe1kxI42ecifxaptkdfbU6xQu63g1YL9D46DEbGq4ywJtxsVuSNYfSLFXNRJbYNcF0c6QILYZfCWmfkxjYvcSZy6MIUKBcwuCvHcrjI7g8mIfcWAw0N/k+MAc313OUzhrEoQOqQlhFsK+GK9jklrqRk/1lLCY4pd1UjulflAnycSsH9WRxubu2rsyUOsPJDiLBnFOoocCITWCChSuSa7/tENqpVgjXeT5PNUzkvVDGsmoR9oycRYNpKHcRbREokebP353h9Lhbz4U4Fy3ibJKVtFqScK8oHksxFg1SWsh7gWb1AD/iQ65l3OQQv//cxLTz1D3XdAlztJdeZlvGd5IzP7+NQqWj6UlPFm8xtEn5/hf1z6Pe6AB3ytjtxLMVUMtMncG+IFG+ZJAmoLGfoG7qCM1KF4wyC4mrB+8D4Z+e1XHasH4VwJSXa0+Y1ey9pMeWgBpPiaezCmB2Yqg/0SMMeUw8ceqOqB7gtK1EC2E5s7Nv8XxryYUrkHpdcVR94Ykrf0Jzd1gNyWV10zickz1uzbrRyW10124nEVqEJah7me4ttjHWKFO5bV7O0ijgmT/gTkMLSVrhLwxN8p1v0Ykdf76z3yN3qWS8giY0G/uNA1/2YBE0Kvp2Cs6+RvKGE73lU5XY4+qq24GH/gpCiEcIcSrQog3hRDnhBD/08bjFSHEN4QQVza+l9/xP39PCHFVCHFJCPH5zZ2S2lOWGtT32fSdlMi3znLBwV0WGG5M6SJEuRS/JjA7SsLG6CU4j60RZyXTP2HQGxKbFW8GoD1iEBYEzUMJ2VkY/XqIFGqq4fUJGvslpdOmWjRd14gKFvFOnygHpceXeKo2yc7BVa6s9SF+/CTf20Z6rM3AsSUMLaUT2rz+H44iJTyWm+R3Lz/Gv778JGK8y8h3oTiVoEUCLYSFL4bkpgylKTAc0+sXrD+Q4vdLirvrxBmJsUm51FtJNwHwrJTyGHAc+AkhxBPA3wW+JaXcC3xr43eEEIeAXwYOAz8B/HMhxKbyfFROCSqS5n7J2hGleuysC9wlje6IxDmZoTMuGHxJWRpOf1ZtkfYGLfiqmvjnrmsYHjcbWTYDp5nSGU8pndHw+gXdIZOBFzRyc5KgIjHbGo2HQjKLgigDrXGT4S9ZfOpnTlCwAupxhmvnh0leLuNX7725aeJIxENNxqt1So7HjXqZuRNDFD+ziBDwb+efJghM4lgjTXSWH9JYP6iTHOxidoA1m+4RH/8BD6OhIx5q4i7omE1B60oZafB28rlFfODTpcJb/jHmxpcEvgj8zsbjvwP83MbPXwR+X0oZSCmngKvAY5s5qdykju4JstMaA6+l2OuqjvmWJmfiKAGwMKdRuiCIK8pxbv5ZuWFpKMjNKwOv1Nx8oK4e1RGVgMZh5Wnq9Wk09gtaEwKjI5CapPqySZxR+vlRRtD4S22udyp8ZuACryyPY63r1M5EyHsvTgmHI3ZW1zH1hNVeluhMEbMtWDo1wF85+CIFy0MTkjTRYN7Brgv8vT61/+wqleuhHpXnbOyzLqVL4N/I4w0mREWJiAX569Ad/RC2UIUQuhDiFLAMfENK+QowIKVcANj43r/x9BFg5h3/Prvx2A++5m8JIV4XQryedLvv+pvXrwrteiiZ+zRYDSWVuHZMUrwKUigFkuY+aO6TZKZMxr4eUH1DTdg74ymLT6l9//K527Bs7ELuDZf8pK5a+vKqS8rwITUhf0Nt46YGhEVoHYw52LfETw+c4V+c/hiNlwZITcnqEZMof2+N/d5IzP6dCzd/X14tkJrg7Q7Z+/gN/vD6wzxbvsgX9p3BOJdFD5X5hIw0RCrJLqa4L+Tw+gWpDc39UJjUKF7SGfme6hlYfzAhM7/JSsytPElKmQDHhRAl4D8JIY68z9Pf6wx+6NOSUv5L4F+CUkp559+0EOY+LUEkFC4ZJA6kpkTWQvwVhzir7G6cyw5mW8nRLD3q4PdJRr8eUbhqo0WSoGwQf3EdXq3cytu8idxMyvJjMPLdlNWjBqkp1SZATtlHen1KLgcBhz97mf9u5Gv8zbO/yrV6Fet8hrD4do+AuyzuWBv0o4Lfn95cOL2FQ2MLMKZ+DhKDoXyL/+mlLyA6BoUWGAtsOKeY1A+Im70ZQVW9htnRSA2lGTvzaR2zLRDjPn4tuymD4k3NFKSUDeC7qLnnkhBiCGDj+/LG02aBd3p77ADmN3UcA8ymjjtrIFLo7glxVjRq37ZJdTA7Avuac5Pu0H8iJLEhrkXc+GkTr18SFgTJw22C1zcXpACN/RppJqEzrCvzCE+Q6hAMR2iRJDXVajgsSK6s9fFH9UfxQ5PoOzUKU6rykFmQhNX0thSkf1yQufhdQfpORInO/sIyZy6Nqh24VN2AQVnQG4/JzKsbs3QtpjOmTIzdJY3ilRS/TxIWlTx6+WKK82qO3Mx7HuZH4gO1p4QQfUAkpWwIIVzg68D/AnwCWJNS/gMhxN8FKlLK/14IcRj496h56TBqobV3Iyu/J+5F7SlpQO7RVQZyKoVcmh+4p/bx3wthSbLn4el3BWsrcJiZrWKumDdNKLYat6I9dStD/xDwOxsrdw34Qynll4UQLwF/KIT4q8A08EsAUspzQog/BM4DMfC33i9I71WIFNKNe/zS/MA9t4//XrAagulvjf/Q43cDUeYDA1VKeRp48D0eXwM+/SP+5+8Df/+Oz+5uRgreyzUmqfHhqC1t4524K3emtrGNH8R2oG7jnsB2oG7jnsB2oG7jnsB2oG7jnsBd04PmjcRUT+gktsCvgT8Yo3c1Ulcy+JzAq2p4A4py6w3FlM4a+P3gjwc4N2yys6qR+b/9zFd4vr6XXmwx2yxSXyogAg3NU652d4qgL8VZ1ohyktw0SE31vWqRYla2H/IpvWSjRbD+QIrZ0jC6AqsFeqiauIVUW7HergD36tud7qktCYZiCn0dxst1UinUnroUeLHJjWW1eZF0TJxZkzgniQsJuUkD3YPhP7zKtd/eTXYO1QwjlE9qd0TZT+anoHlAUriq4fUpLlN3RFCYSmmN39056+4xm/hr/xeCcopd18jMS9YeSSif1unugNiVZOc0OrsScpO6anzoKCEuv09Se2wJLzKQUlBfy4EUiJ5OaazBjmITgHPTQ9gX77wimJuTNA6git8ahKUUq66Rm5WkG11B9aMpmTmd/HRKY5+GFkJwpIc+5ZKZE7hrKc3dGmFBYm4U0b3dISPD65ScW2vUHMk0eH1plHbXIao7DD6v0R7VCKqSylm1zWu2leO1SNQGRe8hDzHrUDkLvQGBXZfKLkiA+WNkIGxVwf8jgTcUU3tNx+qmLHxCIkINPVC+m51RjdxcSuzq9AYluRlBe3dKdkajcA1ajUE6x30KrzlkckpxT+8J/IUqV6kCsEki6o9EZ4cgHgwYfl5j+ic1+l8SrB6XNB2IxgLsqw59r2mIJGXtiIBdHZLJLOVvujT2qay68pCgck6SnwmYf0oZkg4OqZa6tzLoB31/eX4C14qIV1z0UCilGFdZ7HSHoDeaUH1Dw++X5CdBSIgvuHjjEcufkvR/x6S1WwAScQ9wue6afG+0dfyaUE5zCzpD35f0BgR+TTD0YsjiUxKrqXhRzUMx7pJG5xEPb0AQPdmGlrIXBwgHb4N/covQfcidsVnfb1I+rdHYpyGkuqHKL9jkpyUrj6uMaXYE5ps5MgsCqyMZfCWlcUA1rIhUaQS8BbnRD5je4vd//sDv8QtjJ7EHe5T3r9PdkRJUErTPrhGWFb8rLAqiYkJ7p3KDtutQe8lg4FsGy08mFK4p92ireff3It41GVUkathcfVBSPq8ei4qSqJiQGhaFq9A8GGOUQpyLGbQAsidcxMfr+KtZCtd0sgspYVagJRbJh7RdNPhyj7WjLvXjMfaSQWFS0h4XNPdLRAJ6ICid1ag/HJI/bxGUJcFRj+hMBqMHWiR58HMXOPHNg4oxu5Eqlq9ViSY0wtigs6QMUe0lg9RSU7Oo/O5d6F+//F9zcO8cgWfy1w6+wP969vNEmRT/tSpOF3ojKUYP3DkDowftogES2hNqXi0ijfaEYPDllLmfDDetz/VR464JVLMj6B1SC5HEEuihpHANvJqB358SZwXVEzoicWnsl4QliRYLqlZEWAzIzhus/YUe4lKWqJBir3w4g0Vzj4vuQ3bSJCxJWruU6a2zrBYvbzEKRE8nLEncFYE+41J/KiB32iYaiHj51D4qs5K1R2LcaXVHOYs63mINePfeuha+5cj37o8qzkoyB0M+vf8SthYR9UeItlpUtY8GDH/VpL1D4A8mZGZ0clMG7npK44EUzdfQA0F4wGNdumSv6JsSLPtx4C4KVEiuO7QnJAJJb1hXvZ+DKUZXEJUS1j8WY086FCalml9JWJov8ckjl3j+0cNYZ3Iglb3hh4WVp2OsRQOpKwvJ9rggPtRFXsiSn5ZITdAbhOHnwKsqHS3dB3PWwq5LOm2D0kVBe1yJv90uUktyea0PKQWviDFIBO6iRliUONdtlh+WFK9IzIYGAryBlO7REOuGjR4IRr7nsXLcJbuU4pfEpo2CP2rcNXPUzuMeQV9M5bwSw83dgNZuqJ4SSuU50NCWbUqXU4SEzKJADwTDO9aZ7xZJ+0IqFxO8iYg49yGuDhJBaileUXtCkL8hSRZckoxk9UFJ8kibYCBm7vMp9aNq+LU6kv4N/SxnRSOxBGZHkD+6dtunYa9rlFwfr2ezo9jEWjYwO0qRxGwrlZPmXkg21GKKlwXZsw47/3MLqcPVXzGVTNGYRmf0g4/348ZdE6j2WZeJP06pHwBpKu5Tdlaw+niCswruojrVXr9GUFRNu/qRJguX+mkGDpOf/dfs+DtXGB9fwV348MYxd8bAXRIYnsAfjvD6BfnrmhL19QTJ1RyZaYPyCYPSWY3G8YjEEsw9C90R5fTcedQjdmFkQ9PpduANJeSsgKd2XcOLTaKcxOxKjJ6kN6Q0XN0lQd9JibuoEeUEiQ1Ljxfxh2LMckD0WJvueEL++tZdnw8Ld02gBrWUxSctokqC3tGIHUGcgcwNA6lDfjpFS5TSs9WSpJYkulIAXbJ2po9df/TX0ZBkzBBvv483oeR/thpxVqLFqu5YOm3S2RPReaJHVItxlgXlC6r7v3rGQ+qC/EVT0Va0DZ3/tkb2pEu6r8vMH+667fMwmxqGlnJiYZTr031IO6UzJghLAoRED1RgLn5G8ZT0EPyBBG9QYq7rlL6WofTHWfSuxtoT8Qcc7cePuyZQSdWQ5c4YJNmU3pCif2gRtPYmNHdr2KuCoZdj9cFLgbMsMNoaUpcUr2i8dGE3F6aG0c2U0bFVBh5d3PJgdVYFnfGU3GxKa3dKdsokaVm40ybFqYTegEALBPMfyxDloe9UwPpRiVnXMLqQn1TVjcqXXYLSjz5O4kr8vYFyetn4CmpvT2mGH5/Hi0321Zax8wHOooHZhs6+UC3A9nTpezPCWDYRG3GolUPMlppKBSXB+hGBsy7QM3d/oN41iykx4qHfyBC7YK3ruCuSoCyIc+AsK4XnwnXJ0qMmiS3pO5my+MTGNGFFJzeXEOUs3GWJX7NYKrgMPLpIOBBvqZGs0ZNKQO2wRNopXn9K6bRBd4dk7pMCLU5JMilGU8ddEXg1E3dRo7s7wlkz8X6qhT+Tx1kVSFNJXL4TqS2JdvqMDa6TNcN3/3EYFtqKhNX0HIYLLfqcDqWcR6+VRw8he8WiN5JQ/naWzhDEfRFhbJI/K2mu2nQf8Kk8Z1M/LCldVNRvTrqblub8qHHXBGq66NDcn6AF6o5v7oG4GEMKIhEULut0hwVWEyrnI2Z+I8G+4GJ40BtOqe/XqVyIqe83KE4mrO/Xmb3Sz9GjNzjf2Lkl+/wA3RG12NNDQfa6TmZJEhZUHdhZ0RApVJ9LMTshqw/Y9Po1ypcT3I81ic7WSE8XybahtUcy8FrM6uF3fwTBUMzh0cUfuSM1Umjd/P3SYj+/8eCL6ELyjUdyxC1VCy2eN1h/MEYkAq1pMPRSTH2vSX4S/H5bbTp0lCLih1kh2UrcNUO/1dBw53XsNQ3dE+SmBYWLBn2v6ohE0HxQaUkBXP+ihnnFZeRTM6SmKtV094asHDMoX45Z+KJ6orOgM9ssYk503ufIm4O7LKieVt4ChgcilXSHJWZHkFoSLYTFJ3WmP2fhV5SJ2sLTgvjPa0qE2FdDv7Ms6Pa/e9EX5yQTE4rMm0rBai/LhZd3cv7cGGenRjh/ffjmztRP9Z1B11N+f/Ex1sMMf+OB53DmDfKXDXpDEr2r4Szo5K9rrB0yCcqQW0iRQmVtoyfoDaVEOYhzW3Z5PjTcNYEaZ5RcTvkTiwgJdkPJ+vhVQe0NQf6chV9VGpz5qwaFa5LVTpYdn7uBFgqcGYvcjGTmJ8A9+3bJ/K2mlK1CZ0w5pBSvql2mxj4lOen3p4SlFKlDdlaVn6ymoHUwwmxrxBllziZS5QzYOhyh/cDUMCon5K0AgHOTI7Re6MdsCZx5HX3JIl/qoW2IWT1X38fHxq5xenaE2XaJS70B9n56ks7uBKmBiARSh/LlCCRYTQjyAntdMP5nKWFZMv6VmLCSEhbu/s3+u2boNzxB4kpaXx+ksJCy+PEUs64TZ2D9MBuOb4KgqmqD3WFB+nqFy31Fcrub+BdKtCcEGMrND+PtXaLQN7asKaV8ATpjEBXU4k8LILFUlhr6Hiw/Kokf7jBSaTL78gjutGrHc1YF0k3RfZ2wKCj0d+iMlm8udN7CWxnTvfb2lqa3O2RidIWsGd78++szo8jrWfQI5kOd1VaWL+49Q/XhLvUgw4Xnd6EHsHrERIvAG5QktuLgC2niLENjr0XxsqT9w8TTuw53TUbVQlWk9muSXr+Gs6juIb8/oTAFZltJ/YSllNycctvIX5fkrus8OjhDNBgiUjBWTTqHVVYq7F+n4bsYN5z3O/Sm0DigrIVEonaddnyni7czZPRrEqkLhl5MSK7nWPvjHUolJS8pXlH2k6NfFlTPBTQfCYheL+P8gP1UZaRxM2OmtrKrLD6zxOFdczczrSYk3cjCOpXD2mjNy79pM1hqsxyohVY7shGJwGpIsgtKBCM7p4I0d81ACujsi2jtSqkfSW+2Gt7NuGsCNSpIMgtq7tTaH2O1wGoKtFCw/nBM9YKvJBxTWP5kRFAB+curGD3JK/PjlKsdHv2Js5htQflVC28iImeHzF7p35R0zAdB6pLcXILU1bbvlb9iUnvRJMxrdAc1Zj8tsNcEYVGJuZkdwepTEb1BweynBQtP2jjXbKUAs+PdpbP15cLNjFl7bImdj83cdPl76/F2aDN7YhivXw3XdkMQFmB6scKVRh/TnTItX3kchEX1PyIBvwKFiwZhSdIbTimfMNAigVHzNy0B+ePAXROo+SnwakrTKXvDIPf5RXojCdkZDWEnTP5lyMwrIwhrQdUGq/93QwmXnSzSvlDB1hK88Yhf/1tf4+i+GWamazhbvEtVuiCY/5jSuwoqkuGvG1gdyfpBQXdEkp1RgrZ2A/pOhSBh/EsCv6qUCL3dIVIoyZ/Ufnegip6OF6tSWsXtYerJzQwbpxqXF/tY+u4IZkuxBPpORkgdon0etW/bzM1UmTo3zNq1Cu6SxK9J/Kqm1A9rCbqvXsvsCIKKoHIWkkWX9u67Xx/krglUqcHAGxFmR/289sog0pKUr0RkLjhUv2sTFgUj343Vlp+Ayb9YYuXpjZ2XAF748jGcOZN24nDm8uiW1k/fQpQVlC+oHlndF8w/m7L6gCBx1GaAvS4Z/5kpjK4kzqhe1emf0EgHAkrnNLKXLcyH6qwflRjdd2d6Z1Fn+o0RZhqlmxn0ynIfl16ZYPKVMYwzby/Px74Ws3bEJCxIkqZJ66c6aG6M3udjr2tITVA9rSgvwUhI7oZOa7fqxgr6E1JLdfk7y3dNCLwv7h4qyj2mPbWNrcOtUFHujdtpG//FYztQt3FPYDtQt3FPYDtQt3FPYDtQt3FPYDtQt3FPYDtQt3FP4K4OVG93qGgoM4oeHW9Y4RSup5g9pf6RWsp/SktUm1xxMsVugjceEedUT4DZgYHXw/c+xnhE7qmVH9ol+rBQuJHijUd4ExF6AJllSVSQGL6iqnjjEVYL4iNdDA8MH3Kz6r3n5iT+oNq+9QcTnPWPtgZu9sBZk7grEqutCI52ExDqPMNyit1Qmy8A3lhE4UZKnJX4QwnZJ1cJSxJpvP2cW8VdHajmsur88cuC/A1BfkqJpK0f2dBT0tWui1tPMdtQuqgcTcICZK6bmC3FG2o/GODVfnSj2GCujXao/ZG8p+XPhmCmuNdNwqIkyig/LC0C/7CH3lLaWubpLFqk3mNnh8BqqGby4gWd3GOrmP0ezb0fbaB6A5LuCKSWoHk8xFkV9IYkI9/zCCqS7Ky69t2xRDWQv24QOwKxu0vlpIbx7yuqB6Gs2iE3g7s6UKNapIxwS9A4EhOUlMNzVE5o70xxlyXmskljl05QhqAkMHqqvU/q4B301dbs10xaP8K8V3g6YaKj6x9NT6Z70aH2nIXdhL5TKWZHYvX38GsS+6K74ekqsVrQOpDwuf/qZSoXE7Jz6j01Hg3wnq/B1Sy56Y/247PXhLKeDyUD3zLojUj6TqTMPusSZyRRTqnbaKGgNxFRPyhZezIiWszg9QsaezSKV1SvrLZJ1aW7OlDdGxZJRuIPxggnwe9L8foFWi6icEXD7KhGD6sl8QcStARSXXXPZxYl9lWH1JYICdknVt/zGM6yhh9/dHYR1U8uEGdVJlo7vLH/fiZPYitCn9kS9AZV70DllMaffPNxav/NdXb/9Ys88clzDA/VCcuSKCfpjH20Dc+dPRGJA/WnQuxWioih16cRFlMycxr9b0TkpyA/qZEf6FA7KSictjB6guBoDy2C8gWPNJNuWpjtrg7U3IxE9wWVkzrF121SE/yRiPwrLs0DCYmtuuvrR1OsdX2DRs1NYqBIYPTpWWJHYP3u+xuj1XLdj8QOcvnlIRrHVVeV2YG//XNfxh9IqJ1S7oDFq2A9VKf7oMcTv3WCX/rcC5yfH+S3Br/H9y/vYf56jfI5kE5C6ny0gZq/bBLWEjIXbNb+clcREfMCLRZ09kbMfF7H6IFTT9G/WSbKQetwhLMqSHyD7IKkPeFANqY7srlrfdd0+L8XVp5QHH9/b4gxa6thp2FSvhwidQu/BkjFUo03aNGZRUl+NmTlaY3MlMnUyRHcmmB1IsVZ+NHHylvBhiDZh9tELDUJUlB8cBU/MvjGyiEoRLR+MWCw0MF/3KCsJ3xsZJJzjUGerE0ReSZ/99JfQHoG1ppOYz/krpiIFOKP0ARKCiid1YjyoD1XRHuiQ6dgM/A9nbUjBpkFZcOZGhpRDqy2wL2hRqvSGxbNPRK7rmHMW2TnxPvSxX8Qd3dGvWYost+8rfhFUn3Q038poTOR4lclnd0x7pIkOydYfiKhtRta44rGUb6cULog6BwMKV784Nm7rL13ZWAr4a4IcldMNCH52YmzXFzqZ6CvSa/hUnW6ZMyI+bUi3/zKw8y9Mcw35/fzuSPnWJopo/U0qmdU5k1caB/+8M/3nbDakrCoSIGpBbnvZul7wWD5UUUlSk1oPuWT2GpV336qhz+QoPugxZKonNLeH2E1Bb3B+yijxq6at1lNgR5A/0mfleMO7ZKF1RKkBmSmDbx+9bzRr0FQgN6goHDeZP2QIg2WTlqI+IOzZbXSocvW0VZ+EN5YRJiPyGYC8raqz2TdgJwVqv7a39tL42hK/4uC9hiMPD1HO7A5tz6EuW6QuwF+WbFutUBgz320VmzNfRKzKchPavSGNoyJHZUlpQbd0RQaFkYPWntT3NMZwoKk8UCE3tbpe1nD69fx+tS6YTO4qzOq4UP1jCS1lZTPynGH9s4Es6HhLipGZ1hQuqRGVzDzedASSXdHSvyxJn4txWoIrKakfuzH28UeVFP2750naVn80q6T/NUd3+eVtQnaZ6os/+ko/d8xERKK53XsVkqUlySpxieGrjI3W0ELBa09ktQQG3XUGHf5o+U6mU0NuwGdCaVrFeXAXUuxGhvu32ZK/qrO8FfnMNtKM2Dg9RRnwaR4WbD6sNxgNwhGv7m50eCuzqh+VRI7alU59H1JmNOIMjp2HTrjitmZWBAe7TFQadFbKLP8iEZ+UqPlZKicEzQ/2cOvuVhrHzz0VzNd1vsqH4q2qjRVCnGqHv9x6jjN6yWcVQ2ZV9RpcVppVLX2x/yVv/k1/vHJZzleneXr/98nKftKOyApJgRlDS0QlM/otMflR0vM0xTj1lnRCCoQ51Kk0LGaUD0raew1iApw7f8wQpxRcqFBXiPa18MPM5TPgNevEtDkL+m4m3CYvqszqu4LrDYMPy+p79NZfSwlyUjaOyXOviZRfqP09HKGlWYOZ8omLcWEZTDrBmFBoF9zySwKovfhrq91M4SJjqUnpLkPR4fJWdS5/vw4xqt52ldLFC8rBUBSMNYNWntS9FBS3tHkn7z5KX7lyOv8yfcfoTOa0hmFsD9GeBpapKohzX3yQxGBez+EhRR3VdLdHRGWUpK8Khm2dylhiygnCcopzhposcDvS+kOC8SMS1BJ6Q0KtATaEymFS5vLkXd1oOZmJe1dCWsHdcVJWtQJKwlWU+BdLSISxa3vDUnSqSwAmSsWqSGx6oLy1ZiolBIWQAt+9FuNT5Zo+Gr5PD723vXWrYAWgvmxNTLzKiPZ62obuHgZaicEvU91eLB/jrOf+N/4/515BHSJ1dDIzgqcORN0qFxMCEqCzLxG9dRHO/SXzwk6o4LMpImzqqG3dDLzGkZPIHVB6TJKpHhXir0qkIbKngjQPWUhlBpgNTWM7uZusrs6UDPLCZqv4Q8lhJWExJKUzhroPhg9Qd+bMVosiTMSwxdosVpt6gfa2A3J0sNquI+zKWbnR3+owUBCf3brZH/eD63LZey6pP9ERPBsi7iQ0toNld+c5tDgIl5i8sC//m9wzrsgN6jO/ep7ZlqnM6zT2R3TG0pZe+ijraMmjthgz0LxWoruCfIzKfa6YOVhiYiVBuyBf7JInFWBHZQgHfbJzm0QDasp/mCifLA2gbs6UNcOKbMFaaWYDQ0tEohENXAYHvRqutpWbWhE+ZSwINF9SC7kCUpKUtGd1+k7AVbjx/1uFIyuuuSdYQOv4UACVktw6eIIw26L12dGSS3w+1Kkk+KsS6yGaljp7g9vqh2mmRR3/qMV3u8NSZrPeqS6pD2qGLbdYQ2vX82Vg4og6I9Ze3IQf0dEUBaEEz79f2bj9Ss9seIloRbDK/dRRvWGEpxVjeprBgOvJ6CBXxOEeZU51x6NcVckxatqf9nsCLwBiCpKWynKKiOw9YNK1OJugO5D7AqinKD4poU0JEOfneGffu53GbSbaJoy2bCaGkRKW783qLrDat83sVqS6pkNp5TiRztHjcd9xLRLnJWEZUnpksToSqJajNkR+H2SwiWD5SclA99VN5GxYLP4yQS7DqXJiPazXcyW0mfdDO7qVf/uPwqZ+usQVG2au3WK1yRen9hYRAj0zttcdalL/D0B2XM27opOaiqNqqAkqZ2WNPb86LdqL+us9LL0Zbof+nvqPxnhV3SWnpTIXMzAQJPrb+zgt6/8BqXTBtlYqkVHCNgpVhOkLjC60NyrtAO8AYG1InBWVYnoo0L1mw5SA69fGdQ19ikR5fxFk85xn/wJh85ESm5KZ+npmNrrgu6hENE2cFdTbnxBQ647yGM93Nczmzr2XZ1Rr/+Mg3bDwegI+k9I1p8JKE6mZBZUH6ox2qUwCeF4gBzx0VZN4ozShNJCpWSCgKXHwXofuXwRQ5J+NJdi7bBJ4a/OQiFi1+gKY4U6+QPruDMGZk/SGYVgj4/UQWsb2A2J3UAFpACjK9nx7R6GJ2gd/GhHic6owGmkmB1YflwqU4tASdjTMDF6ksIVVZnIzBqsPZiSO22jexrtMY38VR0RC+SCg993Hw39uq8kflITlh5Vrh5eRaNxKKXvzZh0MofhSfJv2rinXDLzGrGr2uH0QJKZEzgrgsoZQVB+/2OtzJU+kvdkNSXzXx1DaJLJG/1cWevD+FKF1ILVR1P0QKAtW3jDMdkZJVdp9CTeoCoBmV2Y+3iGxJGU3/xo56h2Q5l9eP1QPqukgkqX0w1dWOXC4qyn9J7s0puI6HtN7fkj1ZxbJKB7mlpTDGyuz++uDlR3Ccy2GgZLlyA/kxJU1IIkdjSiaozZk5gtidWU9L0ZkJsVVC4kygPAVCK1nVGBNN7/DtbbOvFHkFWDkqD4mUU+vvcq+WoX780yfkUQ51KcRR2pqfeXmzJo74uIXUFrJ2RnNOx1NberXEoQidj0yvmOz/2TLdafCImzKVZbUrmQsPT4RumpmOAuCbw+jahtUTxr3lQIz8wLKmcF3Sd6JI4kLKbwPuXC98JdHaiJq6gPeiAwfMnC5yPsOiChO6Rhz5usH9CpH5bEWUF30KS5P2H9oK76NffEVM8kmG3IzL3/h2qta8w2i8yulj609+MPJez92SuM5eu8cH0nnekChWvQmUgwx7rUzsTYdcjOS/pPBJAIOhMJcsxT7tDDKUEVVo5rZGcl+Rsf7WKq+ntZ3Ks2I99NWXw2Zv7jAmdFw+9PcadNvAGpFrrZGPlsncqFGL8miTOoPuIpF0ohSSZFpJu7yba1pz4KCPCGYw4fUnuGcapx6crwTXvJ/9JxT9mg38+Ic/JmkHqxyfXTw7hrd/VgdtdhO1A/Qsy3CjSvlrG3g3TT2A7UjwBGWzD5jZ0AW+Yl8F8atm/tbdwT2A7UbdwT2A7UbdwT2A7UbdwTuKsC1R9OGHwtJM4pbSOrpTqERKp0jvRHGup7BH1vxtQ+vkBuVhKWU8LDPaKCxBuL8PYElK5uba+mNNW5mD1l2T78go80VPudSBVxz+xBeMjDH0gwO0q6JipKnDX1ZTeUsVtm6cOrXSeuxB9OiA732PO5yZtf6bH2u9ypb+m1HuhgtaByKcHw1ftPMuq9SEM1tnt7AsyuKsFJXb23sJLirCm1l+yCumb+8J1x1u6qVb87qzP9eUF2RlA77dPcZWO2BHZD0htNiGbzyL6EzILB6lGD8M1BnH5BflIgrrmIFFLdpDsq8d9fb2LTGHhNWTVO/5SGvawz/4zqJIpzksSRGHVDyfNccNUHKSCzoNRcmp/wsM9mkEKJpK08AvbKFm9/auCNh9QGWwzklI7WW84qAHv6V5l3Qvz1KtxivEaLGYptSX2v6ilIbEV/6Yxr2GvQ/pkO+vUc2cWUxNHQQkgNSCoRiW3TOh6AFPR/1yTK3llfwl2VUXv7AvKTGlKH+j6b9gSIB5tM/MpVPvnQBbI72pBC8wmf8uWEJJuSnVf7//VHIlq7oLUnhYkeUtvaQJh/xmD20zqFSzpRISU/LZGGxF5Xpm3SkCSm0lQKSoLUhsL1BKstMa5k6I3FaAlEGUHuxtZd9tRWo0iw3+PwnrmbQfpeGC608HbcejNI4ZrG6kMppasJegh9bwjyV3XkuIfVlgTzWQqT0NypkZqQXVB3gGip7VRjxcJYNln7vE9QvbMR7q7KqPa0DUK16EkdSpcky+M2zmCMhmSw0KZYWyZvBpz6jRF257pM18o4L+ew31CiE9nllLVDWQxPbro5930h3qZWWE2N2JGEwyFhv4Zb9jBfKODXFPMgqErMGcHKgxq6L0hsSeGigeGp4THdwqse5SWH989u3Qu+A4kF2Rmd1q826N0oEOU0snMS81yGMA9GT/lueQd89CUbq5Oy8rCG1ucTWRaVUzrrj8RoKzaDL0rq+27/XO6qjBpWEmIHggp0P9ll+XFJ9qzDSxd2c2plGICf7TvFlUYfjw3d4C+NvERfsUPwZJviz83T2pvSnNAJ93p0xrY2oxpt1cUUVNT8sv5UiLFsoWcj/IUsUQb0ELzBlMzchov0olA6pquC1gMhRg/8qmpd3AqE5ZSxI++jU3SH0AN1U2nfLSH6fUQMQVkQFSR+v+r17Y4nOJcdBl5NcWe7lM+r6z7+ZYnVltReMtA9QfvX3qch+BZwVwVqZlZHpIoOnHkuh8wm9L8R4MxYpF+r0fh3O/hffucvMn+5j2+8/AC/N/cEc/MVTDPhn+z7fQq7GnQOhWRPuRhb3KxvtSDsS0BCeMCj/xsmzoqAeYfh70J6rI1flST5hPaehO4OSXeHpPqGRnc0RXR0RCoJi5Lc7J0v9BJXMnpsgay5OSGH8Z0ryFvM6Pm5BC2C1oH4JgVFamq6kZ9UohOFKzqZJUliCZaeKrL2cErSNll8TOkUrB+V6J4g+wfF23iXb+OuCtTejkQpS7tKltFaNJn8NYHhQelKSGuX6uzPTquJ+eXpAfaNL+J5Fn/j4q/yt/d/m//bE1/lk7/yGs4ntpb2XL4ckr9sEFYTzCsuzb3q0pUuCvRQop/K03dS4k6bSEPiLqrM0hlXXK7stM7SJxJy04L6gS3I9oJNBylAzgpuWQdu+UGNoCoxWjpJTmkqpIZiRMQZgbugEWWVJlW08ZmJQJC7ZioKjVBN66mtLNjvBHdVoOau64SllM6o3FDpE2SuWVgNiR6moIGzolybx/8swblhc/2lUUrfclk5NcA//me/yD888TlcPSKIdbwdW0fVmPukorm4cwb2OhSvpCQ2dIcF3QEdbzChNa4RVFPKb+r4/RvZx1QfrtRh778NCcrcckZ7P5gHP3goXerkuTAzeFOz4C1E2Vsrj9l1gdkWWE2BzCRkpzWKkylJJcYbfFs/ylmLSGxBbhoKVzWcVUn/SZ/2uKB+SCJiQWfHfRSo5cuxWj27EsMTIJSMZGccJn/epO9kSuOBGKsJq8fU4inOSPQQSCHcGF3+w7kHeWZkCvStq1e6y4LsvNLb7z3VYf1nPDUPXVJkvB3flqQm5KY1vH5lhZ4aEquhNFy1GJp7MgT9CaVLd3YuQTWlL/+jdQhSKTh3YZTWySppoFNyvHf/3bq162J4EqsJw897EGoUryd4fRrWvEnuhtKWApj9jEXj4VARLx1YfSxh9aiDtyNG9wR2XdWP7wR3VaB2B3ScRYPh5yXeYEKUVZP3sD8GAfMfh/KbOs3DMVIDEUGaS6gfgso5SVBJEYs2hpnwyuLYlp5bqkNzH0hDMvAHLtnvZxEpNPeqDQmRqMK/X1O/d4756IHKIom1wYgtCjRfsPrI7c9RpQn7H5z+kcP+5GqV69+ewJ010P33zmJ2/dY+9tx8gtGTzHzGpXxKp75Xp3o2YOilmDijGBig6qulExZ+vxplnEUDLZLs/oOYqKBGx7BwW2/3Ju6qQG3uUxly4SlB/pqOuyLpDUpET81J82Mt4oxg5BtKyieophh1NY42d2uULgmMsS6OHfHJ4atbem6JA6TgLmgs/FJAZ0wSVCWprUQv2iM6/j6feGNYzZ5zyE2rKgBAfjolKEPtlFo03i788QDtfTQbk0S75YL+B6EzpLP6RKx2nrICbyhl4Wmb+j6T7hGfoLxxk8aC6jmfwjVNyfh0ISwK5p920H3FXXs/FvCt4K4K1F0Pz4Cm2KeZ5ZT6MSUXM/AK2Gs66YtlJblYUe4blbOCzKIAycZEXxAuZKnlulxqD2zpuRWup8TFlDgD9pkMzprSEkCoQnecFRhzNuZwV9kGdSEsCOxVDWcdmns0+k/GdAfVDs7tIlv0f+ix+VaBS/MDrHkZ9LPvT/SfaxZvOZBbeyVG08DogO4paU+pgdQgd9ohKm0I+s4Irv+0jbuSUrya0tkdExYk3qjaXOiOqyrIneCuCtQbL4zirKgJ/OInUtw5HbshWXkIgr6Ezt6IoKzmhL3dIc09kHysSZxPyU4L4ix84vFz5M0AS4sRm2Q6vh8a+wS5SVWKqVxM0EIwOhqZWR2/qtGZSLAagsJXckQVpYkVlCVaDL0BFbgrDxiEJUnhxtZpta55GbqnK5jnMqyd6lcKgRuIs5LhHevvfh+LecQtHl4KyN0QNJ9QOgNGRyWF3qAaRbI3dLKzSpHGWRN0dmgsPQHWio5IIHfVxF4XFC7r2PX7aDEVllKMnipB7fxSqppMBtRd7M7q2PMmVkMgBWgdAy0Gr2uBAPETa3R2xXiJyXynwMkLEziLW8d7zyyA1ycJ84LuoEZnPCU7Jxl+rosUKuNbTYkWSZw5c6MevCEiVhdk55W3UvWsZPEXNukGtoE4L8k57/7fMNbRPRUERkcQZyXezhBvZ0jfQ0uUf2AhtRm4Sxp+H0hfpzMmyS5IokJKUopJNsSVw4ISFvYOq8WlXdfIX4ewliAS5QuGALt+Zxn1rtpCNXoasavkGOefMcnOKp2pwqSG1/f2TklhSrK8I0FbM7EvqkJ08p0qR39ukrNLQ/zynjf4T3/8Kby+rTu3KCswO6A9Uyf5Xhl7TbD+aIRfyxLlVKUiqErykzrOCmQXE1YeMIhdVfDXQw0thtaEhn7t9hwi4nxK7Qdkh95qafAHEqq76thG/EOrfIBuZLHUzGMv3jrzNc6qDQpiVbXoDarRzuwoYQlnTdVOc7MC93mHOAulSylrxwQiFgQVtWVs9ASpeR9l1My8wHuopyTPqymZpQS/PyXMw/ALIaUrCQhYfXijsK5BcNCjcl4NP63AwbUi5oMS3ZGt3UL1hlIKUyndq0WMrmpz09r6zYwhYsHQ80qMrTMmWT9ooCWqZmp2BN1hVZXoDanC+VYha4X4ewP2HlYNKe8VpADXp/vgZGFT8+NgNETaSnQ36oswO1A7nTD8XKCM5wqSJJcSu2rxhITOqEbpolIcHHoxRuqqH8Fs30dzVK9fkn8xQ2tPSmZaZ+kxDWdZw12VrB22aE2oDYHMvNJMzSxIdv/zlMbPdvEHEuJUQwjJydWRTXttfhDMpkZnVEMOBnTGwO+PKV8Q2OvKC2vXl3p0hjScBUOZY3jQG0rJLEpy00q3fvAlibOiUbq4dTdRzgo4NDGPqb/3xLMT2moBdRvzdXvGYuQbAmddYqyaNI7ErBzXuf4FNbVBQv6ajjeYYjVVn2pYlKwdk2QWJMsPmYwcXyAzJ1h/bJNWfT+AuypQtVjNR/veUErMzqogzqnFU+tISHYhJcmmyq5nSsfrF8x9Iot2Jk92R5uPD1wliAyWJmtk57e2OdnwwetPMaYcsrOg+RqprhRAjB4sPJ0lLII/HG1sWEgoh7THBM19EqMnWTuik527/a4uo6Wx2stu6n+mFysEr1Zw5zY/y0scaP56m6AsGPleTO6aQWFKkplTQ7/RU1OzgZfVZ7T8ZIK9LnDG2hRuxCCh/R+HSG3IXbY2ffx34q4K1NiRZJZTmns0tEDVHrVAkL+u5klSE+SmDNaOSdInm3gjMb3RmDgj2V9b5ss3DpOxQ+xlneaerT23t0wWwqqafgipvrQYOrtj9BC0CAoXTMoXJY0DktKLNqklKe5bx+ooq8zUVCYatwOjI1hevrXmjlbgcGWpD/vq7dsRGT1BZzGH1ydpjxpEeUl3SO0+xTmJ35+ixbD0mEZqSwpXDPQAhv6pTXO3SVSUdJ7tkp9O6Q3eR/2ouRnB4lMpWpjiLGssPSmBhOVBSeWEwfJTCea6zsAr0Fwpkg+U0l93Z0w7VB9Io5PBH4jR8hH6xa2ztbNaEGdg7Kuw8BToPUH9iLr4RlN5DOgBmG3J+kFB7ob6QKUmib5TI5iA3DWN9i7lm3W7rX7ONZvLdj/7hpZ/5HNSKZg/PYjVuLMpRjAR4F6xN6ZRynHQ8FRVRl8RlK8kzH8MrIZG9VxCe4dOdiFh7mOOskRf0/DqSoExOwfxHVh43RfaU1FRsv+x6wCcPzeG8xFLhm/jznAr2lN31dB/u9ACNj1328a9hbtq6L9d6L6g9UI/LfgQDSK38ePEfZFRt3H/YztQt3FPYDtQt3FPYDtQt3FPYDtQt3FPYDtQ3wMihcyyJOhLCSupoli4ksyiku95S38qqKrWvcRRtejipGokjooSPdjYBl6//Tq1NKF0VbU7ihTcFXVskSoTjtycxB9IEOmGNlZBMQ4Mj41tXFWcD/pSBt64s732d6JyMaFwIyWopSSO6hyz6+pYaJCfloQl9b7jvGrg8fYEjH29TVhSPQH+UILd4JYj8JYDVQihCyFOCiG+vPF7RQjxDSHElY3v5Xc89+8JIa4KIS4JIT5/65fg7oDVVtI77oKm9KKuaeRmoDcsyM5DUAJ90CM/pRH0JwipPJhWHoGwnJC7IejsihGPNAkLt787ZK/D6s95xIWEzKJitSKVflXiqN2tfUdm1Y01v3FzpAKvXwWH2ikDzRf0+reuEtncqeOXNRDc5LMFZWWf6Q8mrD4s6TuhxOuqpyVBLaFwyubGT+XJzgmkoSSQGofjWyb9bSaj/m3gwjt+/7vAt6SUe4FvbfyOEOIQ8MvAYeAngH8uhPhQtopSC/y9AcVnltjzuUkKTy+THmvj7bwDrscGDF95V+34dkpqQHtccYF6Q5Cdk0Qdi86YRK8ESn/pUEL1lEBmE/JzMbnrBr2OjXkHQhitPQkDf+iQmzSIXUF3BDrjKWsPqGZlsyu5MtevSIb7lDCb0YEknxINhJhdSXc8wWoJ6ofu+JLcRGqCFks0X6C3dew11bDd2x2Sva5TPidYeUhpcgVFQeVNZeymharPoTckcRYM7FWd/OytUdpvKVCFEDuAnwb+93c8/EXgdzZ+/h3g597x+O9LKQMp5RRwFXjsls5mk4h2eRzZOYeUgrNTIyxNV9A0yZ6dS5iP3j4/126kdEdAGoL5Z3SiAhSuQX4mISpI1esqJNlZQdyw6A4Jqm9ohAWBNW+S6oqFkMkFNPfefjOG7mnMf0LQ2R3T3ikJ+xJ0T7kRJk81WX4MtHkHr1/tw5sdQfBwl4Fdq+irFuuf8+h7TVOO287WbZX3JiLqD8VYTYG9poIwGgqx5010H+oH1bURCbhrkt6QMkazWorblliKtYsUdIZuLYfdakb9x8B/z7tpYQNSygWAje/9G4+PADPveN7sxmNbCn8oYWJojbOnx2m92I9zxcadMeBkgZnvj9JpOziPr6l50ybR3KPhrKoWttyMIM5I1j8e0B3SqZ4SOGsgfJ3yT8+TmTWU7M1O8PrVXHH+CzHhgx16LQd7/faXAYmb4ixp6F2N7KygcNFA7O4SVCT2NwukfSF6ANl5AZokPN7lnz767/ml0ZN86uOnqRS7GL7EXZWK77RFMJqG6mbTN6ZJOYkzZaN7grAI+SnVBhiVU9YOK1EOPZK0dqfoPUH1jNLpshuodsBbwAdeRSHEzwDLUso3bvF9vNcV+aHbWQjxW0KI14UQryfdzY+P1Yk68/Wi4kX9wKtrIVhXXerzRXgfavGPgoihOyKx6/ImH9254iASWHksoTckkVbKjav99PaEFD+zSPmSJLUkiSUpvGEjrmWpPG/h3oFor7Ok4w0llM8qi/HeYz24oqgvRg/0RRuRCtp7ElJLMlJr8JXmMb66dBhTS1hZLpD7r2dZ/axP9oF1goO3z596J1JLMvCCRmpJghK4K4Iko4iMeqiypbc7pHhBVwa+4y0lx5RJsVqClcdUQ7nXJ7H/21sTebuV2/1p4GeFENeB3weeFUL8O2BJCDEEsPH9rb6zWWD0Hf+/A5j/wReVUv5LKeUjUspH9OzmGkqCaspArk10/UdTg0UM7oxBakDxmSX8/T9MM/5RMNsq2A0Pamdjqm9KlTXWU4oXDIpXgFhg1XVER2ftxUGWnlEVgMyiRmKr/lVvQNDeuam39i54YxGiEuLXBKXLEHdNwsEIaadKg9WU5KYlZl3j04+dZb2b4WuTB1ls5Rl31ji6a44d2Qa7hlf59I7LDNWat38y70B+UmP9iMBsCaX4XYDSBUWLtj62SlSA/DmLsADFayne1SK5w+sUzxuKbp2NaY8JMguCqVdHP/iA3EKgSin/npRyh5RyArVI+raU8teBPwF+c+Npvwn88cbPfwL8shDCFkLsBPYCr27uUrw/0mJMkBiY3Q8ezpxDDfoyXYb6G7f8+okDzrrA9CSN3QYrj6gFytJPhPSGJY3PePS9opM4kuJ4kzgrlSbVmsDfsF7XNzReDe/2h9zcNRP7gktvOMVdi9G6Os6MhbWi0zoeYDU1Vp+IiUopz9/YhWUkuN/Kkb5W4vf/xWc5Nz1EN7H4a6PP8ec3Dt72efwg7IZECwTOmiTKCcwuNPbD2FdTyv/vHO6SJD+TYrVg/YigeAXqK3nl+P3xFaovWJhdCEsQD93awvdO6qj/APisEOIK8NmN35FSngP+EDgP/Dnwt6SUW0dkBybGVlhq5xAfUBqUJlQyHmeu7mDxfP/7P/kdyM+kGD1JZikgLIAWCPSOhnvRIbEl2qRL4ihef+dSGXdJwIaWvx4qarDRVQuIvpN3INQmISxJctc1bvy0UqoWCeRmQLQN/IEY4SSYfR6/ceBVwm/U6I6okaC5N8WcdDkxs4PZsKoUVLYInRFBblrSOAhhXjGD8zegvtfk2n9l0BsSNHZr6L7EbAmceop73aI7Aq03auiBop5LAfbkrVnEberspZTflVL+zMbPa1LKT0sp9258X3/H8/6+lHK3lHK/lPKrm7oKH4Cgmt6y3GKqSwq2j+jpm7J17AxrtHbBjZ908MdCBl9JKV9Q8jwDr4AWKQvy8sWUOJcQ5VQx3uiqv7V2gTeQklgw8xO3+04hLEqcVYHhSfSeRutAxMh3exi+uknG/1SiL9ik17P80dSDBE+3QYPW4RB0ZW6RywS8WN/FXz7wEnPT1ds/mXfA8FAMW00qSrRQ5bvWkfD/z95/B9l5pemd4O98/nqT3sN7Ghj6Ytku191V1eptu90tjfzuKFbSTGhnJE3Exu7E9EyEFBOzI8VqZ6WVZtXqmZa6q211dVdXFcuw6EkQBOGBTADp3c28/n7+nP3jJEGyimQBIIsA2PlEIBK4uHnNd9/7nnPe93mfB7Nr0NsVEVZ1dSQYlGwe3JoVG4+xuuLGzFiwMySYurnP8p7jo4pbqPZEIzGJNMiOdZArNy8kG5UU+TlB6kJcMtg4bOI29L7VH9AqfUkWVh9TlM9ZWpdKgJEqjKMtCt8oYndh7bH3t5A4LYHxeJ3q/+RRf0KQveix+Cmb1NMHl5VHLTKrguxPrVH2fK76fUw9ukDGivl0/0W+efgQnhkz26xydnkEb+mDcbMWKTQ+38M5m7txdM4drhM/X8WtK6TtEOf0FmFzZ4xbc2l+tod3KYdX0xUUb9oj94pL/N4KRDdwz7VQnbpBJ7qJ5cKAgZEmfmIjT96a2rG3IehOaGlvu2liBluj3NcFvSGFt6H3Xe6GVpF26wq/X1CYT3C+pcsEzl9dRVkKI7j9S2x1IX2+wsojGbzLnpa9zCuioZiB13S7NqwoVq9X2VXY4FcPnuRgaYUzM+M8vbGXvzr6PM0oQ7PjkV798Vulm0Vrj8R+PUd4wKd0VeLvCQlfrtLdkdAbEUhLH0ilLcifdzFicE/lcOrQ3gGjf2KTuor2TsnY999dPvOtuOcCFWDu0hAjhTb+xDvv/9Kswh9JGM63Wdy4dUnu7piiOKPVUaSt6N4XEFUlCF2KicqwcSwlKkv8QUFU0rr2tfts6g+kJBmB/DeDVE6bFKdv/xKbgeYXdMe0X1XjIDgNQf+zNpv7TZTQPX2RCL71naP81kuP82ffPQGhwWaQ4795+edwzQSUIL/wwdVR7a2abP6VDK0dBt51l4HXEsy2SbTbp3V/BIbev3ob+osclhWZDS1v1P6NFpWLCiMRXP6rNzeTcU8GqlszWW4X2Ld3CeuhOv6u6Maf9P4OxQc2MEs6fcSNWx9OGXxFUZhPSD1wawb2nMvEXyisrtbld5r6Nbg1fbhpPxhQuK73qaXzFnERVk8YFGcThLz9Omr9PsX6J/V+06sJktEQ6Wpd/PJ0it0SjDynKE6bZJcF3oKtdVpTgf8fRlBSsNAsYZ/JvUMl+/aRXRb4Iym5VUmc14SUzYMW0pPkT2bIX3CQNgy+GtHcq8t0RgLtCYPOrpR2I8v6F0LKFyA3e3O7z3syUEUC8akKl6+M0mllELZE2Drjyes5Oi/1k7b1fszs3PpbTDKClYdtrcF6xCeuSnoDJp1JgdUVDL7iE+8McBta86nynF7eEk/QPBoiEi0rXrvPRonbz2TZJYPqCw5W28CIYeyPbESsZ8SCqkGaVdTuM2k+EDH6nU28mnZiyV81qR1VmEsuv7TrFHYbLTf/QUHqw1xzp6Hru1kt76NsRXdM4T/o0xtSrB3TAna9yQQjFlqnIRBYyw6ZsxmsQNGbvLl9/D0ZqPBmQd+75N34k5lxsFu6x2wVI9qRi92+9bfYGxbERUn1Qoj3egarZZBbSbCP1lGmYuljGYxFj6Afdv9uRGO/NlMQSrcS3zD/CgYkvdHbD5DO3hhp6j1f6kJYMCjMae38wlxC1J8QFyTFMw5XfqNCkoHeqCLJgd02UBMB//aVjxHnwdx1c3vBm0HrMZ80q0iyiqCqv4hxVWJ2DYozUCz0SAZickuK7JIAS1K5mLL6pCQ/Z9B3RtE7FFB7QNz0aPs9G6jvBWXDSF8TpQTiNsqYcU7hbhhc/xuS7s4E6ShmvwLO18soQ/e3M+sCrwbdURe3bmyxmSAYTnDqBt0dCSIRxOXbJ6UYbZPGgzEyKzFDqD2kC/9WV9DcbVN+3cbq6RaqV9PeVkYIXk2RW1J4ZzL85w9/F3/sAy1jU/2Oh1PXqjHtvQnSAqttsOd3OrT2KoyvVcldcaidkPgDiuIZB2MrsYRlqD0oMNYcqucUSf7mvsgfyUBNHfWuqnY3A7sjyKwpcqcyOJUAaUHllEVrly552W3F+NdWaRyNaOw1cOu6bLZxvyK7YGF3wV2zsHoCZ+P2GY6ylOCsWyhD0R2XWC2D+l4blC6yu039JXA2zBuOgXFFElQF8qfriAT+l29/FpVNcZ8v3Pbr+GHEeUG0x8fZMCletgj7JMVpuP7lAm7N0MZxFYW7blK+rBUNu8MGUUVi+VC4CsPPK9aPQ/HqzT3nRzJQ30DTv70pf2Xqi5tkwTybJzdvkGZ0djbaFvWDcP0XhxCBiT8Va4NgCWlBG6b1Hu5hRNpZue/c7WfUvudtMquCwhULu21gdQV2R+E2FK1DMd1Rg9K0NnvoTEqQULxkkqkpwleqdCdT/sXP/v94ZP9Veg/1bvt1/DAyGxLZscmsCVqHYvKzBm5L0XdGMvUnG9gdTepOs4qoJNjx5asU5xKMRBD2Kdof96k9YCBSQXPvzT3nRzJQhdL6S63W7WlP9XbpfWecVwycSrQKtg3RUIKyFaPPpCgTMktaBduItS3m4DMWVg+817L44wmNA4rG7tu/xK1dWstKOpBkNGN+45GE1BFYDQt/UNLaJbB8QfmCwKsJmvdr82J3U3fJ/u3Sk7z02l7M6Q9OhyuoGPS9YhL0Q/aaTetgTGvSQEi4+stVpINuRCwK4jzM/sEuFn4tJjcvMEJB7vksQgrKFyEdvjl90I+E9tQ7IfNojUYzh3vhg/uA7jkIMI836M93WXh1FLv1AVuvf0D4S6M99U6oz1bYO7p205v1jyKikmKqUidnR6TZD8jT5w7hnuv13yy8ZZO55amP7hu8CTgNwWy9QsaJP1DjjTuBv8yf418KpK+U+eAqqHcOH9mlfxsfLWwH6jbuCWwH6jbuCWwH6jbuCXwkA9Vt6H68MrXmkeVzQ4cpqsitoTv9Jzzo03chJSko4sKWltOxJtWLWkZHuh9OeSv1FP6ekF2fvcaez10l91jtHUUjLF/rYuUXteaT095y2KtI/KmYZMtF0N8V3bivGerHr1xOsQKoXkqRDqgH2+/6eqxA0xYLcwoM8HdGeJv6cYXUjyek5jckR7raJUZqkYmorLC7+tonBYXxAVANPpKBGhU1i8juwNjXtUCESGHwVEKalZr181CXzpQk+3qGxU/D7t+uY7cF499RGC+WaO42SXKC+EOqw0YDKYd36qlyqQRD+bYecfkh+MOK9g6oHdX6UkqAuylw6gb9L1iYoZbXyc44dHaldEfFDSv23pBJ54GA9aMGcVESBe9e9AkGtAVRa7f20bLXbMpXIsxMgj+eUpgFfyTFDBS5Z3L4wynp0TZ2R+E0dEcqf8FFCe3P9X7xkQxUBDfad8tfifCHFCNfmeXgPz3DxO510uEI62wOMxSYwRYR+1NV4sM9Fj9pUJiT+EOSyuWY0pUPp5tTGX37zP1svXKDSf9WuBv6NrstMH2BFSg6exJySwq3JbG6kJs3CKsS5aUMnkwwQ4ER60nZoW849J3RDoLZ0+/etfPWBYVrBtLUA3xWT7B23MW+lCV/1SSsCLx1bUonHdj72z24mKdxUGdxq6edC8eeTm5pzu3d8JEM1DeWpXAwxXYTkh0Bl2ZG+eapIyyfHsaZdcktKZJ8SnkmRhlaREHMZhh6Hpq7DOy2QWO3Ter85ANVmSB+SNEljs13pCjaHW2M1v+60tMFO6F0zqI7KugOa9UYp6ltL3PTDsuPm1gdgbi/RZqB1pRB/YBBklH4I+8eQYkHzQcizEDze+Oi5pAWZrVspJFs6Ufd16Y3pFg/lie3qGWI4pygfSQCBXHWIDf//sPsIxmoQgqa+1PIJ4xUWsiOzdD3TIaeNsktaB5p/bACS7H2130yKwIzguEXUzYPa9MvqwPtXZLWoQ9OV/TdIA93GCm8fb8YrWff8b5hFQZPxdT3atlHuyVo7U9xmxDndTbbeCjFiAVWV08D+GMp2T8v0BtWFGclwWDCxHfC9xwhDwYl+b4eTltTCk1f4E17hBVBcw+0d2nBuOy381iBwO7B5vGEwlUTu6vIXXIYegm6I+Zt6X/9MD6SgRpWJIXJFn39bdT/cxBMhT9gEPQZhBUI+yAtJzx53yXChTzK0h9G7bCFdDT5OKwqRCyovvqTb94Nld8epIZQ2O8irhbsDWjs0qa5yoChl0JQWnDCDLQcZH7G0nqq1la2jgXtHXoWPywJcrMWGwe991ySK+cF4aUSQR9k1rQaSjCQknh63KVy1sBuCYwICtcUUVEw+i2DxAOvLildk/j9Bt1x9Y577VvFRzJQc5MtfmbqHOqP+pj9Kwp7w0JICPqV1uicCqkMtjlWnEPmE3rDCulqhcC4nBKVtMxkUk4/cJfqH0Z0yCfvvP1JpBIkhXc+xKnYwB8CfyxF2orVh1xEIsguGchPNKg9KHA+XiO/mNJ+IMSrgfQkZiC2JIcETlMRVvQkw7uhtVsv7amnNQzCiqJ6xtha8vV4uL8vpDesJwvyiymrj2rC+dov+Gwe0KbGqScJht//sf8jGaid1Txfu36EzfsUo9+0MEKB1dXf7F/91e+wZ3KNBwaX+Nf/8afB0GMlwYgWH/NWLOKSor0L+l4xqT30k2Md+VMxB8ZW8BOb82cnmWuUb/yfPfrOCofuok04HOOumTiHm1g9buwBq/8uT+oqzK/2sfqwSeGUS1iF7LyF5evAs3vgdBThAR/5HnoUfa9rOZ7sktB7zb6EoE8QVhVH/uo50qzCveqS5LUSdmfM1MRxIO44RFVJblWSWzDJLLx/QsxHMlCtpkmv62IksPSFhHA4pv7JgGQg4rmNXcw9N86g28Z7aIPqCzbtXSlDPzCpDLXoO5OSXdJCvJ0JgboN2cqbhq2/BPPrFbxlE7/344U14rzCrmsXZ39Gi11Ie2vrcsRi+Dlo79DTsnFRy5U7TXA/XaN9IMbqKcwI8q9mMH+Mmk7Qr0nbwSEfZ9WiuzeidBnO/IcjJBk9l2UGWqKncjkiP23THRdkrjtkVgzWHxR0dseoD2D39JEMVBT0VTr8p5//FwhL8tiRaQxTQWhy+eQk+z9+jflehc65KklOUD5vUDsK9aUS9f2m1pJaE7gNqLz+k6PHZaZdzl0ex76kD07G1R9P8nZaBqav5YYQuonhj0i8TaXlyYf0ISvJalv23mSqB+qWSww9bTL4bJ3EFbQOxIRD7zH5qPT8ftinkL5F5ZIif8GhvUPQOBGRXTLwBwXZZUXy8SbXflUR9Cm8da2SAuDVBKVz9geyffpIBqpbF8RfH+BXf+cfUHjV4/mzeyh/O8PEN0CMBpy5OMEL0zsRCbQOxcQ5vcernDYpXpMEh3zCqiIqQf3hn+ypPzNr/0gZyniPLG51IRxISfKKwoyh992BoHFE4tUEQb8irEi8DehOJQw+LzTTv2WyeVhw6b/K0hkXDDxvkb3+7mu/P6Alz82eID/QJc4KcisSdxP6f2Dr0ZyiwqtLvL8oIjoWmTVB42hMMKBLWP6QovNoj+K17T3qOyL1IMnpjGAGiv4XLXrDgl6/ye7hdf6HT36VynMuaVbR95JFktU6ASho7TTIn8xgRILcoi6zfNiQ6t1rt0JqIdzMipYmR0FSScjOmRgRhMMJypa0d0hEZGxlRkWak1hdQeUZl8KcJKyI9ywbtfekFOa0pJDx/TKtPVA/qA9O0oHsisLbFNTuN4nKArNrYIaQvWpTvqyl4gdPKkrfy9Dctb1HfUeEAymdnQlmpKg/mBKVdE2x9VNdyq7Pf3fup2nu05pNvSFBMJpQuRKTZIX2keooUle3EOVdRi2PC2CtORTndJYqTcP+f7M1Gi4gd9Vm/FvaHke5ku6IQVwQDD5rElUk3XEIK7qM5G6++/O4Qz3WH5L4Q0J/8UuJlpfcKXEbeu9qt/V1ivMKKxA4Lb0SZNcSkpIkKgj8AV0JeL/4SAZq4YqJu25RP6SonjJpPxiQZGGsr8nVRh/xhSLliwJvQxCVFMULFqlnkF2TmL6uOTptgd028He9fyugDxLSVpQvQntcF9LNENq78kQlReN4hBHr/8usmBQvWCC0bmn9oK4ODD+fkLpQuawnad8NxqsFrLbOyL3xFKNrUrwG+TmDtYe1fFF7h57Lt7oCbw2auwVOC1qT2oxi40SCkcDGJ97/NfxIBmplOibqS8nNG0hb4Fz36O2Kcc0E20yJBhP8fkFY1idgf0ThV/WBqrcjxggFUVHh1GH423dXSk2yivZO8B/pYnUVtWOKzV/sYoYCYkFnR0rQr7cIrQMJSQ7ylx3SiQBlQvIPNlAmrDxqvKc26fBLIf2vK8pXJOVzhu5w9QuSDOQWDC1rXk2IygL5UIvG/bq60totib7YJD9nkJm3QYAze3Oq0u+Fj2SgLj1p4a2YBP1aDTnqT3GXLfq9Lo3vD+PN20RlheVrwwPTF2ycSBGpACWoXNJUQLujaO758C6R6QsuzA9jCEUhG74jza9wzUAZkGx6FH9+GWUrooUc2RVF/opN9XWDvrMp/qCkcMVCJLrL5lzJ0NmZEv9vQ/iDkuL+TYz3OCfO/5RDY7eB32+QOoLcoiCqaG1+f0gRjOgTYFBVGK8UySxaeMs2RiQwvlcmyWrJSbsDcWmbPfWOcDeEHpOe9MnUJCIWhBMRL81NaouZUE9oaic+ReWSpHjBwlsX2KWQtRM6gLtjguzKhzdurUzwshFSCVwrecelOSqBEQlGvwMLp0fIXTfJzRtsHktw2orNI4rmbhMzErT3JviTMXIiwOppFRUjUbqt+vUqmfV3f29xNcHfE9I4GmEk0DygO2FKCApXoe+kCbEgGYz1gS6ju37Fa9B6ICLOazpinIe+U++f2PORDNTU052UJLAI+gz23j/PTx26SLKapXMwwmnqlml7X4JTF2weNPDqWipn4A8zFK9uBUQMfv9Pnj0lHU02DvtTdvbpE07T175WUVm9LbPmFjWROSwaN2iA7YMxfS9bNPYrqmd1/z0paBnO4e+ZZE5l6I1JpK11o3ILBtIUdMbf/b0Zvkn+gouzapO6oFzJwElo7RbEecHmA5KhZw2Krzs4TYXdFtht/fhGy0Lampca9Es6E9uB+o6IypLyazammxIVwTMT1oM8O76ekLnq0BvRnScMzT63O7B+QhL0CTpjBnxxE3cToormVf6kYR5pcuCh6xy6f+7GbVOVOsWjGxx46DrFoxs3bm/t2hKBmw60Jmsehr9j4g8IJp5K8fsF3QmJWQnBkqx8XN4wKYuL2oMgtyxJMxCMvnvBv3DVwNqasw4risyczerHJPYWg99bMfWeflhbSILOnv6AwqkbjD0t8TYUuQW9VXm/+EgGqjK0nqjjxpQ/vkIjzHB+eYjVv+sTF3Rt0Ayh72ULq6vojUiySyZWT9dT2xeqNI4kZBd1Z+YnjTB454LmUL6Nn9jUlt6Udzd9XcBfO5bBDGHoxZjWDoPsqmLtqI109ck8czKLvWYz9IyBSMBuGaSOLsQ3dxl4NYW79u7H/qBf0TyQUj2ntAvMYErpvEV7n5ZaL8wpCleNG2M9YZ/EfKRO5aJu8yohSHKaJ/Bee+GbxUdWewr0nq/v8RWKbsD081NYN2Ggto0PH3+ptacAMLihk7odpPc27q4i4QcMEcPVb70PM9Jt3DX4aGfUbXxksB2o27gnsB2odwDG8Sb9H18mzd75g+y9gu1A/ZCRZhXlrE/Z89+xRbqNd8Z2oL5PDL4a408klK5K+s5rGaA0qyhdlTcM0zLrisK8Ii4q9j5xnc+OXGRHboM0K4mqkvCAjzK54U8VFxVRWVGelvhjiZYbKimsQPsT+ONa1MFINPnE7mhzjNyyIjrcI80qnNZWfbP//ffZ84v6tXO0hZBaQsgfS3Dr+v3lFxTepiLslyT3dQgGJamnULZ+vWGfZOwZH7cJbv32XsN2oL5PLD1pYTVMGvsNmrtMNh6LMWJBY79BZkOCqbtJvSFdqL+wMMwztd302V2cmklmxaD6HQ8McLdGpK2uJigvfzahMGMhTfDWBH1nI5Sp7Xq8TR3M/lhKe5ckdRVJRmBey2DEguqlCGV9MGW59qTAaQjUWT2jlRR1ezbJg7tp4g9p3ql0JZWv5yhcNRh5LiHxFL1dEflZg5mf107SzYO3x/bfDtT3CbchsNuCygWpZ62uOYTVlGiPz8oXI4qXTEaOruiJTwlf3H+eAa/D7197gKg/xR+SbHwiYuilWCuPANlVReE6VF+2CQa0nA4CFj5lY/qQW9AuzQjIjHRAaOWS1AUj1O3hxi4bI9zqZL0PSAcyJzb4yi89w8FPX6G3JyI3Z2K1TOy2YuS5iO6RgKgsGXzBZP1hSZKD9b+le8+Zaw7dCUlx2qB4XVK6dHts/+1AfZ/wH+xRuirpjBvk5xX+eIK7YWLNeuRPe8Q5qP1gBGmBHAsACFKL7FdLuGsW+VkDY91h4VMWpat6mQ4rAn9Qkz+K01ujJB5IV1uzJzmBW9deTr1aFqduEhcV7T2aqByORWTXJcGAeld9gB8HZUJ6fwf3gTrjpSZ5M+TCd/ZSfdGmOybJLQrsDmwecHBnPArXDdqTgsKMSW9PxMNjc6T5LWLMC9r2srHbICre3nXeDtT3icIzWTrjunfe2g3eqkX1fIoyNElDOtryMRiPeWBqgVbiEqQ2zT0G8S6f1vGQ3JLADLXtOkDfOU0WiYqQeoLeiKBwXTH+HUlnUo+AGCHYTQGWwgygckYroKDAm3dY/JkUb0MgbrPPHg6m7B1aZ6Lc4Npmld+7dlSzuUoCb03zB+KcVhPMzytauyWDJ2Pae1IsL2HQbfMrTzyPvzukO2xSuZRiRnr/fTvYDtT3Cbun1e6E0hzX4lXNB7XbWs/KiKD5sQBvwebU+Z18onyZS4tDDJ5MyJ3KYKw7tHdIjEjzXwHmfk5SuK5wN8HbkFTPSzrjgtoRC5Ho+/XG9OHKXrXpTSVsPJSSWVMEgxIjAoSidDWhMHvr78kfT9h7cBGAbuwQnymRKoE/ERP26UNS9JkmQb8gKmsSkOkLeoMWDx+9wi8fOolUgq/PHsZecUgdWD9qkLrQ/9p2oN4R1A/oDyno07P2tWOKsKL3jMGARFkw+gcOwY4Ir89nIapS+b5HZ8SkvS8huyKwW9rNOtpiwls1m+ZPd+mNKlaelKz8dETYJ3GaUJpWKENhdfTzFa9qM9zcNYvUA2/doDgrsVcdmrssyjO3PlQ/MFnHNvWGea2Vx4gF8asVstdsRp5NMX0QL5eIyvoL1RsWpFlJ4wD0EofVsMhTC/voXinjNAR2V5E6gIDNg7cXctuB+j5hxBDv92ndH5Jd1q7U2RVB90iA3TZ48AsXsP7PK9jZiHwm5LfOPMLGIzFBv2D4+wZ2WxH1p+TnjBsHH29DUPh2biszQullDzPU2SvO6+Uzs65AQdAvMGN9W/eTXeTRNnFWP44RwuLHb02hLCorco4exptrlPGX8lrK8lid6qWU5V8LEFLP9CtTW8ZLW9MIJx5axDNjvjezF8dKQShyy4qwom0wi1fle06+vhc+0qSUDwUC1KqL1zKIvlTHfK6CElA45YGCl547gJGCmvQxDQk1l+yqgd2FzSOCdKePiA2a90vsNZukoEhjQAqivoShZ7WYbsYQ+IMKf19E5pKLkHqZl7mU3IxN4Rp04zyZFUX9sK4UpB6EVYm3dvP5yBjvkbP1OExnoYi3YlK5nBKslUlthXE5R1RSZFb1YQ8ByR6fXzn8Cn5q8xezB0lCk7Ur/fSfFURFQXk6pbHHpLnbuG0i+nZGfZ/oO6MwIn3Y4QcV/Ad8kiy0d6bEOW1ee+ixqximpBO4OA0D6UD5Z5cYfXgJz4v5xP4rPHLwKnsem2Xs+BLZZaGHDzsmtQehtUvQmdAykt60e0PLqXJeH6b6P7PE5n2KuKCFJYZehOK0Hk6c+PatFfxH+7TydSoNnK267vpRA6HACiWZNa19IF1dvPfWFWLB46nlfVzr9jFQ6DDwPQehoH5IH/yWHxdIE+KC2j5M3Sms/HRE39k3hSrSlk0wklI+b+DWIXmkzfnlIZQU8GKJqCjZ/alrLKxXiFITw5Acyi/xRHmGhWaJghOSZiDJa0VpI9HapCKFJCext0T+koygfkhRes2h8bVRlCvpPwWjz/ksfz4mGNAM+/X7b23RnL06CIBtphSObaBs6D+t2DiesvxLEc2H9ZYm8cAfgtZnu3z8E2c4VFmlHmaZvTxMZ1Iw8gOFmOjdGFBMPbWlKHh7dd3tQH2/aNqsPaQDx+oqctctlC1p7lO0d0vSmTxx3ePA6CrhAz1kVvJfTHyTv33/M0glCAKb37l6gucbu/jZqXMA9H1hEbdm4DQFZqCzkdMSKFN/IVJHkVmXWmy3ojPn1B8r2pMGG4c88hdcUlvvn513Nz55R2TmLWq9HIZQjBRa7P3UVXb9g4vs2rNCJhviXHf18GBWYRxucWxigZlWP4u9EtevDqKyCVYXlp8QlL+RpbkPZEaSWde138ql22vpbgfq+4QRC7x1g+aRmDSjXVTKr9u4dYPhw2v8gy//KT917BwztT5+/fBL/NXHnuWZzn6OZOZZXSsRb3oYf1zl+dN7+d0LxwBwjBR3U3eaUleXvvwhibdq0R2X5BbRQXkiIRxI6Y4Llj5hYWzN6nWnUpL7OsR59Gn7FrH52gDn5keQSiCVwDZSFjdL/I29z+Pe36D8iRUOP3aVL+0+y0uv7WV+rcLH+meY/JrAsHSHbu9/aFM7LrFbgsHndRcr8aC5c/vUf0egTE0MMXom/oAiySuSjM56rW8N8z++8llWgwJ9+R5fvfogJdPnSneQ/+LlX6b6tEtmSath2y0Tx9GRNnNqnN6oIskodv5hhyQDQy9CvM8nP6vlHjM1hdE1ySyamD7E5ZTOvogkozNv9U+zWlDtNjqWVlfgXspw/twk589N8v3Le+FCgT9YOEq7keVAeY25Zpnn13ZSnmgglj3+3fc+weKnDBwvYfjFiNrxIt66SW9nTGMfdEe13E9v7+3J+2wH6vtEdlGPA0tPUrgOylG6ry+gOJviXvWQSvDY4DX6812OePOcXh0l2fRo7oWopEh+rs4TnzzLgyO6yG51DVJPkV8QrD5SILekcNqSylMeveM++QUtTd53ekvRz4DcdYu+520KC5Kpr+n7S/NNrdJbhgRvycRbMsmd8cguKzp/OEz5BYdXfud+Hhqeo+z5lDIBaiSgfN5g5BmFYyfU9zmE1ttT+AAA5fxJREFUZUGcUwyPb5Juif5KG8a+vt3rvyPo7I/IrGmDB2kLlC1Rx1pbvkwGSEikwe+dPk4qDf6ofpxPjM9QnmiQDkcMHFljd7XGc7M7Gcs0uLg8iNUDp2HQOB7S3i3JL6WsPGLitCWpb2L+wjpxSd3Q2e8ciIhzCret8KuCzQM2li8J+yVW8AFwXsUWXa8iCAYE2XXJq2sTHCkuESQWpWc9jAT8qoF8pkJ3QhEe7zJwSlE7M0huXq8C7b0pS196D/Hg98B2oL5PVF+2iUqC7pGQ1IPyUBv1ehG3rgvy0las/v4Uj+69ymOD18gYES+uTtELHJ7+9P/MRKHBJ6pXME3Jq5sTWOfyWntgWVE+6VK6pE/umTVBnDMoXHBYu9KvHUligfVIHWfZpnAd4qygOJuQOrD8uEP5vGDj+Pvno1pdkLagtzciyShWH4GfnjjH164fofX0EPX7UuKCzu5WoFvJXMvS/pUWpi80IceC0nkTWrdXut8u+L9PNA4o+k6D9YqLtCGIbEQKSmh+ZmHGQLpw6bcO8OKTAY6bcN/IEqe7Y/yV1/8GUsFar4A6W0Q+78EJiIuSYMAkKiqKV9Fa+0ofjKQFQy/A6qOS0mWD3pky0oXMpiTKG6ydsFEGREVJsl/g1N6/iG6S1VkbdAb/rz77Nf7ZK59n79gaV0YL2C2TzpR2XpGWJM1pgbb4dBkz1Nub3CLUH4mY/AOT9ftv/TVsZ9T3iaEXoH4QGocSuuMS9+mClgnq6SVXfGEDaUL9iCR/MoM4XeDMt/Zjnc5j/3aV+rUKm38xSnZJYXf0svgGZ7N6Xtcvk4yuAAipuarpr29QPSPoDWuzB6cuWHnUICoJRALDL0RUzmleat/Z9y9Lnl1ViMCkcNbB8gWnOpN8+fDrXL42TOmSiRIw+BLExRQ5EOGtWvjDinAgJS4pKhd1GzV/wWX5ie096h3ByscUoz9IMX2D7KLmW4ZlRVgW2C2Dzvkq0gZvzdQqzw2ISpK4oOgOGwy+qHVarQCWntSmE609krCqaE0ZTH5d81B7D/j4A4L2lMD5d1U2HkowYu2Yh9AKhUG/dkjZOOzQGxZkVwT1/e8/ozZ3C0rnTMKqphR+7y8e5E8vHcFd0lr+ylKsPqHIX7MY/jOHYDLCbgmcDQMjgsY+gySnKF5LqZ65vT3zR1rSZxv3BrYlfbbxkcF2oG7jnsB2oG7jnsB2oG7jnsB2oG7jnsB2oG7jnsB2oG7jnsA9GajS1cIKwVCqmUUt8Kdikpwiu6roP5fo0QkfkoKiPCPJ1LRuf2FOG0yYkWadJwVF9WJKXNAM9PKMxO5BXNKMH39XRHZFj4H4E8mNKyZdLakjXcXoswFpVj+uMsHfHeHvjqg+uYL1UJ3wgH9nL9gPQSgYfS5g8FTM0Cta+ieqSNKsIqpKsmv6WkRlfX3eoAtaPvRdSAn7tZaUkHwgFuc3g3uy128G2lC3sV/Q2A/SlQw9rZ1Bwgr4gxZhVdvVWL5g7SFQQuFuagUSI9azPOHxLvbpPPX9JkasCAYU4SM94lqG8lmDzGZK5WUbu6cNF6ymid3SvW9laBdrqwdJxqQ4oydEd3zh2tvcoauZHlTgbGMKb+UnZ6l+K7B6sPKIR29Md9S8GmSO14me66NwDXojAndzy4vL37JTH1OUryiWHzPILgpWHs6QZhQowYchOn9PZlSrp/2fpCfpf12PKIP2hkodSB7oUL4kqJwTZJb1Zcwt6DaeP6QYfDXASCDe9PBHUoIBzZp3GnpMuTBtUrmkSRiFhZSgauCPpiTlBOmCdGH4Ra2G0htVbB6y6UwI/tnf/zd0one2UxTlu8dTtXUoJvW0sVr1LAR9it6pPowI/EHNgmoeSLXTSQRhRWsXbB4W2F3Bzi9d5XO/8gLxzoBw4sN5X/dkoIZVLfDQd9Lcck9WrB8Hbw38sYTiN3LUDys6kxBVwG4bRCUoXdEXfOb/aGIkULxiMfiSwAwF3Qlwmoo4JzB9xcKnHZYfN5n/vCAqwuh3wVvU/vZmAIsfN7d4o1oI4v/wV37At1pHKLtvX+Znan1s+lkOTqwQDL9/gsgHAW/Rxt3UXlKtnYLMqiB1FX1fXCTOa3rint/R8/uNoxHehjZiMyJBMJxwsLjC/uwKf/2B5xHW+6cR3gzuyUB1moLuzgR/SFts5xYEMp/SPJQy/AODxn4oXRJEozHZJT3SkV1WdCb0Up2Zs9m4XzPOe0MG3rqg73VFb1TrPdUfiqmeV1TP6alJfySltcNk6GSMu6FHTcqXNeUts6oIDvlMdwfotzvU/NyN13l+cRj1Wonai8OcuzKOEdwdlzsuSoI+CD7ZJhiNiQtQua9GJ9RUxbgoqf7zeUaeVZRfdWgci9g4nmo91kTwZ9cPsRhV+OO5+zGX3r8h783g7rhyt4jsisQsxIRVSf2wZg05KxaZRZPEE5RmIHUE1Rds6kekVhT5qTbVC5LuzoTUVZQvCpCa6WTE0NxlkF3WhywR6L1kkgFn02D8KW2a1hnVA3ThSIy0tJBub1RAzWW2VeGrs0dZe20IgGubVYy5DEitP5W5bmtC8V2ANCOJ+lL6/mOW0adMnKNawidJDbzdLQBOf+sAG7/WJclA5RWbvldMepMJ5BP+4YHv8IfX7qf3bD/ZlQ/nPd2TgdraYSAWPTKrBt6aIMkqhk6mxEVFbiUhLAn8IUXqCTCgMAvBYp6lL8Z4yxapp2fmhdLyOb1RRTAgkbYWfnBXTTYPC5p7FcFUxOInDdJKQmcC2jsF+WmbqCjoTGoXwL5TgocG5jjYt8KTnzxDK/QIL5aw2ndHYP4wRDalcsZg86DJ0iclv77nJQBODM/TbWTw1g2C4YRgMY+RQlAVBP2CHbtX2Tm+zv/ryicILpRxG4roidsdyro13JOBKh1w6oKgX9G+L8SrCRq7LTIrgvWjNp3dCXZb0N4hcWsmzX0KuyPof9rBacLw84rm53qkjzeRtp4YVY7CbisUW+WvrD7pi55J6bKgeNbBWxdEJT00FwxKvHXtYV+cCzlZm2CxW+Z703updXJYnbszSAHsRYc4LyhPSzJLFv/2/OP8zqHf4v78Apmrzo0ZMKeh+bXBUMrATy0yv1bFNlI6XY/yRUhdgfVi4UN5zfdkoMYFiT+aokxFZtol6Nen+eyaxAxh4EWTqKQYelHPLLmbBvlZ6I4L7I6ivs+k8mdZoitFkqwuYY1+Tz920q8FRbPLBu6GwAgFjSMSu61o7U+RrmTwVIy3btAbkxgpzH7RJfiDISwhcb0Y144JByTyNmbqPwxEQ/o9tnYYPPKzZyjnfT7xrX/Iv/zjnybskyRZQVRS5Of1tshbN1lr5fmlQydZbRc4MTlH46AuW4kP5yx1bwaqt25QvGKSnzOIi/piVs7B6hMK/3iP7GqCVxOsPQzRaET1gj4IIKGxH4JDPtLS3vIosDqCjSMmygC7ZuOtC5IsYEBaTeg7aYCC/FWT/HWLzojW1TciQZzTqnlRUbC7WONvHXiOsWKLw0evE47Gd2WwVl+2cVqK0idXuFgf5L6+ZX7nU/8f4pEIMxA0HoyoXFCkjqA3nhDu8wkDhzm/SiINXnh1H3ExJejXztMfBu7Jgn92VbFxXJK7blK4Do3Dku6oiUgE7uksK49COBFhrdtMfdVg47BJdyKlMGOSXQFx3iPJCroTitIlBULXD9s79Ek+t6wt0bsTEtG26I4ISlclvVHtsuxuGpgROIu6NBUXIBhQ1MIczy89zES5AcB9++a5XBqA0x/O8nizCCsCkcLmWhkVG6xtFLnUGORvH3+Gl+o7eP3aOGufj1C+icik7Bqt0YttTi2PEQYOIhVUXzdo7JeY4fZh6l3RmRBUTxmkDrSnYOBFg8KcRHoSt6FI8pLiaYf+04rZX9B7yr5TBlEZGp/xSTIC6UB2SdDeIag9kpBktVRjklOYsSKeCinMmFRfF2RXFb1BLX/ef9Jg47GY3qjUHao3RMBGQjaCHP9g/3c4Nz/Cek+XqSb6GndN/fQNVC6l9EYko39iQyIY7GvhWgnX/H4mc5vYmZifPniOX3vkBX7xvlepdXIc7Vtkd/8GjhvjrRpsfiykcN1g+PkP573dk4EaDuo58uyK1mUCqB8UFK5YdMfAHO/htHQTYPTPbZwm1B7TwTjwNQ/LV6Qu5FYkTgutRVpS+PtDSpcEzR0m1R+4RCVo74LmPjBSaH0sICwL3AWH/KxxYzI0LipKpR6T+Tr/j+e/hLHkUX9piLVunowVs/fQ4l0VrLX7TPb+TpfFL6Y4NYuV2T7mXhjnUmOQzSjHZH+dS61BNuMcUgnKWZ8vVk4z5LWJpwsMvhrhXfYwA0Vjz4ezKN+TS39mwaS1PyE/YxFXY9Y+BlN/AsuPWigb1PUcZqxIs5LWDgunoTAbFvlZQWO/oDij6I2lBAMCp64NIpShUB19Gk63JBURCmUqyhcEm/ensOnQG1UUZ6B+SNF3WhB9uYFxqUyzmeWLB1+ncF/A1585jt0UrF6vYu2SVDM99h5a5Apjd0W/P8kq5j9bwJvVghFWxyLsk9S/NUJdaAcXfzRl2h1m3+5lvjB8nv/uys/QenqITAQIvfJ0x3VH68OocNyTGRXAW7HoTqa4qxZGLiEsmVQuKdwNbd4VFQTeikVpJsUfEtgdLYYbDCcYqUJZirQaY3chHEiRFribgjgPwUBK+aI+ZI19V9LYD+ULBkYoMEJoPBnQ/6rQJmffryCkVpj+r7/3S7y8PklmUtcWM4sWq6/rBoBrJYzuXb+Tl+wG8nNCHyID6OxK6E0lVM4Jwn6FEWnCjbdm8rcffZqhbIvfvX6U6I8G8feHhBXF2jGbgdd0t2/71P8eEBJQYPYMRCpQDQe/32DzkD6tBwPyhmHY0scFRgTBcILdFQy8oPkBxUsW1qpDVISxb+vaqdVVJHt7VM8YdCYEuSXF8hPa4SToEzgNg8yaIHcqo3VKM4rWwZjMiqA7qpiYqhFEtpZAf+PKKohSnUUNoT40Wtx7oTeilZ+VAYPPmeCmNPdqO6C4AP5kzIHPXuH19hiX64MU3IjK5ZCJ3zexulqzXwnw6loi/cPAPRmocV4R9qUMvSypXJK4aybNByPivoSoLCnOGLh1weDJBMvXJ1yzGKOE1mcqzEu8TUV2VVP+lj8msLqCuCgY/gNXq5Mc6GmvqMGYcHdAMJTijybkVnT27Q0LjENtcldt2vtSMuuCpu/xxOg1pip1wn0+0tGUxMVmCYCy5xMM355I2AcJMxRUzoPdAitQlF/RcuvehqJyKcVqWOzO11jtFVjfLLB4aoS1Yx7zn9cmbNkVRVARLH02Jej/cMpT92Sg5udBuZKNQyarPxuSn1c4SzYiMijOGORWUswQ5n4+ZfBlqUkX39buII0HY+p7DXqDgt6QXrr6XwW7I0ht2Dhs4i1ZOGeyICA77aAiE6duUD1tkngG3f0R+UWFv57FW1eYbYPsiiJ6tUKs9CU9PLnMvk9eZd8nr7Kzqq1AurGD3bzze9TJv2jTGxZ0phT1fQZBPwyc1Nei9oDBP/7yH3KuOcL8egW24jDJag9Wt6H5qkG/wFuwiSp3EXtKCHFdCHFGCPGaEOKVrduqQohvCSGubP2svOX+/0QIMS2EuCSE+PwH/aLjgqB40SYqS5zpjPY58hRmzyAqQHvcpL1DUjztsvgZRW5R0ZkSxCVJ30v64NA5qLX3e+O6GTB4MiS7qojKEmXqrNEbEggF+cs2pi8IKwIzUnhzjpbW2TCJygJlKTYeUAQTMT/446NcXBr6kdcslaAX23dF/3/14QKlqymVC+C0ICpL2lMGYUUQjUX89ye/CMDjO69iuwm5BYG3rpCOovNED2kq3AY4DVC5u6889Sml1INvkV75x8BTSqm9wFNb/0YIcQj4FeAw8AXgXwkhPtA0okxAQm7RuPHvzKoBEz7K0q294ecgKsPBgws0ngwIhhOKV3SwlS8JstMOYdmgb0edzphBa8ohKgrMYZ/MuuYTSEdrLRnRluBuBlY+rnCaukRm+mJrvywQsaDvJYvqxRSuZTkzM86mr7Wk2pHLuYsTNJ/50QC+E2jvUCx9GjYeUDQPJlsEcEnqQbYYsGdknZFMix+8vp9oNUtrb0qcF5rPO50htwi9Ie0zlT//4bTebkp7SghxHTihlKq95bZLwCeVUstCiBHge0qp/UKIfwKglPoftu73F8D/XSn1/Ls9/rb21N2DYFBy6AHtS7nYLBG+VP2JP+cHqT2lgG8KIU4KIf7O1m1DSqllgK2fg1u3jwHzb/ndha3b3gYhxN8RQrwihHgl7XZv8mVs4ycNb91guV3AT2w6Fys//hc+JNxsseQJpdSSEGIQ+JYQ4uJ73PedNmE/kraVUv8a+NegM+pNvo5t/KShwH+hHx+w7/RreQtuKqMqpZa2fq4Bfwg8DKxuLfls/VzbuvsCMPGWXx8Hlj6oF7yNv5z4sYEqhMgJIQpv/B34HHAW+BPgr23d7a8Bf7z19z8BfkUI4QohdgJ7gZc+6Be+jb9cuJmlfwj4QyHEG/f/35VS3xBCvAz8rhDibwJzwC8CKKXOCSF+FzgPJMDfU0rdPYyMbdyT+LGBqpS6CjzwDrdvAJ95l9/5TeA33/er28Y2tnBPdqa28ZcP24F6i/AnEpy2Zh6ZEQSjKcrWE61Car2m7JoiLincpr4tU1OEB3yCvSFpRotnCMVttR+VBZl1/fjhgCQqK/KLekrBSCHYGxIMpdqdWkLf+RR/R0ympihe12QZ6Sqm/rT+zvUZIBhO8Sdjco/V+NzPv8TEp+cIRlPyC29qUWVqWvvLbWgn7LBfkmb0z0xNYfe0zld2TWt+yQfat22BDtuBesvIzlo0j8R0xyRJDkQkSB0tKhaXtFCaEWsFFU0ZlBgRyLaNN+OSn9NF9dySonD91i+/EUF3XFP18rMG5YvaPsirKbwNhTftUrxiMvVnAdKBlccMyqdtlIAkq20fM6uCuZ+pEJV/9IsSVSSH7pvj8P4FNjbzfO0HJ0iUgRKKoF+Qm9MTD6mjx1G6E4ryRcjPGkRDCfnruhWbOtDcq/9EZUV6NY+0twP1Q8Mbvf/MqkHqKuy2IM0qOsd8MitaHqczJkge6KAEuJsGtUdSCjMW4T6f5l54/PHzbN6niG5jlMofS8nPgtXTQ41xQeCt63mv1BZEZUlYgfnPehSu6xZw82BKe8cWc+w6ulY6mmrP1h9CWtDn3jg18S55mMM9rl0Y4b/8xF/QO+YTDOiJhyQLG8dSjEh/UfKLKZnrNv6QoncwwG5DfhZy82KLO6EZareL7UC9RQgJnT0J0dEOItVqKtlFg/zJDE5LkV2VRFWJnM2RX9D9cHfVorMjJf9qhkefuMCplXE+9bEz77r0vhfspoFQ2nAtdXXApi6UZqD5cEhuwSDsT0kdCPoFdkuQu2aiTOiOK9KMJtaIRODWf+jjFzA0USdMLWZWBvTznc6TWTbZ7y4x3NfEDKB8AYrXJdXXTLJLYO3uIC2BFUBuSTD4LYfuuMIfErROBAyelEQliEu3z7TaDtRbRJIFIzDIvJhn4DVJ4boma0sHqr+mO8fFaUF2URAVNZPeDLRp2N6fv8xit0TvWpFvnz5Eb+LWuanVc4rmZ3s09hgEfdoRL7esDdiEKekc90GAFeg5MCOG9NEWpWlNZSx8fgV/QGBEmqf7Vkgb+rNdri7145zPAFC6JrHb8Pe++reIUpP0RBshtRHc5vGEJCsIezZhSdA75mNEivUT3JD6yRUDmrv0KHpu/va5SduBeqtQULysB/tqv9SjO6GIKpLhz89z+fIoK59JaByU9EYUTkuRelpc+EsPnSJrRax+bwwMMNsmIr71lLr2ELgnc7h1cBuC3qgkKgqSHFS+7+FdyJBdMMktKAZek4w82yKeKYDShPOlKwOE/ZLcvKB47e0ZLsnpf8vwzYCq7zfwBxXDL0kOVFcJV7K0dgvGvgNj3zIIBhSZi57ev76cwfK1Xpc/pGWTerNFssuK6gVJZ9ftk8a3A/UWYURg+Vsn2ek87obgk4+eJU5N3DWL3BWHzLKJ2xBEJQEC+n59jglvk2eu7MGMobizgdUVFKdvPcMIqbN3+3Gfzi4tXdQ6HOM0oblH6xIoE6KSwO8zWD9ewIi1O3TaH5G/ZlI+L1AWbB56+xdFuluHnejNsAj7Uj3sWDWoBXlQULiuqP9Gh/peE5HokZxgT0gwoChd9bVwh4Pe2khYezJh7SGwWtsZ9UNDVFLkVhPstsJtCDKfXGem1c/sXD9OE4J+SVxSeDVF8pkG/+VX/oTL80P89vTDFE56dHbHDP23JpWL6oYmwK3AremPzJrx9LyTgspJi9aRCDHVI6zqLK6EVjRMsoL8PDSORwgB2TVJa68WPbZ/iMRtt7bCwXrzdF68YpLmUzaOpVy4PkJu3iR1wJ/V5sTShsyKwFxxkDasnshufVl0Rh18BXIzNmlGMvjK9h71Q4MZChp7bKKS4FO/9DIFN2T2+gDDT1nEBXCaBrk5Qeununxpx1meaewFJUheqmhi8qzNpb+dY/VR9Y6n7h+H3r4Q6WgtAf/nG5Qvg7IEfc/Z5L+fJbeoJ27fkG+3O4rWbii95jD8DZu1RxRmT+9Pw/63B44RC8LU4vC+BfyJBOlqvS0sxcBUHdtLsLoQVgVuzUCakF0VRJ9pgqGndpWls7rVFRgJRAWtYetsmix//Pav+10wE3lvwW5DVIB/+Ot/xKDV4jtffQi7pLRUu6MYeTZh9itA3eP3v/4E1eNrGDWbwpykO2JoyaDrFnZbH8xuFdnLWjhX2Qr5QgUrlPiuQJqC1v6E0e8ZtCYNwsEUyzfpToBIoDOhUKaBESjsjm4cmP4P7ZEVJNLANeHwgXmMgwpLpJScgO+/dIjMskkwALkFhT+oZY42jgjMUyUcCdLS8kbSUmRXBJmaZPVRqJwVJFMBbN6+6O92Rr1FeBuKsU/P8+2Ng/yjP/oN/CGJ1RO0HgqIhmJmvwKH9y8gIj1a3f7+EE7TYO0RRedgRFzQ24feqD6R3yoKc3pr0f+SgdWD1pRB/GgbM1JM/amisccgLkDxsokZgjIUhVndtVIGyKxEWVC8llKceftjy0MdCk7IWjdPbUuSyDFTvv/SIYaeFzds0N9YCeoHDIxYf0F7kwlhn8Spox9X6f/PLhqEZUH29QxO4/bDbTtQbwFJQeH+8iqHy8ucmpvQdj8XBUG/hJaNCE0+d/QsrdBDKJ3FvA1FMJCCAHfeoXJRogyI+tLbqqO2dhlklwT1z/u0H9aqhOlMnvZOWHlUK2JLV2EGeh7KWzfw+wVxOSW/KCGnT96bh02642++AH8iYe/QOmevj9J5foDWc4NceHEnp791AEzw/sayXsorivrDMb3JBJEAhlY1FLGgcFVXAfwvtQj6oXoufbPd2q/IrL3Lm7oJbAfqLeDAY9dYb+aphXnKxR5f+dLzNJ4M+Oxjp3nw/qtkRzo8u7ALqQRWW+DsbFM/rPDWzBv11M1DguEXJPmrFnHuxz/nDyMYTGntTyk9lcG9kNHBrrTKS99Zhdw6oJkR2HWT7lRCklMUL1msPyjwpl2kqZdnb33r0GTAwESdRpDBu+LpeQylBSnCvhRRjJhbrSIdxdBLkv4f2FROmwR7Q/pPp0RlLZgclbWhhz9XwNuA9eMGYVlROyY1t6F0+9d+O1BvAe3IJeo6TDf6+dUdr/Bn1w/xxO4Zzm6OcGVjgGMj8xwZWqYVuFi+QJ0tklk2yC1pRxGhIC5L1k4YWJ3bMxPLLJn0vWJQmQ5uGJVVLkB5OmXzoIHTguIVCMt6SfZWLayuICropoMy9RYgLii64/oxg37JYK7D8nrpR4aGqq/rrtv+sVWOnLhGe8yk9lBKWBVYSw5rJ3QIjT4TYobgDwlEAq3dEm9NMHhK4q2aWl8hf/u9/u3D1C1g9XtjmBWJX7X5V9/4HO6GwQ925dixY40wsHnmyh4KRZ92LcfQnKQ9YeBtKqKCIM2lpC0tCCwSQXO/xKvdep6QLmzepwgGMlrxOYS1jyUYPRPpJIQ7U7wrLsrUp/hgKmLyDwxaUxYTTwWsPpyl/WAAHRurZ6AEKEef/o1l70eez9uUdMdMrrw4hRwPMCYUTt0kyegymJAw8Co0dzlEJYXVFeTmDTB0e7nxG214pUxnzNSHuNts928H6i1CuorgbBm3K+hNJnzl2Cm+9v0TuHUDC8hcdWg/DM3d+rAT5wX+oKL/RZOwAk7TRNogHYESt75NjYpSm0H0aclNuyUY/p5JZ9Qg9QxEaml/gTUDZQrMTYv5LyiUldDekWHwpERaHt0JLbyRvqVElhRT7ObbQ6K5y8RpQL4Fn/38SX4nfojyKy7SgtZQgrdo0xmF3pgkP2fgDyiUpcXkkqzCOFUmrkrikpZPMnu3F6nbS/8tojBtUr4MvYkEqxTxtQv3aVmg1xOtiVUxyC0YmCF0J1LMQGG3BElGkF3TlD+7rcjPcVsFfwywu1C+CElWP15jj4GRaJ5qdkXRf0owdDLC6kFxRuAtm+z6vRQMxcZhgdNSFGZMip9eAcDsvncYFOZTwooOsOxlFyEV3XFF5ZSFSKE7lZKbN+gNKzAgu6RIj7bxNgThQEqaT0nd22twvIHtjHqLiPOAEBjFGLXkUbmguz+LvxjCukHQr03XggFJ+bxBazcIqZC2ILemCKrQGdUso+I1RWfs1jKMshWpB24DyucNeiPQ/3rK4mcUmWX9cYoUGntthFKApiEuftwlzSSkJUnYdYjKCuNPhqGihdz0L/7oHrLvXMzSxy3M3W1eb46RX1CYkSLoNwiqUL2Qsik0O8tIdNE/6Nft5eyKpDdkMPR96PVrlRp1m6lxO6PeIoJBTVkzDE187o4LLF9hXfNQpkJaCv/BHt6agT8s6DujP8BwKKU7qNuPVqC9rfz+W18GlaFLPd0RQWFBZ6q1Ewa5WQuRaNK21dW0P68mbuxV7S4UL1v0D7cQiT7RJ1tVB9MXTK/1s2PXGnHh7cG6/JhFXE1w7IQLr07hDwqcTord0W7TnTGTJKtIcjpIsyuK6EiP4lVo7jEozEKUN0CgDdVuE9uBequwFOGuAMtO9UnaVLgthbcpcDdM1J4u9sUs0tHiY2uPpigBRs8gGIDUg76zKZllQXQbjiL9L5kkZa0Xtfgpg51f88nNayO3NKML8m5DZ+rsmsQfknhr2qEwzkPzTB+dwyFOW1cK3oC6nCdKTfY+Mos41sQ43sR9eJPdH5vFyCaopytkl7XvVHPKJuhTmmOag9yCoHBNs8QaB6D/ax5RUe9Jo6Jg8z596Kq+dvtr/01pT/2ksa099ZcbH6T21Da2cUexHajbuCewHajbuCewHajbuCewHajbuCewHajbuCewHajbuCdwVweqsiHJ6UJ2VNWaRv5Egrep6LuQYkaaKpdmFE5bzwAJCamnyC0pwj6JvyfkiS+f5sDnr7Dnc1cxTzRIcne+dgx6otQfT25oR0kHcsuKYCTFn9C3x0WFGWlHbSN58/eSvCZlJ0e6+BPJDXZLmlUMvRLhtPRjCQXepsLfHekZ/S6Yse6WZtYVpavyhkaUt6EozGlW1NDJmGBveOcuzg/hrg5Uq6f70FE1JT+r/aP6XzJJXUFryiR1IdrrY3UFrT0pytQBqIfbBCOH1ihWelSdLr3EQSrBVKXOwLFVUu/OB6tQkJ23qD0ksXr6CxfnhXZ8uWYR5wT5OYEZauOyqKKDNclpoos/JPi1Qy8zsXOd6se2CCY9gbR1y3LjQUlhTpJkBANP2zT2GjhNPR0aViV2V9GZMJC2Nnhr7wQEREMJ139JMfznd484+l0dqCM/6FK5JMkuWPgDitr9FtKCoF8hLW2vQ83F7mqOZ2bFxNvQ7brOpGL5wiAFL+TPrh8ib7+ZHSqej9zt38F3pmE+Wtfs/HWTzpQkPydo7UvJLJtYPpihorVbkmQgzgkmvhmRWdMZLyopgokIuZVKM/bWAJaA5k5NThl/SiEtQftgTO2TEcFQwub9it6IpDgjCKqaH2vEmshi7Oiy9mSC8E2KrzvUD9w94XH3vJJ3wJW/7hBUNbHC3FLNs3sKr6YdoJWA7IJB82hIdsmgejElsynp7E5Ih0P+zk89xWC2TbftcfmP9nHl2R3EW76k+0dXiUt3Nqu2ajkQ+n0YMSDBCAT5RYXb0GPO49+V2I9v4jYUc19wqB8Cr2ZgdwT5vh6/e/kY9V6Gxe9q24Q3pga6YwplCNY/FpObtsm/ridAZS4FJWjtVnQnFN0dKUIqrB64L+YpnbEpXTbIrUi82ru88DuAuzpQUZqhk13RrsfRSEztQehMSfzxGKeps4m97BAMKFYeNohyBr/26PP888e+ytcW70MqA1p6CTN7gstXRm88fFK4s4rthQsO4WGfJC8xfYGRKOy2gbRg8zC4G4Llx0z6/qcs68cF2X0NMmuC3oEQkUKnluP42DyPjs4SDOv3Mv6UT1zQ24r1owZD37PoTiUE/Yodf6KwNi3SYkLhuoFTF9gNg9UnJd6GwmkqWicCOlOK5U+lRMU7ennehrs6UEef0uJacV6LGFibNt66wciziok/F/gjku6ExGlpBz27I1h/MmaPt8o/+s6vECYWl7+1GxEL2gcjAMyOgVQCQyjs8p09LBRnU2TNxW4bCClo79L6pN0xQVJMkQ5M/XnA8uMeo0+nZH+3RFhR0LJwG4riQIeX5iZ56sIBKmf0R7n4yQxxXlGcBreul/fsvIW7Kdg8aGPvaYPacog2NRUws2BhRor2LjDWHdKs1Nd37O6xXrirA7Wx26B2IqX7UI/aYwkjz6akGdg4ZBKUTQrXDDIrBlFZkWT1yfmXjr/Cd+oHKI+0aL3WR5xX2C0Ds6HXRCMSpFIHa9y7s4eFKG9szRgpwomIJCdRppbjqZ4yiYqKtRMZlID6PovGPgO7rffi/oDgF3edYrTaIl/uEVT16uLW9fBe6miScutYSKamcNqK/OdXCOfy+rlLelQkyYC7CRtHBCIRFKcFKpuyecAiN3fnDYbfwF0dqMoCs2cw+p8crLpFVDCQpnqTGa8091LEULpkEFYkZxqjdGKXxloBpyGwu4JwMmT3V3sAJHmJbepMUe7v3Mm3R2Oflr7ZeDxm/GsmmSWT0gWTYCymvRO8DUFQVaRZRTCgSF1FMCiRFpQeXeOp1f2UXZ/2coGwTw/oRQUYflGSZiAqgntd27pvHk0Jf38Iu23gbJgEU6Eu0ynIrqe6DOhqUQzL03Wwzs47b9n+Bu7qQM0ua8HZyf/6MsqAzs+3MRJBb0SLxlq+Qn2hjrcpaO+WVM4Jar0c05v92DWL3LIks6bIXnGZ+UWtn6PeIgDWWL6zm7DMms6C3pzD6sMGyoDG/QmTXxMkYyH+sCTJK/rOKJKxEGnB4Muw8/fW6c92WdwocfrqOIVpi+oZ/Vj+SMriVxKkreWHgtEYEQky8xZOS5FfUMRFSfaKq+ecdsSsPC7IzxmUL4E/KJBrHv6gZOCF7Yx6U5A2ZNYF5377EIwGyJMlCte1yEJ7d0LqQPRClc6ERPZHdCYF9XP98IMKTkMQ9Bn0hgVxQSFSgbJh176VN5/gDpdS7Y6idE1ixFpC3N8VUX7dor7fovCqR1pKkK6kdr/AnnMZelnRGzSY+02H6e/v5M8f+1f0D7QxQ6gf1o9phAall/SUqN1RmE1Lf9m/0STJCHpDgpGnIXW1PFH5tE122aC9U9KZEvj7QuymQWbVYPPInb0+b8VdHaitPUqPOqymyESrw/mDgmBIUrxs0ZnamgbtCewFl/4zKfk5LW4rHeiNKjjSxm4JpKWQBzvk7AhDKAyhyCzc2dnGxgFd53Sa0HwkoP8Zm+BTbaStu0nVl22qr5nEAwlWV9AdNnAbiu5qjl/4yg84Fw3SPNNH44EYORoAuswV/1QTb0M3D8wQ7PEuV369QO0hSViVJBmD1FM0DkKS10YVu78akltQ5M67iENteKR5QzX6bsBdPYWqTN0GXPyMwp118e/zcS9k6HtNgFK0d20JJyT6grbHTXojitzuJq6dkLVjXDNh+ZGE4/1rN7pTbyAuKezmnfswMmsG6ydS8rMmmUse2fWEzpmCFmlQEFYEYUXhLdr0phIm/hx6gyaHD86zGee4FI7gNARmaJNdsvAHoHgV8s9kmf1yilWMGPoDl6ViltK0gRkpekNaNSUpJXgruoFSeyTBH8hgdyEqKNznC1i+ov1kD/vcbUgO/gRwVwdq5aygcUBhdUyUAOdyhvKMpLVDyzfaTQOrJwj7tvanT8Yc3TPLY9Wr3Oct8O9WPoYlJJE0scSPishOHltk9tQYTuPOBKvVA1GNcE95WKGiO2xiBXpO3h+DoWcFYVUQ9qe4qxatKWjtS9ljh1xoDPEXlw7iWiBNRWcSTF+bUMz9jIHdMBDrGTqjgty8PuUHA4rCNYHVUxQuW6SuPrD2v2gBupmSW1bU7jeJ84J0w71rHKbv6qV/80GJ0zAoX9CCsRiw8hjYHQgrCmNLA99IBLWHUwgNrtX7APin53+OV07u5eVnDzC/XmEjeFOR7ML8MIk0yNkRYsy/LQ2oDwJGrDAWPDoTgvoBQeoKpAlWy6B42STOCZyGwNkwNcmmrKsg8+0yhlCoDZeorChe1eJkoGUokdD/miK/oNvN5la5uHRZC/wKxQ0PLG9dUT+kaO0EkSq6QyZuQxN73Nr2YeqmkFk0CYZSbUPTUYR9KSqbgoL+0/oEqyyIq1q1xCjEHOxf5Rsrh+n6+lQLMDFQ5/LsMACXloZwL2eYeWmS9V6O/aOrhNXbl+x+P+gNCey2oHhdEe8KaByLUCbIyYCwqkeN3bqWa+yNCHJLCnNHh+XLA3Qjh/ysXlFau7XxGOhy3eALgjgrSDytVBLn9DaqO6ptd2pHtWpJbsGg84kemVUDIxVsHjYpz0R4Gwp29rB6d+SyvCPu6qU/ySnK57QwbdAnKEybZDYEfh9s3KdVjI1Fj8ysTZKz2PHwIqeWxgnqHt6SjVAwfnSJ6/MDHN0zy+sLYzhbey67Lai/PEjNA+8O7VOjqnxT9bnm4tU166nybY/Gfl2piMra4CKzqmjvBEPAX3niZf7whYdQ9+lUOfpnFoln0J7ULChlCqKiZkj1vWpi+RJpCeK8YOM+UMVIS0DOOSD0PL4RK4I+g9kvm4hQUf5uFjO6dSWXnxTu6ozqNAWNRyPCiqJzf0BnSmL5Wkgh9RTFFzJg6DKLuylwzYR/dORb5C/bTH18FvPjm8ytVvnS/af5mYEzOBfefjAwQnFHD1OFqwZWR9AZ15r4yoDoYI/mXl2Wiw74W8GqebbeuuBvH3yWT5fOo9yUwlmXwhmXpc/JG9pQ4Wisy3YlhdM02LxPYiS6udA8mGK3BdnLLvkzLu6GYPS3XMKybrU2DifkZk0t/jsoCMt3R5DCXR6o5ZkU5Ztk1gRGzcEMBI29WvLbqRsIqShc0yUru6soOgH/3fM/yy/8xveYym9S8EIsO+WbVw/wv/yzvwJ3ZoV/V4QVtKboAa11mrqK8ncz2G2htfA3HTJrCukpitdjoifafG9jH4txFTsfIS204NlFm86kfnPFc/YNU2Cnoeuq6w8K5FSAt6RpkHFJu/0lOQgqJt6mxAwV5XMWQVVTCEUKgyfvPBXyDdzVgbr4Keh71aQzKfF2trHbgqBPERzrkVtWdKa0YK0Ra1+lC+tD/M0Tz/Dni4eIpMVwroVpSjJPF9j45N3DVn8DTgsGXguhaRMXFGlBEvQJvA2F1QVvxSQsC5ShWH7CRimIUpP/8fWfIu45CKX3nmYAlfNbvf6GIipq6mBUhMKsIM1KxIKHGcLG8RRlaD6AEcPaw7D2mCLJCLqjCrsryC3qx5r+tbvlzH+XB6q7rtlThesGff9r7obVt5jLEFYE3prAH09RJvQOhLTnirzeGmNXaYMrjQGuN6t439bOuEPfcO7wu/lRKAErj7kMvqiFUrNzerXojQi8mnaqli7k5ixEKhgut7k8N0yl0MO77mB1NbMs6AOv/pblQsHQS5K4qAj6oHLWwN0UBANap9VuCtQDbW1aZimGf6D5vWYocBo6G/cmEyrvQyvqg8a29tQ27ji2tae28ZHBdqBu457AdqBu457AdqBu457AdqBu457AdqBu457AdqBu457APRGoytKGuWYMlctal8kfS0izmgUkpCawCAn+rghvU5EUFHYPwgFJdMgns651mG5Hd8qfSOj/+DJ7PneV5L4OyQ9bJRpaN8pIdEvU21S4Dd15SjN6IG/4xZCoIjEjzVPI1BRJfouCZ+gOkzjW/ECu108a5RnJ0CsR2TWtBSYkJPd1UJY2jMsvKKqXUpw2lK5pnkJ2RWF3AQH+WMLOfzuDdPRndzO4JwI1t6hIMgqRwMpjAqtpgiuJBhKsHvR2R+QWBdVLKSPftJC2IB6Mae9KMX1B6WmPzoTuBJnRrREtgv0Bhw/MU3QDpBLsG16ndN/Gjf+Xjm5FlmYU3R0JUUl7SnUmFK29KVZPYESCa79kULoi6E6kZNYFzb3qhjcTgLQVycW7SPHhPbB2AlpTNs09kJ216O2NENM5rC5kl8FIFasPa7mgsCRw2rrbVrkck3qK0kWLK39/F3Jfl2Dg5iL1ngjUJCsoXDPIL0ksX2C3BHY2JnvNprMrwa7Z2B3F4pcTNo4ImvdHiJ5Jbl63ANtTelCwMKdHgm8WwXDKocllAM7PjjDz3Z3MfHcnGxv5G/fJLmsdrM6YoO+kiXShO6qQjnbHM+KtrKE0/7QwY2IGCmXogb7eiB5ZLl+GZCr4QK/bTwpmqCcPsksCywd7zSa3AN0JSVQU+H3aY9UMFennG7R2SZQF858ziaop0oK+MwrzQu4G+f3H4Z4I1NaxkMy6XkIm/9zHH0vIP50lzSiMQkzxCkgLEJrVbtZtyucMhp/3NQvoVQkGlK908Wq3kFFLWnhstl7Bm/YQCYgE3Bltbhv2SSq/sEjmkRr+SMrm/ZLcvNBfpraBEW+xm64DStB3PiXO61movtPa6lG6evlv7RK4lzI/8hKkqwiGUuLDPapPrrDnc1exH6rfUTXCwZOS7u4Yu6voTkjM3R38Yb1y5Jb0LFt+XlA/rIhfqaBMKF2RlM8LRD5BfqzJymcSoqrErd/c53FXE6ffwOC3beoHBIXrsPILHnZLj21kVgA8No6nuDUTc9klyQrsth6raO3wSDOK+l4Tuw3N3VmtyxTf3PMODeg9o3+1iPOWFUqkOkhLe+qUHJ+rc4M88MB1DhWX+e7zjxOVDYKRhIHTgs6wSVSCwmWLzgiEB32cKxnNAR1QlM8LeiNsDfNJ3A3tzBwMpTiDPYbLbfJOSCJ1TpFKMFFu0HnEZ+H0yB2Z9/IrBvaGQWZDIqcN0vnClvWmon5Y4DQ1USazYqAscDcNjESy+WhM/nWPqOzSf1WvlFb35r5w90SgdkcNEJrWN/S8XmaNBPwhbZ84+j2D5U9ICldM1CfrWM9VcJuKjfu0Jbe0ID8rCX65gfVs9W2OyjcDkbw9GIKRlL0HF7HNlF7ikK/0aIQZfv9Pn8Ad1fctXLZYfUhht3W2D/skuUWDwa+7tCf1LJOW1DEIq5LCVYNoX0iIS9/+DXblOsSpyczKALLh4K6ZpBlF4fAGIwUdvHIohMaPWpf/pGEFiso5wdoxg7ioAEXpkoEZGBTmJZ0xAzNAZ9xRgTrWpubm2f1bkuZO7T5dO5Ew/AOD2gMfoaW/O56idvcoHNugflDPtrd3SIZeSVACGr/cJnfNpL07RSmhlf/ygiSr949WDzbuF/DNKsltfK5vqP5FVYk41rwRpABXFgcJrpSYnR0gu6JHQPJzeg9avgTyaJtgNKFwVVuXJxmtCRVPhThNg/bxgPycQdCvmBisc/DYrFZBaZaYfn0c53wGb8nUYhlTPUYKbQDC1MKedz+oS3xLaOzXtESrJ8BUmL4gv5LSvS+gsc8gyYDTUfSGBdF9PQb/vSaDL37Cw4wV/hFNyO4NGZRmbu4574mMKqQgbro05rNk2pDZSOk2Ldb+apf0Sp5gI4tdVYx9B5YfL9Ffk1r+50oRBCRZLTkZ9Oll+2bR7Gboy/Ton2xQs0vs27V8I0Dj1OTKxTFy17c4m+s2ytDlqc64IDrkEyx4VP40T7ofmodSShdNpK3oO59QTz2K11Li6y61h1K+/NhJxtw636/t48ryIPa5LG+EoXQgnIg4PLJ247XNrlV1oNwBmL6gPaVnstyaSeopgpJB8RWP1t4UIxJ0pEGaUVgXs6z8Roeo7iG8lF7bxb2UIbusaE+pG5oMPw73REa1RnoYPYPBVxTF2RQjUSQ5hbqQJx5I6H/BYtcftIlzAmUqOuMGxrMlPeYxqJA2hP0pSV7h1m/+eYMVPWI9lG9zeO/CjSBdbhe4cmEMb8mkO5niNLUqXvNQghHpuqt9JUP5oi7LJAWJciTNhwM641Dfb+E8WWPtuEFYFvw3n/lj8mbIb08/zKWXdmCff8tsl4Ds0Q0O71q8cVOtl8O8kONOISprxUEzEMQFSTwQ6xqqDVbXwG4ZCKVlQM0IrNN5dv/HBLGpx2ScBtQ/EeC0hN7b3gTuiYzqvZCnvUPS6zdoHkgpXrHJrG6pNGPp4bUDebqjBk4L/EGpPegFZFcE0gJVSPDmXBoPxGTmbm6TqjJvT79hajF9aQR33cLTcqvk5k26YxCOR2SuOnQnUwzfoDijqB1TZFYEpm9gr5hkVy2SjKC9U2J9ox+7Atk1yR+vPchsvUL2P5WId72ZYZQN6lCb0WLrxm21Xo61K/14d1C6VE34RCseMisZfM6kfsAmtbVGbfEqtKe0YUbxkoU/qEjyksWPe3jrupad5CBzTlc47PbNPec9kVGjIohqSPNwit3U0jStI3q4bfBkSupyo3bnNLTinXT04F/ibYmtTbsE/Yri+R8fpHFJwdEW+3cuv+326blBMosWRvTmbZ2DIblFRXbaoTCv1QfNUI925BaMLX1WQZJXxAW9P5X5FLujiMqSxn6DstPDNCStnQZJXuGPJwT7AyafnGPP4Jv65J3Ipf7qAN7anf3YBr7mkVkzKL9usXG/dq0RUpFkIagKBl+V2A0TI9KTxNlFE5GCGWnhiySrhwulzU2rWt8bgVqSmIse2XmT1NOz62bdwttQtKYsuqMCK1CEVUVvVNHaaSAdRVTUBWmU1rCy24Lu+HuXQ/yJhN0n5tjVv0EsTerBm7XNcrX7I/cf+1MLf1BXIWoPKsxQF8KDqkAJKE3rVqqcDChPa4lyd8XCHxI4TYOoKImkRS9w+G//s99m32PXOXxwnkOTb+6HQWfSxddG3tQBuIMwIz2pWvq5JaSnyC4b1I8nZB+vERcU7maMuyFoHk4pzso3bYn6FOGAxOoJqud1gskv3Fx56p4I1NyiQeooite0IrNIYex7kqBf0BtVujtkCpJyQmZVS3pnlk2sHpSupeQXFLUTKUYK7ua7f9D+RMLB/QtYhiRMLa4t9bPx0tCNYK3melvlmDfRntADeZ09MZVzgty8PvnbbZ0x/CGdRb0zGXr9Jk5Lm2f4A7oiAXDmTw8A8M+nP8dyu8DFxWHOn528IegWpyb1UwN3VIPgrWhPmGRWBevfHaVwZStbNizC7/STTAXM/IpFb1RiVEKauwzy84raYwnupkBmUqQLnVGTcDyise/mnvOeCNTUAeUq1o/rro+3oTWSzECbULR3QDAAuWs27QMx+asm/khKMAjNXSat3TDytBZ44F0+6zeC1BCK6bV+rp8cx7vkYYSCZk8HasaKSfNv700rU48el87Z1I8oBk7prGuGOsN7NUVvRB+2uuP68JBktGBGfk7hNAx6UwnxUo7GKwP4L/TjnM+gTC2NCXDl7DjmHTrhvxvae1Kt/leBzJqiOIN2qqk75K9aeDWDqcFNnBY0dwOJwOpBZsHGCKF1X8TE14yb7hTeE4Ham0woTJukGYlXE9QfTNh4KNXSNVVJ6Qo4Td3eHP9zg86kJLNsIk293Ew8FbDypMIfTd+VPfXWIBVnC2/LXsG1wo3OUGXy7WUDJbSrXlSA7KLBlb/ukLqKjYdSZFZSO5GSlFKddULIrEFxxsDu6Gybeop9/6eXKMwY2ocAHfwDW88jlbjje9IfhltX5GZNesMKq6MbGt1x/Tn1nzRIXd3gWHp6nNYuibchyM5bhBWIC7oKY6/Z1Pdo04ybwd11Bd4F+WsWRgh9pwzaOyS5azbZWYvmCa2OvPmAwh/WfefFr8SYgaA3lWDEWkxh+ldsBl4wULYkLrw3W0ddyf9IrdXZNFhta32Aohe+bfkPT3RQBthd3dfPzNtYvqBy2oREUHndxMjFSFvhDyhaeyXtnZKoDOpYC6cpWPt7j5Nb1Z0rgHBXyGBO+wtcPDPxwV3IDwi2r8gv6gDsTqXUD0E0FmE3TGrHJV5Nb2vioiS7ohsARqKvUepJ4gM90qyieySkeZNL/z1RnnIael8qLXDrBt0dCXbdxFxztONcoMUoUleQNl3CoYTiBZvoiTbNpRzeqqUz3nUbM+Idu1OxNHHNhMrRdVbnqlqNeiselQ2GoQM8Y8VIV/HGHqL8jRy1hyQiElgdQf8ZSVgS9AYF5Qs606imQ1zU9pD563oLkl+UxIsFlKFo7tMHRG9Te6MenNLVhktLQ7ibd48IxBsIKoL64yHOdQ+VTbFrJtlll/yiZP2oQetjAZmzGSqXU9Ye0uUpd90kqSqsrkH2SpbW/hRrySGeuDkFm3siUIM+XRMVEpShMPIx2XMWURnyj67TPNVPVFZ464LMOjgNm8JCyvJ8Tjv+uYrODrA6ukz0Tvu966+Ms+PEAv3ZLv0HulwuDZKmOsNNDm2Ss3VN6sL1ETLrby5EtU+FlF72iIoQFRUrj7JVloH6AynZOQuEQXZJH6Iy65LEE6w8oTBC3aDof01QfbVO/cEynWMxhlAk0kAtZm6aQPNhojMBledcuqMw8AMblCKsCOoHDJwmZC97+IPQ2GsCCqtlUJhVWgozC2EVBp8XtHYJkvbN1bTviUAFfYpOXa1ol+RcmsdCqi84bFzqQ/alWvWuohX6Bl9RmKGielYb0opUU+tST9vlqHfYv9stcSNYXTNh31valW9AKoHRePuFLZzyaE8p0kKKUzPJrBl4m4rmbsXYU4I4q9h4QBH0C9IDXcILOfIL2qEvzisGXhFsHgH/5wW9aTg4sUKcmky/Po57h5SwfxxKV7TsjxlAd0QnEX9IklkxkK4+/BbmJLUHBFZXUJjTfgK9YUVuQR98w7KBvK+NJT9Ch6lgQNIb1bzN3uNdsksGzpJDVNTF9fIZC6vfZ/T7aFfl+wSNXRbtHdDaqdX+uuNSyzq+h//pG8F67uoYYfr27/DFpSFmnpvCrb39krV3Suy2wFu2cJoCf1ix8aDC6gkWv5iycb9i4lspZgDuqZwmGvc0c8r0BfWD2k/KeKqCHNRZe2Zl4Eee525C7TMh68cV0uGG47URQzAoyc8qWnt0uTC7rFeR7oigN6Tfc1zUpKHsuiS9lidduTmPgG3tqbsAygTjgSY7+za5tDSke/13/mP50LCtPXWPQCiwTH1Y6yt3iAt/iaL0JnHP7FE/6mht5li1UhxLe0pt4+3YDtS7ARIyMw7dmX62Joq38UPYXvq3cU9gO1C3cU9ge+l/C6Kq5NCJ6wBMf3PXnX0x23gbtjPqW/BGkM7WK3f2hWzjR3DXBKpQWieqPCMpzOuZqKSgZW+CkZRgb6hd/PaG+Dt1YfyNqVBlaWaSkJr8oG5yHNqMtXqKWwf1YJtGkOHMhUny/3sJZYLb0I8V9mulDzPWf0+zWkepPC3xd8Qg9H1vFdk1Rf+5hPKMJLuiiA/3MCM9zKdMrVtld/X7yi8qguGUvgspqaevjz8Va7JHB9LsR7ukddcs/U4DCnOwcUSQWROkGd1JyqwKzMhEnuiirrm40yZRUQdm5ZwguVpCSEV7B+QWBWGFmy6Wd3YmdMcF2SUTx05YqRdwKgGN3XniAz3CTZf8NZO+M4rukCZCg9YSCB9vE14o4KwJksNdooXsLYtBNPdAWLLI1BSdSYFxNYOQmgrotBTrJxTxZIhKDCzfwfQNNg8IktyWtFEhonLFYOUXQoyZH1VZ+SjhrgnU9i5JIy8pn7XIrEs6jwbkX8zSOhTjLdqU/yBPnFFYwda8eAnCPsisou1rNgSt3RKRQuGacVOzOPkZC6HA/VSNohfQc22iPxqkNwSDf+whLWjtgLVf9IkbHuUzlra4aYJ4poAy9HPnns3R3im55cKSgt1fmsFPbFobFZLIYufja0wvD1J62iOzYjDwDZPOmEX/6TaX/q6LMiyySwZ2V8FCluUnFMZcBuno3vtHFXfN0u809LBY41hEVBCoNQ/LV7irFpl1Re0BQXE+JnUEuSVtdmu3BQOngxv8UHfDQKSC1p6b1zKUNnx67DLHqvOs1YpsHk9Isoq1L4dkagmpp6j8WY78tIU/CN6aoDsmaR5M6I4pMmuCJKMHCm8WSV6R3t/h5z73Alkr4ovDZxHnC1S+73Ht2UmG+5rUDyni+7ssfsKicVBx/Ut5Rr+p/aa6R0J6I9CZ0rI5cV+CsraX/g8FSVaR5GDoKYvaUYWzqWV8Ug/qx2MqJ22u/YKgelILSqSeQsWC9fs97JbCCCGtKhAK5aXczHeweiGhvs/i+8t7WJ2rkp2z6E3FmBHkv+Ox+rCgfEVhRorMOqSunlNHQGbRQpnQ2p+SWTRp7L6573xSUPQ9sEbRCfn95x8mP2Py4sR+bAGbT4bQtlGAt26gNnIoS+GuGpSuJazfb5Fdgo5jU7gGjQPQORSSveLS2xnD5l2Tdz5w3DXvrO+sIrskCMsGmd0tkpyiOwbKVOSuODT3K0RsEFQFnQMRcTXFbuuZHemAP6IoXgGvZtD38s19/zojJgNfXCCVBkYuJskostdtwqEE6YC7obmwflULKiCguT9l4BX9xZKOonTRvDHi/OMQ9kkGH1zFsxJmTo1DCu09CcrRVufujEd2pEMncPFqimBnSOWior0npbHb0jsLAZlVQxOt1wXWqoPdgYFn7pqc8xPBTQWqEKIshPiqEOKiEOKCEOIxIURVCPEtIcSVrZ+Vt9z/nwghpoUQl4QQn7+Z51j5RErzWIgZKdJXy3g1QTgWY4z6pBmQnmTsKV0dyE47ZOYtHUybmmOajgW09mpF6s7kzb35sCpo+B4fH5lm+GsuTkuQW1aUztpIS1C6HmP1FM2DKZ0xQW5RYTcMVj4Xk5+D3Lym+SHBbr43Ez8uKPY8sADAwqujOHWDoRcEVsvEqgb0RhT9Z1IODKwShDZhRZA/77L6mGL374R0xyV2Rxv9unWtbZXkIR2O6A0r6odv7j3fq7jZjPo/A99QSh0AHgAuAP8YeEoptRd4auvfCCEOAb8CHAa+APwrIcSPnaconbPxrrvEOUGSU0RFRfmUA9eyWnzXVCx+Cnqj6RZTXgvgdscVXg0G/8xFxNAeN28Myf04fPoXXmYg1+UPXj7B5gFDOy9nBWEfuE3F+oM29UMKp27obLoP4qIid9Elzgua+xWyL0bZWpT33ZDkFX33r9OJHVbODGG39H0be/XlL38jS2EWFn8uZqFd1jpNUiv+WR2DxU9lsXrihrx7UBWMPe2TeorBb9mYgZ4U+CjjxwaqEKIIfBz4twBKqUgp1QC+Avz7rbv9e+Dntv7+FeA/KqVCpdQ1YBp4+Mc9j7QBAe3dKXZTj0VXz4f60NQSeAs2gy8KSpdMwj5JYSHR9UNXkWY0WdoMBGagbpSR3gv2Q3XaiUeqDMpnrRue9makCCYj2pNaosaIIdoRYsRQuQADJ/WkgJHqw1vuvIvVFjQPvbPGjrKgfH+NghuyenbwbSUsp6EfP84JwrLgZw+d4ZcmX8UIDKQDSV9MZlVLEpUvavE1dxN6eyNWH85QvqBru7nlrT39Rxg38+52AevA/yqEOCWE+P8KIXLAkFJqGWDr5+DW/ceA+bf8/sLWbW+DEOLvCCFeEUK8kna7dMckhYfWEeWI/IJmj28cdrEeqWtFvoyiM2bQOBpRviBI/v4GqadZ+Z2dqdY9ui5pf7qLcRM6/VOVOjsyG0zPDNM8kDJ4MmL8uzHFuRhn0Sa3qAjeUPV42sWIwKun1D4X4DTBq22ZJ6gtcd7aOy8a4WDKcL7N9MywzsxvQVTRkjetvZL/69/8Xc7UR/lXr38cVYkQCVryva2oXJKEFUGc11I42WkHJXRTYPURiAoCp/WOT/+Rwc0EqgUcA/7fSqmjQJetZf5d8E5R8iPrklLqXyulTiilTpi5HKovonatSr4Q6A8lpxj4uXnil7W0djIQk1+UOCs23THY/MEwY1+cJffAJlbbwK0LEk+Q/34O58coitgP6Zn5/3DuYZxVi9x1k+s/ZzD3RYv+/9s1djw+z+6/dQmkwK0LumPixoHGuu5hhoracT3L720o3EsZLR30DlfiwKF5NvwsmfkfTfPm/U06hyLuO3aNzTRPyQmQdRdaNpk1xeArks6kwP3PVkDpRkNnd0LYp6Xeu2M623YnJGH1Pd/yPY+bCdQFYEEp9eLWv7+KDtxVIcQIwNbPtbfc/63D6OPA0o97EuVblC6YxK9U8IcUA68qGr89jj+WEFYkI9+yaE8YRIMJcV7h7wmZfXqKztkqSVYRFbXUdqYmSTPvvl+LqpKpig5U51yWuKhtf8qjLT7/xGv8teFnOVheoWCFnHj8EnEWSjNa0CIqa/sff1BgBIJgPKa1C0789FnEuxz6LUPiRzb8UGlXOvAre06ya3KNI6Ul/sX3Ps/VP96NEQhEKuhMQZwxCEdjFl8dIRjQVYbMgoVbM7DbCqurD5LFGQNp/yXfoyqlVoB5IcT+rZs+A5wH/gT4a1u3/TXgj7f+/ifArwghXCHETmAv8NKPex53zbyxv1S7eqw+Ckf+7lmMyMCtG/h9BuERn+J5m+oZgb3sULoiyc/pone0M8CrS5q7TLKL755R0+qbBhJxUWEN+siPNYkSk5lWP//0X/4Nnixc5rszezn57H6SgtJq1Qr6zqTkdza1NqsJVj5G7vJ58anD+EffKaVCN3ZuiFYoU5eo/B0xlYdX+ebyQWbXqpxtjmK1DIIBRWbFwAgFVk/Q2i0on7JJ+mPCoeSGIkympjBi6E5K4moKEioXPtqBerPFt/8L8L8JIRzgKvDX0UH+u0KIvwnMAb8IoJQ6J4T4XXQwJ8DfU0r9WDXP6nnFyuMKBBReyGGk8PLyfaiJlMFXYxY+aWFe9+hMSoTSuvq55ZjVTwuy0w72nMvmEbCbEN9E+7Q7U0J6CtV2ODC6Sjd2uHJ2HHV/xG9e/CKHxlZYemqnFmIbT/GmDaQtkM9VSMcklfOCXkcP4RWvKRriHVQtFMydHKN4aIO9D88ilcAxU6LUZHp5ECEUlWIPx0xAgFMXOE2Fuwl95zosfCqvD2OnHLof61J7DHLTNklWEFUUIhb0nTXpDUFc+Ggfpm4qUJVSrwHvNCX4mXe5/28Cv3krL2TtIShdNOmNKoJBRVyQeKsmyk3p/edNyn/Ur00bKgbZZUV3XLD8uIuzosh9bJ3Ni30oU8/9G9G7627et6Xc7GwaRGXFk49c5L78Ir9z/QQiBbcY0pgt0+lWGWxKkpxJ9TWD7pjQ+gBVRXFGcwm8GjQPpjQcbWjxTscpuyXwX+jncqVKmpeYHQOrJ3BiwbEvnqcVe7x8djfZpqA3luIPC+yWwcZxj/I5LcAWf6aJ82IJOaCJOr3JhMrrJkG/uNGlM9sf7QGWu6adYfa0xqnT1HpGypZYvom9ZrPR6kPs010qZUv8ERCpQCRQnBbIa/2YO/Tj9EYkxZmtFs57wN8Tki/7ZMyY35s/SsaOqe7bZGO2gtU2qFzQ4sDdcT1/723oBkHpsqIzCXFeYncM8ldNsmuS5q73zmhO3YC3nPr9iYSTixNIqXWqzFDRm1SojMSZN5j4VsT6MU3QWdnM0L+m6O5LiGNbd6c2FO2dWkbT2xBI531+AHc57pr1wowEzX0QVhTlswakgvYRzTs1A/1hGJGgcNWkf9cmRiSQgxFhRYsc5BbAaQjSckJcePcgPXNVV8ru27VIZy3H2c0RSm5ALA2ODc4z9hTYbYGQuoVbnNamZL1RheXDxoOS3KKifEngbkDYp5sEucVb2CMKKI+0mKzWidcyNPdr/VYR6Y+jvSdh9qczxHnoTBiMfNvSqtnXHXKLgvy0xdLntAepZk1pWcuPMu6aQHWa4NY0M6o7BgMvmlRfsBl5LkUZ2vR17Ptasdn5d1XSnKT/Oy5JTlF/LKKz5d1UOO8QvcdcfGbaZfqbuzhzdYyHDl/lM8OXuHR5jNX1Et/75oMsfUyQXVXUv9wjsyrojeie+th3YxIPyhcM6gcVfr/WOnXqgs6Elgy6WcR5RTXXY39pleppg+yCgVACb92geN6m71UTa1+b1N2SxrlfYIZQfWJFKwbWFIaXbpWstBLMX/rO1IeF3qgu8rd2azZ9b1ggFGwesBBS6xnVHrDxaoL2pMnw04KwpCV9rCUHM9KZNSrd3PNlpl1Of28fv3/tAbAltGyMRDD+HUlQFVin8/Q+3tEGCQZc/0Vgy57HWzcIqwqnqWuYAGH/zQdKmpfk7Iivvf6AZu8vSZSh3abjrPZuLf5hHm8Dlj+pH7/9sx2C3xuiNyRo7Adr1iOupHQmdabPz97qFb+3cNcEqhkIMmsK6UmkpRnufr9A2pAUJf6QYPT7XUQK7R2SzUMGUUmPYSgToj0+mXVBOJQQDdycZYi0YeBfZqm+ZGPXDca/69OeMPE2Fb1dEe6LeQpLiXaPft0hqGrStt3VbitRWXeWhl9K37WO+k5wBnsA2NmIoCqoH9DkFDPQe02nKVg/DnEOihcs7I4gvZInt5pqq/VAi48NvGBSugKVKwn+0Ef7MLWtPXUHMPbpeTJWzJkLk2Tm75rz7B3DzWhPbV+lO4CFzTJxbGqx4G3cFLav1B2AeK3AR7ya9IHj/8/ef0fbdab3meDz7bxPPufmfHGRI8EAZhYpVhWrpCq5ypYllS3JcvasbrXc9vLM2D3u6em0umf1dBrbPb3kKNuyLJVUUpVKlTMzCYIgkYEL4OZ8zrkn7ry/+eO7BbJULBIsgCBA3t9aWBc3nn32ec8X3u99n99ts0bd1rbeTtuBuq07QtuBuq07QtuBuq07QtubqTtQiSuJJ3zGBurkrABNSC5v9OB3LWTTwqxpyl37A6TbdkSN86oFWaRgN8A4VsfsgD+ckJ+XREVJkpF4UyGVCwl2XXV6xlllSmHX1bFrZkU1xL2TooLEbKuz87AksRuK5xTlJfl5SW5RYjVVLahIlQN1YTbFG4sJelO8ccWBEink59+73LTfn1K+Z51CwaPhOSw2iiw0ilhGwtTwBgcOzrHjkTn8wYTMmsTf46tmyK3rRlN8LntTuT37wwneRIQ/rNp5/L0+eqSeR9CTYrVUq83bFaPfCt22I6rZEGTnDAxf0h4V+Ct5tEll2uv3CErnJRsPxGQuWwR5SWpD+fAGrRf7yM+ntCY0EleQWUmVD+RbDDBJRhL2xeT6Ovzm3u/zR8t3Y+oJM1/dQWk6wus3MELB2sMxvaOb5P9VmeWH1dl/4qY0dunkpjWspiQomxieeij5Hr79zaZgYzOHdebH3URm82WicszgeI2dB5aY2xwlc85B91QDYZRXZBdQTYyJI8hf1GlPCooXBe2PdMi/kCVx1XPILmh0hyS54y5mR9Lt/7GHvGW6bQO1OxnRNSTuFUv5wYcCd1lj6LmA6V9VoIjyqwatKVUaaH10g9q5HtKelLX7lDFa6XLMyoM6ZuPH/743HnFw78K1z/+XP/5zPPXxE0y4G5w9MMSK45BYkuwi+GMp3Wd72fhUiJmJyL3qEt3jESYZek5JWqPKnj2zoo5Ya2MSq/7eTL26L9DfIkgBjJbAaJl0Byx63C5hb0Kc1xT3wJEYHXXsC9Ael4BEWhIRq45b98Us3pAkKinfLlLQfA2/RxJl39+lxG079ZdeMxn+uoE0Ic6mUIjpTMYsfsRl9GvKeCs1Bakl2bw3pHqlzMBLIJ2E1FawtLW7DTKLgu7kj9rfJRnJ1M7VH/naz3zsJDOdCv/y7MM8umca3YfCVfW9nudN1fG6ahJ1TVYfT0ivZrE3NNojOt6gpHw+JbMeYzWUY/T7odSWiHsaTJTrrHVyYKeIGKKRkMIlge6p0kRnDXLzqoBb8wW5qxqdYdVuk+pgbOo4Kzqar2E1BUlGLXPeT922gWr4igYiNYnZ1Oj/pom5qZNk3vC5T2xw1jRKxy0qpzSWnkgpnDfJX9Hx+yTG4QblSxG9L74xcUgDolKKa0REic7Z1ya4tNqHLiQVu4NpJjx7eSdaBLEjKM7ExFmB7qtNTPGkhTtn0nvXmvJVzahpMiwIZj8t2Dyc3nqqngb+YMLIw4vs6Kmx0s7TeLWX8ssmUoNyTwupq2q0xgHV9pKaYDUF+Rlt675IirMxxWlwaqoLOHVTtAhl3LZ3e0R9S8ktBz4tUCOB16shYgh7ElpjOv5gfM1uW4slhifJzhp0RlPaEyn2hkC+XKS+20S+qUfEH444cHAOgEvnRjC6gt5im1+ovMwrS2P894f+mFzOZ9dnL9GeTOn2Kbvvvtcjxr+eYHYkug/V1/pp7w+QAsVzNQWj3xRIOyG3eGtHn2h/lwOH53CNiDNXh2mc6EX3BO1xZbnefaWXzpjaZGbmDOKMMkLOLqY09qqKrJ5TEr+kE3yiiV+RxJUYEar2Gy0W79iC/l7rtg3U8Oc3cVdUSV2clYRFtT4beEYjzkDlhI43EmO2FG6nuUNtarRIIGJBZzJRFOfcll/nloz8n1kG2BIvNPlbT/9VPj55gf/6/Kc53L/MA+UZpAabe6F0Uf3s0l8LaO6E9s4YOe4x8C2ToCclzqjNSWdAY/jrOt3BW3Nb/eGEwiNr7B1epR3anDk/hnvZvpaaSkZ9OiMSo63uXc+ZkMRV3bCpIbCbKT076kQ5BbFojwiiSMfwBPaKgV3dsjDfonm/n7ptN1Piu2XkR+t450po0RvY89VHU6yqILec0NypOFObj/mkHYPEMihegPaYwFrU8foBoZA33X4BGuRzb7Q1a56GtSnwGj2MP7LEd+Z3k7VDlrsFZpoV8lc00CC7HLK52yL/DYPOqMBdMIg3dTbukciBAC9j4o1KnCWT/KLEG5AY72GznTQh3d/mwMA6UaJzfmkA60yGP9uNUnzOYXN/yvCzKXM/q7H8oEW6u0Puyy5RVrD0iI75Yi/+SEJq6OQWJEE3R1hUM5oegdaGxNniYL2PRm23baA298aUvllG61dU6caDPvqSjVnT0UJBa0RX+dEUsidcDE+yeSilYWrYezcJT5bIrEhiV+BXtm5wCvXlAsMFxb9J3ZQk0ND3tfjPJr/Dl2t38fr6kHr8rkM4JJGGpDVpIJKU8hmBPxYiuvo1cG7hZRctgTAP3lDC2t0GUSnGaL03tzZxJOFoyMGBddY6OWrnen4MFfRDxa7A3tCp79bJzip79mbbojmpZiHDV0un7IJOlJMEJUGUAd2DpKgQSWFW0Ozdws6X3pOndF26bQPVXjcwO5LmTh9wMWdtpKlGVn8oxlk36DuR0h7W1Nd6BKVT6gXrJEXcmuolsuv8xA7NPQcWCBKDrBny35z9FN2uzUempvne9G70OYfsiiC3lLJ6v0bvq1KN1PmQoT+0WbvHRBpqpNEShUvPX9Vp7o7Ruu/N1B8d7DJQaVJyPBYbRTpnylhvM8pZTfVGjfIQH+jgHM/iLJr0nE1YfBLcZZ38ZQ2/V62zo6wKXquh3LFXP+NjXnJxatAdlO/raddtu0Y1O9AeEciGAtVqscBdFsSTPiIReAPQ7dXQYtVPFR9qE5YEQVmQWBDl1OlUd/BHT1TMmkEzULAIQ1O9SwCthQIsuLzwxSOkTZO4mCJSiG2BVRekpiAsSOSVLO0RAy1Ro5vhKY5AqqsRSwu1H9m83Sx5YzE7B9cpOR5nrg7jv1J5x6m4OyTw+1PsOujnsrQOhAS9CcuPqmbA4uUUdyMlrCSkW4NA+EBLdQNXwLzkIhLoDkic6vZm6i0lBWSXJO6CrtanIUR5SLsGohiSmhK/H1oTEm8wJf/dLN4hD6lD/ysSbzShPSZJHEnPKbWBkiZE/REFW+WPtK2z1Y1ulvxoE6MtMNsw+LRGflrHr6h2Zb8/Jd1qyZZCpaI6kzG6JwjKEndVEvQo4G9mSVNr25uoqCg5uE8BEs++NoE7bSOuoy1MCkBINu8NyM+qDt38ZR3dE7jLBrWDgo27BJXXNbqDCkOfTuc4/OA0VhP8oYg4I3HXxXvy5ns3um0DVfdBjyRhUe3cpQbDz/gQC7Kvqm1DnJEk+QR3TSPKCdK2SelSyuZOjeJZHWdDMZy6v7EJdzcp3r/GwZ2KlLLYKJJKNUqkUtC9UFJ3YysQpYDcvMI8ls8K4oxAS1TPPwJEoKGHMPxsoqh6lmTwaYHfK2ntvr7mwuuR35+y9/4ZAC6cH8FZu/6XLLsk2fNvOxRet1l/JKa1L9qaaRS8w2wLBGota7bUc84uCi5/YTdevyR/wWT0uxG5BZXZeD91265RowI074rZ9dsJq/e6BBXJ/MdsKifB6wd3XaBF0JrQSSwQCZg1nZVHUtxlaNzv415w0D1Bs+Owq38DgKqXofpaP0ZHsPFQTG+mA6gXz+hA4qoefT1ALSV6E+Kshl2DxmiKtFOcRROtz6fjmICJFkFqSep7dUrnJHFWJ7jOtu13ktanRv/L6704q+/u5eqMCKrHXDBDSq9YdEYV16owG7P8sE5uDjIr4PWpZUziSsISDD8jyS4Lwjxc/SUgSTBrAqO9vUb9MXmTIaNf1pn5OQe7LonKKVEpvdYWnFjgV8Dw1Bm7uZUrtOo63YkYoUnijOQ/+bU/IVrIEqcaZ66MsPn8gEodpSgcJNCfbZPc2yKobO14HUmUlbTHFYfUbAq8gRR3SWdqxyr+YIxccyicM1Ul1ZEOmq9uZfWovGmjT5yT7Bio0g5tOJV/Vy3ZoN687pxJ/ox1rSjHakn8sk52UdDYI4kdGHjJo3gJ+o6rAwK/pFE7qJL9zrzFvn/SQove/rHea922gWqsm9T26uTmBPVDErOm4azqmC1VzqcHPzwIkFQfiLE3FYhB92Hw+xpi1UYPBJe8AaQluXh6FPey9SNru86lEnVfLSOSWMMfjkhciAZDEgfciRbSUkAMd1UjykuqXxxl+DsauifIrqREOUnUsNEDsJqQWdbQgptzD3IHa9h6zHK9cF1rUlBHxNJUx70iURvKsCjxe0ELBc0pQfUI5JYTdF+wuV+ydq9LUBasPJ7SmpTKqyAniTOC/Kxk9jOV9x1wcdsGqrsmCCoqeV46LxCpQoN37+8iNSjMJcTlGAzJ+JcE6x8LMNsSbzhh7QFIXUlQTvmTs4fRfIGz+uO7AT2EsqMOAA4Mr5CZNUl1RVGRhqSz6SK6Otl5SaortlWUh9oBDaQiEEZ5hX+0NgV6oDYkN2ONGuck5YxHO7SRV7I/8ecSRxL0pqR3tUjvapE7tkHpgVV2PzpDait3w9RQp0vSUMYcVkOw8qCGFglK59R1tydT9I6GFinjDJEINg/H1A9A+VJKa8cNP6Ub0m27Rm1PpFReFwQV5YzX+3pCY4dO3M3gbMDqMQ1jU6NwGeY/nmIsKUeV4gWd1FQplfyBGv25NheboyTem/KAW/z7aOyNoS9F7fgbdwdUnrfwJhNEWx0n+n1bZXDrqhZWJCqXWzhvEha5NoJ2hyDojym9bihP1pske1+DzkL+x78+2KUv36HkvHHaVvUy1FsZ1jYKFO6ukfygAgg1K6wItFAycDxm7ldjjGmX5hQkmZTcrLpvmRVJYqmDjsysgTRg/R6IemLcmet0Q34PdNsGqu4Jqg9H2AuWcibJaRSvJrRGdfxeiHpjjJpB7ApkNmHwO5KV+3VkU1U6DRxdZelSH6NHGgztXKc+kCEK1dN13PAaHh3Ai02uTA+SzUD2vE1QBmfRJMqn2FVBdKxFejlHdThFWim5yyY9LxtEeVUy1x1SCfLsArirBslbMH3frYy2YHa1h30jK+SsAN50vX9W5+YHSX0Da9VAi9Um0wai2KVyOaa63yC3KGlNCHLzsPSogXXRBE0FKaUI92VlsrZxX8rAs4LcjEZnTKE1U0PivIUHwa3UbRuoYqJL8fksTjUldgQb90jyMzpRTuVUc5dMnA2J1CX2osn6UUHxktq1tnakpN8eJJfC+nOT+J/bZKq3+paP04ks5tcqlE4ZaJFa50oNsqsx9d0m3WFJspQFR5Kf1ukOaxht8PrU6GwmW/nZoiQ47MGSgzHeQXvtx0fAdyvrrMuVs+8859o/4etOVTL/FCBiEAZBJSWxBcW7N/C/00d7KsGsa+jrDl6fKrbJXzRo7AKzBcWLAsNTa9XG7vQnHtXeCn0o2FOpLZULoJXQU2nTn22z2s5T28ySdgzcufd3tPiwa5s9tSUtELhX1IF/G5c2fcBPHom2dfvptt31b2tbb9Z2oG7rjtB2oG7rjtB2oG7rjtB2oG7rjtB2oG7rjtAdGahRQSJN1TYhUvXRH0pIHMngiwHeWIzZVf1AcU4SllPyc4pLJVJFSQlLilsV597/PPIPFfSk6IFiV4kUxY4aSnA3lElx4koFmUgVS8ofTkgyEv2+TcQ9Ddx1iTQgLKfEb2NhdD3KrGwxvDZRtp9zkv4TEd7OkKHnA5KMJOhT1xvnVL1BaTolrKTkFtXvZpcleqSex43qjgzU3Jwq0+sOpfh9KVZTYWmicsrCxyzcRWMLsKDhrgrsmrKANDqCxJUMPK3z+GOn0DcN8lff72fzhrRQKMbTgCC9p0XtkMRoarQmAE913GrRlg9qAiPfVoZt2T8oIE8WaU5tucOsadi1G6sdrT4a0h1N1XF1QdIdEqzdZ6LXDOY+aeGsC9JCTGc8Ic5Isgsay4+n5GY1aodVcU53UCA1qN5740U6d2TCPyhBMJAgiiGT/1pj8SM2zqoqKilNR6SWYP2oca19JcpLEltgeBDkJc1JjaP5eZ4e3kkzybwlm+r9kNUUpIY6ws18M4cc3eooAPTAINjpU37BorZfw2qnbBzSMXxAqkKZxJWYsw5aAH6/RO+qboSgkiIGAsb6a+StgPVulvUzfW97JFo4aeMNKRPl3JwgKEP/8Yj2sEF7XNAZkZROWMSOOqoFiUgM5cL9mqA7qGqGExty0wbJDZ6u3JEjauJKhfq57FI9YNN3MsEbSug+3GHjLhOpKY6SFJCfkRQug9j6PM3HGPfV+T/PP0rPl9wbHnlupoKymiLDox3ST9Uxm4LOYZ+gIsnNSXKvOSw9FSu3wn0a5YspsauKx0c/Ooe1qdF3MsWpSbQ9beKshKNNDt47w4HRZfKWKvPqy3So7K++rV1s4qqC8XCnh9cH/kDC8sMG3WHV2cuwT2N/gne3R/XuFLeWELtqYAhzArMl0UNVTeZufEin/qic4vZ10X3IfWqFpccEv/jYi3xm7+s88QuvsPDxrZFgQbL+YEr9oZDMsrpZwtf52YlzDBZbxK64rTxESxdBiyXuS1n8Vyt0Dvs4l1QpVv3jHu0jPmNf1sgsCkQErVGNKCfJLSZcnB7C7Kg1e/1THaLZLMP3LbOrb4Mw0VloFDl1cYwzr0xytVZhMNd6W+ZpZyrC703JHXeJpjzVFBiqFp32ZErhaQcS6Pm6gxYKMovda+tUqUN7HMympHJSv+6i77fTHRmoY1+VhLM5UhsansMnHjvJ51+/h5we8L0/vBc0VaHu9QuMpkbxuE1QEjgPb5AbbXKyNsrP9F+keneKs/F+P5s3lJ+P6YwomIXRhfxJBxFD/iq4r2Ywlm1qew38PrV5MTsSsy3o9msM/EDHWZfU9+g4TkR5X42C7bPcKjD96hj+iz24MyZ2VaNdf+deGWvNID+jYbYlxoxDa0eCNxYRlCSyHBLllV17t1/gbGhs3JWHVNAa1Qh61GisaDbQ2H3j9+aODNS1+wykBg9+8hRSCi41+8iecvgPX36czmRM5VWdtC+kvSuCHR2iHPSejtA1ya7KBkfKi1zt9lK4pNO4O3y/n841LT1m4GwInA21DPD61MzQ2COJs6ryvjMZo+9rkZvVtjgCavQKShrV+1L8I12myjWODcwxWy/TeqUXq6Ze5qgoCQ92yZW7NAMHLXo7eIVQ5ZIm5Ge4xvjSIkHxJYegR+KuCsKS6n6oH1ac2sRWvVrdqYjWmIHRFtfoizeiOzJQOdji5z9ynO+d2M/RwUUuL/ZRno4hVWzP2rEIGQtGv6YRr7uULiXMfy5mfb7Mhpfj6ZWdPPOtwzSOhGiN22c/abRVJ0FQVhkKuTUzWA2N7KKkMyax13X6/k2GKAfVYwl+X4qzLmiPSYwej7966AVqfoZvf+1uouNltDe9D+WIz4GRFXZUaizVCoi3adgLi5LMsgqP2l3KnzZzxUQPoLE3IdXB65dkVgRxXpK7qpGfVQ2VegDmukFYAKeqgv5GdccFampLNC2lEbkcPDDPq8sj9H/dYv4pGHk6IhkMKb1q0fO8yeoDGuamRmNK566JBf7x41+i3nVxjJjSRbCXzPcV/PVWak2pZkJvR6gwO0MhSJXqiV011VuNiN7TMe6igeEpOIQWw89MXcJPTVZqBYqXgT+zBM1k3wC3Rt239w40mwK/R22IzJYyrwhLkt5TMT1TdbQY0p0eeiBx1gSpDWtPhuQWoL0rwl1TnIA4A6XpG1+k3nGBGlZSChmfvOFzYWkAfzbPxlEBTsrsJw1oqX6lxh7QuwI9FOSeXOXc6iD/cekYpp4wO9fL2mMxdk2xV28XFa+kOKsaQSWFSK0PMxdtcvNSeQOYKc6GZO4TDut3GXiTIVZD0DoQUroAFzYH+J0TD9D3BYfawR/922ElZbSo8nDNwMFafPti8dQEZ0OoXG5H4K5BXEjY3GnQ7Dg4G4LKV128TzbpP+ETlCX50zZahGrTyaqu3O5QyvLDH8IR1WhrNLsOX5veT+wb6KFQ67o5i/JZgSiFhEVJbl+d4Y8sUH5khWojy+GhJTbaWX55xwl27Vhl8Dv6NZ797SKnHisXlqsa5dd0avdHSA3q+1XeuHTaoLlT9ZO5q5LxP9LoDqf0PGdSPQJxqlHpa9L91QbGG/1+eGMxAwfXMDQFOZ1brrwj8MxqQPtAQOW0ICwq+nflpE5rd0LuO1m6wyluLSa8VKC+x6Z8VuVMNx4LaU0owkv/iS56IMgs3XiY3XGBCmDoKfuG1sDXkEBnZ0S81ceuLzik/QGmkfDZodfoBhZT/VWW2kV+bedL/Pbvf5z6747SnNQwuhD0vc+E2jdp7W4Ld1Ul0xt7JNlLljoObgmkm5BdTdA9QVSUNPZJagcMsnOKZphdEjzQN0PXt5ko1ZF7O4w9Ocf4R2c5vH+OHrf7xgPJdx7hWjtSSsdtOiOqFdyppzR3Qu9kDa9fUDormHtKJ7cgCEuC1qTKXfc8ZxEVU+y6xtJjGXpPyrfN116v7shABSiYPiIVDBxd5ZFDl8jvr/GR33iRAw9f4RePnGB9pcjT9V18bOwChpay3shxsjWKP5jQHVCnVN5AinYbGYdZDUlqgRZB5ZS6xsqrOlFe0vOcyfrdGmEpvXaE3DkQYNclGw/F6L7kazP78dsWc40SuwfWMXW1Npyplzm7MHTtca6nTc5Z1+iMSjJLkuTuFkFRoenbz/fhrkpqd6W4qxqthzzMlmIoxFmJiEG6iTod60tZfRA6Yx/CNarRFjSX86x6ebSKOml5YWaSZivDol9iNLPJ82s7+H8/9nlOfWMvj+Qvcf6VCX5211meOb4fd0HHrkPxaoy7rFG48j4/oTdJGmrXb3QEfo/AakgyaylxMSHziytoscBsa7QeVlNq4YSt+vVnTWJXEFwp4F62aUyXOTc/yIUXJpn7zgTxy2WS1htrUmPtnU3Y45ykclrS2gFB1aU7JIkLKplfmIvRwi3fqlddrKZiJhSuKF6XXjfpjsVk5zVK5wV29cZRgLdPbuZ6JUEkgqVvjTH8xDKmnpB0DR45MM0Lz+9j13/ssPmRPP9V9lcwfHjUWWXfvbN8+fxhNF9heaIiBBUDvy8lXbl93qtRjmunOIkNnWFBUNYZ+XbC+uoQdk3N2tqpDHoI7fEUu6ZhNcFsS/LzKe1faSJeLOOecxQO/s+oGTgY3XeeRcpnJc0pjcRN6TmuE5YEUQnsGqzdbWI1FB/MaFu0JjRyc5LmDkUQLF2E9qiqtfD6BUZXnfvfiG6fV+ldqOcVDcOD+reGqP3RKP3fN3n+hX0Ks3NvDpGqzUD2iTX+Se1+/p8TX6Kv0gRNlcnFmZQoLxn5niTouX3WqFqkCmjsumJIpaZKO9X26tg1RbfujEq0BGJXTc9IaByMaU6pv9G5VEJL+IlJ9jjVrgu2JjWB35uiRYLWFBRmEqQusZpqYxW7qiglrKTErqR+LMKuCzb3KXslwwM9kGgRN1yQAndooLZ2gNGVaLEiI3eHBJkVjf7jEGfU6Qg/U6fRcbnU7ufv/E9/l+pmjoEXlGeVVVeonvaITnbx9rkF7YkEpGDzSIwWKOO2xFL1qXFGBbGzIWhPxWSWlbuJ1MCdN4gKkvV7NcXMGk1oT/yZP65vFbwk1zcNB2WB2RTYVUFuFhY/mVK4aOA0EvQA3FVB40CCXVU5VnfGIuiRUFYnDLGrDgqCkiQ/9yEtSgnLKX6vguZKDeSDDcyWxPBUsrw7nLC3d41f2H2S58/uYvNIjO1EVA/pJI4ksyxxVyROLb0p7/abJb2rEQxG5C4b6KGqNZUaVA+Duyaxaxo9Z2IKFwyinCA/l+L3p0hTbb6QYNU1+l7WiAZ+9Gg426N2/RvV6yO4aJHE3hS0d8bqXH/DwO+VrN6nE+cADTLzOn6fQnMaXcgsC2Ss0RmGYDQks6RMORo7P4R5VIDyaQ1vRC3m/MGY+HRBYWcmDXLzKX/9ie/TCFyudHoxshGj3xCkL5UQB1poocAbFIRPNYkdcT2ZmlumwlXoOa7WdkZXUr0voTuakp9RPClvIlR1DkJB5LxejZ6TgqAvoXzBRw8EwU6f2IX8qTfegVKH0dLmu7qW5i41iotAI6ioAWHg5YT8DGSWJV6/6jSwa5qqqJpK6A6q5UBmVZC7YGE1JMVLAqt54/fmjgxU/6kmWlejfPc6uatvQMnKlyLW74XfuXCMo5UFqn4W82yGxg4daYB4PU9iS7LzEv0HRVJTVa/fLoqygvp+xc+SQqBlI6QhaU9IOsMCc8OkcFnh2nNzyqwscSA7q7PwpKtmi3MOYVHNNj9U0Jdg6e8uRVS8oFJg0kyJx30yy4L5p9QGz+8R6ug5BXdVYnSBVG1yu0OSzUOxujZL0B1UpYE3qjtv1w/IE0UsoPlsP5jKMj121G4UKRkuN3hmdYqN4wPoEmIH2NozmS1VfQ4qMIz2+/UsflyxC3ZVjR1BGezzbxTLpoZyj26Nq8+jLWTqDz20RKy+n+qAzo/WMLxpOJLh9Y1NP4TAuYsGoI5E1f+3zguk2iT98Oecla21byhwu+rnopy6rpuBib8jA/XtZG0Klr87CsD7bORx20jzBc3AYalaxF66M4FwH7hA3daPy6pprH1/+I5+se/INeq2PnzaDtRt3RHaDtRt3RHaDtRt3RHaDtRt3RH6UASqP5SQW5RYTfBGY4J9nuI5jcTEeUlUkIrzNKhYTnFe4k1E5OekKhJpqCp5b2eIP3TzfE6vR1oMw8/5hCWJ4avrKF9K0UPwBxO0WPELes4mTPxJTdnvHGmjJcpHKzzg4Q8k1yr+/QHF6Ar6UuTRFsF+j8RVjCg9UIwpIbes44UqLkFTZX96gGrTboOWgFO7dYclH4pALb+uERYEUQ5KZwycsy66D0Pf15Bbd6B6b0J2tMXoA4uUz0JmxqR2SJnYbt4dkr9oYK6Z2Ou3LjubWZX4vZLl3wiwmsqMzV00SCxo74kontPpjid4fYLFj8LFv14CAc5zObKLktJ0gn3apefkGy9z6ZxG7miV47/wv9BXaJNGGtkjNbz+FKcqWXs4AamCNnHVKZjZVPfB8ECWI/RQHak2d20H6k1RVFSghuYu9XlQUYUoUoPcckRjSrVxiATsisfjo5dZb2ep7we7LsnNCuIM5M9aBBVJ//GUqHDrygKr9yTk5gXeQp7EkhSnU4JKSmdQo+clg86YJDfSxO+TFM/rWJsa9YOK6Fffjzo61iAoCeIsmMfqTP6ladJU497v/gaHKsv8J8e+R8aKMMY79L5cx9rQ6e4JlH1nj5qJMuspjft9Ng/HFI/beP1Cmfxmb929+EAHqj7ZZu/ROcR4h/YDHmiSxpSGuy6Z/5iJFkN2QRANRAih3PtatSxxPkX3ISyh8JT9ksIV8MsaI9+7dS/O1B8kNHcn6IHAbAs6QxrSlER56AwJ7LogfL2E2RI09iSkhkT2hMQZiAvKurx0WT2XX/+lbzJRrnNqYYTN5QL/7QNfZNSu8/2NPfS6HcLlLBf/oUs0FlJ53lKWmS2d+gHY3KUhNizceYPcUkKclTjrgvzFW3eE8IEO1Imeuuq8lALzkouIVUdlWFINa6kB7kbK4V0LDJcbbARZCDRELIizqo/eXRNMfsUnzAu8XsHGoVv34iw9ZpO/rJOaEm9AsUgHnxYEfTH5OUl7X7hFMQGRKnKfOWcjEpC6WnuvPCIRn6zyz7/xUQD+wv6T/IVjx/lq9TD/6vRDnH19nPFsDYTkyd0X+fSh1/F7BXZNwrBP1BcRllOyixpCwto9Gtl5gdWUtHfeuvX6BzZQo7xECLWG+qsHXyAop/S/IkFIjI6qn9RiEH91DYAep8PxE7sonTHILGiKPOIJmvsiVh5wae1O8Pb7ZJdv3bqsdCkltZTZbuGyRlCB2BFgp3SGBb3PmLTHBOWLEZXXBZUzEi3esi9fNUgGAmQ2wbUikoIKqm/O7+WL37ufi/U+WHLQegO+/tX7cEfanFgbocfs8JlffIb6QYk26+DMW6TZhPZUwtDzPrk5cGopjT3Qc/zWhc8HNlBH7126Vtr2e//6o2TnNZqTGmlvROexNsWZmM5kQrWZ5fTsMM3Q4dBds2iRJLvyxvTe87JBlJMMPi3Y9X8mNHbeuufQHtFI7m+SndfxBiSxq7pUcxcssosS+5dWcdck60dNRKpwj4XLaqPjD8ZohuTe3TPU2xkylS6dyKLj2WihoP1CH4XLgt6vOCQZSXipwN/d/R2WgiKR1KEvIKokkILmxgx/BxYfc4hdQXdAU6bH+VtXzPuBDVTXeAOsFJQUHzXKSUa+ZLDzf4hYekSnZ6LOp3ae4dMHTnFhZojZP56iOyhITKHWrnlJUFIvxuoDcOUXXMxbiADyeyXeRgbdV5X+cTEhtxSje9CaFHS/OEBjt8RsK+RPZj2lPaqI1L/0wEt8et8piqZPJdfFW84x//IIxskcUmfLR1YQZQQDL0icqmAu7OXV9VEmnQ0+e+A17DUdZwPEms3mbp3K+ZTUhu5IioRbiuz8wAbqD1XzMrjr6oUWscDoJCz9TJn8rGBjqcikUyWWOnpVWZonjmTjwZigDMWL0D4YYLYUXr1w6dZ2BNh1wfC31Vo6yksycwaNCZPuiMQfjmhOQd8Jtc5OHFj42RT7oSrO7gbz3TJf/u59rAc5HuibwVnWMdoqiHNzgs4wdEYl7QlY+lhKe2/IS/VJslbIjN9LJHWigqSxL6F0XrW5bO7SkAKSbEpmVZBZ3U5P3TTVOy5ONaW9Iya1JdWDFt6gpDUhQcJ3q3u40OjnscdO46wpXM7ef+EjddAjxX4SiVrT1o8mBL23btdfvpgQlDRKl2PMtiCzIumMSzKLgty0qbpWMwK3qtpKe18wqC8U2Vmpkjd9Dh27yulTE/zgnz2AP6CWQc6GogSmliq0lhoUz5gIX+fn+k4xkGnxBy8d48tnDgMgyiFBSShq4IZaVpROG3h9kii7PfXfNO3srVL7bBfsFLMlKDy1wiNPnCbJJWiZmJOv7sTUErzEpP8X5hh9fJ7C/7rEX/nz32b9EwFWQ1Wqtw6EuAsGRvvW3bLWqE5QEiw9qmN0BO1RQWJJWlMphqea6fxPNFl40kT3oP2zbSZ3r3J+tZ8wNXj99CSiGKqNUaiCKruYkllNVdbDVmk3EcPE3hVebE7xZOU8SCi84mA2NKjalC4naKNd3GqqnFcEWA2FurxV+sAGaiNwrv1/qNzk4M5Fwrs6/O3JHzDTqmAUQ4b6GkhbjZCvLY1wtLLA9OwA616OP106yFBfAz2QZFYkWttQ6aGX3gYqepNVuhyRbCF+opw62hz7doK7qtEeV6Ob+9UC2TlBaoL+Sp7D5SV0PcUQKWZdo/S0Q++rXKMWNndobPz5Lu6GJOhLqD4UsfdXzjO/ViFKdb5T24e1odO8J6BwRZH8ant1mMkS5jTCoqA9kRIVJHHx1qWn7uSi77fVxisDdA/VGSk22Ghn+d37/j2/HPwaX1o/yq+OvshzeXVc1Rmy+LWRF/gd8QCff/0eMtMWtddGkIY6yQqPJdirBu6ywK7Lrb6sW/McqgdN/OEEd0EnsyKpf7xL1ctgbUoSRzXwpZagMJvSHRGMPLhAwfDp1jJ8e+UQmZY6WWvslWTnBVFWdZBGnRztUeh7Uac1YXDcnuCvH36O850BVtolpA7GikWU26Jfl6BwGVoTal2aZFKSvLzGCrgVEvJ6iFnvsZzhMTn5N/7+e/K3g74U3RfYGwK/T91kgAOH51jr5Bgv1Hnl4iT6pqGOBO0EY11R9bqDajccZSXlC5L2qKaKQ1q3aG2mQZyR2DV1zh4WoHIuodun0R5TNJWolFA5qWO2JfUDgskH51lp5eleKOFUBWFe0ntKUjugISL1O1ZDAXa74zF9Y3Xu659nrlPmWHmW3/7BY+Su6niDkrgU486ZiAS84YTsvE5QkdhV1VnaGZWYjRu/Fxf+u7//ipTyvrf7mQ/siPpD2euaAoltSEQq0AOd9rjkzJURPnX4FF5iYq6auKuC3DIsfVTirgjsTamoK5ZCezemBHoI2S3PpVuhwpWU6l0CpyqJcoLueEzs6qCBsybojkgqo5vUwwqUQmSg8+nBUwyP1fkHS5+jXZFkrph4PaB7qpM1LKXEGYHVEJSHG/zNqWe537nKH1n38Huff4JsqDgAVl2QWTTRQ4nZgbCs0Z6KGfquRmdYbcJyc+KmdJhejz6wa9Q3S6SqH710JaY1qYpLDu9c4E9fuYuK1WHi6z7Nu0IaUxq9LxpIHcKi8lOyN9Ua0a6rpcCtClKAtQcliZvSnILmrgR3wUCaoPuC1lRK+QxYv1th8DkQmkRv6Xxp5QjTwQDHjlxm784lpKEwR30nQ+K8pG/vBr/41LOI+xrc3b/I/+/iR/jPL/0y//7U/aSWJHbUiZ27popbUmNrxEyhcN5AjxRmqLUzuaWbqQ/8iApqNDF8WHpUw2wLlh+HRrWHiak1/ui7DyA/CyNfkVQPqHVYvLuL+2qG7oA6628fCej5gUX/yylBSXtLSt57IXNTI3HUG82q61gNyCxDZwzQJK0JjWQr6d77DYfmlGB6ZoCsEbLhZVnbzPGPf+X3+XeLD9H9uMk+t0vGCPnd4w/wS/e9DECjkaF5pYS7oqHF0BlJSTIC3VOGE+6aoDkp0ENVDLN+l06cS7HqGmZL3DCl73r1oRhRUxOcaorZ1tACcJZ1Pjv1OruL60zctYQ0JY0pnaA/IZryKPzARWow9EJCZzQlc86mOQWtcf0aNe9WyK4L8rMK+hZWEjpjkvpB5eIHqjxRbFWAbe6D1JQYbszrV0cx9YR9Q2v89sLDmFrCgfIqAINOk3v3XWXZL/AHZ+6m8n2F/umOJ4p9Oq0R9cREOXUy19itirTNtlAwiaGI/GVlxJY4P/HSb7o+FIGqRZCainvfcy7GG4/wEpOC4bFQLZG/rCte02iTzGsubk1iNSTzn0monBJEBYnZEhgdeUtdVOIMbN4bkORSyqc0xr8eYnQEYVHgrOmYbcjPAVIBjksXQb+UwVywmLnazxM9F/k7499noVHkW68q94mi4XHypV2c+dcHYd0ms5GQ2in9zwuCsqQ7IsldNDG6MPrtFOkkhHd11Oi6LrFWDeKsIrrcSmTnh2Lqj3PQMQSdiZjkEy2cyCBnBLy6Oca9o/O85uyncFWyscvBKEq6fRqbh2L6vm+xfn9C78s6cUYxQ6XOdfFFb4YSR1I4aWO1JBuPRNQfhvxJQWtXgggFhavQ7VdjjeGB1ysUYM0ThL2CLywcZXGpwq6JVZIXy5wOJpke6UULBJsf8dEWHBqTBugR0dYGKyxIgnvbBFeyNPYLnEWDwNcxdOXAJ6SqE2jtgMyidsum/g98euqtFPSlHLx7hvNLA3A1c9t5TX3YdD3pqXec+oUQe4UQJ9/0rymE+M+FEBUhxDeFEJe2Ppbf9Dv/SAgxLYS4IIT4xM14MjdT9rrG9DemME5nt4P0DtE7BqqU8oKU8qiU8ihwL9AF/gj4h8C3pZS7gW9vfY4Q4gDwOeAg8Eng/xBCbPPKtnVDerebqY8Cl6WUs8BngN/e+vpvA5/d+v9ngP8opQyklFeBaeD+m3Ct2/oQ690G6ueA3936/4CUchlg62P/1tdHgPk3/c7C1td+REKIvy2EOC6EOJ50Ou/yMrb1YdN1B6oQwgL+HPD5d/rRt/jaj+3YpJS/JaW8T0p5n57NXu9lbOtDqnczov4scEJKubr1+aoQYghg6+Pa1tcXgLE3/d4osHSjF7qtD7feTaD+Jd6Y9gG+BPz61v9/Hfjim77+OSGELYTYAewGXrrRC93Wh1vXFahCiAzwceALb/ry/wh8XAhxaet7/yOAlPIM8PvAWeBrwH8qpbzpFbb+QEJmRXGh9BDQVH406EsJelPCsmo11gNlKKEH6sw8LKcEPSlJRlKaTolzEndDrUz8vT7+cILZBbOjkDrhAQ+zA0d+7jxHfu483liM1FU1lpYoJlNYSbFagAC7Af7ugMzK+5+ffq8UluQ1JpXhKSdpvz/FH0xILfV9qXONaZUcaaMHEOclaODtDMmsKFeV9DoPDK4rUKWUXSllj5Sy8aavVaWUH5VS7t76WHvT9/57KeVOKeVeKeVX3+V9uC45qzqtSQjz4PdJ4qykfFqQWikTX43RA4WdEVKdhTtV5XJsdDQFamgKjEAdAYqtk0Brxsaqavi9yoM0sQT6jENiw2yzTJxquAsG7pqkPa7K5nRf0P+yqhX1d/ts3hVhX7UJyh/c/Gx+RrEDrE1o7Y7JX9HIzWoYbY1gJCTJJfhjEeXzksSE3j/MYLVU+WTQk6I1DIKKwOyoqrTr0R171q+HypfJaihXFKlBz+kOhYsGUU4n7EnUkWBRlau59QS7plqLK6fEVj+8QeW0pDOkgspsCcJKSmZJKA8qQx1jTv3cFQ5VVri6WUGLIXEURcUfSPBHI5Y/FmNvQuacw65/G5NZkbe0Q/NWK7OW0B1SI+LgDzSaByOkBvkZxeky6zqFsya1A4Jgv4dXERT+4hL7f/E8UpNk5xWcrjss6Q5f3326YwM1caAwG9LaqSh0jHos/EwOPZBUD+hk5hQ4IrsISFi5XxnPZlYlYUFQPSpxNqCxUyM3r26W1y/JT+u0dqT4FUHiQFKKOXVlhKdnp/Cf7yU11XRVvCjIX1EviJGJ6YxKskuSxcddugOCsPDBHVE3jqpZqfWwR2OXRvG0CZrylWrtiYnzqTJXlig3wT+3zt8Z/wGnvryPwiUdPYDilQS7KiheuL7HvGMD1RuKWX7YRgsEQUVgv54hcRUwIrHVUiDOp2zuk7Tu9kktSVCRtEcFndEUa1OjsSfh2M+epvZRH4DSedBCyCxpykNpIMWomgwP1Qnaqm3aG4rJPrFGe1I5JkdZEDMufa+kNHaBU1VGwLeyBO5Wa+S7IdamgCUHvzeluTdBamA3JLlpg9yMTnNvjNjd5sk9F4kSjX/8pc8hJLTHJSKRrHwmJLMqqR67vu3LHRuoztoWauelBJGoprWwnOKNJKQmhD0JfS9r5K9qWLM2WiTofVWBeVNTtWSIWLDcLeKcUdXHrSnQYrURSG1JblZj4MgqppYi9FR1gkYa/tf6QaryQS1S1Uqxq5z0Gnsk0kpvKUXkVmvxcYsoLxl+NiE3p5Gf1unuC9jcA2ZH4g1IjKaObcU8MzdFbbFEdkEQlCRJOaKxV7LjXyrztoFnri8E79hADXd5aDGsPKAjdagdlhgdjdwVnSSf4i4Y1A6A1ydxVxXpI7EEhWkVXO6qxq67FihaHlF+a520S52QhcUtflOvpNftMJht4p53yC6Cu6Lh9auNWViEsCxJD7doTgqkKdECweQXJbmFD+4atXJO8QA6/TpRTrFkrVkbo6MILHFPxG9++iv85t7vIs/mEYmgc7+nSiTNlP6XYe7jNu1xQe3A9S2R7thAzZ5wcdYEpQvgH/DQQlWBHpYkuSvK3z6uxFgtge6rYO0OCZWy8gRhSbKzsMGw2yAuqm1/tOrSmlSdn9lF5SHqJwauHhFnJX6PwBtKyG0VKxenJfaGQH81j9Rh8DlVCR+U9A/0rr81rlF9JCSxBakhqd4lGXg5Rg+gdBGsXMhue4UXmjuRulT2kys2dk1gzTis3y0oXgKjA1r0AQ9UpyaJs8quO22a2HVB4bIKLrMtCYvgLJlq7dRSO9Tskgo2pyYoH9xgzc+xFuTJzKnirsySTlRR4FyrJZnYucY95Xm+d3I/TvWHRdOC9gQYXUFjl2I5WQ3IrEiakxpWE5oTGs29t6i6+lbr7ib7PnWRwcFNohyUz0usTY3aAYOoAPJzG+QyPv/l+c/wrdcO4GwIcvMSd0V1A8e7PBjxqB/Y2myJ65t57tgK/+aUmsLXHpZgSmIXqkclVlUndtW71GgrY9mNIwItBlD5v+aeGJaL7CptcHxuHC2rpuzEhvJJHbeaUjuok0YmL9cmwErJLSQ0J3XMtobRlWiRpPZYiLlg0R6XmG21iUt7VErLWbljb+1P1t1NdvVtcH59gGLGozsWo0UGSGjviNm/b4GMEXJmZYhgJUNhWscbUqm6KK9GT23WQQ8F2QWJ1/chGFFVDxMUz+pkrprEWYnZFJgtgR5sOXhUJD3n1ObKHw0JS6rRLzNnsGfHCu3YxnXDNwh9AtqPd1n62YTYkfzlyZcp213sBZPafp04C62DIY09EreWIj2dntMSOeYz+FJIOBQhUpBOQv7qB2eNKg0I9nuMlTdZaedJXylSf2aQzLxBe19IWEopDTe5utHDKxcm8Rs2ekejuS9GCwT1/QJ3dYv770j8kYjGbjXDies8s7xjA9Xvkegqq0Tm4Q2krgAT3iEPqQnaOxOMrmDtbo3soqB0Uu1Ue08nhEXJ7LNjVL0MAFqiIlUKMM5mMTMhzt4GM34vjh5RPrZGWEkJehN0N6ZwWaM1pmNt6FQPCuwzLt1+A2vVRPcFuUvmB2bXL3XgUIv9oyuYesL6Ygk9VH1Tv/W3/ilmNuKJh0/zualXKGY9Hjt4kcpAk//pF/4dDxyZ5i999nvoB5sq/6zD8NMSa8UgHfaVu8p1LuXv2Plp4HjK8qOC0s4awXd6KXYktaMx5oJDY3+Cva7jjcS4Swa6L6k9GEGiyHhpIYKqiWPE7BtcY/WJTTQhebL3PItBmY8WzvKbL3+OU/VhipZHs+tgtASDL6W0h12CssJQZlYFYR6sFlQPC+J8Qm7GoHIhZuPwHXtrf0T+joCDfRvXPj+4Z4Gj9y/Qa7b4R5d+gb9y8EUAcrrP6lyF+qt9OGuCv7f4l9m/d4F/8/yj9Lyik9gKbBFlBc6GwNccNo6C1D7gJ1NLjwlIwX++F6Mr6Q4JSmcM9F1t+l7UyC5KcBLKF1J1lh8JjJqBSGFgeBO7JrD1mCutHjZ9Nfz9789+nK/OHOB/m/sYn9x9lrwZcPLVneT/KI/uC5pjiugXu2oT1x2QlC6rjEFxGux1A79X0u3TKV6+da3E76UOTy3+yOfNwOHk5ijP1HZR62T4vcv3MGrV+J9feorCOYOwRx2vWus67f99lJ2/F1M7kuL3qCKVoKDR/0oX3RdoERSuXN913LFdqNIELQB7UxLlFQws3sqHDrwAjR0aWgKdiYTCBR0tkgRlVROQ2Cr9FOUE7f0h2bJHkmiEgYF10WXnk1fZ9F0Wlyrs/LcpV/68RXZeozOZkJ3RcTckm3tUot/eKsVJHFWkYnShMybpeU1lAe50+YMJRq9PNhOwuVTAXTBURZQJvadjBv+vl1n3cjQ/P0x+PqYxZaIHauDQPbWZDXrUSWCcVdA5v0cy+t2Y+V+LSQKdub/2D2+8C/V2ldFRG6MoJ+gc9HFqAlmK0D1Bc0LD8FXQmDV1YlS/SznSJa4k7E2Qhio6MaomnbqLOJnHuOqgRXBufpCp4gb37p5h9X4Hq6HwjYWLOu0DIVosifpiel+Pae5K6YxKKucijK4qtHA2BGsPvf8DwM2Qs6JjnM4SvFTBXdhazkhlNXnoH79O2eoyM99H/eGQ9ogyadMDhVEKS5LsslSuLlcldl0FqUhg7imd0ncchr5+fUukOzZQw5IqKukcDCi+7NA66lN8ySazJHA2JM09MbkltaGyGhJnySA/p4olKid1RbYrSuJKhGirm5VkVHB9bO95wtSgG1uKQWoCQpUDOrMWa8egcMZk8XGNsW8k5GahNWbg90minlitX+c/uI23ZlvijLdoxxbf++ZRcmctjBWLzrBARLC5R0E/QJlg6F2NMC8w28qqsnJWlV92hwT1fR/wI1RnXSiaybctzJbEmVZVIIkDpcsBVk2nvkdV6vh9AqcGtUOC4Wdi/IpipcaFlOwli77jW9jwBQ1vOOE3+7/Dlc0eUimIKxFmS53AhAXofT2hclqtr6QOqw+YtJ/o0NqxZWgRaYR56Oy6dWTqW63WBPyzo7/LXKuCFgnaBwOcfZskriQ/B05V4ZOK0+ANJRieUMYW4xKnmlLfJ0h7Ivw+ZZl5Pbpjt6ZBGUQ2pr5fR0hInBTd06ici/H6LQrT0Nwpyc2r6ViLNLILguaEQWqylddTDnTNHVsHAXtjxnas87+tfozJYo28EXC5Nk5qgn14k+TFMsuPClInVea9hkQLQHs9R5SXJDt8ZNNUULGNO/bWvqOSfMp/feXnWX55CC2F3DkbYhs9v3VSaCjIr7um4S7puOtqOSA16AxrDL4Qs3HEpjsZkblqXtdj3rF3c+j5iPmchdkWRDlJ6ayqH7U2Q+p7M8QZVbKnB8rNLixJElvgT4YYdoJ2KoPUYPOBELFpYq9r9L6os748xOKuEntHV3n53BS5NUH7QEh8oYSWlTjrirckDUn2qk5mVbK5V70IvV+xiXICs52SGhrpHXt3317STah2MphNQXJfi/B8DrsuyM1JvD6hzJE3NLrDKZXXBRuPROh1Q3m6NmHusym9z0rcVYPW1PWt5e/YXf+23j/9sM9JC2/O39tGo2/rPdHNCtB39Zi3/iG3ta13r+1A3dYdoe1A3dYdoe1A3dYdoe3N1LaUSdrddSbKdVIpOP/ypOoyvY20PaJui3hvl7HSJqkUbPou2m14qHbbBKo0Ic4qppEeKSOysKRaoJOM6nkqXlGMp/y8OiuODnbJLUr8vT6F2VRROcZixYG6xQp6U/zhBD1Q1x3s966xqaK8JHHVc+DuJmYXvIkIbyoEoSrdw4Nd4pyk93RMYea9KxGMilusLU3xs+KcxHZUZF5a7aP29CBGR6F34pzE8BXnKzrYRUtAv28Tu6Hayb3JWxfRt02g+oMxdk0gJMQONHen6sZ4grCcIFLY3KPRHZLU96uSutgz0AOJPe3QHtXIv+hi1nTCwq2/fmdNo3hWJ3EhsyIY/kOLOKPaY7JLgjgnlUv05Ty6pyxyRNtAD6E9LtC0lPwMLP6MRmf4vXtZUkvZVcZZSXdAYOxukXMCSpaHdi5HnJeEJUlmWSMqJogYhp6GzEsZoqzEv1TE74HcLIjg1oXPbROovS/ppJYqSi5dSsnOa8psd1PgrBokrko02zUBAvzhmIk/EFSPCKyGqnvsjEkFgzjUvuXXX7qc0hmTaKE6TvV6NHRPqPrUUUlaiBVPYE3QnlQligMvQlCSypjXjmnugDQf0x1570ZU0e/T3pEgdnTY/dErJInG/f2zHP/mAZw15YFqNRSby2jptHYnNKaUD1c4EhHnE7KLkuoDMdK8dcXht02gbu5RU2bsSqxWSndQImLVm4MEvz/G7KiR1FkXWBs6G3+ri+4JCvOKljLyfYWElLPXR7AO+tJrU+6bJQ2Fqiw+usrUx68SXkeFT1AUmA1B5cllwqKkM6LqC5qHIqydTbSGQXN/TJRVXaphWbJ2TBVfA4Sni+RnYOQr+ntqs557PoN0Ep7YcYmf6ztFOpvlW188RmZFdYoGPQokt/5oTHZRIGJBYSbF6II9b1G4oLokMldN+p6/dXvx22bXbzUETk0F5dJHdMSIx/h/cRb/qbuoHzAQsaB8MaI5YeD1CrWOmi4w8QOflQcdjC6s3WPS/0rM6n1v/7RSWxLt8Nk/tgJANKBzdb1CtOmgZSP6elr0Zt7wFTDa7/x+7g4KipdTxD/to9ALzSmJtSmQhk43yVK8otGeUDNB8YyB1y/JLKqljJAKXNEdgqigU5iGoHRDt/MnKijD4EidapDl32w8RFyMyS6YbB5M0XxBakuspkbhjLI/t6saqw+kiMEuxe+5tMcVP6p0Adpjqh3oVui2CVSnKhXS0YK4kND3dZcr/829aJEKYntGY/2o8osnVVX0zobg8i8ZaGFKblb5cy49oqwX305RQbJ/bIU41VjvZBkpNNkzuI42JEmlQBNvfDy7OIh1HWfbQU/KehG0SKdyRhL2pMRZDZFC/pKB1Lam+6c1Vh+J6Tmh4/coNkFuQQVmMBBjnzewmgr2drMV5SW5/XV+feIF/r9nf4YwMHEXTKIsaL7AXdHILqc0dwjyc5K1xyP6njbJrAg2E5favTGl10zMjlqieUMx7uKtCaHbJlC7g1uQ11BQOG/Q/FQLx4ppzxfoOaFRvTvF6PMRMxl0T9AZT+g7rtGehNyMRulyTFg0CAsp3o4I98pbo4zjnGT84DIAF64O4V6xODVRRM+pHaxcdZDliAOTS6RSIFevD8tndATOmqC5P8apSex1k9RUxcN+D+gJ5K+qN5PuaXRGBWZLTf1BWQWxu2CQGtDYJW564UfQl5KdbHDf4Dy/Nf0oSaJR+p5Dc0qiBwKjq1pzoqwgsyxZvz9l8FsGnSH19dRNyMyadIdUx4Ozt0HuudKHz11ai8AZaZO/quoZ47ksSaIKb2tHJJXXNXr/2MXaVCCD4nmd9WMpo99U/Uxr9xgElYTiRUHx5E++e1E5IW8FANeC2Z01sc5ksM5ksDc0Du1QnZfn5gexN67vFg3ct6LK38yUzoBB4kqGnk+IcgJ7U9KaTNH9rZEyVU2BUt/yMO1PVSOco0iCYeXmkuTDSsrU4UXGS5t869WDeIGFdjanMJFziiiTm4WgLIkzAjSovKax+rCkfdTH608ZeFZlCqJSil0X8GyJoOfWlYjeNoGaWZGEMzm0ROKsC5JcijxRZOh5H2mphrDlp2KshiTqj9AjSeU1jfaQjhYpjGR2QUcagsLc23Of0p9APUhtiX6fSnxvdLOYV66fIrHxzBDdoZTsRYv6AShegPlPqhaMKCcoXRAkruok0GK1VixdSugc65Kb1WjvjDE6Aj2A7E3stworKQfum8HW1T2xqjo9+Q5aCO1hDa9f0J6KaXzUY+q+eZr7I2qPBYR5QemshvR1hp9JSWy1Lyif0ujsiPGG0vdyz/djum0CtTMqyM1paBG0J1NEJPAmItbudXAXdNrjAkKNxBEUTlk0d0D14Yj2pMTvkRQuaYQFid8LC0+9/fpOE1IZ9v4ZheMhE+U6capRe63vXU2/4mjjWgrKagiqxxJKpzTi3ojW3T6xK2hNJVQuxDjrgqiQ0u3TSDom7R2K1RoWVV9WdJNst4K+lB1HF6+9MS8sDZBZEojf6iPoSWlPpPiDMXpX4xf3n6DsdLlr3xyP77lE5ZNLbO6T6E2dhScFG3ersNx8zCd3xUAOBDgbt+6Y9bYJ1KCcEpQgtxQjEkH5tMbAD3S1ucpJgkrK4NMa3oDEbki0SJC5ZOFsCIqXIbuaEOckzhrvuBN9q7WnP5hwYFLZYV24OoTRfncvgv5skfxZC6lDbkFSOG/Q+ohH/rSFtmpjeJLSGY35j2l4gxJ7QyPOCHpeNCidE+iehuGraTcYD97VY7+Vorxk8tDStZH0wtIAxgXVorPwqYQDx2bITDYRqaC8r8bvff9hLmz0c2mjFy8xWXlhCIoR2miXzKKO2RJ0JyLcMy6koC05tzR6bptAdTY0UluiRSlxLsHrE7RHNbzRBH1vi/6XAQnZeUHtkFQ/G0N3KKUzLAgKGu6KRndEMvb1nzwpVUY2aQYOZvONpy4NcIY6aEIyv1n6iRuxt1NqvvGxOQXegCSJNBIHhp5P8XoFnVHFXup/OcVdl1gN1QhXP5iqlNSUjz8UYS3c2A4lcSRiootrqA3ixZU+jPMZRIzK0UYaC40iDw7P8uDRi9SmK1ibGq2LZUpZj9e/to94l4c9Y6NdzCrGl4TMjHnNw8DY0Saxb+gy35Vum0A1uio1tXaPjbtkkJ+VioOfgt+1qB4WbBwVbB6NGHxekpqKCk1vQG5e4vUrkG/pPKzd+5OTGa3TPaycGPyRad0fiJnqrdIKbboXSj/V9XeHU9pTCe666ny1awJRs4gKko0jOv4Bj/wsNHboVA/p+L2C5i5ILUFuRsfvEYz9voGzaF7jC/y0Grh/hT1DykjxykYPxqncNWpefiHFXjXwQxMvMXl1aRRZDrGO1vnHP/+HtH2b4sOr2KcyJLbEagi6QxJ7U2C11OskUqh8PnuNEnMrdNukpzpj6ti0M6bwL0FJKICWgOzrapoOy5LS6yb1PSC1lKggsS671A+mlE+rde7awwmF8wZR7q0f589O6f5gwuGDc6RSMDPfh9v86dZdk38S0dhpXeO2ppY6vDA6gr6TMRuBS34+wus1sRrqFE7f28LbpaGfziFiaA/rBD0p5k95DW8lr+bywy1h4kjyf2uBUdvj072v8V997y+AhIeOXMLQEo63dtC+UqStQSaG8jlo7ITUUdmA1ABvMCV1UqKcjhZy3djIG9VtE6j5yxpxdisfuS5JLUF2AeyqgsHGuRRpSESib7FQNXQfzDZEe3xqmkP+qiCxdILKOzegpbakfGyNKbfLYrNA50Qv7g3kLjvDFlFWYFdh+Jk2l/6yS/6qTndAsvKgTtgXMdevM/RswtIToLc13O8VtgzdUvQhj+RiltJ58Ht++kD1hxMypnoiS80CesPAmwoZHK7T43YZyWzy/Zld7M+v8JljJ2hFDt+b3k3uFRdvQJKpCXRf1U5IDZwquOsKqiEkFC8IorxBYitnw27/rdlQ3TZTv0glYUESjodk11Ka9/nKlmdTYrQFIhIULhiklkrwu1sWwWZbkn8mg9EVuBspmSN1ipffeeqMdvj0uF0AGhcqN5xgb40Loq0X89KvuGiBhruekj9QI38Fel8wEP0Bxt9exaopZqv2iQ38vpThpyUDv+8QFVJSQ+CN/vRYdalLtC3c+HChyb5jMxzcuXjtuQJEocEXrtzFy+vjPP8nRxj4so1TVWC32IHWZEpmVeL1CjJrKdnl5JoNZ3sS2vtDEkdVX90q3TaBWj+q2PnZszbNcZ2+b9pEWUmcAW80Vr6ZH9nEbAiaexLCIhg+tCYVWz+xJUFRoH+5/I5mZEFfyv6xFXVEOjOMVb/x25BZlTjrkDzcQNopaJKNo9A5VaE7LNjcC/nnXGav9FO8BHEWvJd6EYlg8Wdg5UENLVLBXj750+dR3XmDi2+RegOo+y7f+e5RjBmHwu8WWH91ALsOKw9L1p6IaExpJBnJ4PMSkUr0UHFfF382ISgLOhMJeldQOqFKGK+5ydwC3TZTf+6KQedAgHPJVp5RBQFC0v+Lc2T/yShrn+sSni0SjsUMf09j5SHFPbVqGn4vFC8J6odTzLp2jUT9Ex9rUlm6tkIb8wZ32D+UXxH4vZL8t4t07wuxNjW84ZikkjL5ezD7aZ2gomPWdGoH1QyiRYL8rECLNBJLGVxsPBHAhRvbTltnXa6c3fGW39NtSXYBzFaCiA26wxKrrmMs6Rgd9QZKTcHa/RK9K5GmxJ2xSCwYfFpQOyDRPZCWhOvk798M3TYjqrsm0dcs/MEEr1/QGZEULsPFuUEWPp3gPpsjzkr2/qsuq8cEQ89IUkvi7QpIbGWia25qZJbF25bleWMxE2XlFLt4fBjduzk3u7s/ICnG9L3aoXTcIixIxr4O5qpJlNcpXFRwX6lBXEpAgO6j0EMRtKZSnHrC4NfMaybC74V0TxBnBSsPqTHKagjCSV+VJo5KrKYy5sgu6JQugBwIEIk6+Vt9UOHj60/4ZIbbN3XT90760CF99Ps2mSjXubzeiziZvyWPua231/UgfW6bEfVWyTJiNCEJ1jLv96Vs613oQxeorY6jzr6zH1DDsg+obpvN1K2ScSrHlVM5PsDmzx9IfehG1G3dmdoO1G3dEdoO1G3dEdoO1G3dEbrtAjVxJGYXtAQQUL6UUr6oSnS8HSHeaEx2WVK5kGD4qkA42O/hritkzsSfbqIH6ng1Kty8HHFmVSIN6H81wh9KGHohID8nsesKRxRWUlJb9Twljro+d0OZBofllMnfUw2F2eVbe/QIqlLLasLA8fDa9RZmUvLzyvkZoe5tblFiN8BqqnsX5SWF2ZTMmsRdV/f5Zt7Td6PbLlC1SB1FdnZEpEdabP5yi26/DgKQCjzR2AOtUR13TTGdnDMu9SMpZhsW/1+S/hOB8t68iZ2cQUXQeyph47CJuakx8/MmG/dIGvsTzCa4KxruqiC1JFZLsPQEtMehvkcnKSSc/y/L9J6Oqd4lKZ+7edd1PRIS1aLzpEnQo/xHg5JGY5ey4onykswVi41j6bUqKS2G7JIqSG/sBASUnnYon7211/5D3XaBmjgSEQtGv6ahv5pHe6EIQhWduHMm4WhIYkmCimT9YwGVU4KwLJFmivdom/Zigc4/aHDXp87xd/781/F23JxojTPQHNeRunozOasahcsazpqOU1WV+t6AJCqntCdjKEQYHUFigfA1+vuarDykqxqArdK41ILejyyTuO/tKGVtClJTMvRcQmqoY9DukCS1YOB4QlxICCopekvD70tJdTCbqhC9tTMlLqucc+2+mM7w+4OjvO0CNbugCqaXPqKYUt5gSmdYUri7itRg179MmPqCTzAcYbkRfq/q7DQLIQeHlhneuU6tkeUj5YvMBRUO7lrE3x2oEfkGNPJ9n/x8QpRTNQZRXpn2Jo6kfgCCQx5RXlI6rYGTknvNwfCUO7Pe1VidL2PVFQSuO5yCBtl7Nig53ntu8+MNpMSjAa0xg/yMZMeX2iBBxNAcN5j8oiQ7r5E6kuFnJKkNiQvNXSmZnQ1EqNEdEggrISy/P1P/bZfwj/JgbRU7xFlIe5RlufsvSvi7oHrYVZXlfopczpFbSGmaGnuGVwhTg+Vz/aRuyv/8pT+HuavFUKnJoR2LXJzbgRb89NG6/JBDakJmBTrDEndNvUFSU02V1rSD3ZAsPZGg1w3cNUl3UGCsW/ziJ56laHj88+bHMDuKSNLZE7Kr0FTPsy/CbL13JIee1wSNXTZhYWs5cjBL/qriITQOxqSWSfJwg3Qjg180IIVUV+vt4HSJwVMSRMpq0XrXTY83S7ddoOqhmkJTQ+JchWjOwmoIFh8HdxWCslABM6/QPff83Vc5kp3nW9X9vHZ+HNwUd8EgcST+Yo5NOyRr3vj0b3RVL74eSLREwRqspirs3tyjNltLT6YMPK0RFgRen5r2ew6t8+XZgxScgNRO8YspItSYmli79rftfAC8d4Gq2mMUuMPZEJQupXQGJX4fmDWdzkhK5vkiZkWVVzo1tanyejX6Xo/Y3GkSFqFwGby+W9d+8mbddlO/U5XkZqF0TtAZFhSuqKmyeEHg9adYm2pX6ven+L3QZ7U43RklTnWEr5GZNRQ8t5DS94ogb4dUvQwiubGRoDCb4FQhzgjCoYgkI+mMpoR5QWZF0JzU0Fs61cMCpy4JygqW++nR0xzoW6XWzqBFguIpk0fvPfcjb56RSuMG79rbK84qQh9SUWiqB3SCssBsCey6oHxOlfZFlYTWroSgJKjes5VpqRiEeYiykqAMQf/7UyNx2wVq7RB0RqG5G6wGtEcFYsinfjRh7NsJnTFJYoE71UTqksudPk5WRzj7/BSFSzqlyylmQ6D1BnQHBAXbZ7VaVK3CN6CVBzWyywmZp1bZ9a9i0t5wq05T+XymliTJJ+QWVNCGoyG9jy7zenOEF8/shNcKpKakdb9HO/rRwuj59fKNXdw7KDerkdjqGguXFM3brqv1delyQpjfsoVcNrA3dIIeid7RyKxKGrsE/lCiGhZNSeXV98c1+7YLVCSkOpTPQlgAbzgm82KG/CWDxoRJ//FUpZ5+UCKx4cx/OED7K4OIBLw+SWyr1hRt1kU8UidMdKxz14/m+UmK8wmtcZ3a8X6m/5pB6QUbZ10DKTC6it5n1nUF8e1PKb9k0Z9pcWGjH2fJxK7C0aNXsN2I82s/2iqSxO/tyxDlt9bSiQJ1IGBzf4pdFSw8pdrOdV85aPsToWo3Oa8ctM0WmA0NZ0MtuRp7tvOogKo4l6akPaJw4s6qQfJ4g/ZUgt1IqR5WrKnW3T6VM6D7EqspsasCq6WWC3Yd9EDQl+uw3sneFDBu//M6UXbLBzQRNHarHK7ZhupdEr2tEWckXp/CitfviRnNbNK+UmTiI7P8xn/2BXbn1/mb+57jr+17nktrfSy3FMM97b63WwU9gOyiovZt7jIoXo3IzqtU29D3NLQEel+PSRzQzAQ01bufW1IeA7lZ1c4+8v0Y/QY2pDei2y5Qu2MxzrqGP5DSdyJVULFnizjLOrXDAhGpNWruNYfGTrVpqR6VtPbE+BVJZ2dElIOHP/k6U/kqtdWbA/S3mwlWA+SoT2bGRCQCu6oQ4sVLGpkVgczH9L6eYnSAFP7k7GH+/if/lJwZcLY7rJ5favFacwxO5Wl1t5YA6Xv74scZ1foc7PXILSbMfhbMljrJa49o2HVJc9wgKcYUXnCJ8pL2uGDmUyb5+ZTagxHuqsbKA+YtbT95s267Xb+IBfnZFKlptMa3UIyawLh7E/P7JVILWjtS1bHZE+MsmeiewBnr4HfzmFUDIWHZK1D3XdyZm7OblprA75P0f8lm5VM+ph0T13N0xlU7txbC2Bc1goJGUAF71aBw7yb/6vJDfGz0Ii9tTNAJLXozHS4/N4EGDJVVespovLfrPmdj6znEGkICsaA1BWZTdfhmPlqj+2ofzqKJ1NSJnhaAu6ax/HhM6YSF1wfpng5cyCLeB3uf225EzV3V6Qypy4odcDaEsr95pUT3/i7ZJ9YoXBZEpQR72SR2JYVpyH8hT+m8olB7fZKMEeKH5k3j4Xs9GkZHsPx4ysgfm9gv5QgqKYVLgmA8wG7AwpMa6w+mZJckUUHy8OBVdparfGn6MFkz5L/Y81XOzw0it+IybwVs+u57Pp2GeZXWcy/abO7SycwbWHVBUFEUweTzfUQ5ibOh1rNmS+Dt9xVSCRCxJLsocY5nGXrmxgFuP41uuxG1dCVm7R4dd1WweTBG9nmkyxmshk7/vzOp7+lHmKq92qmqjUB7XBCWUwrTGu27fB7ceZXlboHWdOmmZSer9yWYmzr9L+g0x1U9QmZZw2qn6BsWTjUlLGhbkDQJKeikVP0sgWdy9rUJ/t6lUbKXTToTMVqootWLzPfcVjy11BG0UxUgoTuS0nNS4K4LWhMCv1eQmxW41ZSgpCEElJ+xqd0TX2MM6CHUpxLaOwyc5ff2et9Kt92IWt1vKNRkBvLTBn7VJTerY3iSzV0mUQ68fkl3JKU9JgjKAnGgxdg3VdAWi12WuwVmLg9QOnfzRqp9/7RJqiuEe2qos/DUhO6ARu9JSe2AeqzcvOrXpy/gq194kNofj9L7bZvSOUHluEHxckJmziDov3VZ86iQklkWGB3J0DMtSmcFtUOq8CQqpbT3hlhNSfWgwGpCWFTw4ZFvqiWC3ZD4PYLKSY389HZ6CoCgIjG6gu4RjygP2asGmdWU+uM+mfWUsKQ2AfaGRn5W0plIcL6XZ+1ug+79XY4OLLLeypGbNmi9NYPhp9LKExVyc2oklYY6No0zkvZEQn2/oHRRcbCaO9Uue+BPbbQEtFBSPSqJs4L6IwGrD4gfOS/vBubNu8ifoOycRuIqsMTCx/KERaFcWAYFVl0jf86i/rgPApp7E+KRgMQBv6whYlj7uQCzLWmPCqzmdnoKgOFnE8LdHtqSQ+GKZOg5Dy0C+4KL4adYTcHYN33KF1PaI4LiOeUuEuzy+X/c8xUubvYRXizg98ubajxrdBVeqHBZvZmcDXUWbm1q9JySCClpP+AR9iR0JmOkUB5Zm/sk1qa6zaNfMMjOa0hdUhhUPpjd1ZuEl34beYMS/2iX7rCkfDGhtT8CCYmjfLvKF2Nyr7gUL0HptIaoWWRWJM5fXMVsCbRFBxGrFFdY3E5PAdAZ0Cm84OJsCKp3SdpjNvW9GiJRlT7+YML0rxk0JzXMrfP3woxEVC3+h9c+ydJyGaMtyCwKlSa6SUoN5VlauyuheBGqD0ZYdYG7Jmjs1AjzgtHfMcheNdA89Xn1WEzxorg2zXd7dPw+Sf6KhqGrr9nr7/02IeqNMS5mKJ2H5oSOWTUISylBT0LljCSxBe0JBURu7JHkL2vUD0ga3xkksyIpn4fOiCB2BfGNn538VLrtAlWLVFHKjk9fwWxrhDlV7idSaB3zKJ5RznbtfSHNgyEiVZ5HH3vkNTRNUih3ySyrwpHs6s1bB5pdiR5JChcNlUCfttASaO5MlZNJQdAeMuhMRaSlCKlB9opJexKMlkZ7PKV+UBJWEpq7U0a2KqduhcwNZWuUOOD3SuKsskky+z20GGr7dcymRlSUZBc1vH5J/3Fo7w0REtYfD3GqkNrKFOT90G236/d7BVJIzrwyiSynZOeV96l3yENfcNi8O8KdM9E9VawSHOrihTrfvbKbaNOBZQPdUvUAix+XuPM357qCkkZQVuBfLdaJ8iq/K2JVYNwdT5CaDomg92mL7sCWP5OhWj3yVzQQ4G6A9WurAHjxe8uZ+qGkBmkuoXaXIDdjIBKNsDfBmMlSPawGh2A0RK8beP1qo9geVjZDQUnQ/x1F7/OOenTqNs7Srd9Q3fHsqSgv2f3ALF5ssnh8+H2rl3w3igqS/Q9e5fJGD/JE8f2+nPddHwr2lNSVHc/cqXfvZPJ+KS4krHezeAvbkLbr1W039b9bWZuCK9/cwS006LhhuQsGjYWBbazQu9AdP6Ju68Oh7UDd1h2h7UDd1h2hO36N+n4oKkrMqRZhaKDNunfMJu5O1nag/hQyG4KsE+LaIV53m1x9K7Q99W8prKSElZSec4myeNQg6EmJc/JaUl6PVDosPy8ZyTewjIQoq74f5yTeVMj412/sxMnehORIm8SRDBwPyS5LwnKK2VVFL4kjCUuS4pUUBDg1ZbMD4D64gT+YoEfK8M1uqOdg+DD0fIDZUSd8/nBClJdqZmirMsDMmiTOSsIDHt6ugPSuljrUMMGbUJXS3uT7UDG9pe1A3VJmUcNd1Vj4dLLVRgwDL6n0V3SkQ++ZmMJMSvFKSpSBK/UKI7kGWqxat/OHqpirJos/UyC1f/pDFD2UFL+SpXAF1u+2CAuCpKzs1c2O+rtOVaDFkvKFBL9X0N4dERUktcUSWqCa8oKBmFRXTojlizGrx2z8XonflyKtFHtTqLP/owGliymNXWB0BcO9m2hmSuY7OUqHqrirkkJ/W705TpkE+7z3HEH0VtoO1C15A6rYufyyyaGfP4/fA8sfj+mOJJS+nmH+E+CXBY1dqp0jeamMo8cce/Icfa9AbbGEvalqVZE//Zo1dgVhURBUBH5vitmW6DWT7qAkygplorvfZ+OIhtVKkQJyl0zsuiA3rcBS+fmEygmDyvmQcChi/YhB7+sRUofMikbmionfI+kMCdzLWxloCf5QwsYPhsiecGlPQvDdXoKSIDxRVuyC+3wKBY+wL77lkbMdqFsSCXRHU+xNybnP7wOgeNJC5mNa44Lh72ps7lcEFy1UU+pzr+7l+MIYq0/GiFDQ3q/qOG/EdbkwFxNllRdpdkFRV/Kzyhc2qIA3lGJfdohzkpk/p+gnQUWSWZEkrpraa3t1avdHLD1qojcM7PtrzP9KTOmCauV2NyT5WYE3Hik/0wENd98mVl+X+FCHfX/hAvaGoOdMhN+fEm+5Xcu2QW+uw+F983jjt3YZsB2oWzI6gtyMRuwKmntiBTjrhd6nLUQK63cLChc1+k6myrO+N8RoaKSpoNzbQu9qmCsW9ia07/N+6uvYnDLo7lDVV62dCakFUXbLgtOH4kXB6Hc9tEBQfl3H8ARGW1A9ItEitVQxPIUJspqCzKKGF5hY0y6tcYHZ1KgdkkRZQAo6Q8rm/Ej/Mscf/ufkcx6vvLyb7kjK0mMG/S9B8TJ0jnoYbZ3LC30AaM6tJaZs7/q3ZHhq+pe6oO8lne4AuOuw8WhE5pJF4TIYXkprVJUZylAne7COEJIo0Rn7VsjMX5F0TAv74k9ftOkNSLKXTbStAeuH/VSZWYOgonrzVx5yiftD0kULkUCUk2QXNFp7Yoa+p2HXY6oiT3dINe9Zr+bxB1LylzX6X+kw/Zcdco+vkfm9PuKM4lE9m9vDsa8dIOhJEVJVWzmzJqsPpWiBII109BhKZVXkm7bf+86EN2t7RN1SlIM4lxI7kFmPMTvQuC+g92mTwmyKHsLqR5JrDtLWmkHnTJmfGz+L71ksfcQi/4pD+azAvIGNf2ZFULyaqp6xyzrtyQSrIbFaCsnT2h2T2FB4zUIaKhPQc0pSOR9hVXUSU+D3GEihIBzZJQXycJc1GkdDpn/ZJTOvszpXYVOtcGgcSDCKIakpsYY6FC8K7JKPfKJOfrzJX//Ed/i/P/hVtF1thrfqaLXc9tT/vsipgrOiE1ZSZj8t8Huh8IqN1y8wuynVw+AsmwQ9CimkewoyNueVKRW6mE0FR6sdlnTuV1N/lFfpntzD6+QeXscbe+fpsrk3ZnOnhtTVlJ9d0JEG+D0o6Nslg6An3SJwg1NTubP5j+u4KwIhJY0pDT2EgZcTzLak70SqUlDTFs6aRliUyhD5lZQoK8hd0ck/7TL+8AJBzaX8FxfJugFPjE6TcwJ+sLGLMauKZcXXLNap3toyoO1A3VJnTDXtkQqcVdWqYbVUOmfpUY2+E6q3XSSqLSWzIhn7+CyPlqbpyXRU1kDC7t9pY5928ff6jN67xIHRZQZzLQZzLfRCSJx9+9ROZtag/9WIoCchs6rWnVoIUT5l/V5J524PegL8XlWQvblbIzUFVkOj81CX1UdT5F0trKYkyOu0xwX1vTphMaU7ERPnJakt0TIxtf06iQ2tXTHtcVj9yhiF8waLtSKagK9N7+fBvhkyRsi/Xn6UewcXANj03VtOTNkO1C0ZHYE/ElG6CH5/TOms2mhUTgvsqkb1iKD6UETpvJqaDU9yT3mey34/l14fU5uviuDib9iEZUmu4JG3FKyhGTg0A4d9w6vkDtWuASjeSmFJMvPLKVZdJ3GESiONCvJXNYb2rpE96SJrNlIDo6OgEWuPxOqNdSrD+J+C8XIeq5Xi1mLMlhrZe18FZ8Vg6NmE1JKM/46BtQnxoQ7mpk7UG6s8a68kCg06vsU9Ywu8uD7JoNviUrWPH1zaRZxqLK6XbgiK/NPoujZTQoi/B/xNFHfkFPDXgAzwe8AkMAP8kpSyvvXz/wj4G0AC/KaU8us3+8JvtrwdIc6MhdeLatdIJWYHOkMCs63o1/aCSVgQBEWB2ZZ8bX4/PdkupXOC2rGIxDLJnLPJrEr8PervXljpR389B8DsaIyWj5A9Kc7aW48RZlPgrtqklurHMkc62D/I03qkS/iDQTI1SetATOmEiTcgiLJgNAzKZ6E9Bu1hne4RD5G4WE2N1u4Ye9WgMyIYeElBeYe/nxDmNbXGfT1LVJTkLphqBqkIrGxA/GqJV67kkZqk1hgksSSVq9D9nEVfpUXDcG8Y5flu9I4jqhBiBPhN4D4p5SFABz4H/EPg21LK3cC3tz5HCHFg6/sHgU8C/4cQ4v2hFrwLFU5ZRHmJ0YXOIZ/EFjSOhgQ9KU4tvWYy0RlV6Sk9gE+OnePKqRHSn6vjzioydulyQvGyD6iR9IdBCqpgWqy+/drOOFanPalarke+l2I/l8fvAZYduqMx3UFB/qyF1ydIdQj6YooXILFA95UhxI5/qfq4OiOCygkdo6vOIJYfNsispYgUGjt1/F6JtzegeFG1gDd2gTecEJ4tEuz0ySwJel4TRDmJ4Qlak6BrKUXbJzVv7enU9U79BuAKIQzUSLoEfAb47a3v/zbw2a3/fwb4j1LKQEp5FZgG7r9pV/weKdxiLjX3xdiXHfRAUjphkTqS9WOKH2ptcm0kbOyG5aDIsWMXaTQy2FXw9vs0dugsfFSlp0w9+RFPKX84oe/AOhR/8o45eamMXVPv6+WHdcISuGuKIWA0dVKbaw4sZgcwFdWkMyLw+iUYKfNP2rR2JSS24gqYLejsUr38YUHQ83dnyCyrBsixP9RpfkKlnIrTkJvRkQLMeZvK+YiNeyR2XSBSCEYiTC0hRdw0ptf16h0DVUq5CPx/gDlgGWhIKb8BDEgpl7d+Zhno3/qVEeDNvZ8LW1/7EQkh/rYQ4rgQ4njSuYkN+D+lEldtlOw1g7C81eN+zIcUrLqGU1XAhu54jN+Xkl0QfO/cHk6vDtHb0yLOQuUHNlFekl2QBKdKLG0WGL9vkehgl+Kjq+w7ME9fpsO+iWX8wbdu5dYiKF1M0boacW9EairTitxVDashiLISqyHJLAuCChg1Q5FbEnBXBZqZYniC4gUdOeGRFhX31F42SDKS1ji8dmGcjXuUu0u3V0c7k8OuqRoBKRSj1lkXrN2t6H6dyZji5QRjw8TQUjbaWbQbRM2/W13P1F9GjZI7gGEgK4T41bf7lbf42o+9/6SUvyWlvE9KeZ+efe9pIe+kOCPxhxMGX4rILGokGUn+hEN2QcepQnYlwWxJel7RSTMpIpVomyafnXqdZsdh7OdmqB2WhEVJ7SMBuaNVBostsmbI/pEV+jIdDC2l5mU4Nz2CVX/rW5+aUNsvSDMpI1/RFTOqq0wunIc3SIoxfq+gtS8idhUNxq4pz4KgV2JedhULtQQsuOTPWjhVSfmcREQCPRBYawalc+IaPyvOSsKyJLOuEOhCgje0xfK6oKF5Gs7fWcaYapNKQetS6ZajJ69n6v8YcFVKuS6ljIAvAA8Dq0KIIYCtjz+0+VgAxt70+6OopcJtLWmmuAs6c7+cYHRUOV2Ug/aOWBWoPCpITUHtsFTNeY/6SEsSpAb/7N7/wEy1AuWQ0X2rGFbMSKH5Y24sp86Ns3ZiAHvZ+Im75rCg3Fb0lk5jh05nRHkEBGWB/e8r5M+rE6Hhb2oMP5NQOZ+gJZCb0zDaArOjHFzCckqcT/B7JfWDsPKUKkpJXJVGs9qSntOS3FKCtblVM1DQcOoSrz9FRKq87+hfOcX/5ePfpGh5/NKeE8xvltDiW18ofj2BOgc8KITICCEE8FHgHPAl4Ne3fubXgS9u/f9LwOeEELYQYgewG3jp5l72zVd2zsCuS2Sg055QcN7MmiR3xUD3oTCtEWdh/BsJUoPerztUTmr88fm7+L+d/QX8rsVfP/oc9/TOU8r/6Fl/J7I4dW4cZ9nA6Ii3Te2UtjY2oPxgRQKZNVUss3FUYLVU/jTKaPhlnY0jW+SY8RRvKlTr0AdbuKsaY19Tp1P5KzD8FYO0PyCzqHCT6/eA1UyoHlIHCPkrGvX9gvUnQ1JXMvTAMvn+NkudIqdaI/zK4It8/g8eJ361hNG69YH6jukpKeWLQog/AE4AMfAq8FtADvh9IcTfQAXzL279/BkhxO8DZ7d+/j+VUr4PzkTvTl5/ihZpiEAjNyeI8rC5L8VdVtOj2ZZ0B6E5ZmB2FH7SbEv6/8imOZFB75X8G/0hEBLTTKg3ssRdVXbnzphc7+l/Z0jgrEP/qz6LH3FIDdg4qk7B4qEALlv4fQnmvU3aV4qULqiNXc9rgupdBmFRIqdzeH2SzoRg9Jtb9pD7DGRX0tibovvKTGL2s8pY7r/95Of5eu0gFavL16/sJ7Fj7u2Z48v1QwwPNnigcIV/8J3P0bMi6fa/P203dzwp5WYp6EnpeU3Q3KnWiXZN1ZZqkUpFtSdS7JoC9VpNZYimewKnKileCVl40sJdFdgNyca9Kc7KT5eRc2oSv1eh1p2qxOtT03mcUW4x7oZExFB9OMK9Yl07li2cN4jy6hRLf6iO84US1bskqSkpnteJs2A1JN0hgbOhnmP0aJMHRmcZcTY52xxk2G3y0to4T42c5/XGCEeKi/yH7z+CzCQIT2PwOcHmzpt/RvShIKXcLJktjfaY4oZaDYGzoQpBDB9aUyl6IPBGttavNkRTPkFPSmoJ5p+ysDcFhi+xm+kN3VWpCYKeFG8wJXEE7oakcj4kcZR9efvJDpv7VN4XwKzrGA0db0C1knTHEpobWVoTAt0XGB2NOKf8UGtHE+w6bB6NMFsS9xt5np3ZwZ/MHMLSE/5i5WXW58ucbQ5y4Zkd/Om/eIzeEwLhaVh1nW7f+xcu2yPqtt53bY+o2/rAaDtQt3VHaDtQt3VHaDtQt3VHaDtQt3VHaDtQt3VHaDtQt3VH6AMXqKkt4e4mhUfWiA91FEfqTYqzkmC/Bxq466rgwxuJiYqSKC/xJqJrfCk9hNJ0ircjJLVUx6e3I0RL1N/xdryzN6S7LkkcVZll1yE84NF/IkKkqkXb3lQfE0ciJAy8EiGkYly5G1Ihg4YS9ACGn/Vvyj3KLSpeVtCTkmxVjSWupO+1mOhgFwB/j0/lfIK7IckuqcMEbyTG2xlyz6fOUnhkDW9HiPHTIwzelT5wgRpWUnb1beAYMWN9dUYfWKTy2ArxoQ5BX4pdE5hXHIJySmoIuqMpzqqB2RQk2ZTMVVM18Omq/lSLJfmzFsVpFTTF1y0SR/Gf8ufe2Wk1cQTRWIhV1dg8HGOddZn/lZiwJAkfbKH7kvaEgrEhYfEjBtlFqQKoIuiOJ8q3qihpj94cZ1ctkqQ22DUNqy7QusqQeOUBHfNMhsSV9H/DYulx1QfWGRb0n0jBlDx16AwXav0c7lnmv3r0S7QOhj/G2gor6Q3xt95KH1gAxdy5QayGMvO1BrrsG16FYTgtJnFXNLRQ4D3Rwn0ljzeozvGl/kMImSpW9oZilvoF0owJenSsBphNNfL6Qwk9LxvEztsXabR2pBRO2KRP1jGfK9MZT8i+5iJ16ORdMiY4I22cb+VJHPD2B6z36vQ+b7C5T9L7snatmrc5cXPGleoRZRZXvpgSFATSEAQliTRVy4kWCmoHVSBLA0QMxt9epezbdGKbeiPLt+YO8uzSXRRaEBXe+Nv+YMKBw3OcPT2Os3zzOpA+sIFqeKqczlnRYTXPNHnVz26pKdbaBP9sHpGA2dQI9no4F120CMKnmnRrLtlLFt39AZkL9taZf4LZ1Mhf1jC7go1j6Tt6LhldQW4pwf/TksLoJILOeILZ0MjMG9TvCzGm80STkAz7yK6B1tVoTwjK5yRhXhBn1ZsntbgpTtRJLqVwWaM9otEdkAy+lBBmVdt1/UBK6bwqyAlLqiCnMyzpddvsKFT5/isH2PXvfab/kqpJyM8Kopx6s8ZZyfje1Ru/wLfQB27qt/rVGku++ZlJ9c/aFLgrGvE9LYpXImJHkpoQ9MfkX3LJrEraeyPky0UKpyyCHkn+NRupK8e7wiUdUkBAe0zQ98I73z451WX1mLaFdZRkljSsuhqprAYQasTFBMY9Rv/QYPjbqjBGCyAoCrxB1fPUHRDEuZvjnqZ3NLU271OIIK+i0ditkJf9x8HrF8RZQfeAT3dAUtxXJZWCKNVxVnQWPpZFxIprpYdvTPFxMSW31SJ+s/WBC1TTVKWvcfGte3m7u0PsZ/MsP2yQn4Hhp7vkrho07g7QQuh5wSAsS5r7YwrToHsqmK26wHukjR4Kmg96FK6khPl3rs10X8jirKu2j86oou31vp7irAv8Puh5VWf0G4L+P3aY/7Rk5SFBYqv+/s5YqlitjZT8nCSzeHNervxVZctZnN7qZMgL7Jqg2688XFMThp5uYU87JEMBtbUCNT/Lc9NTIFXpo9VQNJaVR7dIfwaUx+sALDaKWNWb23j8gQvUzpIyGTPrb72qsectWlMp+asqaK78RQejA9aSxeZeqB1R1f3ZGYPOKNhNlRmwWpLxf6qTmpLy9x3WHpC0J955w9DavUXky0uyC6oO1C8rE2KRgNcnWHlIo3pQo3LcQCQCMd4lcSRJLqWzO8TsqFZmp3ZzNijegFB9VxOCwhVo7o7x+yTFqwlBWeCuSi79agYhoed7Nua6iakn5Aoe/lBC4kBUUGgje10FZOJIhvItUinwTlZuyhLlzfrABeo7tfG6q5Cb1WiPC1pTijQtUoneVV2Yuqc+/pDW3O1T02TsCi7/DQ1nQ1B7NKR4QX/rNsY/o+ysTlhUaJ72BHR3B0QZQetuH7OtrrdyCswWtCZBi8E4m8VZ03AXDbIXLVpjOokFUfbmVNfHrko3Ze6u0pyCqT9MyCwJwrxG7EDiCuyqTs+ZhPaYwlpmjZD/9fDvY/d38ScC7A2N2IVgQM1gyZhPlOicPzV204MUPoiB+g5qT0r8Pom7rqbzOKuMgoO+FGnC4IspYUnSeyYmt7hVBC2hM5riTNs09yYUXrFpHEiu6wUZOO7T87pEiwXuiuD/396bBel53Wd+v/Pu3770ju5GLwAaAAFwBTdxkUVKskPbsuMqj12TzMXMlG9SNTO5yFVqKpWp8lWSm1QlFzOZZFSOnczIsssq23IsU7JIUdxJkCD2tff925d3PycXp0mZI4qEiG4KgPqpYpHs7nr77fd73nP+5788T/U1h7gI1ppLumPd15kSRCU9URoPJARHAoQEaWuCh2WYfnSZ/oHdWVGTYordFnQuVCldg8WvOSRfbdL9rTaWD+37Q5QBjSMmThu8bUUkTf71ld8GYHysgXqkTTAT4q2aBMOS8aEmq+0i3sbeaI3cs0RNxj45qM8v6pHh7qTCH5MkGYU/G2JEArcm2HjM0ITJGGz9aog/opPjaUWL+4pIYMSKwjUtMPZZmH/BZePLKZlNRetUTFQUSFvhbQmdL/V0Qt/uCorzEm/VpvS6h388wOoLnJYe5lv+0eSnalb9PKiMt3A6eixamjB4VpGcKdNvexgx5M+7JFlFbyohv5LSPwALfzdN/O1hotBiNKdHWCYP1Jn+6jz3PbBA0Q3Ie6E2ptgDVt2zRP1Z27IRKZyGjqlKl/XYCehtvjub4LQEwYjEqycYax5xOcUfUeQvOhQWU0pXBEaih/0Qn73CFW4KMgs2SVaQv2rDU02ikqJ8LWHzSynhcIK3ZpJ6sHlak7czo7CXXLwt7c5id3QuM8nuzqk/fG2A7oQiHo0wYkhcQeWKJHvVxQz14dGY7OFum2zfb+gBxQ1Fdwry72Q4Mz/Jj394kv9l7lt8efCqfq5CMZDpc2Jumdnnb2I80tLzXLs0C3jvEnUH/ngCD7U/etPbh3R+0K0bNE7HeNuK4vsu3haUz1n4J3xMX1A/5jD2aoqI9cx7bzqlftwkyQqaxxSdKXBrn/347J4iGEkJK4okC93NHGYgWH/CxNmR7slsKsLZADPQ061mBIV5qD2RkNqC6IkOCLD6u/NxGamOhYd+6BAOCJQFtZMCkWr9KqcNmdfySAsKi1q5pXY6JbespY8mvmWTeor/+j/+S/7tO88CsNIucnl1hIVGBakE09U6J44t4R/anXTVPUvUqbEa4TGfUycWOTy0zamjS8hTHWb/rIsyILOhEJakOwlmoIjKOveqWg7xaESch43TJplVE5HC2EsQlyS5NUluycBIBPIWKprSApEIRCpAwJE/irF8weTfRUz9VZ+hN0yi/6JF8W1PazwlWowtKgqMnokyIWx4KENPxu4G/Af7xAdDag8oUhfqT4dIG/xRyfbjKZavta4GHtqkcxBaRwyKlyzMSPcg+FUTt2ZQvgT3z2jN1PpaSZdf3y5z7Y0p5utVAI5Orevy8G3iniOqUzNJpEHBCTlxcI2rm0NcfGOGC8tjzI1sceUPPNKMzh2W3vQQR7v4o3rVTLJg+Ablt1wtxbikU1hpVlI/YZJdMVh/LsEf0bqkH5ZcPw2dGXYmW6F6PmX7/gxD7yW0pxyu/VcejWPQXc/jjyh6D/soC7Kruu4/8rpe/bF0imy3ZB7Lf58BKRg4K/DHE7IXPLwtgbdpULhq0T6kd4KNq4OMvpGSXVNkalIXBQKon1KkjqIzLcjbesW0mj9JB1odgTxT4vyFSSxDcuCRtc8UMP4s3HNENSJohd5PvnCugN0SOBcyXN0c4rlTF0mGIswA/GHwXtF519zKTq5zVdA7qLDbgrAs6E0l2C2Tye/1kDZkbzi4DUEwAOHwZ+tqJFlFNBXSOR6z9gyEZcHyVwz6owKRaI19u2ESVVOyZzMYEQQDAjOA7gEDpLYRMgMIB3bn1F97KMW94ZI6YJcD0ozO0fYPRRQWU4xQ/37lStpTFu0Z2H5A4G0K4ge6uDUDq681sca8FqBtNz8GCWLHbyvvhKTePlF/CmsbZaQSfHB1AvEPuBQFFrUwx9dPXCAqQTzr0z+gSLKKzoz+mc5DAXEx1b5T4ynFKxapp2geyaJMvcK6TS3sIOJbqExtGhTf8XA2LLwtk/6sbuMLByS5RYP+kz0QcOhPteKzkej6uhnr//YnE3IbkmBAUbixO89n4IxJVJGIFJz38uSWFb0XOpTfdaidNJGODgkG3rKI87qCJZRe3bOv5nGbiv5sTDCk+N3Km1zdGPrE36Oyu6f0e08S1b3hcfnNaTI3nZ8qAHhWzPfOniC7prAWPdyaIKkkZFcE3rbAXnARidAmYMWE3oTE6glqXw3oz8S4DUXt8ZjcCri3UCbMrin6o4qh9zQxCpdsyhcFKEFnLsF7K0f5Eiz8qkt2TdA5HtGZS6g9lNI/oMhfswjKguyawN8lOZ36KYlbM3C6Cn9U0nzOJ/+XBYIhmH5mgcI8dA9Cb0LQPxFgRpBZF2TXFUasiEqC0lmbZDDmH7/2B6SJ+TEmSVdhP9rg+PQaAJdXR3B/hnrhreKeJGowlvxU/BgMS45ObPDOwkGEb5Jf1eTs3+8z8IZeOaIiO1uagdMR2MsO5YsCIxYYKx7CSelMw5H/kNA8KTFu4UAblQTxSEx/yMDuQuVyQlgROG2B1TIJhhW2ryhf1jV0e9vGqZm42yaFG1BYloQV3RdwK4e3W8Hoq4LM49s0jhu42wb2pSzdSZ0au3xjjNZhrbWaZBWGJanfp0u/iScozidYH8rZBgZfP3KJY+PrRMd8/IMx8Yk+s88sMFluYgjFWqeAfT6rm3luA/dkm9/RuRUAriyO6i8IxdzkBq3Qw76YxVaaQNKG7PsZmkcVZqC0qnQbQJB4OgvQHxOYIeRWgasu/ohg9eks499PaU9B/Bku6CKB7HWH9jN93HNZ1p8wsXoQ53TVKc0o+sMG/TEFSh+i3LrQnU2/tU375SHSjI697a5OD90uwpKgcaOKGk7wNi0y60pXpA7B4W8mrD3l4LQVhdck609mkI7C7unWv7BsEg5AbkXhrVv87Y8f5PCpZY6Nr//U77m8OoJ9cXds4u9Jol45N8HU8XUmxuqUPT0rMd+oEFwoEx+MySzaBP+4gfenVV2vPt1D3ciRXVc7OXxBZ1qR2dAHBulAc06veNXzkmDAYON3A+KmS2b50x+hP6KbT4ovZ2ieSMktmrhNhdMSSFe/MGFZp6SSrEIZivyKYuK7NRZ+e4jcuqL1Qg8J9BsemaXb/8iigsBpCspXISxBpq5ozBlYfVh/MktvJiGsmCSDKeUzJp1p6I9oS3QwsNvQnRAoS+HWDK6em6Aw1SLvhVQ8n0sro6hNF7du3PZK+iH2taf28QvHvvbUPu4Z7BN1H3cF9om6j7sC+0Tdx12BfaLu467APlH3cVdgn6j7uCtwzxE1ySvioqJyJcXuQm5V4bT1+LF0tYhDfkXtuNVJ8iv631FF6uZqdII/GNF6T7eDylVJbk3htnQTSjgoyWxrLaokr0gKCrfJTjVM61r54wn5FT1+ElUlRgpmBHb/9u7l58GHv09I3WBdfGoT+UBHtzx6imA0Jc0q/MkEY6fpRyjdgWUkgAHZdT2d8A+9YG8H91xlKruq/Z96Y6buXH9YklsyUWMB7tvaftGIFdIWKBM6B/WsVGZLERdsPVN/2Idtl3BAj05/Xmw/IHZGXnRPajiY0h+xkI6idAXqD0i8uiDJCFrHU/KXHJI81O6XWD5kNvQ68vNMdaZZRTwecd/0T5slnr8yAYCzaWH6P/vvKi5I6vfpke6gKunfHKB4ycRrSDYfV9hNg7ioyM1bSFM3ocSzAf2+hQgM3JpJ45RC5hJE38TeBQO1e25F7czInV5P/f8TLyq8LUXhjQxJFhCQOtpgbPhNCEYTooqiNyHoHpQYMQz/pUtm3cAMbu8BeyeazH7b18ITqQBHknqK3LIgzQiM0GD7IYEZKrAUvakUI4TcikHhBnRnE9yaojSfYPdvbWVKcuoTSQpwYm6ZE3PLjD22RuWZdSrPrMNDbaLqx+ucq78iUfd1MGLILRuMvKKndTNbCe6WiZACTEX1YoKy9Ko5+h0HZ8Miu2pid2DsR4rSWRtl704N9Z5bUc1QMPR+THvKInUEzcMWhSVJVBIUFiTNowKvpuUew5Jg+juS1adNwpmAzCWPAy+1WX6+iJFoq0en+fnJql6u0DqiyKwL+uMKb9HBiMF/usuBaosZt8/5Hx3mgX/1PqNum2W/wojb5j+99CWsnqD6rklvQtA4ZVC4fmv3YbcF8/UqQ4UuQfLxj9ezEnJ29DHZnYrnE1earLRKJOeKGKFg6E2T+qkcloLOoYTeuEHpimLtKZvidUV3UpC/YaLMFATk5w3cVkSaFdg9QXcmJcmZZNeUfkF3Afdcrd/uAQriAvgHEoqXLbrTkvLhOsmLg1SuxbSmbfKrKau/E1F4K7PTFKK78c1QT5magW4miYqf9Rt/NrIbWhVQ2vp6hWc2eWx4ge8vzPHIgSV+dO4owpZMjW/Tixz+m0Mv8W53CqkM1oMC71yeJntd9/Zl1xW9A7f2oSsbEk/91JYbFxRJZcfpb6TLZLn5se9v9vK03x7CDCEYlEz8QLLylZ2dRYHd0UZxbkeROuAPGJTmE/pDJu1D2kJdTgQUX83QPB1BZHwUHnwafilr/f6w0gIPPmDog5FyFMGrg6QeNGdt+mOKzdMGxdczGLEiOJDozikp8CcSopLWfUpu0509GBQEQ1JrWv3KGq8/+G3eq00wXOxytTnE3OwahXKf5XcOsLVW4g//+newRcrf/OghprJ1vvPV/43+4YiorAgGbn1lEjGfGBfaHUFm0SazaBO/VeHG381w5fVprqwNAzCc62KebJFbVZQvC+rHLdKMJB6PMBJtdQnQmjHojRkkWeiNmmTqkuINGH5HInsWSQ6yVx28dYvc6v5h6hPhNgT+qMSIDEoXLKIC5OZNrD50phXSlXibJmYE7SMSZSmspj54ZbbAaVkEQ1qQrHwjYfvE539E/ckUd8Pk8Ndv8KtD5/mni89Q9nzOfTCFUzfYqErcLRM3ACO1STKKv/ru4zAT8MbWNC8uHeUfPfw2337pCZy21qnabVgdQXIzC2M/+VrrkM6eOE1BdskiHDRIMormyRRny9TZAJ0goXFK0m1pQeDmMUHlPQt/CMLZEBWYhAPic/vC/kPccyuq1dcHF39UB/HBaIpbV7QPa1JWzxp42wplQG7JoPKBQZqX1B+QNB/QsuhxKcUfVqx8+fYej8omyBNd6n6Wt9vTdGKX69sDZJdN7Lb+AKsXUvrjOseTXTe0D+v5DGuXhsl7Ie/WJzl4Yo3uwdt+NLcM6YDTMHAb4B/VOlNpTlL+wNITvEUJSs925eZN4oIkLoAZCOKCwO5C9qLL0Gsmzm2OoHyIe46oqatjpdyyQWZbMvi2QX49IbdsYI76+MOC1BU8/FvnsLuK9gzYLQPlSvJXbaw+eEM+VqCFgG8HpXddktik0cuQMyPOvnKE/naW4rxE2pC6ivoJE7du4rQEvYl0R9dfwUDIH0y/wjdGz7K0WSUajXfpCX02ijd0rN8+kpI75xLf32P8B9A+pM2LRSLILyusns6fmoFBkpPEBUn3eMjIOz7K0BO3dnt3doF7buvvHEn05OhOsC8UrM1ZZNcgXcvgSAir8OM37kN82SdtOQy+aeK+a7D+hMLKgzhfIDgY4S3c3pCSMuDRmQXmW1Xeq41j9QTelo00JdKB8lXoj+ify61KkqxJa07iNA0cL+EvNh7CMRMMQ5Im9i49oY8jzSqqD24BWpYnvlzEsQXd2ZT8dZM4D+W/zdLekai0u4qwIth6IqF83qI3KTECgdMxtHjGosm1308Qccroq4LaSfGpOdtbxT23ototk+yGxAygN6mI8+jttC5xGwYD5xOEhNIlQTYbkl2y6E4JNh81SAcjTF8QjCaQCCqXbi8H2DmScnl7GMdMOVhoYPcgrCjaswaFBUXpah9vW2F3oTthoAxFbsUgGkwpZAMsI+VafRDzUg6ruTcqeblTdQaz+pR0aXUEyxf4w4rcgkmSBenobENY1i9Ud1wLDWcXLby6xOoKht5T5JcUmW2FsuDQtxKyqyabjwjiyX1Jn0/EyBuS9a8mJFkoXdUyNWFFsX2/QDzYojFn4bSg86xP5s/LKAHBRMTo6ym5yy52F8ofWJTO2fSHb+/xjL0E0etVnh6+zrOVK1pjQEBUlnQnBavP5vBHBP6I0oIPQpPhf/7a/0vBDTmzMEljrUh6rEduZXcPUtLVJdChXO+jr8W+DRLyi5BbkVpdcFsQFxWDZ1MKSynl65KwrIhO9QmqBiLVq2Z3XNAfFtgd2LrfI7OpyK0IjO1dcnLZlavcQVj+zZTxvzZJM3qLkrkUqyuw+gJ/LY9IIc6B80GWsLQT+N9wWHtSi+W2TiQEg1o63PJvL7Wy9bBB7uktxt0GP6gfQzxf17qnjhawiMqKwqJEpIKwqijMw+Bzq/zp1mn+xdQPeHRmgcyijftOTkv77CLi2YATx5awTX2Qu7Y5SGYnZ9s7IJC2IKga9MYV1uEOrVmT9kGTxBMkIxH5V7N0phRJXuseeHXdt5BkoDsjac9CZksikt15we45opbec1j5msKta+XosRdNwiGJUPrQFJWgsKh1ScMq9A7qE6wx3dOK0z2D1NWaqM3jt0dUpyXYvlnl/7z2JbqxS7udYeqpJayeQWZdC16sPysJh1PC4ZToN5uEqcmWn+d/nX+ehXYF6SqsPqTO7hVm/MmE4xPrGDuymZdXRzDe/8kc9uibMdunpR7NFgp/M0vqaEmh7rhg/K8sggGoXACnbpDZ1p5YQ2d69Mclpq8NMwoLwa6pT99zh6ljv3eJMz88SvtoAgr8YZOR1xQbT0isvu4BCAZ1/BlnBf6wQTAsGf2zLGu/EVB4x6M3odXscosGcf7z30tcUFQ+MIhvDrD9nOT3Tr7DVlTgyvAoTz3/AWW7z5+8/TjHjq7y5cGrxMpkO85jInm3PkneiYiWoTOrsFvi404vnxP+ZMLcEd0LIJXg0soo9uXMx35m41EblYlQlokaCRl42aP2ZASJLhi0p028LT3q7R+KcJsO2XXFwgtZUAppaRXDtaeyiF2qfN5zJVRl74je5rUuf25N0TrERxLn0lGUD9dpXq1idwVCCpwGZGqSsKhXXDOC7kyKPeRjnv38TA0HJHZXn3r701qfNTfQp5rr0w0dil5IL3L4o5PfRCL4N8u/wVKnTBDZ5NyI9fPDFG4aFOcT/EGT7vjn20ZTT794zrE246XWR9s9wFKzTCo//gb0mhlE36RwzSQqw+DZlN6wQWtO4TYM/IMx5fdsoiJEFQUS0oJEGbq2Xz1j4PQUWw/qlfWzfBVupYR6z62o0lYoD5LRCCsf4sdFSlcVYVXL93gXMjQG8rgtAyPSq15YFWS3tDBZ/2DK4FsmccEkDrPczlm7cl5QvhFSP+riH08Z/luHjWdyWOdLjP6XC6y1i0TvVvhn5j9h6/wQpSsCfrNG1o1IpIHdMsivpmw9qIXazM/pO2qfaDNdrX/i9/7zej/A1QvTREMJ0jIJxmO2lY1b/0kLpd22kc6OAuBchOhb2E2D6EBE/gOXcABqj6aM/50gLBq33KPwabjniGr1tCxPK2/hve5Qmo9pHrIJK4rB73lsPhsz8COX1IXuhG5QHnw3pXvApHQjIb5sYfel9kKtJFD//KfWYFDQsFwydUWzbdP4Rp/Cm3nCCqx8d4redEq+B9tnh0krCc1jJjRz/PZ97/P61jRxUeJXtYKJ5WuZoZ8HcUGhDvoc2yGpIRRyRwpyqVmm29UXlJH50UEKwDCheNGmfSyBVNt0JjlB6kmyKybhgCK/IOjMQO6yixnoQkt21QUBwZMdsu8WWHk+wQgV7vY+UX8aCjLbkm7XoHVMIR0bISG3LNg6LTE6JsGAwG0oSlehNynoDxq0Twe0j9gUrsPWNwLsC1mM1u09nt5MgnnJYuPZgOLrGdLFPImnu6oST1B9z8Af0uGIvW1hdwS+4/D61jT90CHNSoIBS2s+DUrc7VsLUtOsYvCRDRwzJWf/5DRzozZAcL2oD42RwP0Zxa5wKMWITUZeMYjygkxNYvmS3qhJZ0rh1nReNT8PSU43dsc5aN4fY3ZMKt/VL6NTM7H6uxNb33OnfiOFxjFD65huGbTuS+iPKXoTisEzOmayn6zTPK6oPZoSVrRaHgpEJSLOgVjQproy99lCvZ8Ff1Qx+ucOnRlJagNCm6u174s/yqvKYkL5MvgTKUY15GR1jScPzGP2DaSty5k/T5onORhQ8fyPSLrRLXDxzBTynRJOU2D1BOJTKrJ2W9Ni41cSzEiXSdcfN7Vce6InI7KbimBQT0Z0pxR2H+yaRfWc/n4wrC2LxL721L0JfzzhxH1LXFgYw7vyc+71H15jIuHE8SUAzt88gLPi7EoZc6/wS3mYutvhDuoTk7H1+WPjzLLFjWUtoZ35jJ+9W7BP1DsM6XyeC4GN17znorLbwj5R7zDYLYHd+nxb/r2M/dd2H3cF9om6j7sC+0Tdx12Be4OoAoKjAbNfu7lrziH7uLNwRxI1HJD4B2OiE32CYUk4JMluKsxYj0GnniIc1LpO/sEY+3SD07MLVN0eRrLjc9TXzSXh0C5lnD8BSV53tJuhToAneUVhSZv0BqMp/uGQzJaisKiIS1r3yumAfyiifH1HtmdL20cWlr64fHZwJGT2azeZfG4RHmrr3t0hqbW2PtSLMiC3pjW67D74h0OSgqJ8XbdMKkv/I11Fdl0bUYQDe/es70iimpGgeMlm8K897LYgt2jgD+rqUerqaUdv06A3JvBWbKYqDWZyNbqxSzAokXM9opLult8tV45PhIA4rwgf6dE+BFZX0BsTUEzILZnYG7rXIHVh9HWJOtalO6nIFAPqv9OjP6aIioLeVMrWo3t5ox/H2GiDa5uDXFkcIQxsKg9v8c+f+3t60yn+ZEx2TTBwPmX70ZTMpqAzk+JkY5SA5iFDN3snULkkUSYkOYHVg8rFvSsq3JHpqSQrya0JNn49JP9uhvbJiMEf2/QOaPPbqAxRQbeulY7VaIceUgneX5zADATW+TxRUVtAqmoMNXdP7jMupeQWLEKVRaBLjSKFwhmXzqEUp2lgRJIkK2gMmnAtR2kRktUiQVWhCorcukIok974F0fU1bUK3rxLZkd5b2vA5ZutJ1DZFGFJ/DGLuGhSfU/hNSSdw2C9l9fO0gdjohMxY992qR81cRrQfjgke9Elzu4dUe/IFdXqG2w8Ac61DGagGH7Jpjeujcl6k4rMpiKzJRg6uUnOiVlvFsiaEV4mwu5q4y5lKvzxhIGX9y5oLV6ykDYf9VvaXbCeqYOh6+VRWbL0vEt3Upuf2W1B/eGE9lyCkQjMvqBxVNfG+QIrnJlr7kciclYfitcM4oZLvtKHlk3xujZs6x0QbN8vyKyY+McD4oLCblhk387SHTPxx1OiR7uo0GDo/Ri1h8veHUlUpvrk5w2CiRjxQo3tBzXp3Lpi4H1F85mAp3//Xf7p9GsU3YCvz17iT7/1ZdJzJfyJGLcBhXmD8nlLN5zsEeyudtoTSg/mxXmwvlMhGFCUrsHYKwJl686t4NEuvbmI6jsWRmCQW9HjMhjgNBWFG7+YjyKzpQgrILIpM9U65akmXl1Sug7haMzgWUVxQZI971E9B9LSs15mqFDZFPfNPAioH7fJbO5dnH1Hbv1x2yGZTRGBQaOeZ+CcoDdm4Y8ACEYHW7y0eAh/3GY82+SvX3kEUVYoS5FZtol3xn+ya4rWcYm3tjejxnFBMHguYf0Jk6EzirWvxYRVi9yyQecgGInA2wS7r3D/JocZK1qHBMrRqoJpOcbIJPRSj+INHa9+0eiNCfJLCm/L4aGHllhqlln9Cig3ITNvI9IUTEGShfh36xgXqsi5Ho1ilmzJp3/AYvx7BpunFan7Sxajmh2T0hWB01HUHnAJBgT+ZEJ2wSI81edLwzd5JDfPv/6r32PixZTCEYPOjKR43aA/oj1N88uK+ilF/qb5czcc3yo6syl218RuCTYf1bKS+UWFkIrOEYkz1Md4qwASWoehfEXgjyfallKBtWZjdy3ivKD2oMTd+uKJaoYQVgRRCf747GM4VzOo6Yj8RQe7o4jyumXS7kH3QpXMhkBt5hESOmYeuy/oTBh6Vspmzw6vdyRRcysGVqBoTxvkFxRxDuymSX8mppwL+ItLD/BnjceoXBEsfd3ErWmNzvbjPuVXPIRUNE7o6/TG5a7pH/3nUJ4kLujVOrNm0JuUpBmBtBTFSybxcuEj6cncqmLrtMLZMvHHE8rnLMIqpJ7AH0+x67+Yrd+IdciSP71N7WaF6sWU9IZF7aQCBP4wFOcVwYAe524dlXgbBuGgZOQ1gRVI/AGDpKPPBnv1qt2RMWrncEp/WOgcXk/r74uZHn/45T/HD23+98f+hMHpOt3nenz1qfcJ5gLM52vkCgHmN7bpTurmjtSF7Nre/YmZJZugqgiGJOGAIrNuULymO/ZbJxLyy4rEg95USvcgjLwOTkfgbehDmFtDK4wsm4y9muzZfQLERUVyskd6f5dgJEXZeiLVfmGLf/PP/5jtlRJm36B+n8nGVxLSjNJTEDckSQaUpWjNKbwNg+EzMaYvaM4Z9IcMwpLAeLTJ8Lt79zfckUQdfEtrxCOgPSMoXockMvlx+whCwB9e/w3ixGRmqMZKv0y53CNKLL4xfQ7TkEw+tYzla/mc4sLtd+n/LGQ2dZhhRoIkL4nKitojKd6mSfGiRX9M4HQV5BKyq4LuhEF/VBLP9TEiLXCx9ZjORS785t5u+8lwxLEDGxwd3eTkAwsc+coNTh1fpOL5/Hcv/j6l81rbymkCiaB8SRDnBZ2DBqXrkQ5VulrvtTGnfzbJKroHoT8hKf9RgdVn9+YsAHcoUesn9TFa2noep3kcfv34OV68PscLs+dZvjLM8cENslbE9R/MoJSgV8/wJ68+yVSxgYHWnLI7gt7o3j28xklJMJbqAsS6lpIsXLO0o0qkCKuK+gnB0PcdLShsglszcC5kiQvQnRQMvKu3USH3lqjeTZdLr85w/sw05xYOsNCoANCOXIQSdKYl5RM1wgHF1Hf0uEwwpOgfilh+3mHoHYirEqdl4A9p7VRrpsvQu5L8TYPNR4zbkpH/LNyRRC1dEYQHYkpX9Eju7CNLXGkP82uHLwLwf73wf/DGjWnOvnGY3JrC/MsKRsdi6vAmjSDL9Y1BnBY4LT10tlfILZkoQ+Ft7ciR57XCit02SF3B9Hf6GKEgrAi60ylGpEul2XVFXFBMfr9P7RH9QWeX9+6FAl2IsLoCd8vAu+wRv13hwlvT/NqBi/yPz/8ZMiOpXa9i9QQLvw3NOUFUTXHWbIbOKOKsYOAdfbAaPqP1UKPQonbSJL8m8WoCt7536ak7kqiNEwoMPTwWFQUj2TaWIWnGGd5rTPA/XPstiq9lcNqC5hykrkBVYo6WN6h4fWRs0Dsod3yR9u4+47xi6E2Th//JWZwWOj61wdsW9McUS1/LkeR0BY1STJyHrecirdgcCRZ+LYPVNvAait7sF6d/CoDSYr3/94vP8ucbjyByCeVLAsvXo9JOSzDwtklckWw8KujMwvZjKcUbkvoxk+INhYxMnDZ0xw2svqLx8C9ZjGp3BMMv2fijkuLz67yzOsnBXIMfnTvKs0PX2H55jPacJBjV8WfzwZgvzV3ne+dOcKU2hLnukmYk0tJ2NnuF/KLuJ3jvP5wiyUD9IbnjzaSwfMHomzHTp5fpTEtmvilI8grvukvtEcnQ+wlWX4u3dScEA2/8YhIwTtOgE7u4mZjOtDZB84cV8qkW3SkgHyMzEiMSWC2T9oyB83idJAv2uk37aEJ/RNE6ohDu3p0H7kiiWn1B/dd8SpcFXxm9yn9/8v/jlZUZhl+2+KPvP4s/G5FbNFCmIrsmyCzYjHktygNdGqslkop2QzEjhbcL4gc/C2FZx9HBgCA47lM+Z1C9EJNf0oeS+X+kaP7xBMZQQHfCQdq6lFq4YtI9YBJWtdxkWJX0x774HKqyIfPYNqcHFinlfISEqABxJaW3mSPJKtzrHtUzBsrQZekkr+idrxCVBElRImI9Ej1wVkBrb8SG4Q4lqkhBpoLeBHy1cJ4XG/eRvlEhGBAUrxlU37DpPhggvJTuTIo/G/HD1SMcH9zAKkYUz9vYXYXXUB/VtPcCvUlJdxLyy4rh77rkNiXLX9Xz78NnfOx1W+dQ38iy/ZDCaRn07wswUuiN69JrkofsqqHDgy8YwXCCaSheXj9E//vDKBNG3go1+bJaTt4KoHItIBpIsTsGxo69UTAodXJf8dGuYPX2jk53ZMJ/8GzMSsVDHerxl80HeW1xmv/pn32T//bV3+fh2UWylhZWOLM+QWk0YDTXZtjr8jfvn4QPT88C1p9WKHOnErQHcBsGTgt6B8D0BdXLMVbPIvVg9V9GmGczhFWFf2BH2jIEe8ElGNTGbdJRRCVJ6ul7tnbBivFW4U8mHJ1bYauXY2ulyoFFTcTWIQenLjA2TJ0/NWDl6QxmV/fNOi2FEoLWYxEYCmvV3fHogsI8+IN7c7935IoqHYE66COu5/jB8hxPHpzn368+y9SBGok0eaH6AfUwh2MlrG2X6EQef/vKg3iLDu66hfO1bWpPRZhdA6y9i1FH3tS9psrQH9T6EzZOcyet9nqJymXtv2R1DUQi6E8lVC4pvE1wG5BfFGRXDaSlbSf3FIZucg4HJZVn1jl1fBHLkDRqBQqXbLYfNGgfgiSra/+ZLYVX1w0obgOMmR5mCO1Dis6sxFlwsRddBs8ohIKBc3sXn8IduqL2h0xkZCJLkuBMlVeyFX79+bd4eeUQJSfg71vHOH9jnMwNh2wEyxcOYmW0s3HxmiJeGsCZBrcm6Azs3X2uPWWRW1ZaomfGAAmdwwlCCkRq0hs1MGIYfF+y8TjMfislLkBnyqDw9Cb2vx+gNaP1meyuIiztLllTTxENptjlkIFyl6Edrf4wsbi+PYC/mqd83qDzbJ/q32SwAknrkElU0mVgqycQUtF8MCZzLo8SWlrIaQktTdQS+AMCt6ZoHjE/U17ydnBHElWkYK05ZFcFIgV/RPDDbz5Gey5FXhng4rDCtrSHVP6miZmAP6IwYh0/tQ7D0BnF5mmFs753f6K3LehMK+y2gT+WMHDGxGlpp5D8kqL2iMRuGcR5g+E3Jdv3u7SPJrhbCv6fQWr3GeRW9X13J9mVhg5lg/Ngg0rWxzUTbDPlZq1Ku++xsVGGnoWzbWJE4KFFgks/yND8jR5JZFJ4K0PvaEjxfZdgUJFmJM6Whd0FM9AjKlFJ21dGxQ81YA2cBnSnJG5tbzbpfe2pffzC8UvphbqPexP7RN3HXYF9ou7jrsAdEaMKIbaAHrC9i5cd3L/eHXGtW7nelFJq6NMucEcQFUAI8fZnBdT71/tirncn3tv+1r+PuwL7RN3HXYE7iaj/bv96d8z17rh7u2Ni1H3s49NwJ62o+9jHz8Q+UfdxV2CfqPu4K7BP1H3cFdgn6j7uCvz/QsZJwQuZCewAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "for batch in train_loader:\n", + " data, targets = get_batch(batch, seq_len, digits_per_batch)\n", + "\n", + " plt.matshow((data.view(-1, data.shape[-1]) * mnist_std + mnist_mean).numpy())\n", + " plt.show()\n", + " break" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 2.281727, + "end_time": "2021-01-23T09:51:11.120768", + "exception": false, + "start_time": "2021-01-23T09:51:08.839041", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "### Average pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-23T09:51:15.248225Z", + "iopub.status.busy": "2021-01-23T09:51:15.247689Z", + "iopub.status.idle": "2021-01-23T12:54:16.476910Z", + "shell.execute_reply": "2021-01-23T12:54:16.477374Z" + }, + "papermill": { + "duration": 10983.306344, + "end_time": "2021-01-23T12:54:16.477533", + "exception": false, + "start_time": "2021-01-23T09:51:13.171189", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 69.36 | loss 73.51 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 69.02 | loss 30.81 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 69.20 | loss 20.69 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 69.32 | loss 18.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 69.95s | valid loss 29.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 69.48 | loss 14.34 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 74.37 | loss 13.69 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 69.12 | loss 11.75 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 69.11 | loss 10.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 71.00s | valid loss 14.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 69.54 | loss 10.58 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 69.19 | loss 9.54 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 69.31 | loss 9.61 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 69.20 | loss 8.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 70.03s | valid loss 9.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 69.70 | loss 8.42 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 69.27 | loss 8.39 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 69.24 | loss 8.23 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 74.44 | loss 8.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 71.11s | valid loss 11.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 69.62 | loss 6.83 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 74.42 | loss 7.24 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 69.25 | loss 7.46 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 69.23 | loss 6.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 72.13s | valid loss 9.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 69.43 | loss 6.40 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 69.09 | loss 7.03 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 69.10 | loss 6.76 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 69.09 | loss 7.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 69.89s | valid loss 8.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 69.48 | loss 6.21 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 69.03 | loss 6.84 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 68.97 | loss 6.14 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 69.10 | loss 6.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 69.86s | valid loss 8.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 69.32 | loss 5.48 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 69.18 | loss 7.07 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 69.16 | loss 5.54 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 68.96 | loss 6.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 69.89s | valid loss 7.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 69.44 | loss 5.11 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 69.04 | loss 5.74 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 69.13 | loss 6.27 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 69.07 | loss 5.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 69.89s | valid loss 8.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 74.50 | loss 5.18 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 68.84 | loss 4.98 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 68.99 | loss 5.41 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 74.17 | loss 5.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 71.83s | valid loss 7.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 69.35 | loss 6.19 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 74.05 | loss 5.32 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 68.99 | loss 5.55 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 68.66 | loss 4.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 70.76s | valid loss 5.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 69.13 | loss 4.32 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 68.76 | loss 4.48 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 68.81 | loss 5.40 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 68.82 | loss 5.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 69.59s | valid loss 8.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 69.18 | loss 4.55 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 68.73 | loss 4.54 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 68.60 | loss 4.57 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 73.98 | loss 4.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 70.60s | valid loss 6.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 69.10 | loss 3.82 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 73.79 | loss 4.63 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 68.67 | loss 4.43 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 68.59 | loss 3.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 71.55s | valid loss 8.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 68.91 | loss 3.87 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 68.56 | loss 3.22 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 73.83 | loss 3.88 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 68.75 | loss 4.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 70.49s | valid loss 7.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 74.08 | loss 4.22 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 68.53 | loss 3.49 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 68.55 | loss 2.92 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 73.70 | loss 4.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 71.43s | valid loss 9.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 68.81 | loss 2.94 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 73.84 | loss 3.27 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 68.49 | loss 3.68 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 68.45 | loss 2.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 70.39s | valid loss 7.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 68.63 | loss 3.49 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 68.25 | loss 2.88 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 68.42 | loss 3.66 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 68.37 | loss 3.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 69.17s | valid loss 6.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 68.98 | loss 3.54 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 68.33 | loss 2.78 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 68.43 | loss 3.19 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 68.57 | loss 2.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 69.31s | valid loss 8.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 68.75 | loss 3.07 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 68.40 | loss 2.70 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 68.41 | loss 2.67 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 68.35 | loss 3.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 69.23s | valid loss 7.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 68.59 | loss 2.84 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 68.29 | loss 2.67 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 68.24 | loss 1.90 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 68.12 | loss 3.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 69.07s | valid loss 6.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 68.59 | loss 2.66 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 68.03 | loss 2.89 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 68.09 | loss 2.16 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 68.30 | loss 2.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 69.01s | valid loss 6.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 68.48 | loss 2.90 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 68.26 | loss 2.12 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 68.22 | loss 2.18 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 68.21 | loss 1.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 69.08s | valid loss 8.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 68.43 | loss 2.46 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 68.17 | loss 2.59 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 68.15 | loss 1.96 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 68.10 | loss 2.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 68.97s | valid loss 8.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 68.43 | loss 2.03 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 68.08 | loss 1.99 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 68.10 | loss 2.58 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 68.15 | loss 2.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 68.94s | valid loss 6.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 68.34 | loss 2.09 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 68.10 | loss 1.90 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 68.00 | loss 2.16 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 68.02 | loss 2.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 68.93s | valid loss 8.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 68.41 | loss 2.04 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 68.11 | loss 2.06 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 73.07 | loss 1.47 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 68.01 | loss 2.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 69.90s | valid loss 7.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 68.31 | loss 1.76 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 67.88 | loss 2.34 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 67.97 | loss 1.87 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 68.01 | loss 2.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 68.81s | valid loss 7.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 68.16 | loss 1.87 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 67.96 | loss 1.84 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 67.96 | loss 1.67 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 67.84 | loss 1.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 68.81s | valid loss 7.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 68.21 | loss 1.54 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 67.83 | loss 1.50 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 73.04 | loss 1.38 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 67.95 | loss 1.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 69.78s | valid loss 7.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.78 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 69.47 | loss 78.99 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 69.11 | loss 31.47 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 69.13 | loss 21.00 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 74.43 | loss 17.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 70.99s | valid loss 28.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 69.49 | loss 14.31 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 74.43 | loss 13.40 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 69.12 | loss 12.08 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 69.13 | loss 11.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 71.22s | valid loss 27.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 70.66 | loss 10.58 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 70.21 | loss 9.36 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 68.67 | loss 9.53 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 68.58 | loss 9.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 70.06s | valid loss 9.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 69.00 | loss 8.14 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 68.58 | loss 8.36 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 68.60 | loss 8.10 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 68.72 | loss 7.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 69.42s | valid loss 10.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 68.95 | loss 7.87 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 68.68 | loss 8.05 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 68.55 | loss 6.76 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 68.60 | loss 7.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 69.41s | valid loss 12.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 68.89 | loss 7.44 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 68.60 | loss 7.13 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 69.18 | loss 6.29 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 68.93 | loss 6.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 69.65s | valid loss 9.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 69.44 | loss 6.30 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 68.96 | loss 5.93 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 69.00 | loss 6.19 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 69.17 | loss 6.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 69.85s | valid loss 8.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 69.29 | loss 5.72 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 68.94 | loss 6.22 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 68.85 | loss 5.49 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 68.91 | loss 6.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 69.75s | valid loss 9.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 69.26 | loss 5.65 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 68.79 | loss 5.58 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 69.01 | loss 5.07 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 68.93 | loss 6.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 69.72s | valid loss 9.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 69.09 | loss 5.51 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 68.89 | loss 6.10 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 68.81 | loss 5.27 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 69.07 | loss 5.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 69.69s | valid loss 9.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 69.05 | loss 5.37 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 68.81 | loss 4.87 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 68.77 | loss 5.89 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 68.82 | loss 4.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 69.61s | valid loss 8.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 68.92 | loss 3.98 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 68.62 | loss 4.03 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 68.91 | loss 5.31 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 68.69 | loss 4.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 69.51s | valid loss 8.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 68.98 | loss 4.82 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 68.60 | loss 5.09 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 68.49 | loss 4.63 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 68.72 | loss 5.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 69.46s | valid loss 9.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 68.78 | loss 3.98 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 68.69 | loss 3.86 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 68.54 | loss 4.62 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 68.46 | loss 4.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 69.41s | valid loss 8.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 69.01 | loss 4.67 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 68.39 | loss 3.74 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 68.52 | loss 4.14 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 68.64 | loss 3.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 69.37s | valid loss 6.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 69.09 | loss 4.37 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 68.33 | loss 3.52 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 68.40 | loss 3.54 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 68.67 | loss 3.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 69.36s | valid loss 6.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 68.64 | loss 3.37 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 73.66 | loss 3.72 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 68.28 | loss 3.44 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 68.56 | loss 4.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 71.33s | valid loss 8.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 68.80 | loss 3.75 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 68.32 | loss 3.52 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 73.51 | loss 3.22 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 68.35 | loss 3.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 70.23s | valid loss 8.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 73.89 | loss 3.28 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 68.27 | loss 2.85 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 68.24 | loss 3.19 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 73.63 | loss 3.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 71.24s | valid loss 8.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 68.62 | loss 3.50 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 73.91 | loss 3.00 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 68.10 | loss 2.70 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 68.22 | loss 2.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 71.27s | valid loss 7.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 68.67 | loss 2.52 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 68.05 | loss 2.75 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 73.43 | loss 2.86 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 68.26 | loss 3.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 70.10s | valid loss 8.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 68.47 | loss 2.17 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 68.00 | loss 1.81 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 68.15 | loss 2.74 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 73.36 | loss 2.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 70.02s | valid loss 7.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 68.48 | loss 2.69 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 68.16 | loss 2.03 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 68.03 | loss 2.37 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 67.86 | loss 2.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 68.91s | valid loss 7.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 68.34 | loss 2.18 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 68.21 | loss 2.89 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 73.43 | loss 2.53 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 68.01 | loss 2.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 69.97s | valid loss 8.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 68.42 | loss 2.14 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 68.00 | loss 1.59 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 67.91 | loss 3.08 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 68.03 | loss 1.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 68.87s | valid loss 8.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 68.29 | loss 1.84 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 68.19 | loss 2.03 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 67.93 | loss 2.02 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 67.86 | loss 2.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 68.84s | valid loss 7.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 68.08 | loss 1.70 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 67.93 | loss 2.28 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 67.99 | loss 1.97 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 67.81 | loss 2.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 68.75s | valid loss 7.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 68.30 | loss 1.94 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 67.91 | loss 2.25 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 67.78 | loss 1.61 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 67.79 | loss 1.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 68.72s | valid loss 7.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 68.13 | loss 1.64 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 68.00 | loss 2.55 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 67.81 | loss 1.66 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 67.66 | loss 1.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 68.71s | valid loss 9.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 68.27 | loss 2.11 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 67.99 | loss 2.10 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 68.05 | loss 2.04 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 67.76 | loss 1.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 68.76s | valid loss 8.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.55 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 69.52 | loss 66.84 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 69.18 | loss 27.53 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 69.25 | loss 19.27 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 74.64 | loss 17.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 71.09s | valid loss 32.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 69.66 | loss 13.71 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 74.69 | loss 12.44 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 69.19 | loss 12.14 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 69.18 | loss 11.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 71.14s | valid loss 18.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 69.54 | loss 9.92 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 69.20 | loss 10.19 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 69.30 | loss 9.49 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 69.22 | loss 8.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 70.02s | valid loss 16.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 69.68 | loss 8.12 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 69.20 | loss 7.57 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 69.20 | loss 8.40 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 69.38 | loss 8.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 70.06s | valid loss 10.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 69.46 | loss 7.58 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 69.24 | loss 7.40 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 69.15 | loss 7.65 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 69.20 | loss 7.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 70.01s | valid loss 13.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 69.47 | loss 7.60 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 69.12 | loss 6.23 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 74.57 | loss 6.33 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 69.28 | loss 7.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 71.04s | valid loss 13.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 69.58 | loss 6.47 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 69.07 | loss 5.87 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 69.07 | loss 6.59 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 69.20 | loss 6.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 69.94s | valid loss 8.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 69.50 | loss 5.47 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 74.67 | loss 5.79 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 69.20 | loss 6.10 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 69.14 | loss 5.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 71.09s | valid loss 6.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 69.36 | loss 5.08 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 68.95 | loss 4.87 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 69.65 | loss 5.58 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 68.71 | loss 5.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 69.78s | valid loss 10.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 68.91 | loss 4.60 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 68.48 | loss 5.55 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 68.52 | loss 5.48 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 68.55 | loss 5.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 69.30s | valid loss 9.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 68.66 | loss 4.83 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 68.25 | loss 4.83 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 68.22 | loss 5.18 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 68.37 | loss 4.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 69.11s | valid loss 9.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 68.63 | loss 4.81 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 68.12 | loss 3.86 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 68.33 | loss 4.66 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 68.27 | loss 4.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 69.03s | valid loss 9.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 68.70 | loss 5.06 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 68.19 | loss 4.17 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 68.15 | loss 4.66 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 68.24 | loss 3.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 69.02s | valid loss 7.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 68.44 | loss 4.32 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 68.27 | loss 3.82 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 68.21 | loss 3.67 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 68.15 | loss 5.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 69.02s | valid loss 9.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 68.42 | loss 4.02 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 68.13 | loss 4.19 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 68.10 | loss 3.68 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 67.93 | loss 3.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 68.86s | valid loss 7.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 68.49 | loss 3.47 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 68.08 | loss 4.59 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 68.03 | loss 4.22 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 68.11 | loss 3.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 68.90s | valid loss 8.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 68.29 | loss 2.55 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 70.02 | loss 4.31 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 69.27 | loss 3.43 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 69.29 | loss 3.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 70.88s | valid loss 8.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 69.68 | loss 3.82 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 69.07 | loss 3.15 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 68.11 | loss 3.88 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 68.22 | loss 3.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 69.39s | valid loss 7.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 68.50 | loss 3.01 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 67.86 | loss 3.11 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 68.02 | loss 3.00 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 68.21 | loss 3.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 68.88s | valid loss 5.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 68.22 | loss 2.71 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 67.99 | loss 3.03 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 68.04 | loss 3.14 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 67.94 | loss 2.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 68.82s | valid loss 7.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 68.37 | loss 2.51 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 67.80 | loss 2.78 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 67.96 | loss 2.78 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 68.15 | loss 3.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 68.81s | valid loss 8.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 68.41 | loss 2.86 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 67.86 | loss 2.42 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 67.90 | loss 2.74 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 67.88 | loss 2.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 68.76s | valid loss 6.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 68.18 | loss 2.08 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 67.84 | loss 2.26 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 67.81 | loss 2.46 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 67.74 | loss 2.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 68.94s | valid loss 8.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 69.28 | loss 2.29 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 68.10 | loss 1.91 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 68.18 | loss 2.33 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 68.21 | loss 2.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 69.21s | valid loss 8.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 68.61 | loss 2.45 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 68.16 | loss 1.93 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 68.22 | loss 2.33 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 68.26 | loss 2.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 69.07s | valid loss 7.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 68.45 | loss 1.72 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 68.20 | loss 1.82 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 68.01 | loss 1.60 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 68.22 | loss 2.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 69.00s | valid loss 9.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 68.36 | loss 1.97 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 68.07 | loss 2.15 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 68.13 | loss 1.89 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 68.12 | loss 2.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 68.95s | valid loss 8.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 68.51 | loss 1.96 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 68.05 | loss 1.79 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 68.12 | loss 1.76 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 68.24 | loss 1.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 69.00s | valid loss 7.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 68.25 | loss 1.40 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 68.07 | loss 1.48 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 68.10 | loss 2.31 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 67.90 | loss 1.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 68.88s | valid loss 9.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 68.42 | loss 1.58 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 67.73 | loss 1.33 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 67.96 | loss 1.90 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 68.02 | loss 1.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 68.82s | valid loss 7.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.97 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 69.61 | loss 80.35 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 69.23 | loss 31.61 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 69.24 | loss 22.87 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 69.32 | loss 18.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 70.04s | valid loss 19.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 69.51 | loss 14.07 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 69.29 | loss 12.74 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 69.21 | loss 12.35 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 69.19 | loss 11.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 70.03s | valid loss 12.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 69.50 | loss 9.51 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 69.15 | loss 10.17 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 70.63 | loss 9.44 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 69.04 | loss 8.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 70.20s | valid loss 16.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 75.07 | loss 8.69 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 69.11 | loss 7.30 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 69.09 | loss 8.27 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 74.62 | loss 8.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 72.13s | valid loss 9.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 69.50 | loss 6.98 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 74.66 | loss 7.28 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 69.14 | loss 7.97 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 69.16 | loss 7.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 72.13s | valid loss 10.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 69.47 | loss 6.57 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 69.11 | loss 6.62 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 69.69 | loss 6.51 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 69.88 | loss 6.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 70.25s | valid loss 10.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 71.59 | loss 6.88 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 68.54 | loss 5.69 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 68.54 | loss 7.01 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 73.85 | loss 5.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 70.93s | valid loss 8.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 68.82 | loss 6.48 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 73.73 | loss 6.72 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 68.54 | loss 6.25 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 68.55 | loss 5.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 71.41s | valid loss 8.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 68.71 | loss 5.31 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 68.79 | loss 5.90 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 75.12 | loss 5.76 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 69.95 | loss 5.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 71.10s | valid loss 8.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 74.68 | loss 5.59 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 68.94 | loss 5.59 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 68.84 | loss 5.77 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 68.98 | loss 5.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 70.82s | valid loss 7.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 69.01 | loss 4.52 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 69.02 | loss 4.94 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 68.76 | loss 5.35 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 68.64 | loss 5.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 69.62s | valid loss 9.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 69.03 | loss 4.87 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 68.76 | loss 4.73 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 68.76 | loss 5.25 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 68.83 | loss 4.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 69.58s | valid loss 9.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 68.99 | loss 4.47 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 68.56 | loss 4.61 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 68.71 | loss 4.82 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 69.26 | loss 4.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 69.59s | valid loss 6.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 69.00 | loss 3.95 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 68.72 | loss 4.81 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 68.52 | loss 3.93 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 68.58 | loss 4.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 69.50s | valid loss 10.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 68.82 | loss 3.28 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 68.41 | loss 3.81 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 68.65 | loss 3.62 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 68.47 | loss 4.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 69.33s | valid loss 6.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 72.84 | loss 3.90 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 68.23 | loss 3.79 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 68.54 | loss 4.48 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 68.40 | loss 3.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 70.02s | valid loss 8.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 68.55 | loss 2.58 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 68.55 | loss 4.50 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 68.26 | loss 3.94 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 68.40 | loss 3.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 69.17s | valid loss 7.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 68.56 | loss 2.65 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 68.26 | loss 3.58 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 68.30 | loss 3.28 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 68.34 | loss 4.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 69.14s | valid loss 7.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 68.67 | loss 2.88 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 68.21 | loss 2.48 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 68.11 | loss 3.34 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 68.14 | loss 3.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 69.06s | valid loss 7.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 68.63 | loss 3.00 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 69.59 | loss 2.85 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 68.13 | loss 2.82 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 68.11 | loss 2.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 69.29s | valid loss 9.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 68.34 | loss 2.46 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 68.13 | loss 2.84 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 68.26 | loss 2.95 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 68.06 | loss 2.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 68.96s | valid loss 7.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 68.68 | loss 2.94 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 68.23 | loss 2.83 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 68.13 | loss 2.53 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 73.57 | loss 2.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 70.15s | valid loss 9.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 68.50 | loss 2.67 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 73.56 | loss 2.09 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 68.16 | loss 2.72 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 68.22 | loss 2.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 71.16s | valid loss 8.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 68.40 | loss 2.23 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 68.16 | loss 2.14 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 68.16 | loss 2.23 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 68.09 | loss 2.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 68.97s | valid loss 8.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 73.94 | loss 2.51 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 68.00 | loss 2.40 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 68.17 | loss 2.21 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 73.88 | loss 2.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 71.20s | valid loss 9.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 68.42 | loss 2.47 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 68.00 | loss 1.36 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 67.80 | loss 1.55 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 68.04 | loss 2.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 68.89s | valid loss 6.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 68.31 | loss 1.55 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 68.01 | loss 2.02 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 73.47 | loss 1.92 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 68.00 | loss 1.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 69.94s | valid loss 7.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 68.45 | loss 1.52 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 67.88 | loss 1.62 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 68.00 | loss 1.72 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 68.14 | loss 2.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 68.91s | valid loss 7.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 68.33 | loss 1.52 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 68.13 | loss 1.98 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 68.00 | loss 1.89 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 68.01 | loss 1.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 68.90s | valid loss 8.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 68.40 | loss 1.69 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 67.92 | loss 1.45 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 72.13 | loss 2.14 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 68.04 | loss 1.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 69.67s | valid loss 7.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.67 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 69.63 | loss 63.69 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 69.26 | loss 29.02 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 69.24 | loss 22.43 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 69.37 | loss 17.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 70.07s | valid loss 25.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 69.60 | loss 14.39 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 74.84 | loss 13.49 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 69.27 | loss 13.11 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 69.24 | loss 11.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 71.18s | valid loss 15.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 69.63 | loss 10.23 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 69.25 | loss 10.12 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 74.84 | loss 8.89 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 69.25 | loss 9.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 71.15s | valid loss 13.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 69.73 | loss 8.20 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 69.28 | loss 7.44 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 69.28 | loss 8.09 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 69.40 | loss 7.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 70.10s | valid loss 9.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 69.54 | loss 6.96 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 69.35 | loss 8.05 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 69.20 | loss 7.75 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 69.19 | loss 7.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 70.03s | valid loss 9.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 69.59 | loss 6.37 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 69.23 | loss 7.15 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 74.80 | loss 7.51 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 69.21 | loss 6.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 71.11s | valid loss 7.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 69.67 | loss 7.21 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 69.14 | loss 5.48 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 69.19 | loss 6.98 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 69.27 | loss 5.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 70.01s | valid loss 7.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 69.47 | loss 6.29 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 69.45 | loss 5.78 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 69.22 | loss 5.85 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 69.03 | loss 6.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 70.00s | valid loss 8.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 69.40 | loss 5.96 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 69.07 | loss 5.77 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 74.66 | loss 6.10 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 68.94 | loss 5.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 70.95s | valid loss 9.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 69.44 | loss 4.66 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 68.86 | loss 4.81 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 69.01 | loss 6.68 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 69.16 | loss 5.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 69.82s | valid loss 9.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 69.28 | loss 4.84 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 74.57 | loss 4.85 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 68.91 | loss 5.59 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 68.87 | loss 4.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 72.03s | valid loss 7.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 69.32 | loss 5.00 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 68.87 | loss 5.15 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 68.96 | loss 4.23 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 68.80 | loss 4.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 69.68s | valid loss 9.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 69.23 | loss 4.68 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 68.74 | loss 3.81 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 68.74 | loss 4.01 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 74.44 | loss 5.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 70.72s | valid loss 7.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 68.96 | loss 3.53 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 74.37 | loss 4.67 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 68.68 | loss 3.95 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 68.50 | loss 3.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 70.62s | valid loss 8.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 69.03 | loss 3.82 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 68.60 | loss 4.11 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 68.68 | loss 3.74 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 68.73 | loss 4.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 69.50s | valid loss 6.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 70.37 | loss 3.51 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 68.46 | loss 4.04 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 68.55 | loss 3.72 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 68.57 | loss 3.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 69.65s | valid loss 8.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 68.71 | loss 3.27 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 68.53 | loss 3.66 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 68.32 | loss 3.45 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 68.49 | loss 3.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 69.29s | valid loss 8.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 68.68 | loss 2.75 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 68.27 | loss 2.73 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 68.52 | loss 4.08 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 68.35 | loss 3.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 69.21s | valid loss 8.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 68.67 | loss 2.76 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 68.31 | loss 3.53 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 68.41 | loss 3.10 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 68.25 | loss 2.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 69.15s | valid loss 10.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 68.38 | loss 2.69 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 68.27 | loss 3.03 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 68.21 | loss 2.86 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 68.46 | loss 2.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 70.18s | valid loss 6.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 68.58 | loss 2.21 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 68.31 | loss 2.69 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 73.69 | loss 2.58 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 68.45 | loss 3.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 70.23s | valid loss 6.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 68.47 | loss 1.82 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 68.22 | loss 2.27 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 68.35 | loss 2.86 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 68.31 | loss 2.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 69.09s | valid loss 7.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 68.53 | loss 2.49 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 68.28 | loss 2.29 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 68.09 | loss 2.29 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 68.29 | loss 3.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 70.15s | valid loss 8.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 68.36 | loss 2.47 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 68.03 | loss 2.27 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 69.63 | loss 2.14 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 68.08 | loss 2.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 69.22s | valid loss 5.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 68.47 | loss 1.75 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 68.04 | loss 2.24 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 68.02 | loss 2.31 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 68.14 | loss 2.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 68.92s | valid loss 8.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 68.35 | loss 1.91 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 68.15 | loss 2.12 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 67.99 | loss 2.07 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 67.93 | loss 1.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 68.91s | valid loss 7.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 68.22 | loss 1.79 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 67.95 | loss 1.96 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 68.04 | loss 1.77 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 67.91 | loss 1.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 68.80s | valid loss 8.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 68.35 | loss 1.91 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 67.94 | loss 2.44 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 67.88 | loss 1.97 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 68.04 | loss 1.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 68.80s | valid loss 7.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 68.18 | loss 1.95 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 68.07 | loss 1.50 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 67.79 | loss 1.81 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 67.94 | loss 1.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 68.79s | valid loss 6.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 68.13 | loss 1.49 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 67.90 | loss 2.11 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 68.07 | loss 1.84 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 67.76 | loss 1.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 68.73s | valid loss 6.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 31 | 200/ 938 batches | lr 0.00021 | ms/batch 68.40 | loss 2.01 |\n", + "| epoch 31 | 400/ 938 batches | lr 0.00021 | ms/batch 67.75 | loss 1.40 |\n", + "| epoch 31 | 600/ 938 batches | lr 0.00021 | ms/batch 67.74 | loss 1.39 |\n", + "| epoch 31 | 800/ 938 batches | lr 0.00021 | ms/batch 68.20 | loss 2.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 31 | time: 68.77s | valid loss 7.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 32 | 200/ 938 batches | lr 0.00020 | ms/batch 68.23 | loss 1.48 |\n", + "| epoch 32 | 400/ 938 batches | lr 0.00020 | ms/batch 68.10 | loss 2.09 |\n", + "| epoch 32 | 600/ 938 batches | lr 0.00020 | ms/batch 67.75 | loss 1.10 |\n", + "| epoch 32 | 800/ 938 batches | lr 0.00020 | ms/batch 67.88 | loss 1.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 32 | time: 68.79s | valid loss 7.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 33 | 200/ 938 batches | lr 0.00019 | ms/batch 68.17 | loss 1.41 |\n", + "| epoch 33 | 400/ 938 batches | lr 0.00019 | ms/batch 67.88 | loss 1.40 |\n", + "| epoch 33 | 600/ 938 batches | lr 0.00019 | ms/batch 68.08 | loss 1.85 |\n", + "| epoch 33 | 800/ 938 batches | lr 0.00019 | ms/batch 67.82 | loss 1.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 33 | time: 68.77s | valid loss 6.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 34 | 200/ 938 batches | lr 0.00018 | ms/batch 68.39 | loss 1.12 |\n", + "| epoch 34 | 400/ 938 batches | lr 0.00018 | ms/batch 67.90 | loss 1.84 |\n", + "| epoch 34 | 600/ 938 batches | lr 0.00018 | ms/batch 67.80 | loss 1.23 |\n", + "| epoch 34 | 800/ 938 batches | lr 0.00018 | ms/batch 67.95 | loss 1.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 34 | time: 68.78s | valid loss 6.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 35 | 200/ 938 batches | lr 0.00017 | ms/batch 68.15 | loss 1.21 |\n", + "| epoch 35 | 400/ 938 batches | lr 0.00017 | ms/batch 67.91 | loss 1.12 |\n", + "| epoch 35 | 600/ 938 batches | lr 0.00017 | ms/batch 67.79 | loss 1.47 |\n", + "| epoch 35 | 800/ 938 batches | lr 0.00017 | ms/batch 67.82 | loss 1.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 35 | time: 68.74s | valid loss 6.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 36 | 200/ 938 batches | lr 0.00017 | ms/batch 68.11 | loss 1.85 |\n", + "| epoch 36 | 400/ 938 batches | lr 0.00017 | ms/batch 67.82 | loss 1.05 |\n", + "| epoch 36 | 600/ 938 batches | lr 0.00017 | ms/batch 68.03 | loss 1.10 |\n", + "| epoch 36 | 800/ 938 batches | lr 0.00017 | ms/batch 67.68 | loss 1.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 36 | time: 68.67s | valid loss 7.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 37 | 200/ 938 batches | lr 0.00016 | ms/batch 68.06 | loss 1.03 |\n", + "| epoch 37 | 400/ 938 batches | lr 0.00016 | ms/batch 67.74 | loss 1.59 |\n", + "| epoch 37 | 600/ 938 batches | lr 0.00016 | ms/batch 67.66 | loss 0.91 |\n", + "| epoch 37 | 800/ 938 batches | lr 0.00016 | ms/batch 67.71 | loss 1.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 37 | time: 68.57s | valid loss 7.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.88 |\n", + "=========================================================================================\n", + "Result over 5 runs: 5.778070449829102\n" + ] + } + ], + "source": [ + "num_runs = 5\n", + "best_model = None\n", + "best_result = 1000000\n", + "results = []\n", + "\n", + "for i in range(num_runs):\n", + " features = nn.Sequential(nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(64),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + "\n", + " nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(128),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + "\n", + " nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(256),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + "\n", + " nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=(0,1)),\n", + " nn.BatchNorm2d(512),\n", + " nn.ReLU()\n", + " )\n", + "\n", + " classifier = nn.Sequential(nn.Linear(512, 1028),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=1028, bias=True),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=10, bias=True))\n", + "\n", + " conv = nn.Sequential(nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.AvgPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.AvgPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.AvgPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.AvgPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.AvgPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.AvgPool1d(kernel_size=2, stride=2))\n", + "\n", + " model = ConvMNIST(features, conv, classifier)\n", + " model = model.to(device)\n", + "\n", + " optimizer = torch.optim.Adam(model.parameters())\n", + " scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "\n", + " epochs = 30\n", + " model, result = train_model(model, epochs, train_loader, test_loader, seq_len, digits_per_batch, optimizer, scheduler)\n", + " \n", + " if result < best_result:\n", + " best_result = result\n", + " best_model = model\n", + " results.append(result)\n", + " \n", + "model = best_model\n", + "print(\"Result over {} runs: {}\".format(num_runs, torch.tensor(results).mean().item()))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 2.275384, + "end_time": "2021-01-23T12:54:21.054888", + "exception": false, + "start_time": "2021-01-23T12:54:18.779504", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "### Max pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-23T12:54:25.620899Z", + "iopub.status.busy": "2021-01-23T12:54:25.620371Z", + "iopub.status.idle": "2021-01-23T16:52:13.614240Z", + "shell.execute_reply": "2021-01-23T16:52:13.614893Z" + }, + "papermill": { + "duration": 14270.285659, + "end_time": "2021-01-23T16:52:13.615099", + "exception": false, + "start_time": "2021-01-23T12:54:23.329440", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 69.01 | loss 93.40 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 68.58 | loss 34.42 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 68.58 | loss 21.78 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 74.21 | loss 18.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 70.50s | valid loss 48.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 68.97 | loss 14.34 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 74.21 | loss 13.80 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 68.64 | loss 12.19 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 68.63 | loss 11.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 71.64s | valid loss 13.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 68.97 | loss 11.29 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 68.66 | loss 10.51 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 70.58 | loss 9.33 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 68.59 | loss 9.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 70.05s | valid loss 17.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 70.72 | loss 9.11 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 68.70 | loss 8.48 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 68.69 | loss 8.37 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 74.29 | loss 9.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 70.93s | valid loss 11.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 69.00 | loss 7.80 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 74.29 | loss 7.32 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 68.67 | loss 7.13 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 68.67 | loss 7.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 71.69s | valid loss 12.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 69.00 | loss 6.70 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 68.62 | loss 7.31 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 68.78 | loss 7.18 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 68.61 | loss 7.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 69.45s | valid loss 9.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 69.05 | loss 7.64 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 68.65 | loss 6.12 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 68.65 | loss 6.68 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 68.72 | loss 6.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 69.46s | valid loss 8.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 68.93 | loss 6.81 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 70.91 | loss 6.17 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 69.09 | loss 5.82 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 69.02 | loss 6.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 70.36s | valid loss 7.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 69.48 | loss 6.13 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 68.98 | loss 5.68 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 69.07 | loss 5.26 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 69.21 | loss 7.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 69.90s | valid loss 6.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 69.46 | loss 5.68 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 68.86 | loss 5.85 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 68.95 | loss 5.33 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 68.98 | loss 4.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 69.76s | valid loss 8.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 69.10 | loss 4.88 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 68.93 | loss 5.12 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 68.93 | loss 4.85 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 68.74 | loss 5.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 69.69s | valid loss 8.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 68.99 | loss 4.47 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 68.61 | loss 4.34 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 68.79 | loss 4.82 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 68.88 | loss 5.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 69.58s | valid loss 6.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 71.21 | loss 4.65 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 68.51 | loss 4.49 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 68.61 | loss 5.23 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 68.80 | loss 5.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 69.95s | valid loss 7.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 69.01 | loss 4.83 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 68.71 | loss 3.92 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 68.73 | loss 4.79 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 68.54 | loss 4.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 69.52s | valid loss 8.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 68.80 | loss 4.32 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 68.59 | loss 4.52 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 69.14 | loss 4.23 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 68.67 | loss 5.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 69.53s | valid loss 7.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 69.09 | loss 3.41 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 68.31 | loss 3.63 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 68.46 | loss 4.00 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 68.60 | loss 4.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 69.34s | valid loss 7.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 68.83 | loss 3.91 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 69.87 | loss 3.47 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 68.30 | loss 4.12 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 68.28 | loss 2.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 69.53s | valid loss 8.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 68.82 | loss 3.27 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 68.35 | loss 3.93 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 68.44 | loss 3.82 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 68.31 | loss 3.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 69.27s | valid loss 7.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 69.94 | loss 3.26 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 68.48 | loss 2.82 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 69.04 | loss 2.80 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 73.23 | loss 3.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 70.47s | valid loss 7.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 68.03 | loss 2.89 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 73.52 | loss 3.99 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 67.83 | loss 2.82 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 67.53 | loss 2.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 70.80s | valid loss 6.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 68.01 | loss 2.78 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 67.72 | loss 3.49 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 70.48 | loss 2.55 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 69.10 | loss 3.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 69.52s | valid loss 7.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 74.04 | loss 2.45 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 67.40 | loss 2.70 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 67.44 | loss 3.19 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 67.69 | loss 2.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 69.61s | valid loss 5.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 67.80 | loss 2.11 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 67.62 | loss 2.97 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 67.54 | loss 2.85 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 67.45 | loss 2.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 68.39s | valid loss 7.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 67.92 | loss 3.08 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 67.32 | loss 1.85 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 67.41 | loss 2.00 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 67.43 | loss 2.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 68.27s | valid loss 6.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 67.90 | loss 2.60 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 67.45 | loss 2.19 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 67.59 | loss 2.86 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 67.59 | loss 2.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 68.39s | valid loss 7.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 67.70 | loss 2.41 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 67.53 | loss 2.61 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 68.83 | loss 2.51 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 67.96 | loss 2.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 68.81s | valid loss 6.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 68.29 | loss 2.11 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 67.86 | loss 1.64 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 67.93 | loss 2.27 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 67.81 | loss 2.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 68.77s | valid loss 7.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 68.32 | loss 2.09 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 67.95 | loss 2.23 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 67.95 | loss 2.24 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 67.95 | loss 1.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 68.81s | valid loss 6.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 67.94 | loss 1.07 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 67.97 | loss 2.71 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 67.61 | loss 1.07 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 67.81 | loss 2.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 68.65s | valid loss 5.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 68.12 | loss 2.06 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 67.77 | loss 1.49 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 68.18 | loss 1.55 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 67.65 | loss 1.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 68.70s | valid loss 6.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 31 | 200/ 938 batches | lr 0.00021 | ms/batch 73.81 | loss 1.58 |\n", + "| epoch 31 | 400/ 938 batches | lr 0.00021 | ms/batch 67.69 | loss 1.88 |\n", + "| epoch 31 | 600/ 938 batches | lr 0.00021 | ms/batch 67.80 | loss 2.05 |\n", + "| epoch 31 | 800/ 938 batches | lr 0.00021 | ms/batch 67.90 | loss 1.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 31 | time: 69.79s | valid loss 6.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 32 | 200/ 938 batches | lr 0.00020 | ms/batch 68.11 | loss 1.88 |\n", + "| epoch 32 | 400/ 938 batches | lr 0.00020 | ms/batch 67.79 | loss 1.51 |\n", + "| epoch 32 | 600/ 938 batches | lr 0.00020 | ms/batch 67.76 | loss 1.50 |\n", + "| epoch 32 | 800/ 938 batches | lr 0.00020 | ms/batch 67.61 | loss 1.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 32 | time: 68.64s | valid loss 7.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 33 | 200/ 938 batches | lr 0.00019 | ms/batch 67.97 | loss 1.80 |\n", + "| epoch 33 | 400/ 938 batches | lr 0.00019 | ms/batch 67.81 | loss 1.71 |\n", + "| epoch 33 | 600/ 938 batches | lr 0.00019 | ms/batch 67.72 | loss 1.03 |\n", + "| epoch 33 | 800/ 938 batches | lr 0.00019 | ms/batch 67.54 | loss 1.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 33 | time: 68.55s | valid loss 7.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 34 | 200/ 938 batches | lr 0.00018 | ms/batch 68.07 | loss 1.34 |\n", + "| epoch 34 | 400/ 938 batches | lr 0.00018 | ms/batch 67.48 | loss 1.11 |\n", + "| epoch 34 | 600/ 938 batches | lr 0.00018 | ms/batch 67.63 | loss 1.74 |\n", + "| epoch 34 | 800/ 938 batches | lr 0.00018 | ms/batch 67.84 | loss 1.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 34 | time: 68.55s | valid loss 7.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 35 | 200/ 938 batches | lr 0.00017 | ms/batch 68.03 | loss 1.51 |\n", + "| epoch 35 | 400/ 938 batches | lr 0.00017 | ms/batch 67.76 | loss 1.43 |\n", + "| epoch 35 | 600/ 938 batches | lr 0.00017 | ms/batch 67.56 | loss 1.47 |\n", + "| epoch 35 | 800/ 938 batches | lr 0.00017 | ms/batch 67.71 | loss 1.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 35 | time: 68.59s | valid loss 7.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 36 | 200/ 938 batches | lr 0.00017 | ms/batch 67.97 | loss 1.36 |\n", + "| epoch 36 | 400/ 938 batches | lr 0.00017 | ms/batch 67.65 | loss 1.19 |\n", + "| epoch 36 | 600/ 938 batches | lr 0.00017 | ms/batch 67.73 | loss 1.31 |\n", + "| epoch 36 | 800/ 938 batches | lr 0.00017 | ms/batch 67.55 | loss 1.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 36 | time: 68.51s | valid loss 8.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 37 | 200/ 938 batches | lr 0.00016 | ms/batch 68.05 | loss 0.99 |\n", + "| epoch 37 | 400/ 938 batches | lr 0.00016 | ms/batch 67.58 | loss 1.69 |\n", + "| epoch 37 | 600/ 938 batches | lr 0.00016 | ms/batch 67.61 | loss 1.87 |\n", + "| epoch 37 | 800/ 938 batches | lr 0.00016 | ms/batch 67.60 | loss 0.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 37 | time: 68.50s | valid loss 6.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 38 | 200/ 938 batches | lr 0.00015 | ms/batch 67.77 | loss 1.43 |\n", + "| epoch 38 | 400/ 938 batches | lr 0.00015 | ms/batch 67.81 | loss 1.31 |\n", + "| epoch 38 | 600/ 938 batches | lr 0.00015 | ms/batch 67.65 | loss 1.04 |\n", + "| epoch 38 | 800/ 938 batches | lr 0.00015 | ms/batch 67.44 | loss 0.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 38 | time: 68.50s | valid loss 6.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 39 | 200/ 938 batches | lr 0.00014 | ms/batch 67.98 | loss 1.19 |\n", + "| epoch 39 | 400/ 938 batches | lr 0.00014 | ms/batch 67.61 | loss 1.57 |\n", + "| epoch 39 | 600/ 938 batches | lr 0.00014 | ms/batch 67.61 | loss 0.84 |\n", + "| epoch 39 | 800/ 938 batches | lr 0.00014 | ms/batch 67.47 | loss 0.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 39 | time: 68.45s | valid loss 7.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 40 | 200/ 938 batches | lr 0.00014 | ms/batch 68.07 | loss 1.59 |\n", + "| epoch 40 | 400/ 938 batches | lr 0.00014 | ms/batch 67.42 | loss 0.64 |\n", + "| epoch 40 | 600/ 938 batches | lr 0.00014 | ms/batch 67.57 | loss 1.35 |\n", + "| epoch 40 | 800/ 938 batches | lr 0.00014 | ms/batch 67.55 | loss 0.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 40 | time: 68.42s | valid loss 7.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 41 | 200/ 938 batches | lr 0.00013 | ms/batch 67.74 | loss 1.12 |\n", + "| epoch 41 | 400/ 938 batches | lr 0.00013 | ms/batch 67.67 | loss 1.43 |\n", + "| epoch 41 | 600/ 938 batches | lr 0.00013 | ms/batch 67.58 | loss 1.45 |\n", + "| epoch 41 | 800/ 938 batches | lr 0.00013 | ms/batch 67.47 | loss 0.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 41 | time: 68.43s | valid loss 7.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 42 | 200/ 938 batches | lr 0.00012 | ms/batch 67.83 | loss 1.19 |\n", + "| epoch 42 | 400/ 938 batches | lr 0.00012 | ms/batch 67.49 | loss 1.23 |\n", + "| epoch 42 | 600/ 938 batches | lr 0.00012 | ms/batch 67.60 | loss 1.48 |\n", + "| epoch 42 | 800/ 938 batches | lr 0.00012 | ms/batch 67.54 | loss 1.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 42 | time: 68.40s | valid loss 7.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 43 | 200/ 938 batches | lr 0.00012 | ms/batch 68.03 | loss 1.45 |\n", + "| epoch 43 | 400/ 938 batches | lr 0.00012 | ms/batch 67.48 | loss 1.04 |\n", + "| epoch 43 | 600/ 938 batches | lr 0.00012 | ms/batch 67.57 | loss 0.82 |\n", + "| epoch 43 | 800/ 938 batches | lr 0.00012 | ms/batch 67.66 | loss 0.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 43 | time: 68.46s | valid loss 8.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 44 | 200/ 938 batches | lr 0.00011 | ms/batch 67.69 | loss 1.10 |\n", + "| epoch 44 | 400/ 938 batches | lr 0.00011 | ms/batch 67.50 | loss 0.85 |\n", + "| epoch 44 | 600/ 938 batches | lr 0.00011 | ms/batch 67.45 | loss 1.34 |\n", + "| epoch 44 | 800/ 938 batches | lr 0.00011 | ms/batch 67.54 | loss 1.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 44 | time: 68.39s | valid loss 7.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.67 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 69.47 | loss 81.93 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 69.20 | loss 32.17 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 69.30 | loss 21.82 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 69.14 | loss 17.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 69.99s | valid loss 24.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 69.65 | loss 14.61 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 69.19 | loss 13.06 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 69.21 | loss 12.93 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 69.22 | loss 11.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 70.01s | valid loss 18.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 69.44 | loss 10.90 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 69.20 | loss 9.79 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 69.10 | loss 10.06 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 69.07 | loss 9.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 69.94s | valid loss 10.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 69.48 | loss 9.15 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 69.10 | loss 8.58 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 69.21 | loss 8.29 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 69.08 | loss 8.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 69.93s | valid loss 8.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 69.58 | loss 8.12 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 69.10 | loss 7.66 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 69.10 | loss 7.87 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 69.23 | loss 7.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 69.95s | valid loss 13.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 69.40 | loss 6.90 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 69.18 | loss 7.26 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 69.10 | loss 6.97 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 69.09 | loss 6.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 69.93s | valid loss 9.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 69.43 | loss 6.72 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 69.07 | loss 7.17 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 69.22 | loss 7.14 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 69.05 | loss 5.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 69.91s | valid loss 8.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 69.48 | loss 5.72 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 68.92 | loss 6.43 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 69.02 | loss 5.78 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 75.31 | loss 6.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 71.09s | valid loss 9.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 69.43 | loss 5.69 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 74.93 | loss 6.50 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 68.93 | loss 5.08 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 68.95 | loss 5.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 72.16s | valid loss 14.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 69.25 | loss 6.27 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 68.86 | loss 5.03 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 69.05 | loss 5.78 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 68.76 | loss 5.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 69.72s | valid loss 7.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 70.64 | loss 5.33 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 69.61 | loss 5.12 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 69.67 | loss 5.15 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 74.84 | loss 4.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 71.54s | valid loss 8.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 69.84 | loss 4.53 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 71.09 | loss 4.89 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 69.35 | loss 3.65 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 69.53 | loss 5.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 71.58s | valid loss 8.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 68.50 | loss 4.65 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 68.10 | loss 4.64 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 68.30 | loss 4.91 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 68.12 | loss 4.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 68.99s | valid loss 7.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 68.57 | loss 3.88 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 68.18 | loss 4.58 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 68.18 | loss 4.50 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 68.39 | loss 4.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 69.04s | valid loss 7.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 68.45 | loss 4.41 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 68.29 | loss 4.23 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 68.02 | loss 4.42 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 68.10 | loss 4.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 68.96s | valid loss 9.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 68.29 | loss 3.35 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 67.72 | loss 3.35 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 67.94 | loss 3.83 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 67.92 | loss 5.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 68.75s | valid loss 7.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 68.42 | loss 3.69 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 67.98 | loss 3.91 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 67.87 | loss 3.63 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 67.90 | loss 3.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 68.80s | valid loss 7.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 68.05 | loss 3.47 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 67.92 | loss 3.70 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 67.82 | loss 3.51 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 67.74 | loss 2.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 68.66s | valid loss 7.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 68.03 | loss 4.01 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 67.76 | loss 3.33 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 67.72 | loss 3.47 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 67.68 | loss 3.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 68.55s | valid loss 8.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 68.02 | loss 2.96 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 67.64 | loss 3.17 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 67.69 | loss 3.05 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 67.81 | loss 3.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 68.54s | valid loss 7.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 68.01 | loss 2.76 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 67.60 | loss 2.74 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 67.55 | loss 3.16 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 67.55 | loss 3.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 68.46s | valid loss 8.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 67.79 | loss 2.67 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 67.59 | loss 3.18 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 67.60 | loss 2.72 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 67.40 | loss 2.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 68.37s | valid loss 10.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 67.92 | loss 2.88 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 67.40 | loss 2.42 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 67.44 | loss 3.20 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 73.60 | loss 2.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 69.59s | valid loss 8.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 67.70 | loss 1.96 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 67.56 | loss 2.35 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 67.60 | loss 3.01 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 67.39 | loss 2.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 68.36s | valid loss 7.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 67.80 | loss 2.68 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 67.43 | loss 2.62 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 67.54 | loss 2.45 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 67.28 | loss 2.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 68.29s | valid loss 7.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 69.43 | loss 1.67 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 67.46 | loss 1.83 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 67.37 | loss 2.22 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 73.11 | loss 2.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 69.74s | valid loss 8.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 67.76 | loss 2.32 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 73.06 | loss 1.92 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 67.34 | loss 2.28 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 67.46 | loss 2.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 70.30s | valid loss 6.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 67.69 | loss 1.75 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 67.33 | loss 1.71 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 73.22 | loss 2.18 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 67.37 | loss 2.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 69.39s | valid loss 8.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 73.36 | loss 1.87 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 67.38 | loss 1.46 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 67.45 | loss 2.56 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 72.94 | loss 1.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 70.52s | valid loss 7.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 67.65 | loss 1.10 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 72.81 | loss 1.02 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 67.24 | loss 2.35 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 67.33 | loss 1.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 70.35s | valid loss 7.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 31 | 200/ 938 batches | lr 0.00021 | ms/batch 67.60 | loss 1.85 |\n", + "| epoch 31 | 400/ 938 batches | lr 0.00021 | ms/batch 67.30 | loss 2.09 |\n", + "| epoch 31 | 600/ 938 batches | lr 0.00021 | ms/batch 67.33 | loss 1.55 |\n", + "| epoch 31 | 800/ 938 batches | lr 0.00021 | ms/batch 67.26 | loss 1.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 31 | time: 68.15s | valid loss 6.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 32 | 200/ 938 batches | lr 0.00020 | ms/batch 67.65 | loss 2.00 |\n", + "| epoch 32 | 400/ 938 batches | lr 0.00020 | ms/batch 67.20 | loss 1.64 |\n", + "| epoch 32 | 600/ 938 batches | lr 0.00020 | ms/batch 67.20 | loss 1.98 |\n", + "| epoch 32 | 800/ 938 batches | lr 0.00020 | ms/batch 67.23 | loss 1.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 32 | time: 68.11s | valid loss 7.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 33 | 200/ 938 batches | lr 0.00019 | ms/batch 67.42 | loss 1.56 |\n", + "| epoch 33 | 400/ 938 batches | lr 0.00019 | ms/batch 67.36 | loss 1.72 |\n", + "| epoch 33 | 600/ 938 batches | lr 0.00019 | ms/batch 67.22 | loss 1.95 |\n", + "| epoch 33 | 800/ 938 batches | lr 0.00019 | ms/batch 67.09 | loss 1.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 33 | time: 68.11s | valid loss 7.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 34 | 200/ 938 batches | lr 0.00018 | ms/batch 67.47 | loss 1.36 |\n", + "| epoch 34 | 400/ 938 batches | lr 0.00018 | ms/batch 67.12 | loss 0.73 |\n", + "| epoch 34 | 600/ 938 batches | lr 0.00018 | ms/batch 67.22 | loss 1.37 |\n", + "| epoch 34 | 800/ 938 batches | lr 0.00018 | ms/batch 67.22 | loss 1.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 34 | time: 68.06s | valid loss 7.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 35 | 200/ 938 batches | lr 0.00017 | ms/batch 67.62 | loss 0.93 |\n", + "| epoch 35 | 400/ 938 batches | lr 0.00017 | ms/batch 67.22 | loss 1.74 |\n", + "| epoch 35 | 600/ 938 batches | lr 0.00017 | ms/batch 67.18 | loss 1.61 |\n", + "| epoch 35 | 800/ 938 batches | lr 0.00017 | ms/batch 67.27 | loss 1.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 35 | time: 68.11s | valid loss 6.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 36 | 200/ 938 batches | lr 0.00017 | ms/batch 67.46 | loss 1.27 |\n", + "| epoch 36 | 400/ 938 batches | lr 0.00017 | ms/batch 67.46 | loss 1.30 |\n", + "| epoch 36 | 600/ 938 batches | lr 0.00017 | ms/batch 67.68 | loss 1.21 |\n", + "| epoch 36 | 800/ 938 batches | lr 0.00017 | ms/batch 67.12 | loss 1.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 36 | time: 68.22s | valid loss 7.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 37 | 200/ 938 batches | lr 0.00016 | ms/batch 67.48 | loss 1.11 |\n", + "| epoch 37 | 400/ 938 batches | lr 0.00016 | ms/batch 67.18 | loss 1.46 |\n", + "| epoch 37 | 600/ 938 batches | lr 0.00016 | ms/batch 67.28 | loss 1.03 |\n", + "| epoch 37 | 800/ 938 batches | lr 0.00016 | ms/batch 67.26 | loss 1.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 37 | time: 68.09s | valid loss 8.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 38 | 200/ 938 batches | lr 0.00015 | ms/batch 67.47 | loss 0.85 |\n", + "| epoch 38 | 400/ 938 batches | lr 0.00015 | ms/batch 67.11 | loss 1.26 |\n", + "| epoch 38 | 600/ 938 batches | lr 0.00015 | ms/batch 67.12 | loss 1.07 |\n", + "| epoch 38 | 800/ 938 batches | lr 0.00015 | ms/batch 67.37 | loss 1.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 38 | time: 68.05s | valid loss 8.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 39 | 200/ 938 batches | lr 0.00014 | ms/batch 67.44 | loss 1.48 |\n", + "| epoch 39 | 400/ 938 batches | lr 0.00014 | ms/batch 67.16 | loss 0.73 |\n", + "| epoch 39 | 600/ 938 batches | lr 0.00014 | ms/batch 67.08 | loss 1.24 |\n", + "| epoch 39 | 800/ 938 batches | lr 0.00014 | ms/batch 67.13 | loss 1.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 39 | time: 68.02s | valid loss 7.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 40 | 200/ 938 batches | lr 0.00014 | ms/batch 67.57 | loss 1.72 |\n", + "| epoch 40 | 400/ 938 batches | lr 0.00014 | ms/batch 67.07 | loss 0.99 |\n", + "| epoch 40 | 600/ 938 batches | lr 0.00014 | ms/batch 67.08 | loss 1.11 |\n", + "| epoch 40 | 800/ 938 batches | lr 0.00014 | ms/batch 67.17 | loss 1.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 40 | time: 68.02s | valid loss 7.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 41 | 200/ 938 batches | lr 0.00013 | ms/batch 67.44 | loss 0.62 |\n", + "| epoch 41 | 400/ 938 batches | lr 0.00013 | ms/batch 67.21 | loss 1.39 |\n", + "| epoch 41 | 600/ 938 batches | lr 0.00013 | ms/batch 67.08 | loss 1.71 |\n", + "| epoch 41 | 800/ 938 batches | lr 0.00013 | ms/batch 67.05 | loss 1.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 41 | time: 67.98s | valid loss 7.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 42 | 200/ 938 batches | lr 0.00012 | ms/batch 67.28 | loss 1.22 |\n", + "| epoch 42 | 400/ 938 batches | lr 0.00012 | ms/batch 67.02 | loss 0.71 |\n", + "| epoch 42 | 600/ 938 batches | lr 0.00012 | ms/batch 66.92 | loss 0.84 |\n", + "| epoch 42 | 800/ 938 batches | lr 0.00012 | ms/batch 66.87 | loss 1.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 42 | time: 67.84s | valid loss 7.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 43 | 200/ 938 batches | lr 0.00012 | ms/batch 67.26 | loss 0.98 |\n", + "| epoch 43 | 400/ 938 batches | lr 0.00012 | ms/batch 67.02 | loss 1.20 |\n", + "| epoch 43 | 600/ 938 batches | lr 0.00012 | ms/batch 67.01 | loss 0.61 |\n", + "| epoch 43 | 800/ 938 batches | lr 0.00012 | ms/batch 66.92 | loss 0.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 43 | time: 67.88s | valid loss 7.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 44 | 200/ 938 batches | lr 0.00011 | ms/batch 67.53 | loss 1.14 |\n", + "| epoch 44 | 400/ 938 batches | lr 0.00011 | ms/batch 67.08 | loss 1.12 |\n", + "| epoch 44 | 600/ 938 batches | lr 0.00011 | ms/batch 66.93 | loss 0.85 |\n", + "| epoch 44 | 800/ 938 batches | lr 0.00011 | ms/batch 67.19 | loss 1.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 44 | time: 67.99s | valid loss 8.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 45 | 200/ 938 batches | lr 0.00010 | ms/batch 67.48 | loss 0.87 |\n", + "| epoch 45 | 400/ 938 batches | lr 0.00010 | ms/batch 67.24 | loss 1.00 |\n", + "| epoch 45 | 600/ 938 batches | lr 0.00010 | ms/batch 67.14 | loss 1.01 |\n", + "| epoch 45 | 800/ 938 batches | lr 0.00010 | ms/batch 66.90 | loss 0.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 45 | time: 69.13s | valid loss 8.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 46 | 200/ 938 batches | lr 0.00010 | ms/batch 67.44 | loss 1.12 |\n", + "| epoch 46 | 400/ 938 batches | lr 0.00010 | ms/batch 67.01 | loss 1.24 |\n", + "| epoch 46 | 600/ 938 batches | lr 0.00010 | ms/batch 72.80 | loss 1.06 |\n", + "| epoch 46 | 800/ 938 batches | lr 0.00010 | ms/batch 66.99 | loss 0.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 46 | time: 69.07s | valid loss 7.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 47 | 200/ 938 batches | lr 0.00009 | ms/batch 72.97 | loss 0.74 |\n", + "| epoch 47 | 400/ 938 batches | lr 0.00009 | ms/batch 67.02 | loss 0.93 |\n", + "| epoch 47 | 600/ 938 batches | lr 0.00009 | ms/batch 66.99 | loss 0.97 |\n", + "| epoch 47 | 800/ 938 batches | lr 0.00009 | ms/batch 72.76 | loss 0.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 47 | time: 70.17s | valid loss 7.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.78 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 68.87 | loss 82.90 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 74.43 | loss 33.73 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 68.49 | loss 21.23 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 68.51 | loss 17.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 71.65s | valid loss 32.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 68.85 | loss 13.97 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 68.50 | loss 13.35 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 74.40 | loss 12.54 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 68.43 | loss 11.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 70.43s | valid loss 20.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 68.93 | loss 9.68 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 68.43 | loss 10.65 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 68.45 | loss 9.48 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 68.55 | loss 8.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 69.27s | valid loss 13.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 68.71 | loss 8.66 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 68.50 | loss 8.09 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 68.37 | loss 8.49 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 68.40 | loss 9.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 69.21s | valid loss 15.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 68.78 | loss 7.90 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 68.52 | loss 7.48 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 68.62 | loss 7.55 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 68.51 | loss 7.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 69.29s | valid loss 8.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 68.91 | loss 7.58 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 68.45 | loss 6.81 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 68.51 | loss 7.10 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 68.60 | loss 7.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 69.31s | valid loss 7.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 68.89 | loss 6.62 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 68.60 | loss 7.02 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 68.39 | loss 6.55 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 68.40 | loss 6.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 69.26s | valid loss 8.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 68.53 | loss 6.84 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 68.35 | loss 6.78 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 68.48 | loss 5.90 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 68.35 | loss 6.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 69.14s | valid loss 7.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 68.91 | loss 6.37 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 68.47 | loss 5.40 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 68.49 | loss 6.33 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 74.33 | loss 6.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 70.45s | valid loss 7.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 68.71 | loss 5.59 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 74.27 | loss 6.00 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 68.40 | loss 5.62 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 68.31 | loss 5.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 71.50s | valid loss 11.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 68.64 | loss 5.56 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 68.29 | loss 5.40 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 74.25 | loss 5.59 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 68.25 | loss 5.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 70.29s | valid loss 9.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 74.25 | loss 4.52 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 68.10 | loss 4.97 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 68.05 | loss 4.48 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 74.06 | loss 5.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 71.25s | valid loss 10.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 68.41 | loss 4.50 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 73.95 | loss 5.32 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 68.05 | loss 4.24 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 68.18 | loss 4.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 71.23s | valid loss 12.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 68.39 | loss 5.26 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 68.04 | loss 4.68 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 73.78 | loss 4.29 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 68.09 | loss 5.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 70.00s | valid loss 10.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 74.26 | loss 3.82 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 67.98 | loss 3.58 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 68.08 | loss 4.83 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 73.90 | loss 4.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 71.20s | valid loss 7.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 68.40 | loss 4.01 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 68.13 | loss 3.68 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 67.96 | loss 3.96 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 67.93 | loss 3.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 68.81s | valid loss 12.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 68.27 | loss 4.57 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 67.77 | loss 3.56 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 67.89 | loss 3.95 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 67.76 | loss 3.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 68.63s | valid loss 9.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 68.17 | loss 4.08 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 67.67 | loss 3.51 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 67.97 | loss 4.03 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 67.71 | loss 3.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 68.61s | valid loss 7.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 67.92 | loss 3.57 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 67.92 | loss 3.86 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 67.66 | loss 2.99 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 67.80 | loss 3.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 68.60s | valid loss 8.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 68.07 | loss 3.30 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 67.82 | loss 4.01 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 67.91 | loss 3.17 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 67.75 | loss 3.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 68.62s | valid loss 8.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 68.30 | loss 3.33 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 67.66 | loss 2.63 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 67.73 | loss 2.92 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 67.78 | loss 2.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 68.60s | valid loss 8.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 68.04 | loss 2.74 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 67.77 | loss 2.92 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 67.67 | loss 3.51 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 67.77 | loss 2.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 68.57s | valid loss 8.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 68.01 | loss 1.97 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 67.57 | loss 2.60 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 67.82 | loss 2.41 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 67.63 | loss 2.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 68.50s | valid loss 7.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 68.25 | loss 2.33 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 67.58 | loss 2.67 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 67.50 | loss 2.18 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 67.74 | loss 2.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 68.48s | valid loss 7.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 67.88 | loss 2.13 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 67.59 | loss 2.41 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 67.61 | loss 2.45 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 67.40 | loss 2.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 68.41s | valid loss 7.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 67.70 | loss 2.11 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 67.51 | loss 2.03 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 67.56 | loss 2.04 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 67.41 | loss 2.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 68.32s | valid loss 7.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 68.13 | loss 2.43 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 67.35 | loss 1.87 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 67.47 | loss 2.09 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 67.62 | loss 2.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 68.37s | valid loss 8.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 67.76 | loss 1.72 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 73.53 | loss 3.10 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 67.55 | loss 1.74 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 67.51 | loss 2.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 69.58s | valid loss 7.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 67.81 | loss 2.19 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 67.39 | loss 2.17 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 67.50 | loss 2.12 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 67.50 | loss 2.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 68.29s | valid loss 6.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 67.84 | loss 1.46 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 67.41 | loss 2.13 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 67.49 | loss 2.26 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 67.52 | loss 1.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 68.29s | valid loss 6.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 31 | 200/ 938 batches | lr 0.00021 | ms/batch 67.57 | loss 1.41 |\n", + "| epoch 31 | 400/ 938 batches | lr 0.00021 | ms/batch 67.39 | loss 1.64 |\n", + "| epoch 31 | 600/ 938 batches | lr 0.00021 | ms/batch 67.36 | loss 1.92 |\n", + "| epoch 31 | 800/ 938 batches | lr 0.00021 | ms/batch 67.27 | loss 1.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 31 | time: 68.16s | valid loss 7.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 32 | 200/ 938 batches | lr 0.00020 | ms/batch 67.66 | loss 2.06 |\n", + "| epoch 32 | 400/ 938 batches | lr 0.00020 | ms/batch 67.26 | loss 1.70 |\n", + "| epoch 32 | 600/ 938 batches | lr 0.00020 | ms/batch 67.31 | loss 1.55 |\n", + "| epoch 32 | 800/ 938 batches | lr 0.00020 | ms/batch 67.38 | loss 2.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 32 | time: 68.19s | valid loss 7.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 33 | 200/ 938 batches | lr 0.00019 | ms/batch 67.70 | loss 1.64 |\n", + "| epoch 33 | 400/ 938 batches | lr 0.00019 | ms/batch 67.14 | loss 1.45 |\n", + "| epoch 33 | 600/ 938 batches | lr 0.00019 | ms/batch 67.23 | loss 1.36 |\n", + "| epoch 33 | 800/ 938 batches | lr 0.00019 | ms/batch 67.38 | loss 1.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 33 | time: 68.12s | valid loss 7.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 34 | 200/ 938 batches | lr 0.00018 | ms/batch 67.57 | loss 1.38 |\n", + "| epoch 34 | 400/ 938 batches | lr 0.00018 | ms/batch 67.36 | loss 1.14 |\n", + "| epoch 34 | 600/ 938 batches | lr 0.00018 | ms/batch 67.15 | loss 1.58 |\n", + "| epoch 34 | 800/ 938 batches | lr 0.00018 | ms/batch 67.13 | loss 1.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 34 | time: 68.08s | valid loss 6.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 35 | 200/ 938 batches | lr 0.00017 | ms/batch 67.52 | loss 1.22 |\n", + "| epoch 35 | 400/ 938 batches | lr 0.00017 | ms/batch 67.29 | loss 1.94 |\n", + "| epoch 35 | 600/ 938 batches | lr 0.00017 | ms/batch 67.41 | loss 1.44 |\n", + "| epoch 35 | 800/ 938 batches | lr 0.00017 | ms/batch 67.21 | loss 1.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 35 | time: 68.11s | valid loss 6.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 36 | 200/ 938 batches | lr 0.00017 | ms/batch 68.48 | loss 1.24 |\n", + "| epoch 36 | 400/ 938 batches | lr 0.00017 | ms/batch 68.24 | loss 0.94 |\n", + "| epoch 36 | 600/ 938 batches | lr 0.00017 | ms/batch 67.65 | loss 2.14 |\n", + "| epoch 36 | 800/ 938 batches | lr 0.00017 | ms/batch 73.13 | loss 1.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 36 | time: 70.00s | valid loss 7.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 37 | 200/ 938 batches | lr 0.00016 | ms/batch 68.45 | loss 1.03 |\n", + "| epoch 37 | 400/ 938 batches | lr 0.00016 | ms/batch 67.84 | loss 1.63 |\n", + "| epoch 37 | 600/ 938 batches | lr 0.00016 | ms/batch 67.65 | loss 1.95 |\n", + "| epoch 37 | 800/ 938 batches | lr 0.00016 | ms/batch 67.58 | loss 1.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 37 | time: 68.64s | valid loss 7.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 38 | 200/ 938 batches | lr 0.00015 | ms/batch 67.88 | loss 0.99 |\n", + "| epoch 38 | 400/ 938 batches | lr 0.00015 | ms/batch 67.49 | loss 1.54 |\n", + "| epoch 38 | 600/ 938 batches | lr 0.00015 | ms/batch 67.75 | loss 1.53 |\n", + "| epoch 38 | 800/ 938 batches | lr 0.00015 | ms/batch 67.58 | loss 1.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 38 | time: 68.46s | valid loss 6.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 39 | 200/ 938 batches | lr 0.00014 | ms/batch 74.10 | loss 1.05 |\n", + "| epoch 39 | 400/ 938 batches | lr 0.00014 | ms/batch 67.65 | loss 1.32 |\n", + "| epoch 39 | 600/ 938 batches | lr 0.00014 | ms/batch 67.54 | loss 1.68 |\n", + "| epoch 39 | 800/ 938 batches | lr 0.00014 | ms/batch 73.61 | loss 1.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 39 | time: 70.88s | valid loss 7.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 40 | 200/ 938 batches | lr 0.00014 | ms/batch 67.95 | loss 0.95 |\n", + "| epoch 40 | 400/ 938 batches | lr 0.00014 | ms/batch 73.42 | loss 0.91 |\n", + "| epoch 40 | 600/ 938 batches | lr 0.00014 | ms/batch 67.55 | loss 1.89 |\n", + "| epoch 40 | 800/ 938 batches | lr 0.00014 | ms/batch 67.49 | loss 0.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 40 | time: 69.58s | valid loss 7.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 41 | 200/ 938 batches | lr 0.00013 | ms/batch 67.80 | loss 1.07 |\n", + "| epoch 41 | 400/ 938 batches | lr 0.00013 | ms/batch 67.53 | loss 1.79 |\n", + "| epoch 41 | 600/ 938 batches | lr 0.00013 | ms/batch 67.65 | loss 1.46 |\n", + "| epoch 41 | 800/ 938 batches | lr 0.00013 | ms/batch 67.72 | loss 1.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 41 | time: 68.61s | valid loss 7.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 42 | 200/ 938 batches | lr 0.00012 | ms/batch 70.49 | loss 0.76 |\n", + "| epoch 42 | 400/ 938 batches | lr 0.00012 | ms/batch 68.04 | loss 1.24 |\n", + "| epoch 42 | 600/ 938 batches | lr 0.00012 | ms/batch 68.23 | loss 0.66 |\n", + "| epoch 42 | 800/ 938 batches | lr 0.00012 | ms/batch 67.66 | loss 0.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 42 | time: 69.28s | valid loss 7.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 43 | 200/ 938 batches | lr 0.00012 | ms/batch 68.71 | loss 0.99 |\n", + "| epoch 43 | 400/ 938 batches | lr 0.00012 | ms/batch 69.97 | loss 0.89 |\n", + "| epoch 43 | 600/ 938 batches | lr 0.00012 | ms/batch 67.40 | loss 1.05 |\n", + "| epoch 43 | 800/ 938 batches | lr 0.00012 | ms/batch 67.02 | loss 0.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 43 | time: 68.85s | valid loss 8.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 44 | 200/ 938 batches | lr 0.00011 | ms/batch 67.45 | loss 0.96 |\n", + "| epoch 44 | 400/ 938 batches | lr 0.00011 | ms/batch 67.13 | loss 1.03 |\n", + "| epoch 44 | 600/ 938 batches | lr 0.00011 | ms/batch 67.07 | loss 0.72 |\n", + "| epoch 44 | 800/ 938 batches | lr 0.00011 | ms/batch 67.15 | loss 1.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 44 | time: 67.97s | valid loss 6.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 45 | 200/ 938 batches | lr 0.00010 | ms/batch 73.35 | loss 1.14 |\n", + "| epoch 45 | 400/ 938 batches | lr 0.00010 | ms/batch 67.09 | loss 1.13 |\n", + "| epoch 45 | 600/ 938 batches | lr 0.00010 | ms/batch 67.02 | loss 1.19 |\n", + "| epoch 45 | 800/ 938 batches | lr 0.00010 | ms/batch 72.88 | loss 0.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 45 | time: 70.27s | valid loss 6.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 46 | 200/ 938 batches | lr 0.00010 | ms/batch 67.36 | loss 1.00 |\n", + "| epoch 46 | 400/ 938 batches | lr 0.00010 | ms/batch 72.98 | loss 0.99 |\n", + "| epoch 46 | 600/ 938 batches | lr 0.00010 | ms/batch 67.02 | loss 1.38 |\n", + "| epoch 46 | 800/ 938 batches | lr 0.00010 | ms/batch 67.01 | loss 1.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 46 | time: 70.27s | valid loss 8.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.34 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 69.00 | loss 85.55 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 68.65 | loss 31.99 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 68.79 | loss 22.73 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 68.67 | loss 17.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 69.47s | valid loss 30.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 75.02 | loss 14.33 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 69.00 | loss 13.08 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 69.33 | loss 12.55 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 70.96 | loss 11.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 71.42s | valid loss 14.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 69.54 | loss 10.15 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 75.37 | loss 10.87 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 69.15 | loss 9.39 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 69.18 | loss 8.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 72.45s | valid loss 9.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 69.57 | loss 8.64 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 69.20 | loss 8.94 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 75.41 | loss 7.45 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 69.19 | loss 8.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 71.24s | valid loss 8.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 75.71 | loss 8.22 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 69.17 | loss 8.12 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 69.08 | loss 6.77 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 75.33 | loss 7.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 72.41s | valid loss 8.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 69.43 | loss 7.11 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 70.88 | loss 7.37 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 69.07 | loss 7.30 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 68.99 | loss 7.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 70.22s | valid loss 9.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 69.31 | loss 6.55 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 69.17 | loss 7.30 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 69.06 | loss 6.64 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 68.98 | loss 6.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 69.82s | valid loss 7.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 69.47 | loss 5.59 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 68.96 | loss 6.31 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 68.96 | loss 6.20 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 69.61 | loss 6.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 69.94s | valid loss 9.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 69.32 | loss 6.54 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 69.06 | loss 5.49 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 68.99 | loss 5.40 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 68.88 | loss 5.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 69.80s | valid loss 8.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 69.22 | loss 5.82 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 68.88 | loss 5.57 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 69.03 | loss 5.92 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 68.83 | loss 5.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 69.71s | valid loss 8.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 69.18 | loss 4.68 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 68.83 | loss 5.31 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 68.82 | loss 5.59 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 68.83 | loss 4.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 69.62s | valid loss 9.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 69.11 | loss 4.72 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 68.69 | loss 4.62 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 68.71 | loss 5.71 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 68.63 | loss 5.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 69.55s | valid loss 11.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 68.89 | loss 4.46 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 68.47 | loss 4.71 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 68.70 | loss 4.52 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 68.55 | loss 3.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 69.39s | valid loss 7.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 68.70 | loss 4.38 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 68.52 | loss 4.64 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 68.42 | loss 4.45 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 68.42 | loss 4.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 69.24s | valid loss 8.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 68.58 | loss 4.29 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 68.47 | loss 4.23 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 68.41 | loss 4.31 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 68.38 | loss 4.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 69.20s | valid loss 7.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 68.69 | loss 3.63 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 68.30 | loss 3.48 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 68.49 | loss 5.08 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 68.44 | loss 3.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 69.23s | valid loss 8.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 68.81 | loss 3.48 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 68.27 | loss 3.40 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 68.28 | loss 4.16 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 74.70 | loss 3.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 70.46s | valid loss 7.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 68.67 | loss 3.62 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 74.32 | loss 3.28 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 68.13 | loss 3.23 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 68.17 | loss 3.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 71.52s | valid loss 7.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 68.66 | loss 3.04 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 68.07 | loss 3.17 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 74.32 | loss 4.20 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 68.09 | loss 3.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 70.21s | valid loss 6.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 73.94 | loss 3.18 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 68.06 | loss 3.34 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 68.10 | loss 2.86 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 74.42 | loss 3.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 71.28s | valid loss 9.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 68.09 | loss 2.54 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 68.00 | loss 2.65 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 67.88 | loss 3.08 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 67.93 | loss 2.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 68.78s | valid loss 8.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 68.11 | loss 2.47 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 67.93 | loss 2.85 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 68.07 | loss 2.55 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 67.89 | loss 3.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 68.79s | valid loss 7.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 68.16 | loss 2.49 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 67.92 | loss 3.11 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 67.77 | loss 2.92 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 67.90 | loss 2.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 68.74s | valid loss 7.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 68.12 | loss 2.17 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 67.85 | loss 2.66 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 67.82 | loss 2.64 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 67.86 | loss 2.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 68.73s | valid loss 9.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 68.26 | loss 3.11 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 67.77 | loss 2.14 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 68.00 | loss 2.62 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 67.93 | loss 2.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 68.76s | valid loss 6.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 68.33 | loss 2.19 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 67.77 | loss 2.17 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 67.75 | loss 1.73 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 67.93 | loss 1.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 68.72s | valid loss 8.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 68.17 | loss 2.20 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 67.95 | loss 1.85 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 67.94 | loss 2.29 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 67.90 | loss 1.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 68.83s | valid loss 6.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 68.27 | loss 1.83 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 67.87 | loss 2.03 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 68.07 | loss 2.05 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 67.85 | loss 1.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 68.81s | valid loss 8.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 68.32 | loss 1.54 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 67.73 | loss 1.83 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 67.70 | loss 1.69 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 67.80 | loss 1.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 68.68s | valid loss 7.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 68.02 | loss 1.43 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 67.77 | loss 2.08 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 67.71 | loss 2.11 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 67.67 | loss 1.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 68.63s | valid loss 7.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 31 | 200/ 938 batches | lr 0.00021 | ms/batch 67.92 | loss 1.04 |\n", + "| epoch 31 | 400/ 938 batches | lr 0.00021 | ms/batch 67.73 | loss 1.90 |\n", + "| epoch 31 | 600/ 938 batches | lr 0.00021 | ms/batch 73.85 | loss 2.00 |\n", + "| epoch 31 | 800/ 938 batches | lr 0.00021 | ms/batch 67.59 | loss 1.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 31 | time: 69.79s | valid loss 7.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 32 | 200/ 938 batches | lr 0.00020 | ms/batch 74.38 | loss 1.60 |\n", + "| epoch 32 | 400/ 938 batches | lr 0.00020 | ms/batch 67.56 | loss 1.02 |\n", + "| epoch 32 | 600/ 938 batches | lr 0.00020 | ms/batch 67.75 | loss 2.05 |\n", + "| epoch 32 | 800/ 938 batches | lr 0.00020 | ms/batch 73.83 | loss 1.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 32 | time: 71.06s | valid loss 7.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 33 | 200/ 938 batches | lr 0.00019 | ms/batch 68.11 | loss 2.20 |\n", + "| epoch 33 | 400/ 938 batches | lr 0.00019 | ms/batch 73.95 | loss 1.23 |\n", + "| epoch 33 | 600/ 938 batches | lr 0.00019 | ms/batch 67.53 | loss 1.40 |\n", + "| epoch 33 | 800/ 938 batches | lr 0.00019 | ms/batch 67.65 | loss 1.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 33 | time: 69.84s | valid loss 7.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 34 | 200/ 938 batches | lr 0.00018 | ms/batch 68.06 | loss 1.90 |\n", + "| epoch 34 | 400/ 938 batches | lr 0.00018 | ms/batch 67.62 | loss 1.23 |\n", + "| epoch 34 | 600/ 938 batches | lr 0.00018 | ms/batch 73.86 | loss 1.54 |\n", + "| epoch 34 | 800/ 938 batches | lr 0.00018 | ms/batch 67.96 | loss 1.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 34 | time: 69.85s | valid loss 7.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 35 | 200/ 938 batches | lr 0.00017 | ms/batch 70.24 | loss 1.54 |\n", + "| epoch 35 | 400/ 938 batches | lr 0.00017 | ms/batch 67.62 | loss 0.98 |\n", + "| epoch 35 | 600/ 938 batches | lr 0.00017 | ms/batch 67.61 | loss 1.10 |\n", + "| epoch 35 | 800/ 938 batches | lr 0.00017 | ms/batch 73.77 | loss 1.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 35 | time: 70.20s | valid loss 7.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 36 | 200/ 938 batches | lr 0.00017 | ms/batch 67.93 | loss 1.14 |\n", + "| epoch 36 | 400/ 938 batches | lr 0.00017 | ms/batch 73.78 | loss 1.57 |\n", + "| epoch 36 | 600/ 938 batches | lr 0.00017 | ms/batch 67.69 | loss 1.08 |\n", + "| epoch 36 | 800/ 938 batches | lr 0.00017 | ms/batch 67.63 | loss 0.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 36 | time: 71.06s | valid loss 7.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 37 | 200/ 938 batches | lr 0.00016 | ms/batch 67.91 | loss 1.13 |\n", + "| epoch 37 | 400/ 938 batches | lr 0.00016 | ms/batch 67.61 | loss 1.14 |\n", + "| epoch 37 | 600/ 938 batches | lr 0.00016 | ms/batch 67.85 | loss 1.61 |\n", + "| epoch 37 | 800/ 938 batches | lr 0.00016 | ms/batch 67.61 | loss 1.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 37 | time: 68.54s | valid loss 7.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 38 | 200/ 938 batches | lr 0.00015 | ms/batch 68.11 | loss 1.18 |\n", + "| epoch 38 | 400/ 938 batches | lr 0.00015 | ms/batch 67.58 | loss 1.14 |\n", + "| epoch 38 | 600/ 938 batches | lr 0.00015 | ms/batch 67.52 | loss 1.29 |\n", + "| epoch 38 | 800/ 938 batches | lr 0.00015 | ms/batch 67.85 | loss 1.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 38 | time: 68.55s | valid loss 7.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.39 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 69.51 | loss 89.65 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 69.31 | loss 30.61 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 69.17 | loss 21.08 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 69.16 | loss 18.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 70.01s | valid loss 21.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 69.55 | loss 14.34 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 69.18 | loss 13.30 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 69.33 | loss 12.44 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 69.18 | loss 11.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 70.04s | valid loss 15.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 69.70 | loss 10.05 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 69.19 | loss 10.42 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 69.23 | loss 9.14 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 70.74 | loss 9.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 70.31s | valid loss 11.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 69.40 | loss 8.61 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 71.62 | loss 8.54 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 69.07 | loss 8.71 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 69.07 | loss 8.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 71.69s | valid loss 10.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 69.54 | loss 7.53 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 69.01 | loss 7.51 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 69.16 | loss 8.13 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 69.01 | loss 7.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 69.88s | valid loss 9.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 69.65 | loss 6.85 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 69.01 | loss 6.16 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 69.14 | loss 7.22 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 69.29 | loss 6.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 69.97s | valid loss 8.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 69.40 | loss 6.43 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 69.14 | loss 6.44 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 69.07 | loss 6.80 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 69.03 | loss 5.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 69.90s | valid loss 10.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 69.28 | loss 5.93 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 68.95 | loss 6.11 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 69.21 | loss 6.35 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 69.00 | loss 5.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 69.83s | valid loss 9.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 69.35 | loss 5.46 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 68.90 | loss 5.28 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 69.50 | loss 7.14 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 75.24 | loss 6.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 71.11s | valid loss 8.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 69.24 | loss 5.44 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 69.12 | loss 5.75 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 68.94 | loss 4.67 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 69.01 | loss 4.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 69.80s | valid loss 7.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 69.21 | loss 5.19 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 68.86 | loss 6.13 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 70.36 | loss 4.94 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 69.06 | loss 5.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 69.97s | valid loss 9.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 69.18 | loss 5.62 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 68.63 | loss 4.56 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 68.78 | loss 5.46 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 68.98 | loss 5.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 69.59s | valid loss 9.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 69.12 | loss 5.37 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 68.75 | loss 4.72 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 68.62 | loss 4.47 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 68.58 | loss 3.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 69.53s | valid loss 10.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 68.85 | loss 4.79 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 68.57 | loss 4.70 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 68.78 | loss 4.81 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 68.69 | loss 4.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 69.43s | valid loss 8.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 68.84 | loss 4.46 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 68.47 | loss 4.87 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 68.24 | loss 3.61 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 68.66 | loss 4.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 69.30s | valid loss 6.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 68.67 | loss 3.28 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 68.50 | loss 4.05 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 68.53 | loss 4.35 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 68.34 | loss 3.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 69.29s | valid loss 7.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 68.86 | loss 3.67 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 68.24 | loss 3.78 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 74.47 | loss 3.28 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 68.40 | loss 4.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 70.46s | valid loss 6.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 75.03 | loss 3.18 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 68.32 | loss 3.57 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 68.51 | loss 4.72 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 74.78 | loss 3.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 71.79s | valid loss 6.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 68.63 | loss 3.40 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 74.69 | loss 3.23 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 68.31 | loss 3.40 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 68.31 | loss 2.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 71.71s | valid loss 7.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 68.58 | loss 2.96 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 68.35 | loss 3.38 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 74.43 | loss 2.87 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 68.17 | loss 2.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 70.32s | valid loss 9.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 74.84 | loss 3.56 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 68.17 | loss 2.53 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 67.98 | loss 2.33 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 74.37 | loss 2.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 71.50s | valid loss 8.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 68.45 | loss 2.64 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 74.33 | loss 2.89 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 68.11 | loss 2.78 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 68.14 | loss 3.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 71.51s | valid loss 6.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 68.39 | loss 2.34 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 68.11 | loss 2.44 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 74.34 | loss 2.56 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 68.17 | loss 2.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 70.25s | valid loss 6.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 68.47 | loss 2.42 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 68.19 | loss 2.73 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 68.12 | loss 3.07 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 74.20 | loss 2.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 70.21s | valid loss 8.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 68.27 | loss 2.19 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 68.14 | loss 2.49 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 68.15 | loss 2.85 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 67.90 | loss 2.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 70.07s | valid loss 8.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 68.10 | loss 1.95 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 67.79 | loss 1.33 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 68.19 | loss 2.86 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 67.89 | loss 2.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 68.77s | valid loss 6.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 68.21 | loss 1.79 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 67.78 | loss 1.90 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 68.05 | loss 2.27 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 67.89 | loss 1.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 68.78s | valid loss 6.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 68.28 | loss 2.19 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 68.11 | loss 2.41 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 67.87 | loss 1.72 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 67.82 | loss 1.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 68.84s | valid loss 6.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 68.20 | loss 2.01 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 67.83 | loss 1.96 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 74.05 | loss 1.58 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 67.82 | loss 2.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 69.96s | valid loss 7.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 74.32 | loss 1.93 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 67.79 | loss 2.12 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 67.79 | loss 2.23 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 67.93 | loss 1.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 69.90s | valid loss 7.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 6.01 |\n", + "=========================================================================================\n", + "Result over 5 runs: 5.668732166290283\n" + ] + } + ], + "source": [ + "num_runs = 5\n", + "best_model = None\n", + "best_result = 1000000\n", + "results = []\n", + "\n", + "for i in range(num_runs):\n", + " features = nn.Sequential(nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(64),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(128),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(256),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=(0,1)),\n", + " nn.BatchNorm2d(512),\n", + " nn.ReLU()\n", + " )\n", + "\n", + " classifier = nn.Sequential(nn.Linear(512, 1028),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=1028, bias=True),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=10, bias=True))\n", + "\n", + " conv = nn.Sequential(nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.MaxPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.MaxPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.MaxPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.MaxPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.MaxPool1d(kernel_size=2, stride=2),\n", + "\n", + " nn.Conv1d(512, 512, kernel_size=3, stride=1, padding=1),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.MaxPool1d(kernel_size=2, stride=2))\n", + "\n", + " model = ConvMNIST(features, conv, classifier)\n", + " model = model.to(device)\n", + "\n", + " optimizer = torch.optim.Adam(model.parameters())\n", + " scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "\n", + " epochs = 30\n", + " model, result = train_model(model, epochs, train_loader, test_loader, seq_len, digits_per_batch, optimizer, scheduler)\n", + " \n", + " if result < best_result:\n", + " best_result = result\n", + " best_model = model\n", + " results.append(result)\n", + " \n", + "model = best_model\n", + "print(\"Result over {} runs: {}\".format(num_runs, torch.tensor(results).mean().item()))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 2.544268, + "end_time": "2021-01-23T16:52:18.751793", + "exception": false, + "start_time": "2021-01-23T16:52:16.207525", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "### Smart pooling" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-23T16:52:23.879235Z", + "iopub.status.busy": "2021-01-23T16:52:23.877476Z", + "iopub.status.idle": "2021-01-23T23:15:18.171100Z", + "shell.execute_reply": "2021-01-23T23:15:18.171617Z" + }, + "papermill": { + "duration": 22976.883221, + "end_time": "2021-01-23T23:15:18.171789", + "exception": false, + "start_time": "2021-01-23T16:52:21.288568", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 152.96 | loss 64.30 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 152.25 | loss 18.84 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 152.08 | loss 16.68 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 152.22 | loss 11.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 153.21s | valid loss 10.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 153.17 | loss 10.62 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 152.28 | loss 11.47 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 152.45 | loss 9.04 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 152.29 | loss 7.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 154.56s | valid loss 17.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 153.56 | loss 7.84 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 152.36 | loss 6.74 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 152.20 | loss 7.53 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 152.34 | loss 6.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 153.47s | valid loss 11.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 153.12 | loss 6.56 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 152.25 | loss 5.59 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 152.38 | loss 7.45 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 152.23 | loss 6.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 153.39s | valid loss 8.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 152.93 | loss 5.55 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 152.27 | loss 5.43 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 152.17 | loss 5.43 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 152.27 | loss 5.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 153.27s | valid loss 6.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 153.08 | loss 4.89 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 152.20 | loss 5.36 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 152.28 | loss 4.72 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 152.12 | loss 5.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 153.32s | valid loss 7.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 152.91 | loss 4.15 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 152.30 | loss 5.13 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 152.15 | loss 4.78 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 152.29 | loss 5.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 153.29s | valid loss 8.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 153.03 | loss 5.37 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 152.12 | loss 4.24 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 152.27 | loss 4.72 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 152.14 | loss 4.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 153.29s | valid loss 6.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 152.91 | loss 4.38 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 152.12 | loss 4.39 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 152.08 | loss 4.29 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 152.15 | loss 4.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 153.20s | valid loss 6.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 152.77 | loss 3.94 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 151.95 | loss 4.23 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 152.04 | loss 3.66 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 162.39 | loss 4.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 155.30s | valid loss 7.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 150.59 | loss 4.35 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 149.91 | loss 3.55 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 149.82 | loss 5.50 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 149.88 | loss 3.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 150.88s | valid loss 5.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 150.72 | loss 3.52 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 149.81 | loss 3.85 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 149.90 | loss 3.43 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 149.82 | loss 3.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 150.95s | valid loss 4.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 150.61 | loss 16.44 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 149.71 | loss 3.44 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 149.60 | loss 3.32 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 149.73 | loss 2.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 150.77s | valid loss 7.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 150.40 | loss 3.03 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 153.32 | loss 3.45 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 151.77 | loss 3.15 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 151.68 | loss 3.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 152.72s | valid loss 5.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 152.45 | loss 2.90 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 151.82 | loss 3.21 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 151.75 | loss 3.57 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 151.71 | loss 2.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 152.81s | valid loss 4.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 152.43 | loss 2.14 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 151.69 | loss 2.44 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 151.69 | loss 2.69 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 151.66 | loss 3.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 152.78s | valid loss 6.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 152.34 | loss 2.57 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 151.80 | loss 3.24 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 151.53 | loss 1.93 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 151.67 | loss 3.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 152.73s | valid loss 7.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 152.57 | loss 2.79 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 151.63 | loss 2.42 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 151.67 | loss 2.96 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 151.38 | loss 1.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 152.70s | valid loss 5.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 152.25 | loss 1.98 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 160.25 | loss 2.23 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 151.56 | loss 2.61 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 151.84 | loss 2.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 154.45s | valid loss 5.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 152.45 | loss 2.04 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 151.62 | loss 2.23 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 151.74 | loss 2.27 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 151.60 | loss 2.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 152.78s | valid loss 5.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 152.31 | loss 1.58 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 151.67 | loss 1.98 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 151.59 | loss 2.53 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 151.78 | loss 2.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 152.73s | valid loss 7.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 152.44 | loss 1.71 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 151.64 | loss 2.33 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 151.69 | loss 2.04 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 151.63 | loss 2.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 152.77s | valid loss 6.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 152.26 | loss 1.54 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 151.62 | loss 1.56 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 151.55 | loss 2.22 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 151.65 | loss 1.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 152.66s | valid loss 5.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 152.31 | loss 1.56 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 151.56 | loss 2.06 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 151.62 | loss 1.62 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 151.48 | loss 1.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 152.67s | valid loss 6.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 152.15 | loss 1.43 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 159.57 | loss 1.81 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 163.05 | loss 1.50 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 153.28 | loss 1.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 156.40s | valid loss 7.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 150.09 | loss 1.46 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 149.35 | loss 1.77 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 149.34 | loss 1.43 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 149.20 | loss 1.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 150.40s | valid loss 7.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 149.92 | loss 1.06 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 149.33 | loss 1.60 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 149.34 | loss 1.89 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 149.28 | loss 1.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 150.36s | valid loss 6.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 150.10 | loss 1.41 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 149.14 | loss 0.90 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 154.45 | loss 1.11 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 149.22 | loss 1.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 151.40s | valid loss 6.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 149.86 | loss 1.09 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 149.38 | loss 1.50 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 149.18 | loss 1.07 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 149.38 | loss 1.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 150.33s | valid loss 6.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 149.99 | loss 1.07 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 149.18 | loss 1.05 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 149.25 | loss 1.19 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 149.20 | loss 1.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 150.32s | valid loss 6.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 4.54 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 150.64 | loss 49.62 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 149.83 | loss 15.17 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 149.93 | loss 11.72 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 149.84 | loss 10.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 150.93s | valid loss 15.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 150.62 | loss 12.83 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 149.99 | loss 10.44 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 149.87 | loss 7.86 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 149.96 | loss 8.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 151.73s | valid loss 13.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 150.79 | loss 9.52 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 149.94 | loss 6.58 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 150.03 | loss 6.47 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 149.92 | loss 6.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 151.03s | valid loss 8.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 150.74 | loss 6.11 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 150.05 | loss 5.15 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 149.90 | loss 6.01 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 149.99 | loss 5.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 151.00s | valid loss 7.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 150.71 | loss 5.65 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 149.85 | loss 4.67 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 149.97 | loss 5.47 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 149.81 | loss 5.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 150.97s | valid loss 7.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 150.56 | loss 4.56 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 149.85 | loss 4.01 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 164.08 | loss 5.13 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 166.73 | loss 5.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 159.50s | valid loss 8.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 166.66 | loss 3.56 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 155.45 | loss 4.25 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 149.82 | loss 6.38 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 150.03 | loss 5.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 155.28s | valid loss 5.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 150.63 | loss 3.63 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 149.77 | loss 4.07 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 149.89 | loss 4.68 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 149.91 | loss 5.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 150.92s | valid loss 10.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 150.57 | loss 5.14 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 149.90 | loss 4.22 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 149.76 | loss 3.87 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 149.94 | loss 5.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 150.91s | valid loss 4.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 166.58 | loss 3.75 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 157.13 | loss 4.11 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 149.77 | loss 2.98 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 149.71 | loss 3.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 155.50s | valid loss 9.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 150.40 | loss 3.97 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 149.73 | loss 3.82 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 149.61 | loss 3.55 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 149.68 | loss 3.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 150.73s | valid loss 7.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 150.61 | loss 3.91 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 149.56 | loss 2.96 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 149.71 | loss 3.01 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 149.79 | loss 3.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 150.80s | valid loss 5.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 150.27 | loss 2.44 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 149.70 | loss 3.28 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 149.62 | loss 3.72 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 155.29 | loss 2.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 151.84s | valid loss 5.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 150.34 | loss 2.81 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 149.49 | loss 2.59 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 156.06 | loss 3.30 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 149.57 | loss 2.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 151.96s | valid loss 6.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 150.24 | loss 2.72 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 149.65 | loss 2.88 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 149.48 | loss 2.58 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 149.65 | loss 2.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 150.63s | valid loss 6.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 150.36 | loss 2.27 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 157.84 | loss 2.67 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 156.00 | loss 2.99 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 151.49 | loss 2.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 154.41s | valid loss 6.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 152.19 | loss 1.81 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 151.57 | loss 2.30 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 151.56 | loss 2.34 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 151.53 | loss 2.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 152.64s | valid loss 5.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 152.30 | loss 1.78 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 151.41 | loss 1.73 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 151.55 | loss 2.40 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 151.51 | loss 2.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 152.63s | valid loss 5.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 152.21 | loss 1.65 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 151.55 | loss 1.92 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 151.40 | loss 2.57 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 151.53 | loss 2.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 152.57s | valid loss 6.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 152.19 | loss 1.75 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 151.35 | loss 2.08 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 151.53 | loss 2.73 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 151.38 | loss 1.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 152.56s | valid loss 6.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 152.09 | loss 1.66 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 151.46 | loss 1.63 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 151.27 | loss 1.87 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 151.46 | loss 2.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 152.48s | valid loss 5.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 152.13 | loss 6.67 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 151.30 | loss 1.52 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 151.36 | loss 1.33 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 151.34 | loss 1.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 152.47s | valid loss 5.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 152.12 | loss 1.63 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 151.36 | loss 1.36 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 151.31 | loss 1.81 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 151.33 | loss 1.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 152.44s | valid loss 6.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 152.19 | loss 1.66 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 151.30 | loss 1.52 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 157.14 | loss 1.58 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 151.48 | loss 1.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 153.69s | valid loss 5.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 152.07 | loss 1.29 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 151.55 | loss 1.29 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 151.31 | loss 1.25 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 151.40 | loss 1.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 152.50s | valid loss 7.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 152.23 | loss 1.53 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 151.42 | loss 1.54 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 151.45 | loss 1.24 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 151.31 | loss 0.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 152.53s | valid loss 5.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 163.06 | loss 1.80 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 155.19 | loss 1.76 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 149.29 | loss 1.21 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 149.68 | loss 1.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 156.09s | valid loss 6.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 152.24 | loss 1.57 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 151.52 | loss 2.10 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 151.51 | loss 1.79 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 151.36 | loss 1.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 152.61s | valid loss 6.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 152.18 | loss 1.47 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 151.43 | loss 1.48 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 151.31 | loss 1.25 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 151.43 | loss 1.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 152.52s | valid loss 5.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 152.16 | loss 1.17 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 151.29 | loss 0.97 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 151.37 | loss 0.71 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 151.25 | loss 0.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 152.47s | valid loss 5.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 4.40 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 152.99 | loss 55.93 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 152.08 | loss 15.50 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 152.18 | loss 12.21 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 152.03 | loss 9.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 153.21s | valid loss 24.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 152.92 | loss 8.73 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 152.28 | loss 7.73 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 152.14 | loss 6.93 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 152.28 | loss 6.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 153.30s | valid loss 47.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 154.42 | loss 8.29 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 149.66 | loss 6.58 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 149.77 | loss 26.53 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 149.66 | loss 27.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 151.56s | valid loss 45.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 150.41 | loss 25.24 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 149.75 | loss 13.39 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 149.65 | loss 5.94 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 149.77 | loss 5.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 152.36s | valid loss 9.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 166.82 | loss 5.29 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 153.08 | loss 5.77 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 150.12 | loss 5.99 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 155.02 | loss 5.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 156.33s | valid loss 9.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 152.90 | loss 4.97 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 152.00 | loss 4.53 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 163.53 | loss 5.35 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 152.15 | loss 5.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 155.48s | valid loss 38.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 152.96 | loss 20.15 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 152.28 | loss 5.53 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 152.07 | loss 4.63 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 152.18 | loss 4.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 153.24s | valid loss 7.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 152.98 | loss 3.63 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 152.07 | loss 4.19 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 152.13 | loss 4.14 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 152.08 | loss 4.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 153.22s | valid loss 6.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 152.71 | loss 4.06 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 152.17 | loss 4.41 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 151.96 | loss 4.40 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 152.14 | loss 4.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 153.12s | valid loss 6.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 152.80 | loss 3.95 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 151.97 | loss 3.48 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 152.06 | loss 4.48 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 151.92 | loss 3.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 153.12s | valid loss 7.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 152.64 | loss 3.28 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 151.86 | loss 2.97 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 151.87 | loss 4.14 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 152.10 | loss 3.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 153.02s | valid loss 6.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 152.79 | loss 3.01 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 151.93 | loss 3.28 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 152.08 | loss 3.71 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 152.02 | loss 3.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 153.11s | valid loss 6.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 152.59 | loss 3.38 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 156.56 | loss 2.81 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 151.79 | loss 3.40 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 151.96 | loss 3.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 153.88s | valid loss 5.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 152.65 | loss 2.62 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 151.83 | loss 2.96 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 155.38 | loss 2.55 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 151.77 | loss 3.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 153.67s | valid loss 8.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 152.48 | loss 2.38 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 151.87 | loss 3.24 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 151.68 | loss 2.53 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 151.87 | loss 2.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 152.88s | valid loss 7.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 152.59 | loss 2.29 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 151.64 | loss 2.97 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 151.84 | loss 3.04 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 152.00 | loss 4.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 152.97s | valid loss 6.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 153.29 | loss 3.32 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 157.26 | loss 2.37 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 151.85 | loss 3.77 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 151.89 | loss 3.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 154.19s | valid loss 7.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 152.55 | loss 1.66 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 151.62 | loss 2.07 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 151.80 | loss 3.01 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 151.80 | loss 3.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 152.88s | valid loss 5.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 152.35 | loss 1.92 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 151.75 | loss 2.07 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 151.56 | loss 2.11 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 151.81 | loss 3.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 152.78s | valid loss 6.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 155.22 | loss 1.86 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 151.48 | loss 2.44 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 156.63 | loss 1.81 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 151.49 | loss 2.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 154.25s | valid loss 6.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 152.64 | loss 2.76 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 156.97 | loss 1.96 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 151.41 | loss 1.56 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 151.61 | loss 2.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 153.80s | valid loss 6.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 152.43 | loss 2.44 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 151.45 | loss 1.44 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 160.07 | loss 1.97 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 158.50 | loss 1.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 155.75s | valid loss 5.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 152.18 | loss 1.08 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 151.61 | loss 2.00 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 151.39 | loss 1.65 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 151.57 | loss 1.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 152.60s | valid loss 7.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 152.29 | loss 2.00 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 151.40 | loss 1.61 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 151.48 | loss 1.39 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 151.43 | loss 1.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 152.59s | valid loss 7.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 152.16 | loss 1.37 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 151.48 | loss 2.14 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 151.42 | loss 1.86 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 151.53 | loss 1.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 152.57s | valid loss 5.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 152.27 | loss 1.45 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 151.29 | loss 1.12 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 151.48 | loss 1.16 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 151.41 | loss 1.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 152.56s | valid loss 5.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 152.17 | loss 1.44 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 151.49 | loss 1.44 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 151.34 | loss 1.43 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 152.04 | loss 1.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 152.67s | valid loss 6.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 152.23 | loss 1.04 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 151.29 | loss 1.04 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 151.46 | loss 1.05 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 153.04 | loss 1.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 152.87s | valid loss 6.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 152.05 | loss 1.40 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 151.45 | loss 1.26 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 151.33 | loss 1.78 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 151.45 | loss 1.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 152.51s | valid loss 5.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 152.15 | loss 1.37 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 151.28 | loss 0.80 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 151.41 | loss 1.16 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 151.30 | loss 0.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 152.46s | valid loss 6.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.39 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 158.29 | loss 59.43 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 151.94 | loss 15.53 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 152.01 | loss 10.87 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 151.93 | loss 10.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 154.17s | valid loss 27.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 152.87 | loss 8.85 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 152.21 | loss 7.77 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 152.12 | loss 7.76 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 152.18 | loss 6.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 153.23s | valid loss 7.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 153.03 | loss 6.46 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 152.14 | loss 5.81 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 152.26 | loss 4.99 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 152.15 | loss 5.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 153.28s | valid loss 9.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 152.83 | loss 5.02 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 152.15 | loss 4.79 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 152.06 | loss 5.73 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 152.19 | loss 5.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 153.17s | valid loss 8.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 152.90 | loss 4.89 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 152.04 | loss 4.78 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 152.15 | loss 4.58 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 152.02 | loss 4.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 153.16s | valid loss 6.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 152.74 | loss 4.58 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 152.13 | loss 4.40 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 151.98 | loss 4.94 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 152.06 | loss 4.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 153.11s | valid loss 7.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 152.81 | loss 4.47 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 151.97 | loss 3.87 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 152.05 | loss 3.11 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 151.98 | loss 4.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 153.73s | valid loss 5.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 154.98 | loss 3.23 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 151.95 | loss 3.50 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 151.91 | loss 4.41 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 151.95 | loss 3.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 153.47s | valid loss 6.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 152.68 | loss 3.02 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 151.90 | loss 3.82 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 151.88 | loss 3.83 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 151.92 | loss 3.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 152.96s | valid loss 9.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 152.58 | loss 3.03 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 151.87 | loss 3.58 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 151.79 | loss 4.04 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 151.94 | loss 3.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 152.94s | valid loss 5.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 152.66 | loss 2.99 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 151.73 | loss 2.86 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 151.84 | loss 3.38 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 151.73 | loss 3.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 152.91s | valid loss 7.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 152.44 | loss 3.32 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 151.76 | loss 3.17 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 151.73 | loss 2.97 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 151.81 | loss 3.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 152.82s | valid loss 5.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 152.58 | loss 2.70 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 151.74 | loss 2.62 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 151.89 | loss 3.02 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 151.24 | loss 2.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 152.25s | valid loss 6.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 149.82 | loss 1.81 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 149.23 | loss 2.47 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 149.16 | loss 3.17 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 149.28 | loss 2.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 150.22s | valid loss 7.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 149.97 | loss 2.68 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 149.10 | loss 2.20 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 149.16 | loss 2.66 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 149.13 | loss 2.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 150.24s | valid loss 5.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 149.89 | loss 1.84 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 149.17 | loss 2.72 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 149.10 | loss 2.03 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 149.14 | loss 2.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 150.19s | valid loss 5.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 149.98 | loss 1.79 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 148.92 | loss 1.33 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 149.15 | loss 2.25 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 148.98 | loss 2.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 150.16s | valid loss 7.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 149.73 | loss 2.30 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 149.17 | loss 1.95 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 148.99 | loss 2.19 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 149.14 | loss 2.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 150.12s | valid loss 6.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 149.89 | loss 1.75 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 148.92 | loss 1.34 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 149.08 | loss 2.01 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 148.92 | loss 1.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 150.10s | valid loss 6.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 149.66 | loss 1.40 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 149.07 | loss 2.13 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 148.85 | loss 1.40 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 149.04 | loss 2.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 150.04s | valid loss 6.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 149.81 | loss 1.33 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 148.90 | loss 1.10 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 149.00 | loss 1.64 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 148.85 | loss 1.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 150.04s | valid loss 7.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 149.62 | loss 1.54 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 149.03 | loss 1.38 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 148.82 | loss 0.71 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 148.94 | loss 1.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 149.99s | valid loss 6.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 149.74 | loss 1.64 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 148.82 | loss 1.57 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 148.93 | loss 1.54 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 148.89 | loss 1.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 150.00s | valid loss 6.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 149.56 | loss 0.97 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 148.97 | loss 1.46 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 148.85 | loss 1.16 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 148.96 | loss 1.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 149.97s | valid loss 6.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 149.65 | loss 1.29 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 148.82 | loss 1.06 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 148.85 | loss 1.41 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 148.78 | loss 1.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 149.93s | valid loss 8.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 149.52 | loss 0.98 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 148.82 | loss 1.37 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 148.79 | loss 1.45 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 148.89 | loss 0.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 149.89s | valid loss 7.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 149.60 | loss 1.58 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 148.71 | loss 1.02 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 148.82 | loss 0.82 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 148.86 | loss 0.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 149.90s | valid loss 7.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 149.46 | loss 0.90 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 161.07 | loss 0.91 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 149.03 | loss 0.60 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 149.15 | loss 1.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 152.49s | valid loss 6.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 149.87 | loss 0.94 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 149.00 | loss 0.78 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 153.29 | loss 0.85 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 151.19 | loss 1.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 151.91s | valid loss 8.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 151.97 | loss 1.07 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 151.33 | loss 1.00 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 151.23 | loss 1.32 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 151.34 | loss 1.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 152.39s | valid loss 9.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.17 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 152.73 | loss 55.23 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 152.10 | loss 16.53 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 151.98 | loss 12.60 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 152.12 | loss 11.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 153.07s | valid loss 14.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 153.05 | loss 8.57 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 152.14 | loss 10.76 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 152.26 | loss 8.15 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 152.14 | loss 7.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 153.29s | valid loss 9.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 152.88 | loss 6.29 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 152.21 | loss 7.04 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 152.11 | loss 6.45 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 152.22 | loss 5.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 153.20s | valid loss 7.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 155.51 | loss 5.58 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 163.98 | loss 6.21 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 155.73 | loss 5.50 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 150.02 | loss 6.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 155.96s | valid loss 12.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 150.66 | loss 5.41 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 150.00 | loss 4.19 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 149.93 | loss 11.22 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 150.03 | loss 12.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 151.00s | valid loss 25.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 167.96 | loss 20.25 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 163.67 | loss 26.08 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 150.06 | loss 24.01 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 149.96 | loss 23.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 157.20s | valid loss 38.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 150.65 | loss 20.68 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 149.97 | loss 20.29 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 149.87 | loss 20.52 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 149.98 | loss 19.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 150.95s | valid loss 37.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 150.75 | loss 19.71 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 152.54 | loss 18.53 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 166.19 | loss 18.67 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 166.94 | loss 18.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 159.78s | valid loss 33.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 152.85 | loss 17.64 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 152.20 | loss 17.83 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 152.09 | loss 17.04 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 152.20 | loss 17.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 153.19s | valid loss 29.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 152.97 | loss 17.02 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 152.08 | loss 16.92 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 152.20 | loss 16.34 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 164.79 | loss 16.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 157.16s | valid loss 29.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 152.68 | loss 16.11 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 152.02 | loss 16.86 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 151.97 | loss 16.15 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 152.02 | loss 16.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 153.03s | valid loss 29.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 152.83 | loss 15.32 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 151.97 | loss 15.23 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 152.06 | loss 16.03 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 151.98 | loss 14.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 153.08s | valid loss 26.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 152.74 | loss 15.25 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 152.07 | loss 14.21 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 151.98 | loss 15.20 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 152.09 | loss 14.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 153.08s | valid loss 27.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 152.78 | loss 15.66 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 151.91 | loss 14.47 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 151.99 | loss 15.42 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 151.93 | loss 14.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 153.04s | valid loss 26.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 152.70 | loss 14.74 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 152.15 | loss 15.01 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 151.95 | loss 14.02 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 152.07 | loss 14.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 153.06s | valid loss 25.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 152.81 | loss 13.53 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 151.97 | loss 13.84 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 152.09 | loss 13.75 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 151.98 | loss 13.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 153.08s | valid loss 24.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 152.71 | loss 14.69 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 152.07 | loss 13.49 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 151.97 | loss 13.69 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 152.07 | loss 14.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 153.05s | valid loss 24.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 152.82 | loss 13.62 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 151.94 | loss 13.07 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 152.03 | loss 13.77 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 151.93 | loss 13.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 153.04s | valid loss 24.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 152.72 | loss 12.71 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 152.05 | loss 12.85 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 151.96 | loss 12.18 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 152.04 | loss 13.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 153.07s | valid loss 24.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 152.80 | loss 12.96 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 151.93 | loss 12.80 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 152.06 | loss 13.30 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 151.96 | loss 12.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 153.07s | valid loss 23.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 152.70 | loss 12.77 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 152.08 | loss 12.55 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 151.94 | loss 12.49 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 152.10 | loss 13.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 153.05s | valid loss 23.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 152.84 | loss 13.29 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 151.96 | loss 12.48 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 152.07 | loss 12.35 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 151.95 | loss 11.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 153.08s | valid loss 22.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 152.69 | loss 12.26 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 152.05 | loss 11.80 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 151.92 | loss 12.24 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 152.03 | loss 11.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 153.02s | valid loss 23.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 152.80 | loss 11.45 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 151.95 | loss 12.44 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 152.05 | loss 12.90 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 152.06 | loss 12.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 153.08s | valid loss 22.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 152.73 | loss 13.00 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 152.08 | loss 12.59 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 155.10 | loss 12.27 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 160.25 | loss 12.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 155.34s | valid loss 22.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 152.91 | loss 12.23 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 152.06 | loss 12.03 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 152.17 | loss 11.24 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 152.08 | loss 12.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 153.18s | valid loss 22.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 152.84 | loss 12.31 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 152.18 | loss 11.10 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 152.06 | loss 11.93 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 152.16 | loss 11.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 153.17s | valid loss 22.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 152.91 | loss 11.60 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 152.07 | loss 11.85 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 152.16 | loss 11.04 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 152.03 | loss 10.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 153.17s | valid loss 22.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 154.41 | loss 11.55 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 163.51 | loss 11.12 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 149.60 | loss 11.56 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 149.68 | loss 10.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 154.24s | valid loss 22.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 150.46 | loss 11.02 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 149.62 | loss 12.11 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 149.71 | loss 11.33 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 164.12 | loss 10.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 155.82s | valid loss 21.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 7.49 |\n", + "=========================================================================================\n", + "Result over 5 runs: 4.4731035232543945\n" + ] + } + ], + "source": [ + "num_runs = 5\n", + "best_model = None\n", + "best_result = 1000000\n", + "results = []\n", + "\n", + "for i in range(num_runs):\n", + " features = nn.Sequential(nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(64),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(128),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(256),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=(0,1)),\n", + " nn.BatchNorm2d(512),\n", + " nn.ReLU()\n", + " )\n", + "\n", + " classifier = nn.Sequential(nn.Linear(512, 1028),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=1028, bias=True),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=10, bias=True))\n", + "\n", + " model = DoXTimes(Smartpool(divider, 0.3), classifier, features=features)\n", + " model = model.to(device)\n", + "\n", + " optimizer = torch.optim.Adam(model.parameters())\n", + " scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "\n", + " epochs = 30\n", + " model, result = train_model(model, epochs, train_loader, test_loader, seq_len, digits_per_batch, optimizer, scheduler)\n", + " \n", + " if result < best_result:\n", + " best_result = result\n", + " best_model = model\n", + " results.append(result)\n", + " \n", + "model = best_model\n", + "print(\"Result over {} runs: {}\".format(num_runs, torch.tensor(results).mean().item()))\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-23T23:15:23.736533Z", + "iopub.status.busy": "2021-01-23T23:15:23.735281Z", + "iopub.status.idle": "2021-01-23T23:15:24.084948Z", + "shell.execute_reply": "2021-01-23T23:15:24.085353Z" + }, + "papermill": { + "duration": 3.132231, + "end_time": "2021-01-23T23:15:24.085504", + "exception": false, + "start_time": "2021-01-23T23:15:20.953273", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQcAAAMwCAYAAAAtS6WbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAC8VUlEQVR4nOz9d5wlyXXfif7iel+3vG/vprunx/VYzAyAgRuAJIxIUCAhCloZrvZRK2mhfbvk7tu3q6eHj7RvJUprRGohiSJXpEhhCZKACMIMZgCM99Mz7X11l/e36nob74/fiXu7erJ9VXfV4Hw/n/rcupmRmRGReU+cOOfESWOthaIoyuX47nQFFEVZn6hwUBTFExUOiqJ4osJBURRPVDgoiuKJCgdFUTxZF8LBGPO0MeakMeaMMebXb+N1h40xPzLGHDfGHDXG/D3Z3mGMecYYc1o+229jnfzGmHeMMX9+J+tijEkbY/7YGHNC+ufRO1iX/0ruzxFjzB8aYyK3qy7GmN8xxswYY45csu2K1zbG/IY8xyeNMZ+6DXX5X+QevWeM+VNjTHq16nLHhYMxxg/gXwL4NIC9AH7JGLP3Nl2+BuAfWGvvAvAIgF+Ta/86gGettTsBPCvfbxd/D8DxS77fqbr8rwC+Z63dA+AeqdNtr4sxZhDA3wVw0Fq7H4AfwJduY11+F8DTl23zvLY8O18CsE+O+S15vteyLs8A2G+tPQDgFIDfWLW6WGvv6B+ARwF8/5LvvwHgN+5QXb4F4BMATgLol239AE7epusPgQ/bUwD+XLbd9roASAE4D8Bctv1O1GUQwCiADgABAH8O4JO3sy4AtgA4cq1+uPzZBfB9AI+uZV0u2/cFAH+wWnW545oDWjffMSbbbivGmC0A7gPwGoBea+0kAMhnz22qxr8A8N8AaFyy7U7UZRuAWQD/TqY4/8YYE78TdbHWjgP4pwAuApgEsGSt/cGdqMslXOnad/pZ/usAvrtadVkPwsF4bLutMd3GmASAbwL4+9ba5dt57Uvq8LMAZqy1b92J619GAMD9AH7bWnsfgDxu79SqicznPwdgK4ABAHFjzF+5E3W5Du7Ys2yM+e/BafIfrFZd1oNwGAMwfMn3IQATt+vixpggKBj+wFr7J7J52hjTL/v7Aczchqp8CMBnjTEjAP4IwFPGmN+/Q3UZAzBmrX1Nvv8xKCzuRF0+DuC8tXbWWlsF8CcAHrtDdXFc6dp35Fk2xnwFwM8C+LKVOcRq1GU9CIc3AOw0xmw1xoRAI8q3b8eFjTEGwL8FcNxa+5uX7Po2gK/I/18BbRFrirX2N6y1Q9baLWAfPGet/St3qC5TAEaNMbtl08cAHLsTdQGnE48YY2Jyvz4GGkfvRF0cV7r2twF8yRgTNsZsBbATwOtrWRFjzNMA/lsAn7XWFi6r463VZa0NStdpZPkMaGk9C+C/v43XfRxUtd4DcEj+PgOgEzQMnpbPjtvcHx9ByyB5R+oC4F4Ab0rf/BmA9jtYl38I4ASAIwD+PYDw7aoLgD8EbR1VcDT+G1e7NoD/Xp7jkwA+fRvqcga0Lbjn91+tVl2MnERRFGUF62FaoSjKOkSFg6IonqhwUBTFExUOiqJ4osJBURRP1kw43OhKS2PMr65VXW4UrYs3Whdv1ktdVrseayIcbnKl5broYEHr4o3WxZv1Upf1LxwAPATgjLX2nLW2AoYDf26NrqUoyhoQWKPzeq0Ie/iKlYjGbTDZjmjf8LqIyNK6eKN18Wa91OVm61GaHpuz1nZfvn2thMM1V4TJ/OhXATZq99/6H9aoKoqiXI0jv/nVC17b12pacc0VYdbar1trD1prD/pj8TWqhqIoN8taCYc7ttJSUZTVYU2mFdbamjHm74CpqfwAfsdae3QtrqUo65Xs9joAwPot/KkqAGBz7/yKMucu9sC/yJ9hfGx9hR2tlc0B1tq/APAXa3V+RVHWljUTDhsdf8kilKUNNZRjSsexT9DO2vmmD/MfrgAABr/NLpz+xSKPO55AJc3yXW+zfHSuBgAY/aQfva/y/LEZHj/1cITXyFj0f59mmYmfYaq/Yg+vH170su+SRgCopFiuEeK2WhdHKf9CEOmT3FaP8ByBAstWUgY+VgulLn4G8vxMjDUwdy/Lxyb4ubyHhaNjAURneY5iL/fFx+Sc6SvX81JymyVFpnykTxjUojy27QKvM7eP/RpeAmoRKR7kZ0gS+ZmGRW6zbJM+yu9g2wOLAQQK3FaLsX5tp1g2WLDI9/vkHNdV5Rsi9wCfha1989coCWzbNIPFrigAoDzfCQAIFFe/TjfD+tJjFEVZN6jmcAUaQYNSO0eexT38DE/LaDpbQ+Rb7LqlrXwVQOpZelzqIaDrPdv8HwDm93PIS1wAlrfItgNhAEByhGVrEYMLf5kaQ9dhjn6NgAyVVxmQI3MW7ac4tx39LIfB2BleODZtmyPj0pCMnqf5PX22htkDbINfRqpCvxw/ZTD0HOtQ7GaZxjl+RmctqglWyMciWJTY1/h1ZiiMTvmkvGgcyZYWs7SZ17HyhoXs1gYa8pS2H3P9z3pOPmaa53JaRfeLLFwPAwuPUDsLJ8ps55EEAGD2PoPUOblHkevTdq4HpzFs8dAY5nJ8PpZnWQd/XGwQPQtoj/G4c0PshNTp9fGzXB+1WKeUuvkAxcf43V8RdbrT31SpjTzUgSL3lToNFvbwyfbzmUQtJp9xi2CWx7mHvxqTH1odiE3xHAu7+aRH51rnvBL5QYPu15d4vYWO5rkAoNRhmlOjoR/xYZzfT8GRHGugFpcfyHCJbZmi/r5wNzAjgnHTt/hDXNwt05K8QXSO29Kn2fiJx4NXrJ9nnbexLr4Kj7M+oCbe7HIXK9/xHn/0y3sLCJzkD8oJ24nHRUhM+prtK/Rzm7tHoZxFqoPzJPMcX0iVlSlIfBwo9Lsp2w1V/YqU08Cm3oUV22ZFINRfb28K0pQ8L9bHts88XkFPMseNwTseR7UCnVYoiuLJB1ZzcCOD2ZRHrSyqap2yMDROqe0vGwTz3scH8hZ9ZynmfVWeKzfA4+oRwMpg7kbpzM6Wgavcw42heW5LSPxZLWYQKK0cHfID/AzmWmq60xjaTjOZcKnzykFi/jKwcC9HxuSIjJ5l29yXkfzR5TSH3Uq7jLS9QYQXxKB4VKY4Y9Qgzn82jOhFtjUr05HmlMFazO93mkxQ2iVtWrg+FX3zn/Lzwi9StQpOhjDwgmghT8ojKaeMvZxA/DNTAICZQ70AgOg0+zV/dwnR51j3xAXRuqRu7ceA1O+lAADVGDWdepjH1aJXN/LeDPVw6742NYbXeF8CpfeXz25nezfHWwmjQzPr6+eomoOiKJ6sL1G1iqTuolEoHfUQ25v4Ua4FsJCLeR4fjpbxpa0vAwB+78IjAIClZY4IQ50ZnDvTBwDofpXGA6eB5IcMIpPc5lx+tXhrlGqIrcEGuC0yJ3VppwERAKYfc/4177pdir8MZHZRxodlyltJObdly+0XEW2kIdct9LVcoDA8PjdMl1rqjIVfNJzcJpavtDdknw+JUdHKpJoL99zYXHnuHmocwUkeZwOXGm1FA7hbtJ8SYP6cfd2edTYYadPpCDr+4E0AwLl/9AAAwEcbJGYetjB11w+m2S4AqCYM8oP8Pza9OhpEbNpg5kc0KBvRJgOVK5dPDmQBAH7fGvhSVwnVHBRF8eQDqzlkD3N4WUw1EMjKXDMpUlpCWTs7cuhPc2hdLtFSn4pQ04gHK6iL7JzP0FoeCteaZf0pDgvlz3PeXDyeBgAM/riCcprdungXj0+f4nUX9pmmW85soqoRf4naSDVpMf0x1mvTn1C9yF7Ha0/rEY6uANBxgnUydY6KIz8XRHKEdQjmuS0i2kVstoGFu0SNkcEzvNhyqxZ7pZy4G4ee5UUW90SRPss2n/tLYrvJ3dgYE2kGUfEzdcbX9Og4z0zva6K5DBlkt7D/TEOC0N5r9ef43zsIAGgM0x0YfZ3aT3je3/JcZJ2XSZ6DODD4PO/l4u4b87RcjesJXnIh1ZtircKTGdpGQqtsB7lVPrDCITorHT3rb22ccf/zsxiKYFlU69ASyy+lRWW2QPX/pmsw9mQrhgEAKo8vYqvEyJ851Q8A6D/E47Kbgk1jZUjcZLkBPpTdb9eRuEChcPFpPhD9P5gGAMw91oNskcY16xdDqLi96ldpZ/Jio2kgLHXydrroxrZTpiko3A/DNPh97m4/Kpv5I7cl9kfnW/xMn6th5j4RcPukXVuizX6a7KYg7XhX3IFL/LHmhi7p66sw/xAbFlyQNQXTdUw9zPpF5s2K+qIBdBzhtjq7Z4Vb1U3Fgi+yTrlN/N79jkVukOco+6R/urhv+NkyFvaEr6uuq4UThJt28X676US+EkL9RBIAELzajb4D6LRCURRPPrCaw/XgrwD+uZWqnNM46iFg7r42AEC5Q3buoxHpC5uP4jsX9gEAul/jaJkfcGsQqkicleAeObUb8SpJHyY+TI3BGQ/P/LUelg20jJMLewJyXTGaTV65DbP3GSQu8EJuylKPclSKLBrM0k7XVFlTEpEZO9zAQo0Vq8ade1PaHgzAiI3RX5Lo0IwYMgca2PwX4nb8kLhH29gH17tOYbO8djYrGT+KHT60nZG6i3ZW7GtFdC7t4LbkCD8DxVYgWWYPt7lAtfQJfs7v88EGZJokrlbbzmnbhafD6L+P7tGl7/VfX6VvkWqaasHlBsjpiTRS8+trOuFQzUFRFE9+qjWHq1HqaTSNcg0xYN7dMwsAOLbcD98PGOCyvM25xzgibPpzg+kHnQuTn1v/jHP7kZ+LIDLDc1rpeWdXqEYs6iGOIP2viC1ABpT5u688P+44AlSo4CA3xAPSR93ahTJmHuJQXNwk6zVCvPDWP1nG0rY090lMfz3q3LK+1kpLcYu6FZ8d7xos7gjJ+VtrI4CrLgFZweJOalau3o2wbbY1nOGn03QyeywaEt5dmxFjoxhOI4u26RqefoL9H55m+4JZoO9VGv1m76e1M3AkJO00WFygxnA7Rsf8gwVs7sqs2HZ+nEthEydDt6EGN4dqDoqieKKawxXoPGSaK/3mHqIM3RSj+yEdLODI4xx5ki9zVIrOcKTLDhr4ZU5c7uWIfObLPFHylIFfAmNCS60QZ4Cjfnyc2/L9LL+0TcK9s1euZ2YP0HHUeQtc+LTkMYj7EZsQL0WNny40e+IjaRR7eFzfTzj8hpblPAOmubAsING9wZws0gq3Fkm5AKveNzmyz94XuXJFL8HZM5z9Y+7e1urRoLS1LLkhhn5YwdTD1BicdpEfap4J8SlZSXqBj3ItIX04bDFT571xfT0vwVo9r1sU+iQIbQ3n+y4PRXd7doV3AgAip7jTaY7rERUOV6ARBCpt8gBNsZsmijQmvvG/3Y+kqNvOWOli62NTBiEukkTiJR5X7GbZ7nfLuPBp/vBNjdu2/QlX5C3eHUN+gD/SGsMqUEvIGo3slV2EwWWDYEHSkQUoAHxVqVvK34zWc0KhFpX2hYDkOZYPlFho4km5TgPY9Ayl1sTjnNL4KmIE9AG1u1nn5J/xx3fuiyL8zlyxmivofz4DADj7lzmvaD9usHiXqx+vkxzlj2n6wXBz5aRzw7q+K3YDYXFBt52X2Ie9/B5aMs3ENu6cd/2vtOwuPNKHapsYbeevz/16M9gHKD1jwWpz2/Q4p6Opguch6wqdViiK4olqDlcgWLBI/BKXIt6VZuDK86PbAQAJa5HZz9G2/V2OPIsH+D2QD6Dnbc4dpsUY6FyMFz8ZwtCz1COrCcrlsY8zAGbomRoy4rLb/B2qHlOPp3n8VYL44pMWoz8raeleYl0yuzliltt9CEqqgLpLsSbRggsP1GCqsrpxSDSInFO1gfOfY6V7Xue5s5ta05LQt2NyTpbveYXnLr7vtSjeTD7Bdrn5RWym3gzgik/yevP3iEZw0qLtfFmKu6kN1Z/O4xXkZArWNJwGXQ6OluawuJefF3+eS2B73yih7WREyl9fnW+EggQ8DaZyzW1j82kAQOLUGlxwjVDNQVEUT1RzuAJTH7IIvcKRZssnGbFUGOUon3+sgbZjYgAT45xL7bW8r4L6cVnPIAOHC50NZQyym1guNSLZkKoc7Zc3BZDbQu0js5e2jcz91EBSh6/s7iq3GZgiz5GYpFZSSXF08tWBrnfpzpvfz5EyMyzjQaCG6Hk5TtyWy9tkV9EiKfucLSDKmCH4KkApLXkqJlnfuQPOoHnFaq6gLnaP6CRH+6mH/c31IbFp9kvqT9j2iSeTiErqdith0OlzbOfYR4LNIK3UeRfKLfWsNVDolrUUAzz50I/4PTscbhopS12rZ5B0AVyhneyISwOeKsu03UTWsQHyclRzUBTFE9UcrsCm3dNIhznqvvj8fgBA+rzstP7m6FfhII+aBEElj4cwew+3lTs5sibPSYj1QwX0f120igxHs0I/b4GvZhC/yHINv7jcfkwNwOUv8MJfsc18i9MPyPEyglXbGshInoSu17jNhWjHR4OopOUkMng2QuLyu7eB6ORKK352u+SNPG6aXoOsLLRy3gR7nYb/smSjCuTF5TpYRuxNajaTj3KE7TzOfgllLcY+Lh6MM7xA/wtUDxIXUyhI9PPydnkNgCzEyg+Zpos38DLPOXO/eFweXII9RE9JsGUWuGVcbsxtbSt9zxemOzeUrcGhwuEy3BSgw1hM5TmNiE6JoW5BjGV3G0QkwbD7YbiM09MP22Z0XyAvauxOConI8RgaIarLsw/w4XRxBLUtJZhpSpzFu7gtfera9V3aZdH1jpSXZCbJ83nZF0clIcu/t7KMS4ZS2t1o/ticwbTrENtX6L70Vy7LpA+32ufWKMSP8Ed3edzDtXDxCs2pRLKMRjAibWhllgYYKRmedZGbrMOZX+J9MY1WAhfnsq2L29JfBIoyZUhMcOfSh2nYjL/QhvbTbMPirtX70W7eOuu5PXwkuq7jGa6ETisURfFENQfBGZOS+6gSzOdjCIhBybn/6uHWG5TSr8m7In6en4UBnmD4BxVkN/H/qrgG/edlKlFoYGE391mXrkBU+bYXI81tTj0v9qx8P4QX1odm1GVsmiPjqb/FUTiw0Hr/RHYbR8jud1jf7JK/Of1w0Yi1GC9cTbQ0DBe5OPlJDn2x0yGEzlBjcNOd69UYHC4q0QV7df1+AnMH+H9Jokudy3f0U0BCEtZkN608j7/USntXuWzqVU3a5urNiQ9zW/gEXbDpMzXM3Cup6m5DMFIjCNSvkknP3V/X181n4xKlxml3xW0euefk3InjvKG+VcoLoZqDoiie/NRrDm7+i4dp5HIJacfm07BvcGhLypqD2QfElXbC4OKneVj6DY6iZUbFotwRRM+PGDw19nnmeVvY71I6GcRGKY8Lm7mt/ZDYBLYB9ThFfkwMk+2nWGZp65Vvk69sML9Pci1002Yx8Azru7zZ4MJnZb4utpG5e/g9Pm6R7ZFzuHDrkEtC21qf4VKfRc+FpH0WUWfUHOenCzM3VxkdL6UqGoM7IN/rQ1JckQFJq7+8hW2OjZmmPaHzqLMvSNlSA+MfYX92HpJsTx1iq8gaZA5UpT9YZkHezJUd8q/JOzKvxMCTY1fdPzJFtachGbl8YTZ4S/+137V5KefKTMS7Wm/MUs1BURRPfuo1B5fYtD+10v2U/k4csRn3zkp5/do4u6vQBww+x22Tj3MI8vdyiF26v4bph+hfc6+gS4xRBk98vIH+V6iZzOVoFyjK6B2bBExdcjie5XH53mvfnsGf1DAr8+fkGDWNqYfl/Z3nLRIvsZ5ucdbiDgneihq0iaV/ebvkR5CsWF3vlRFcZD2X9tAzUOh176Q0yMmqSDeiO/do4zrTMna/ywODOX4u7Ak13aPLW0SzGuS+tuOtRLFTj0swmSyWik4D6ePclphwRhLJTpUyCKfZhsxOtqHjOO9VZoev+Ubx1Rwex+bSAPj+yxvB692ajnrDu4KjM+2wmZXBceGF1R3rf6qFQy0KxB+ZW7Ht/An+sKODBoUe/uicQTK7jQ9Xz+tM0AoA7aLqFmYli3TUYtOP+aAW+ty7IGWdwFF/MwlrcowPfyXN7/UQkN8q2a3l7VlWsl2njl/Z3Ta/r7VvWV5CG5NoxkrqEsOivO2pyt8JQuMW2c1i8JRl40PfvAgAuPDLm+Cv8Jee28Q2uzRsqXMWy1xi0hQGYYk2LPZc38MZyNel7q21J2464Bj6obhOD1pE5E1azqXp2lRpAwafp1DODVPYumli5v4Kou+xsS7pTiVumt/dVGo1XYzxNzjSnN8hiYI95lm+JAW/15Th/BgXp9hCy5UcnZTkNZfFYySw9ui0QlEUT36qNYfC5ir6IivfiOVU1uFnljHzoKjUfc7gJisa9xn0vM3Rb3kTy5f6JYFowYfpByUHgqjd7u3Z4UWLUgflcVFcb5U0R+but1rTj9kHuc+/eO3bY2xrVFne4UZ5nqf/xTymH6JGM3uXGDsviNo+YGB9kuJut6jfj9CAan0tN9rQczzn3N3yVq1sA35Zy9FMdXc1P50HGUkzFxSNbP5gHcFFibYULaEa4/V63mlg7GnWofMNXjCc4fdGwGB+P42wtYhzx/IaXS8EUWImtuY0xrmit/9f8xj5IudzaxGc5ILLvJHXB2DofXuSq1+VW0I1B0VRPDHW3pjUXwuifcN2x5e/etuuV+qUF58cnEY4sHLoGH+JEt0lOlWUDzpHfvOrb1lrD16+XTUHRVE8+am0OVQ6OAe9VGtYLHDuuhEXyCjKWvBTKRwuZSZLp5B9NQ0ACF7ltemK8tOETisURfHkp1JzcLHnc6eHVDoqyhXQ34aiKJ6sC83B+oCqBK+4UFeHSyV2LS4/bj1wvXX/IOPuy3rri6vV63rqvB6et7XuU9UcFEXxRIWDoiieqHBQFMUTFQ6KoniiwkFRFE9UOCiK4okKB0VRPFHhoCiKJyocFEXxRIWDoiieqHBQFMUTFQ6KoniiwkFRFE9UOCiK4okKB0VRPFHhoCiKJyocFEXxRIWDoiieqHBQFMUTFQ6KoniiwkFRFE9UOCiK4okKB0VRPFHhoCiKJyocFEXxRIWDoiieqHBQFMUTFQ6KoniiwkFRFE9UOCiK4okKB0VRPFHhoCiKJyocFEXxRIWDoiieqHBQFMWTwK0cbIwZAZAFUAdQs9YeNMZ0APiPALYAGAHwi9baxVurpqIot5vV0Bw+aq2911p7UL7/OoBnrbU7ATwr3xVF2WCsxbTicwB+T/7/PQCfX4NrKIqyxtyqcLAAfmCMecsY86uyrddaOwkA8tlzi9dQFOUOcEs2BwAfstZOGGN6ADxjjDlxvQeKMPlVAAik2m+xGoqirDa3pDlYayfkcwbAnwJ4CMC0MaYfAORz5grHft1ae9Bae9Afj99KNRRFWQNuWjgYY+LGmKT7H8AnARwB8G0AX5FiXwHwrVutpKIot59bmVb0AvhTY4w7z3+w1n7PGPMGgG8YY/4GgIsAvnjr1VQU5XZz08LBWnsOwD0e2+cBfOxWKqUoyp1HIyQVRfFEhYOiKJ6ocFAUxRMVDoqieHKrQVCrgmkAwQL/DxbMTZ3jZo9T1pb1el+uVq/rqfN6bddqopqDoiieqHBQFMUTFQ6KoniiwkFRFE9UOCiK4okKB0VRPFkXrszrJXtvCQCw+5/zc+LDaQBAbnMDiVHKuUDBAgBqUbqaKmmg1FcDAMQusrk+fkV8vIG5+/h/ZJbHN4L83n6qgehMBQCQ7w/JPp6z4xvvYOK/uP+66x2dbQAAEmMVZHaGAQALB7gtMu3n57xFeIl1X97Mugz/YAkAMHdfCqEs91kR54ZfMXePgb/EesUnuNHUua+SMohP8TrVmPRH2+q74Kqy4r7/I2M4f6wfAJA871/169xpOg+XkRvksxDKs18zO1r3rxpn38amuc/+yhwAIOBrIFOI8v8ftQEAqkmes5qwSJ3j//4yP0tdPE/fS1nMH0jw/Bmes9jBB6AWX3tX6oYSDrbCG3Hyb7JnQ/OyPWTfJxSSo/yFzPT4EB1fKRTaT/EfX9XCX+S+wiDL97zO4yc/bBGeiQAAtvwZ8+Oe+Nu8biNw3w3VOz/AG7q8JdL84Ybn2ZbBHzPAI7cpgkI3y3UfqgIALvwMH6TUiMXsA6xX4iI/iz1sb3DZoNhfl3rxeF+dZQJ5wF9muemHeN3E6Oo/VJU2XqNhDfzFD64yOr8/jHCGbV24i/cvMcrv1YRBfpD/Z3fy83/Z8QMAwNdOfBpbOhYAAGc7eU+T51mm2GOQ2S33Mst74+PtR35TDPFp3ttiJ69X6GOZUHYtWriSD+6dVBTllthQmkP0InX+5AglbaGP27vfBUJLnALM76Xanu+npA1ngIZouPlhGbYbbHZ01sI0RCUfY6HlrSySPNOaDox8gWnsYhelItbeUL37XisCAC58OoKeN3jOUppyeeSzVDf7Xmlg4EccXUY+3wEAiPArqnEgPsZ6BnO8tlNdl7b74S/xXNV2but5lWWrMYPph7nPjUprQT1FTaxa9yM29cGNHOw4UcXkY3x2IrPcVucsA7GZBgr9orXu4nTwXKUbALA4mcL+7kkAwMkY79/iXTxu6MdVFLvcs+emJfJ8d/mw9Din0GaSZaKeedXWBtUcFEXxZENpDpU2joyz8oaMznf5OfZxi9goNYZgjtvCYsAp9BoEc5TInW9TFpa6WKYeAboPUZsot3Hfwn5K7fC8D7UIjyttolYSPcthwle/sXpPPEbtoPe1OsYlDU5gmZ+hRV5j7NMNdL1CDSU6I/NRmV+G5w2M2Eti0/wn38db1/VuFaFlTlIX9vI68/tZ1jSApBi7Kukbq/P1UEnxc3gLDW+j451Irf5l1g2VlB+ROXkmnuCkf+DrfCYmngij0sfn5GeHTgEA/OB9HNw8j8MzAwCAelierwU+b5OPBjH8Q2qWtShtXLlhXsNfAgLnuc1X5ba2c3z4spvW3uCrmoOiKJ5sKM0hNklZlhijVrC4W6Tp8UDTKu8svc4duPXbLbPu5IfobQgvtlx+ObFNZO7miLz522J5DzcQzFJKh7K0dSxt43lmHq0jder6u67tHOs79pkGtvwxz19Os+7lv0KXS2E0jVCe+0oh7nPaAr/wY2Ev6+LnYINKyo/RT7INw8/wgGCO3xf2GeSGWc71S6h63dW+JrW4WNn90k8TwdU7+Toks92Hwib2ceg0XYzz+3hj9n70NLbGeS9TAdoJvjl2LwDAbyx+ZfvrAIDn2nYDAGoNPp+DsSW8sGs7ACD9fV7H3ffi/iJih6kNRsUOsbyF99Y0Vr99l7OhhIOLQcjKD9+p36kLNczcz50VMcpZH/flh2PIbGeH1qmhoR42UgYod7J8xzssc/FpHtd+1CAaYLnckHMR8vjY6I112/JWHp8+5EN4nvOJxV1UwMtvcY7Te6YljApdbIt7ALLbG2iEJC5iitfue5UuUF+ljmqcD+r4k9xXHxCH+WII3W+yDTUxhNViq2cwDO9gWzJFdmxo6YNrjASAxLhFLcHnpOMI+3Pu5yilK40AvjdCK+NAmv0ynMwAAN59Zg9+O0LrudmSZ5kOlrk7OYbqFj4fbw5yPlgcpHTwT0WabvfJR3jd2PQaNc4DnVYoiuLJhtIcIvOtgBMAqCb5mdkRxKa/oPsou4Oj6MQnONJOPxiAn1oeGkEe33WYI/TsPQGkzojRb5n7Bl6Qa8QMxuUcRiIQt3zHSfHQDdXbaTxLd9XRNkI1MbLAc4fExVgLG0w9KsFaezmqhH9M7SK8YFBNcl9AkuIs7Ik2+6L9FOcKJdE4/OMcyfNDtjl9yW3m9eLjtz6613hp9KU4ZVvIx1jv2pWO+GBgGhbVFJ8PF5RWL/C+1Bo+FHM0ip8rUhts38KbVYtaJM/zHJkkO29geBQA8Mej9zenZaVefibO8Zydx6tY2MN76tzazQjZ2zCtUM1BURRPNpTmkN3MTxd66ubPhcfyWJqnsXHqKQ5f8TMc3fObarDGhbqy/MIeGaGrLXdhZj9FcdebElAUMwjNsZwVr9GFp/l94MU6ljdfvyup6zDrtLwUQJ2DS3OUnf443V+pd8OAuL5Sf0btJ0/vF+oPZdH7bzjiZLZzJMlJXwSzwMWnV86Dy+2ijfRUEDjNfnAaUj163dW+Is5O4yhVWKcb06c2HtZnkDwna3CezAAAfmn7WwCAqvXjn237Y2479NcBAMdmaGeITRlkn6QWETzPG3DkT2if+Gv/2ffwu2ceBgD4OvksBE6yzMz9QdQj8kycYR3cc+PWX6wlqjkoiuLJhtIcIrMSgMKoVPS9RmPC+Yd9CBTFPVniKBoft3KMv2mbWHqY5Qe+xZGulPZhmV4kJE/zuO6fjAMApj8+0LxuUKzwKcb6YGG3D+Ktui5m7ue5/UVgWYJX/CXWr+c5jrdznyrCyGAwH6bNoPd1WTyWTqLc5rQl13YOIWMfCaDtJGW8C0py4eJdL4RQ7JGQ3osSFBa99RGnnloZBVY7S03ng645LG8xCIgL2dkJGuJjfn52B/743L0AgIF/zBtw4WeozdbbgOibvHHO/lWWF8v/yx9+EqaHG+/fzPj8th38/uOfHMDQc7zPTmOssqvhr6x2697PhhIOncfZIwt7+BjO75XosTGLQJGdOPx93qy5/RKnPmXRkKe2/UXq9FOP8PuO/5hFdgtvoDPYzT1OodB2tgJ/iQfOPsQHITYty2VjFoHS9f/IYgyrR2ZPA+kTsnT6kzQ6dv1Tmbo8E8Ps42zD1mdpYKyK26z9RMsA1vke6zn5iBit3qPBCwAMvWQoPsAnuNwRRb2N50qNrE5EXbndYmCY/vzpZfadixr8oBOZh5v5YSDF+/dvX30CABDIBJrRrvP3iDv8hFv/4kP8E/RB5n/YCwDoOClG8YgPlQifszcOc6Tyt/E5737HYvRjvM/uGcpvl/t5fO1jSnRaoSiKJxtKc1jcSQkbn6TUnf48JaydCWP8I2xKG8PaUWmn9K5HDMKLcgIZ4NwKx+y2BLoOs1zul8UVOk99LzcUbkYVJs9x1A1JkFL3oRuLbQ/KiN522ofel+iTOr2N1xn9JOuy9Q9nEM52AgAmH+W5XQRirbOKnb9DrWLublnFKVOOatzXDApzo5rvHMtUBivo/gn7rJS+7upelXJ3HZEA6zKV57kTN7jWZKNSTbambMUaR+5fOPgmAOBb33+kGZS3uI+fiQu8L7Epi9ljnAvHxaAYKDpXth/BfXxA3FQlX2S/Ln4hD1zgPMK5w7f9kWi473uF9eqjmoOiKJ5sKM3BGWMybRIOLQEonccMyp/hyO87ykw7PW9Sek9+yMJXEfdk3BnnuM9fbqCU5rmqb3Akd6P14PM1ZId4/vyQuEDlug2/bQalXA+zD1Lab/5OAyf/FusXE+2l/yW6uOrtMSzu4vnbzkrOB0kJ1vNnVeQHJfZbtJ9ySlaYdvqawWGVFHemRRuyx4LI7JQgrxuo79Xwp1qLM3wTkdU56QahsLcE3wxH9YOdNB5+4zWm2EpNG9QkXZ4LVAuI0Tk3bBCXDF4uk1RF7EnhBYtwjDYip5F1xHmCyR8PoT7M/m5/mfd79l7azVyawLVENQdFUTzZUJqDJHBCdFbyHQy0ksqWX04DQCto5DzdQZHZGFIXOBJnN7F8ftDIZ6C5GGvwxyxfbpcVmFsDzeSsJcnf567rgoyul76XWH7mviDikk3KjS4XfkZW3c0YFAY454zMs55lNgkTT8RbJ3OXlpGj82gZF/4665l8meda2MvjO4/UmwuuXN9FFm7Os1CTfupqzzbDpT/IWZ+8iJ6IIP3kFADg3cVBAMC/+cS/BQD83Qv/OXwPZQAAkZ+kAQC+Cvu+7/Uq5vbzuVrcx3M5G4Jtr8DMUZvcOcA0T9kKtYPyniICEgqfG3S5I1cuIVhLNpRwKO3mD7jUx57tfl1+RO2mlSBF/PoL+/hDSZ9uYOpJWc4tRqCh5/gjtD7TTNE1ex9vgsukXGmzWN7Oc/W+zuPCGap91gRRTV3/zZl6nNff9J0qyu28Xm7QJ+dimVoE8JW5rR5amUTUVwZ8dVnO3SEJZiWyM7oQBPi8Ij/EMp3v8XN+vx+hDKSt111dT4pD7LO+UAUzswyoSN7aKTccvgows8C2V9toRPx303RlNkJA4500AKDrolu7w3u9+Ikqou9IFKm4Ozs/wpu2JTWPikju147QlTnwQ8k+nmtg/MO8totvcYPK7UCnFYqieLKhNIfUmxzdXRr6ZprujMXSZyjJ/e/RaumMj+2nygguUk1rP8HjXLTZ8u46jKh+W75Dt2iljV0yc78P7Sd4Xaem+yo0Rll/Kx3d9dD+Ho9f3ONHdIZaiIuUazvNz8hSHZF5Gp9mRItxUx5EmJ4eABbu46jU8TZHpeyQD7aXBq3oG9SWit2SLKYORBbsim03i422/JXuFQE/bUTmLXwx5sqYE+0pW+BNio9bzN/PPsot8BmqdEgSnDNRlLpkuruXAWT3do4BAM5ku3F8hO/6+IUH6RZ95RkaObPDAdQT8ryMs8/TZ/ngzNy/CotkroFqDoqieLKhNAc3P3cSOjIjqysrBpV5SvCExJynZN6X2RFG/0uSWafsAkioAfS9YJoGvslHqF24tODhBQNfjf8HJIjJraH3VQB7Az2XlfRysXFgca/LzCQnk+uHFvzof5V1dvkSonOtdHZL2ynHd/whR66zPy82hyk//KNse7Gf5bf9MY0VhcEYahHRWh7jccFDN+d+7O5dav4fmtlQj82qEco1sHw0DQCwA3zQ/uE93wYA/B+//5dROct+WX6Emlw6RQNBaHsdT/SdBQC8OM2H4eQSw6gvvLgJZjPvzTO/+ygAoHQ3rxfMAe2H3fgtLyd6gA+HujIVRbljbKghIHsfJWzfD2gzWJAw1UqbQWxMMvLIqOsCipIXGxj7OP/vkjyR/S/Lewe7fE0Xn0sCO3/AzdctKpJFyVmIy528XnTyxubvLly7EQTaj4nt4G5u2/YfuYDHlyti6mMcTZxW4uwmifEGlne4bfJSFXnHZmLMYmEv5DgJ392blP4BYlLX7h9SM3Jp+a+XkrS5I3QblgGucxb2+rHvCRqJjkzQTvCPjv4MAOC+Xz+B0xmGSN+b5htvhiIZAMAPJ3bjL87zJu3uprvyzAJvRKW9jtAF3pvkmLz6rre1yta9UzMsXo7sfsn/cXjt18BuKOGQOMxO9JfZicGsuP7CrR+weyOUi2Uo9PqQOs1tSzvk7VbjrZfS1uXdFAvyrocdvy/vNPzljubbsJwA2fxdGoPy/WHk+69f6XLTodRIHWOflph6Uc0nnqKPO5BPNeMn3FoQlyV7absPIdlWScqSdHlpbqndoP24a/PKd290HAGC+bqcg8f5bjD7dKVX3isqeuz0chLh+Z+u+AZHNWFxeIzxDf6ArIO4SMPkvh2TeOGdPQCA6Tp/+KbGfvr4h97F8UUmfnFuy8LpNAAgOWaa6QYmH+X9i09I/9rW/VraJVG2/zf3Le5Z/fZdjk4rFEXxZENpDvkhl3Ze0qKd4Kg28SE/uiTPwdQXOfVIvCxRI7YVGOW0i6WdsssACYlYdOp+YTNHgui0aSaBnZcpwPJmMfx1mxtK8OlUw6mHfUDTyCkJZC5wBFrY7UdQ6pfdJu0838ofUenjEFJJc0rl0oa1jdQwdzdvYyPEc4cyEpH5aA3J09yX28q+ut73bbjVh+mela9zzk/FkboNxrD1SHzCoLrM56osr0BwCXp+/8yD+H99lMbJ//nQpwAAkSinAOeyXeiLc/p4/Ft8b0V9J++n70IQQXlfSedh95ZtnrsaM1jaxfNHp9w7T12WIc3noCjKHWJDaQ7b/5hSc/YeSu/RnxXpXTaoJCUceZwWyUKfrLwsmua8res9ybAUZ9lCjw/FHu6Ly8tCZu+lRG6EgLq4ATuOOkMfyyTPWdTi1z/vdqHLyQtAMC9vLKqz7i5Rrb/KMGlWlP9E36CNpZr0oed51qsoiUUrNFUgtymAxJisn5DRvvfVDABg/GPpptbS97y0ue/66uxcZeUqH5ELM3zzt0ub/tOIv2RRlaTG6ZOSsFdekBR7NYWvfZrGyd7eDAA0cziciSebGlz7WbHhSFLeYo9FfEoC1SQ0Pjkn9oz+YDMDVGiJZS58RtbizK5BAy9DNQdFUTwx1t75CWS0b9ju+PJX73Q1FOWnkiO/+dW3rLUHL9+umoOiKJ6ocFAUxRMVDoqieKLCQVEUT1Q4KIriiQoHRVE8WRcRLdYHVCXauXpZbG5w+adzkc/txPW5V19ffj8uxZW/Whmv8huN623f7eDSPrzafVsNVHNQFMUTFQ6KoniiwkFRFE9UOCiK4okKB0VRPFHhoCiKJyocFEXxRIWDoiieqHBQFMUTFQ6KoniiwkFRFE9UOCiK4okKB0VRPFHhoCiKJyocFEXxRIWDoiieqHBQFMUTFQ6KoniiwkFRFE9UOCiK4okKB0VRPFHhoCiKJyocFEXxRIWDoiieXFM4GGN+xxgzY4w5csm2DmPMM8aY0/LZfsm+3zDGnDHGnDTGfGqtKq4oytpyPZrD7wJ4+rJtvw7gWWvtTgDPyncYY/YC+BKAfXLMbxlj/KtWW0VRbhvXFA7W2ucBLFy2+XMAfk/+/z0An79k+x9Za8vW2vMAzgB4aHWqqijK7eRmbQ691tpJAJDPHtk+CGD0knJjsk1RlA3Gahskvd7o6fkWUmPMrxpj3jTGvFnP51e5Goqi3Co3KxymjTH9ACCfM7J9DMDwJeWGAEx4ncBa+3Vr7UFr7UF/PH6T1VAUZa24WeHwbQBfkf+/AuBbl2z/kjEmbIzZCmAngNdvrYqKotwJAtcqYIz5QwAfAdBljBkD8D8C+CcAvmGM+RsALgL4IgBYa48aY74B4BiAGoBfs9bW16juiqKsIdcUDtbaX7rCro9dofzXAHztViqlKMqdRyMkFUXxRIWDoiieqHBQFMUTFQ6KoniiwkFRFE9UOCiK4okKB0VRPFHhoCiKJyocFEXxRIWDoiieqHBQFMUTFQ6KoniiwkFRFE9UOCiK4sk1l2zfDkwDCBb4f7DglWlOWUuu1ufXcz8+6PdsvbZvreulmoOiKJ6ocFAUxRMVDoqieKLCQVEUT1Q4KIriiQoHRVE8WReuTEfPm0VMPxgFAFgRW7W4+7RoBPnyrO63uK3UwUKJ8Tpm7+P/ld4at50O8nvSwlc3cn7ui5+YBQAc/+86ER5jucRFyLlYZnFPcNXbd6N0vVsGAMwdCKPzCP/PD4QAAMvb2KZQBvBVWD5QZP8s7eb39mPA7Md4XOR0BABQS7BMww8MvMi3BjSCPJe/xH2z9wWafW1qcp3s6rdvrUmf4b2cPsjHfPB59sX5r1hETrI/rLzmOTrD9i4+XEHieBgAEMpIf+7iZ2TWh7bzDQBAMM++W9zB5yR1sY7cX1viuf4wDQAIZ1hmfl8QprEGDVxjVHNQFMWTdaU5wFpUU/y3kqaotekqAKD9lRCK3ZRluSGWiU2KtN/tR2JU/h/icf4Sy3Sdb6Cc5nFulMg80AsACE0Y+EscGUud3Df/AL+nTq1247yptPEzcXAOlWe7VuybfogjWPvJOiaepMZQ54CH3tc5Kk38bA2Jw2E5F+se5ACGuXsbMPM8buAldsjIz/J757sGMw+wQ6zE0piGHJ8FYlPctnA3rxPK+m+9sbeZWoTtqUdFI/ovGWmXeKEdFXnOBl6i2jXyM/wpxE6FUUmxfPIiP20fNQ7/aBTZIT5LRlRbp91NfiiM0LMdAIC5e3icr0KtIj5hUYutz0Cqq6Gag6IonqwrzWHiyRiiU5S6TtKGTnJUTI5WYX2srptjl7pYxtSB/AD/7/0+pfX0hzjiWZ8f5XaW95c4+hnLa6TOARmZT3a9x89gjtdohNaggR6UejgvbvO9f1JaTsvI5QN8VbELFLlv4S62JZLII/DhHACg8RLVn9g0jyv2W9gIz5vZxn50GtXMEzVExsWuIoNaaJmfHcermL2H+wL5jTt+ZHawj2op9vFzD/wbAMD/d/jD+MnvPAQAGP2YjO6j7IRN3xzHxV8YBAAsbeW25OtiBzNAnd2I8KKzz4Sb+3xVsdPIrXTabH7QNPt2I7Fx77yiKGvKutIc4hMWs0/QxtD5MiW6m2NntgeRPsd9S1u5r9BHyRwfMwgt8/+AWNzDcxw1YtMNBLMcARbv4rkCBcrE0s4SOn9CyZ8b5LZyJ4+PTq/tHNF5YyLdxSuWSchoVg8ZpE9xOJp6THb6WU+MJ1BqY78kRSuYu59lOw/5kO9nP+Q5GGLoWc6RC71BVJKiLRXEKr+dlaom/AiLpb6wjaNudObOe29uFDeCB9PsmE+8/TcAAMVyEOV72Gfxs2yXe35O/e0B+LfSNTP029w3+pRoB0GgERBvT0jsWHIfE2MWDTHLNORX5avJ95BFUz3bQKwv4TBVg/kxb0i5nZ25vJ13uPc1YO5uUXVlBadTkaPzDUB+K9MP8W4NvMibX+gOwM9/kTrHz8xTPEHA10Chj9Kn3MnrJM/z+MYa/xZKIoQG09Q3R871IHVZmVCWZXJDPtRi3NZ+VKZdcTHODjfw4M7zAIBzP6IPs+f/FiPbz0XQdrql2gKADfBz/m7Tcq+JRbL/VXbU9MEgAiKzTHFjKpfWB5QOsBGmwTZkppIAgOB8AH2H5UcuLttgXgzgfh/qUq4a5687Msf+qbQDwy+wb7PDnHc69/HUkxbBhZUCo5Licb7qWrRw7dmYd15RlDVnXWkOpmGRPp0HAIx9NAEA6DhM6RsoNpouNzeqOyPd0g6DuBh/0ie4z1/kSFAPtbQQJ9Fjb3EYDi1bQIyT9fDt0RicC63zvhkAQKZAY1fizPsvPP8pqka2ASTfYDk3CmW30uDau3MO7/yYGkP/DEe6Ure4PeMNlDqcrsuP6Qe4L3XOouelOQDA1IfpQnUBPW1nG6iH2Gf5oVtp7Z2jkgLqYoC+e8cYAKAmGsTFxXbMhqgdhJZcMBnLpkZrGH2a54jI1DS3hZ0XnfZh8lFOMbqOsK8XdvMn5M9atJ+U6dk2XsdX4ffQotmQw/AGrLKiKLeDdaU5ZLYHUU1yZKvFxC3EARKldh+K/fzSdooSvdwh88Vlg7oEvFTEgLm0MyTHm+a5et7kZ3ZYDG8x07QTOeNROc3PcGZ12+ao7OQ8OOyX8NtDDMgK195f1s5wlIrM+pquM2c7SJ5lHywPRvDFn3kRAPDN8hM8UNoUHQPyW3ni4ALLp8/wPJmdPpT7qcYs3s8yvhg/a0ejKO1lPVOisdgNFgNV7qviib2MZLuYZXDShZFuAED3SwGUxTjd+wZVsewwfwq1iA+p43w+Fvexr1ygXNfhKmYPULvKDrJ8VQKmglmDhf08Z4DKL8Ji5Cx2mqbWupHYgFVWFOV2sK40B18NyInrzC34qaRbc+bINP938/bwAssUeyyqnEI2Q2UTFyj3wou2uUjJBUoNf3saADD9kR746dlDcpTzypKEWldTq+96KvRb9HcxtnlBbA1X01C63haPzVYgdZH2h1qUqlH85xnfnI4U8R/eexAA4N9DL0zHd6PNc0TnXJ/xXIXelgV9cTe1q443WdbU+TgEyg1EZ3mdauJmWnrniXYWUapzlB9KZAAAc11cxVePpJq2hpn7WMbKL6GSMmg7T60uNcpt7pmY3xdEYozPV9sZ9vXpfdTuOi74UV/mOZ1naeojPE9k3Lchg6DWlXDIDxrER1glt0ous6flcgrPiaIjbstCf8tQFF6QaEK/qN1jLXWx/STLTTMoDue/1Ns8p7/M8m3n1t4P3RgoISjTidwFLqpIXqW8v+qiNg0mH+UPviqrKiFx+2Onepr9EZCpQ5HaM3w1oPHUIgCgfDjN641wn6mZZhRq+0kK5Nl7RDiUDIq9Ei8yL3Wp3GBj7xAlmWoOpZcR8rGv709xye2p/5NzifTpAhZ38xc89xDLpA+z7TYAjP2cW70rUaWXtL3UIWtxHqGg8eeln5YtMtslilWMxhFZ8VuPWmB548U56LRCURRP1pXm0HGsgUKXGIM+ToOYb0RcfRO+ZtCOM1I6Y1JuKITuZ6kDjvzKJgDAwn2iERwzWNzD/90omBrhiYo9PkRn+H++f6Ur0003VgN3zo50vrktOXJtuRxerEndQigfoBp71yCnE8fH+wAANtxAzwu8jRVRQzqPcwqytDWM/FtcWGJlupXdxjLWZ5E+yf9nJN9BaZhDZHQkhDbZN/8w+zh1fGNESFbTrTUqmQqfnf/95Y8BAOL9Mg3tjaPYw3Kxi2JYlL5rBICtf8D/x54SLeTH7IPxDweRlEA6twLW4atalPr4YLroXDeVCF8Ayh2r0rzbimoOiqJ4sq40h+VNvqZrqOdbNIjNH+C+woBFLUXJHFgSyZylQS3fb5D7CjWG4mZK+fAER7pgrhXX7txJTpMwDSB9miNIdrOsrejg9/j46snNZvx9uIwLk1w5eTVbgyOzg+1Ln66hFuEc+XBuGABwz27Oo4++sq25yrQoxsbiLNs+91AdPS9JcM8ItYnsZs6jC70+5GS9hVsJ6y/yevEJ21x92POiaBWdN9DgO0lbK1Z5W4JBXieymwGgufah43Qdvgq/uLU7lbTL9mSQG5IcGC+KG3iZGlXH0QAKPRIiLb+clvblQzDD/424peOToqF2bswxeGPWWlGUNWddaQ75TXXEL1KiL21zczpZaJSow5+iBA+PcBR1VvnEuEVmF/8PTckcsk00gq0GqXOS2WeUx5/7y5SJ/qUAph90VmpnnV99eelGktlsAokUbSn1EIesq3kBXE6J8EIZRoaqzz3wDgCgKG66M7uWkZ+n56PS6bI2se3t7waQG2I/VhO8nsuH6CtbBPOy6lMCyEJLYpfYbFreIslHGZm74WbfVpyXYnP/fHPb8SXaZYyE2cclc9j0wz4MPUcNwy3mc2Hp/hJQ7JbFaffwWWw/Ii7QsEF0VvJj7BavmGSLiu/NoP48DQvO09P+yjgAYOF/SSP8yvXoiuuLdSUcOg77MP8YLYGmwKqlD/PHGloKoNAnqn+7xKxn5OEOAh2yWnFG3JWBnPic4xbW8P+JJ6grx7iIEfVIa2VnfIKfNTHc1aOr53oSjxpyhRA29S5IPdmWwtnL12K2aAywcsOfm8FgnQ/qS5O0KIYDYqyciWPwnLhHt7Puy5vdVMA0jbcuAjQyw+vGJy0W75KkN+dXRo7i3mWU3mK9ArmbafHtp7nextjmtkKVG10in8wutq/z3QaWtsig4MYC24pRqEmYSO+r/Jz8JCVHx+tBLPwcDcPRVxgAkh2W1HrfbUeVsgh9r1CAnP+rnAJWpuoIr0orby86rVAUxZN1pTlEFhvwz1Pau/TzC3v5OfxsCec/Sz27TZK/FhnLhNSFSjMZa2JETubW1CcNlna7dGuSyEUCUspdjWbQVLBAab+4lwe6BKurSfhkFBdB1XOwOwMA8N935aF5X5qVOLncg50pptMv1zk8/fq27wIA/of/+Nfhq0lE33HeTjetWLivDlOWxDY9PGfX2/ystBl0Hma5jncYKFVqp9XRvpBCbjP3hRc3xvhRH1jpey7VAsg8z74qPCUGxdclheDHq0hKUl6Xt8IlaLl0mtf+Bvt/8uOcvxZ7DDq/zSltSVIP9hziCS5+MoKet3gfssPU8qKSrq8e2Rh9eDkbs9aKoqw560pzmN/rR//LIn0H3TJASt9zfynYTPfm0poV4bIahZurBl2gVLFH5p6NlkEwwWX9zYCUeryB/hfFrSnuQLdeYy0I5oDgW5zQzrbx060Y9eJisB8A4C8anG0bAAB0buYo//88/PMAgI7ZesugKJmj3FoAX7yMtkM0RLpkvJVLgn0akhVq5AvUGOphcefNGyRG5SVBbTff3ttBTVyRlxoiASBbCqPjBJ8lf4kap0s0nDwSRn6TGG0lX0hJ3grQKJqme3P08+zz6Ai/RxZs06aVPsa+mtvH+1gZKGOuSm2k7YxklRJbV+/rdSxtW1c/tetCNQdFUTxZV+Ks0t5AZhtVgA5ZDBSblfx/2/3oPM6RYOoRbus4QuldSRj4xO0XEK2i612xUO/wISZBPm70jE+45KkNLNzFA521uyIuUDdyrhWhJfd5ZU2lQMUB4UWD2iAt5oNJHji6zCF98stlhA7R1dYM5BG3o5mKILOX7el6S9y3spir0ONDKCd5M2UFa6AkSWw/ZBCSfIhxWYVYTa7PhUMup8elXgoAyE4lURKPUN+rfJbm94qbOww0JFhqcR9vvOkVm8WZaFObc1qkezZ6Xl1Gcox9Pbef54rIgj+U/ahFWjk/gUu0QhtYs/wga8m6Eg7J8z5U5d2Yzh3nL0v0XgWYFb/zwAu82U4tnnnMYsufsdzI5yVZ6o/FVbitgogYOZ3RKbtJ1PDJICJz8qas/fxhRGbXT1YTf7GVZKTUKW+12iTGriD7IDsSa76nAsapyBLXccGgkpR3N8iD2v4W13dUEglUo+wjt5Q9fYanSZz3IZhfGefgXL3rDWZ2buHcl/GRAILyfs/sgNx42zqm7R32Z3ab3Pf3OD3oPFaDabDg/F6JspXzTD7Z1ophOC7vrXiQxw9/F6jIitlij0x/cy7V3sbMPq3TCkVRPFlXmkNivN58k1Nmn7wvYZxV7HmriukHKclnD3BbXUbDgedsMxglMinBLGJcC00FUZDVeNWkJJ2NNprndqvxUqd5XTctWYtkLzdKcoR1WdgTBMQNm6twxKtblxjVYEmCnypd7LO7/gUDrUa+2IPOY5yKVRIcB8afYoPTZ+rNFbBOe8oNSGKYNiC7TQKkzq58z+h6w9ddWvE9W6Q10VdtrQdx2oXLZVHPGuAT7KPoy7ROu7Uk5TYfym3OGOu0Ereas4G4TDeXt3OPewfK8iaD6ByfK2f0dcFlybE6lrauq5/adaGag6IonqwrcTa/zw/fPTS4RQ7T4OZGtfGPBhCbZLnkRVk3L/kOZg6Gm2GwTjsoyajYfryByALLTT2yMieBrw7k72MQS6PIrtjx+3LO+6O405Ql61A5bXHXw4z5Dkg8dCxAg9q79V5UurnNaVkzjzNoJzprMb/Xe8jP7PCjKm+86pQXvMwf5Hm2/FkDpU43T2d/utD19YT1AbHYyuCnhrwp3N9As33RmZVp9rverWP6KP2aHeOyBkdCx7PDvuZLkyKzKw2S/qJphqFv/QMGSNV6JEnvriimH5NgO7EHbfv3TEd4+h+3Ifraxsu3t/7uuKIo64J1pTkAgO81agxtFyQEeJ97kw1QlcSdLuR5aRvdkLm7yuh8SVLaiwU+KAFBtajBzEGX24HHu3dQVtoAu8DjQss87uwX+Zk8u/ptu1HiE+wDU/fhofYRAMB/PHs/ACC3wM7w9zWQOsHbuLxL3L9T1BaCOdvME+lWE7p8BI0g0P02ty1vZvm4uDQvfLaOaB9N9JXTHBlj02vRwlvEArklanjdCXphSnnezyBaC946j8oKTEkrX+j2NQPq5u6VHKURFt76pzWMfZTnGHyex83ew++JsZZNavYJxu47b87WPyugEeI9ichrBBYeoS+6utjAnddDbxxj7ZUj9G4X0b5hu+PLX73T1VjXFB6krus7z8fMvb/xpx0XIVncLdOLJQqA60nDp5Ajv/nVt6y1By/frj2oKIon625aoXgTeyN2p6uwLgmIJzP57kbMmLC+Uc1BURRPVDgoiuKJCgdFUTxR4aAoiifrwiBpfa0YhrWk+br063hvYTVlr6vcrdblVgkum+s61+VtufSYG23n5ddby366kXpcztXafCPHrSVr/ZzdCqo5KIriiQoHRVE8UeGgKIonKhwURfFEhYOiKJ6ocFAUxRMVDoqieKLCQVEUT1Q4KIriiQoHRVE8UeGgKIonKhwURfFEhYOiKJ6ocFAUxRMVDoqieHJN4WCM+R1jzIwx5sgl2/4nY8y4MeaQ/H3mkn2/YYw5Y4w5aYz51FpVXFGUteV6NIffBfC0x/Z/bq29V/7+AgCMMXsBfAnAPjnmt4wx6/QVrIqiXI1rCgdr7fMAFq7zfJ8D8EfW2rK19jyAMwAeuoX6KYpyh7gVm8PfMca8J9OOdtk2CGD0kjJjsk1RlA3GzQqH3wawHcC9ACYB/DPZ7pUMzzNxnzHmV40xbxpj3qzn8zdZDUVR1oqbEg7W2mlrbd1a2wDwr9GaOowBGL6k6BCAiSuc4+vW2oPW2oP+ePxmqqEoyhpyU8LBGNN/ydcvAHCejG8D+JIxJmyM2QpgJ4DXb62KiqLcCa6Zmt4Y84cAPgKgyxgzBuB/BPARY8y94JRhBMB/DgDW2qPGmG8AOAagBuDXrLX1Nam5oihryjWFg7X2lzw2/9urlP8agK/dSqUURbnzaISkoiieqHBQFMUTFQ6KoniiwkFRFE9UOCiK4okKB0VRPFHhoCiKJyocFEXxRIWDoiieqHBQFMUTFQ6KoniiwkFRFE9UOCiK4okKB0VRPFHhoCiKJyocFEXxRIWDoiieqHBQFMUTFQ6KoniiwkFRFE9UOCiK4okKB0VRPFHhoCiKJyocFEXx5JovtbkdmAYQLKz9dYIFr/f83nrZm2E1z38z57qV669131wvN1qP9VLvS1mPdXKo5qAoiicqHBRF8USFg6IonqhwUBTFExUOiqJ4osJBURRP1oUr80aph/gZnbXN740gXUKlbm6zfikzbRDIc1upi2XSZ+sAgPn9flQTDQBAfIxystjLsrFJg9g09zWklwq9LFNJW0TmzIrrbES63isDAOoRtmvyMTY0PG8QnZN+FE/b/D383og20PmWX47jzkIf94UXTbM/Oo9UAQCZHUEAQKnLwr9vGQAQ+UEKAFCL8vhGEPDVVrdt9RDQfpr3udDN9pU6eL3USAP1EP9f3sbP7ndZNrPNj+QY73s4w21WhtDlzQEUu1k+mOO2cjvb7qsbxMfkeZR+yTzA/k29F17dxt0mVHNQFMWTDak5NDgYNaV4dMai0M9tTquITXNfeNFi5gkOS6njPHDmfsrEyBzQfpzlpx/haBGd4j5f1WLqcY4EbSc5HIaWWyPDwDOzAIDxp7vfV7+6DBS1+7MAgPJihHW6EESgeGttLnWwDtFZts80bu58AFBNsF1zd/Mx6DjKk0Xmq8gN8ILFXl4nOcLPWtSHaoLHu3Zu/8MFAMD4xzubmtvEkwGpt5xz1gffS22sc100jYyVephm+1YLXxWY28/2+Svc1n2I2gwMEMpwYy0WAwCUUy2tcDEhz8eMT8rwsMiCRWxatFDRQiq9fLYSp4OoxUTNYhF0vsSHsZpc3bbdLlRzUBTFkw2pObj5qRvJ/RWLcIZSOzLPfcvb+Jm9p9KU4BWR4MkRfha7gewmysfN3+VIsrSVQ5iptTQGX4XXWd4u180AF7/Gc/pfWFm3eghIfniadQmwohcr7OZA8eaGx0YQCD7OhrWFOPotvNgHAAjmb+qUAADDKTUSMlee388+7DgWQOYu7ovM8LMa52d0zqIotptqiseN/kwnAKDY24CpiqZxAbKN/Zs630BmB//3l1lm/qDcyGADyWPsz1vRhC6lkraodIg2OM77uLib/R+dbSDXT3Ugs5dlul9n3ToPW2R2sH6+GtvXcYIdle/1I7IgNqop7ut7jd9zQxbRObbHaU1hsUv56qvTptvNhhQOyYu8Ie6Hndlj0Xaa+xb38jN9QlTXhTDK7dyWGhFD0wJ/YNVkGEaez+hpThMy2wYAAMV+0zQ6LW/jubrf5vdGAAj9QULqwG1Oxa7el0MsyPOfu9DD6x4J3VJ7czuq2BbjfGTkMOuXuAWh4FjYw9ufu4uCcesfsp2ZHSF0vicGSHlCqnH50V+sIlDkRusTw9tO6Z83AX+Ffby0lT/Icgd/GYGyReIxCrjsa5yKnf/s1wEAP3vq0xg5tRVAawpwq8THgfYTrn7c5svKZw0IiZG67QTrGVnkgxAo1psDTDXFdk5+mYbF4OE4stv4zPW9ynYt7OWND2Ztc2CJj8l1ZFCpxdfv+omrodMKRVE82ZCaw/wBSuLYBL+bBhBaopROjVCiz9zPpqXPNLC8RYxNIsHzfZT2uU0NBPLcNvKlQQBAudMZJk3TEOUvyag5WgIAnPkrAaQPrZwiVNp4/eHOTHNb/PStaQxuGrRt2zQuTFN1j02snjyv006K6DnW8/zPc/QceK6BxT0ypRIbXt+rbPvsPREkx+vyv0wv0uyzqY82gAa3Jc7yuFCG9Z272yAuflG7n0P4N3I0UFbrfhT3UzOKHokCuHUNIjcE9LzDe9J+gvVbvIt1MXWD5Cjb4J6J2Xv4vCTG/ViS6WNiVNrwLudUvgpQ6uE5F+5ieWcgDhQMjBginTu33M3rJs9sTH+3ag6KoniyITWH4BKlfW4zJXP/SxYLeyjnZj8mgSs1MRw938DcPdQUArJ2PreF+4Z/2EC+l1J9ebvMQU+7IJcG0idpdLj4aQbtXPwEh1r/km0aprKbed1qX2uoG3lP7ALlm2uf0xjaHplpbZxiG1Y7WAgArDwFXa+zL7KbTNMW49x/ox/j9f2llj0BPvbZwI/5deaBAPpfFq3iPilSEZdfuoHSAu00X7ybxpvfn3wUAHDq9AC27ZgCAJS6ZZ5/hnaJxHn/zRkpDbA8zHrmN8mzMNraPb+f1+l/kYlEAmXe29yQD/2vsM3TD1I7DEiukdSFOgr9K8fT1Dm5XMNi4QnecFtjmcTJW9Mc7zSqOSiK4smG1Bxc8IyvStm2vBlIjHJb/8ss49xWE48bhBmj07S8J87zuHrYYmkHt4VEG4nNcOTLDfox+slUsxzQGmHbjxmEchKau5caw7aBOQDA9HISsclbk7nOBRcP8dznx7uQHFt9OR5e5KcLQHJz5vCCbQY/jXyB27Z+k6NpeCqHc79I909DFIiZg2LBf62O3CA3utHWX5JzPpjBvm5qB988fi8AoF6QDrWtOjn3b8828WyM9yBQuvG2RWYNyh38PzotgW2iDSVH65jp5bXHPk7DUi0mnoV0FbFJsVed5n1wNqvMTj/i4zzH8h6ezFfkvtikD6k3qX24EPxQZmN7KzakcFjaLU9To+ViXBb1PruF6m9pkA9z90sB+MWltCR+9vxW/rCDeV9zqtH3GlXCSlp81JkGGgGWj9LLiUKvixJsGbCAlXp+uRRE9CbU4IYfyO0Qg+CWuRX7kociN37C67mmW6Mywwq7tSP1MNB2XoxpF/ljr0mjxn+xvRknsnAPt/W+wn6ZuS/QiruQPljaxX+CpRAqIk0GvsELTz3kIk8NFgdpiGwXl60TjNmDS6i/TsPljRgpG0Fg8Hne07m7+UyU09w3+9cKMO9Q8Jd2ytxPJGPsZKQZATq3h3VPnOf3ahJoP8VnpxEQN/BOPmftpxqITPNcYx+nATM5zn2Lu1Y5/PM2odMKRVE82ZCaw+Y/p0Qu9FIiL2/zwSejSinNEcCUKfcCZduMzItNusAejljF7pY6efFpjmbOtTn04yKymziawYjGIGV73qkheISj/NldK1XGTT0LGH+Yo0utJu7AC9ErtqXawfNEO4vYll5esW9kiu7LxJW74taQ0T11jqP18haOeP4KUOwQVVzWQSxt56Oy6XsFzN1LVTyQZZncIPvAXwLSZ9j2xZ0yvejhuQc6lnD0O7sBAPZuXrfttKytSAK5PLUjpzk46nUfgjejiQWBiSeoMTQCMg0V42jopRT8JVH5J3jfQ8vc1/VeFTMP8LlKH5Ox07p6GlRk3YWb6qQPs+zYJ+owlahch+XdKtfISkVww6Cag6IonmxIzaHSJuG7flmZWGcsPQDU26lVdL5CiT79sEXvqzJ3HKO4D5Qi8r2C2Xv5f2RB1hfc48J/W/N8Fxrdebg1IlTbKFfD58T918lCyXAZgx1LKyvcc2PtWyxwBAofv7LGsRrkh9ie2Ro1hu73qMXM3B9Axwn5/wH2dXkb+26iFmsaLnvflOCnh9gXPW+1hnhn/G2Msw2fO/AT/NaBNAAg9Sw1j0KfuI3zQGc651nH8mIE4Ztw31baLLokBLzQI7YUuaXLB8pIHBXXsAR5RWdYduzLNbT9hJWvpN06EcnrETIoy7aOE1RVF3dR82g75oevynNUUrIiONPSjDYiqjkoiuLJhtQcJj7Mz/iYhOqOWVQl1HmxnfJu/kHOfYd+YLCwW4JhBljIaQLlVKSZfyAoi3JcqHRmF1Af5vw38SZHP5c9qO1U6yU8uSF+Fp5n0M5SyqLSzaFu2zauzsxXOLpMj7W/ry2RMWo4A0+ONbdlFsXavcYv+nHZrOLT7KuJJ8ROkGuFAPvFBNDxPDvN+i0K/Txu8uc5evb9KdtX7PBhWVzDbp4OMcmcKfSiIeHT5faWxgAAS3tqaBcXZqkm83T53rdpAZnxXgBAKHv9bQuUDLKbxMYgofVxsTm1nQvCGVzm7mYZ90zUKz405D6nzrNf5u51XpxGM0ju4qdYT7fIKrxomxqDo5nfYYOyIYWDjfGmFSX9V2gJyOyXqcM5cTFtd2nKAug8Jg9emk98OCvLbPv9qHTyXFlxs7Wd4jUyewC7LMuyxcDkfP+VpEUou3LtgXOz+ecMInP8wc8dH1pR75RHW5xQA4CZLE2PseNr47q8HDc9cFGiva+Jmy5oUGyXVZWdLBSRWJHFuxsIz3FfY46/qMx2WbPQaK1WzA2wTE5Wrf7gzB7US7w3+z5/AgCwP8nFMQeiF/Ffv/ULAIDAEfZB+EFesCNewNRWughDN5BuLT5q37dUevZ+3r/+V+oodLN+Cflxu1WnvsUgChKnUItIv7whCV2OzGLyk8wqFJsQA+a7rFtmR6iZLq/7ED/zfRtbMd/YtVcUZc3YkJrDjt+lJM8Psfr1kEFoQUYvMV61HeHoHZttYPY+UVVlqYL1iWtzpg4cXhmlt7TTRV8C3a+xnIsEdMlWaxGD8LwYm1K3pjqWdrXC/8rHGewTvck1GTdKMCsakaTbyxygRtV2NIjUBXZkTqIYnetv17/LYeRzogP5nUYlK1mnfTANp7pLzow470M+FUaonW19oI2ZYH44zYwyv/vOR1rp+WS0z0zRitexvYCOLs4nijHJnXAd061GCKiLC9oZItuPs24z9/uRkGQ0LumwS98XmfU1A+KWN1NznD0g6z0e60NsEtJm0Q76g3I9g77X2GdTj/CBSZ1ZnWfkTqGag6IonmxIzWFpOw2ELmW4DbRGB5dWvNzmRoQG/BL/7jJCVZPcl93ib068XUhw/0uU/tlNAWSHXfYjyZB0QBLVngjccrx8oZ/n3DrICJlyLQB/5faOMNE59lWpkyOdich6kf4ACmIzcCsZnQtv7v4U/EX+3/6eJGIVl93yJmBpi2gaZQlHloSsphFAPUJ7wr859UkAwOOfOAwAqP6gD/P7Vq5gTJzmiHwh2YHNPbQ/LIqWFbyOcPJazCBQWGkvWbib34d+WMeFz7INm/+Tsw+I+7LNNLUBl2qwUxLvFrp9WN7FPvIX2PbpD/P7wDMWE0+IphmUwLGdsl5n+prVXZeo5qAoiicbUnNw7jG3SrLUadAIiMYgWoEbNaYe9SMg8TXFAQlVnuCBptFyXc7fyxHAJ660crtpXidYkFWSI9yX3VlHxzuSxyF5c6N9vXelYWFytg2JpSsUXiNctqeuw+yXhqTur4cspp5if5QKktNATCOBUssF6ZcmzDzAPtj2pzlc/BS1Azfa5vp5DesDqkkZyWdY/vC/Yhx18V7T9Po4mp6G8WgziKyzkzeyGJK8GldZiFXstc08EEnJuZAY4T0r9Bp0vsltzpXZfYgny/eHsHiXuHidJ0Nckss7GggtunT1bMvmP2OZ6Yf8qCfZiLaj7DNno9qoQ/CGFA4ulj+c4fdq0iJ1Rm48kzIjfVbcUbEG2iVGvhYXP74Yn3IHSuh5hupsWeIj8uLD91UuTWQrKxPlgTCpCmLirlxK3lwX9nRzHUW1LisTz65tNKQXW77JKU12D+dbzv2YH7To/yH/n3mQfeCvsH/ik3Us7JU2izAL5sT3/3QCPhEYk4/LlEx+mDB8ixjQWgU6+yB3dR5qxT5cTmzSYKSPa0y29HMZ99wWmd6dunLfd71jUQ+LoZVLOmBlClldMuiStTGL+2R68KDEamyrNKeaoQwNoNlu992HTd+jZKykWX55i3tLGJASN/qsxNgEuihRo6/Hr1jP9cwGlWmKoqw1G1JzqLRLinlR8dpOA9F5jgSz7S4fA6V3eC6IYhePq3ZS7Qsui6o8GUbqLP1i5ZSsSBRDWiMEzP+cqBgjjFRybtKhbwQRnucQubT1xrrQpbbrknwFLnoydJunFAAw9ml2TFFGxt432WfJMWD6Fznqbfp37KvQPPti+uG2Zj+4iMC2s2yTaQDRWXmvxl0cdV3Ku8iCbQYa5QflXRbyOoFi95XT3xnb0qqqPdRmtu1k0pjJ8aErvrdj/m6DuAQqBWWxa0Tcz4sH6pj18765JMWJcXkz11ywOV1Kjss96mMf1KIWMwf5nAQ/Ra1r+QQzynQdsljeKs/jcZmuvSd5Lh70ruN6RzUHRVE82ZCaQ2ROZJpMU31Vi/EPu7cpiSHyYUr7nrdrzbl0/Jxk5HFh/zVg5LPUCtwI4rIjLd5TR/oljhLFHhcWy9GlkvBh+kEe51KtXS+xzStzNszMMPApiUtWMko1b/a9mtdLoZ/tcWntspLiLbuzjsAFtm9O3oIVyrJS/qpFTN7APXfAJePl5+JeIDpLjWHwaUYZZf/VMABg+iGg4wivm7zIEdllaMoPNZAcufI45bSqsdO0TG7dw0ikyl1F+N+RHAqXhUq3HweWt6483rmfw3N+tJ9i2yc+KmH3kvJ/8Z46IlPijpU3lc0fkDerFQ163pZcHX9AjSEs2sL8Adt8hty6lKlHYp512yio5qAoiicbUnPIi7W6/8citfeb5gpD5z5ywTeFbn/zjchufb2bp9Yen0f5Rc67287znPWIjIKNVti0T97tOP5RfvfnDQZe5HCQ2XlrXegWL+YHG4hskzDhrOSGWKPckY7e1yRxro+jp3s3pK/mR0hCqxf2t96EzX2tVwC6d17OPci+Sx8LoOdNuhtzZ7noLCvJWZPnAF+V18nspHrmXo5zebr3KxEbk3deSoauzb3zmIjzOqGVChlmnqwhcUoyOp1l/Qp/OwMAKB3qbHa8e4ammSUfQz8wWN7M/8ttLjGteCuWWx6VthO8V5nd1PwGnq83X3QD0T6bz88G1Rw2pHBwGX/dKrv0cWDhKergXc/wB+Vi/PN9PlTaxb8u70AML/FuRf7PNIyspFvezK5wyUW73rSoyBICJ0xi78oq0JzFxc9wmzOq3SzbhiV77TBwbpTLvmOnrn/14a1QklRw7SdpgZt6WOIHSi0hGx+VZfGTEvfQ5msuRV7ewR+7cylWE8DIz3Aq1vsWyztDX6ndNJPzOLdlPcxfz6Znypg7cO02O0Nh+VW6NvHUGCo7eN9Db690BSePB5sJgNzbz+J/Jm8NCxs0AqzDwj739jRJBRhpNI3GbSdlUJmXJdwXGpi9T6avJTcd5TXGP+prvp/VvWg4vCgJiKK6tkJRlA8Qxjq99g4S7Ru2O7781TtdjdtCSfIjmF1Uv8sLHPFiFwLNiL/Veg39B53iwzn0tVO9n32JeRZuJCGMQo785lffstYevHy7ag6KoniyIW0OG5mIzF/xCqODbo914YNJ+O0E8k8wWMuFtoeyG3N+vx5RzUFRFE9Uc1A2LL4qUHuOrujYNcoqN45qDoqieKLCQVEUT9bFtML6Winaa/GVrlX37sr1xOV1XG/cyT67Ut8E8mZN+s21dSM8N0Crnuu1fpeimoOiKJ6ocFAUxRMVDoqieKLCQVEUT64pHIwxw8aYHxljjhtjjhpj/p5s7zDGPGOMOS2f7Zcc8xvGmDPGmJPGmE+tZQMURVkbrkdzqAH4B9bauwA8AuDXjDF7Afw6gGettTsBPCvfIfu+BGAfgKcB/JYxxr8WlVcUZe24pnCw1k5aa9+W/7MAjgMYBPA5AL8nxX4PwOfl/88B+CNrbdlaex7AGQAPrXK9FUVZY27I5mCM2QLgPgCvAei11k4CFCBovnoEgwBGLzlsTLYpirKBuG7hYIxJAPgmgL9vrV2+WlGPbe+LfjHG/Kox5k1jzJv1/BXyiyuKcse4LuFgjAmCguEPrLV/IpunjTH9sr8fgLzgHmMAhi85fAjAxOXntNZ+3Vp70Fp70B/fmG8EUpQPMtfjrTAA/i2A49ba37xk17cBfEX+/wqAb12y/UvGmLAxZiuAnQBeX70qK4pyO7ietRUfAvArAA4bYw7Jtv8OwD8B8A1jzN8AcBHAFwHAWnvUGPMNAMdAT8evWWs3aP5dRfnp5ZrCwVr7IrztCADwsSsc8zUAX7uFeimKcofRCElFUTxR4aAoiicqHBRF8USFg6IonqhwUBTFExUOiqJ4osJBURRPVDgoiuKJCgdFUTxR4aAoiicqHBRF8USFg6IonqhwUBTFExUOiqJ4osJBURRPVDgoiuKJCgdFUTxR4aAoiicqHBRF8USFg6IonqhwUBTFExUOiqJ4osJBURRPVDgoiuKJCgdFUTxR4aAoiicqHBRF8USFg6IonqhwUBTFExUOiqJ4osJBURRPVDgoiuKJCgdFUTxR4aAoiicqHBRF8SRwpysAAKYBBAv8P1gwd7Yy18FGqOOd4mp9s5b9tlHuyUapJ6Cag6IoV0CFg6IonqhwUBTFExUOiqJ4osJBURRPVDgoiuLJunBl3izRmQYAYO4+YPiZGgCgFvMDACoJyr1GAAiULAAgs5Pb4uP8nh82COR4rvTZOgBg8jGW6X29gdwAz9V2geee+eUiACD0ahK+Ko/reZs+2OmHYmvQwrWlHpLPyPv3BdhU5Lax7YFl9kUgbxBavva5l+9iBw08w/7M7PDDytMWm2T/N+R7I2iQGuV1lrZyo7u3y9ta41dA3N3+Ej9LnYBltVAergAAEsfYqHDGou0ct83dE752ha+TSpKfkXn5XGQ9Z+8H6nH+nzrFNtSiLNMItY7vOMrnrBpjuwr9BoVBbotfZGNc+/JDFj1v8pylDpavh+kKXb6/jOS7q9cuL1RzUBTFkw2tOeSGKNusr4HspiAAoNRJyZrfzJEoPBNA+jRHKicKI0uUxrnNPvS9QTG9sJvDZ/I8jw+ULGqiDCxtYTdFXuCwYSxQTnPf6S9zWEidXu3WrQ3FXvZFtbeCdEceANAbL7yv3EKejd8cZf/4fY3mvrnvDF3zOtGLvB+FHn4vdzaQOiOj3g5uc6MiLDDxOG9O99uiDR5g2d436yi1cV94mfvKKdEAJ4D8IMv5ghx9S91sX6BkcPFp3pvY5DWre934711iXb7JZ8GN6AMv1DD5KEf+/h+zzNlfSgEAEhcMEpOsn7/ENhQ7eVylzSK47J5jXiO7lWViEz5MfYjtS53mZ7mdZdKvhlCPrl67vNjQwiE5yk6MzBmUO9h54QU+HKbOptUjwPJW9rqvzONKaffdoJIMSjken90iN683AOvjuaIzPM49nNMPGyTP8RyxST4Q1dQaNPAWqcaA4haq98Ob5gAAHfIj9xl71WM7PATGjZAa4XXqIfZrZNYHY3nN0CK3xS/yGhMfSSJ1RoRWnMdv/ZbsezLenE4Ue9jXQz/iXHDhrjjqUR7X8SMKd19NKmAt2k7y39W8N/aNNgDAzCP8sXe9yesXevzoedvKPl6w/ahcPwkECuyPhT183mLTMkBZA79ETbpb0nGY3+sRi85D/D+Uq0sN2Ac2sPaRljqtUBTFkw2tOWSHKdsCBSA2TbE7f4Cfu/71LAAgc28Xph7nti3/idI3kOfwUg/HMHc3u6D9NPc1AqIJJIDEGK/jE6HtDJvhBT+Kffw/mBW11o1Y64h61GLb1ukbPm4uF0cu52GlFBLXcQ4/bYFNA15hUx3+Mvs2tMy+q8c5ikanLXw1bktepHo3+imqELWIhfWxj0s9HG1n7k/I95bxrtDLz/ZTor6XLfwVlp/fd4lF8BYJyPWc8TBzF+tdi1r4urkzGefn0hHOAWLjwOy9bGswy+OXt/DZHf5hGRc/yfoVu6nl5WQfAhbRUR5XkalU+hSvt7DfIjmytmO7ag6KoniyoTWHtvMyMuw3zTlYaImfF/4SLWGVuwto/wktN3N3O3ccpXH7yTIW7qI7aPJxHucviHawaFCNc1vxYRru8oc5mgUKQCjDfU031R1abFcPA/ktVFuCi2xfdEbmsA2gVOMtrshnKMCyU3NtaGSDK44LLctxdSBxi5rQpGhrm7/DE5XbQ4DMqd1oP//3aUzIng4hccG5nqmxRGgiQSNgEFqSeyJ9XnRGzq462g87wx73BfPO9edHdphtdlrMrVJJAaVBju7/45PfAgB8b34/AODt0SF8Yfd7AIC5MjWb52aoNn3sk4fxk4vbAQC5stgcXqPBd/rBMBqD9BtHTvM5NaKpJkctggV+mblfXJlBtrPjCLXbtUQ1B0VRPNnQmkNugLItMgfExVU08VHxMEzKSHk4huwWbvPJCFK4n0NXeCncDABKneG5avLdBoDlHRwl2l6mxuACXkK5BpY38fwFsT3Epm+v6uACiMzDGWxL0np/8c3BFWXCiwaZ5/sAtIKaCuL+ipda1vG1oO9l9kdmO1Wr9JkGSu2inVV54dKLHQCAZIEBTQAQpJKGYg/LJC4CobwEsfW5ebe4O2M+FHp5zu53qaFUUtSCph8Get4Qz1P3zY2BFfFylLbQDjLYv4hokM/E8eIA2xlhRFhtJoo/rt3H+rWxEYE4y+6KTeGH1V0AgHbZN3tA3OMXQwiM8KGzAedpY5uqMSA77LRd1sW5LztOVFY1uMuLDS0ckqMUCAt3+bG8nR0aG5XOFNU1lLEoiM4fzrDzc0HejHIaSIzxAcpuEl/6ooveM0icpgq4tJcPXvUiu8v6fOg5xBuflviGRXFRrTXOF166jyr5pmQO585QACTm3i+gnFC40ve1YmGvqPkSgZoL+pAfkr4V96NzAwYLFvWI+Pol4jE2wc/lHUBiRFyfEyzvjHnJ8xaBomwTYR2QaWH6BLC4h+dw0YzXy/IBCoO+vgzrIgIBADJF/jq/8fZBAEDnK7zvvt1A+7N8rswvSKMvsuz/Vn0KvqBEUo7SSPno3Xxw3j1zF9yc1NVzaSfLDrxcgU+Mqhc+w3M1RB6c+6IfqVM31q4bRacViqJ4sqE1h1kx0iRHLNpPcsSYfYCfxd2cQyTfCSN5UQJOJKKy73VqAmMf9SFxUYKZxBXqL/PT+i1guK/nFQ5nZca/IFCwKLeJK2sXy4SW1qKFLdw0IidTnW09CwCAfCWExDnudC7X9YAVJabQJ1GQPqDjXRfcI0bEJL+3nS1h7oBocy4CUEbF1Fkg3y/uYhnAE2OyNmbANKeMPRJZuSQBb0PPLuNi1/VHPzVEY8ltr2Hb8OyKffkKp0bTFzvQ9zzP3yYRjskxVio/FGxOjQI1nswZUOuFCNKP0qX8wE76x89ku3jdIND/EjWVQq9on20858VPhVFt403teYW7it0859BzNczet7Y/X9UcFEXxZENrDrEJiTs/X8b8fk7GQhnu6zokwSNJ2wyfdvP18Q+LYeu4QedRTsKXtnPkmr9H1lZkTXN+Hh6TuP0OdtfSTiCY5TlKfdRCQktr25VFGYG3becIlCmyvqWXuxCoXvGwO0ZyhJ8hmX7XIgbZzTKSis2hEZIQ62gUyQtiO9jK8pFF9nl20I9Ke2utAQCEl7mv0B9oupCLEj5fTfE8E0+mUOriccHctcfAciePG97W0hqyZT5Ty292AwBSGSDfz30dJ9jp0w/Imp7BKpInxEaVkUU5e2n42jU0jY4wbUSbo/TRfueNe/j9rSomnqRmEpegOysGye63GljYx+eqJBpVUdaOTD4aadrV1grVHBRF8WRDaw4VsQE0gj60n5TQ0yHnUeC++YM1RMYp0Y1Mf7vf4md22GD8I7QCD/6Ikj0+xbKz9wSb6/HnRXp3v8NrtJ9qWccj85xfrvUKuXr7yqikpSxHp/gajx43i8vd4LwPsx+pIHWII3EuLV6KHEfIWqzlNnQBQJltPEG506L9mAQ/cQDH4l+jOmJeSDfd00t7xA1YZdnc5gYCN5IGXoo2bOuYeoP/V9pZqXJ/A4lu+hQvbmGAk1vc9ejdx/HKyAFuS/NZ+i92PQ8AOFnow2A4AwD47feeBAAkz7B9M/cDw89SRZ29hw9ReIyaxOIeoNIm2pUsDHT9uZZuaMeGFg5pWQ9R7A40V/MVRO1z8fip48FmhF2hj9uWdlBymDpQTXDf4l28MdktcrzfIjopfvmK+OU7eGdMA8iK+pu4eHviG/r6F1d8H+rKAAAu7A4hMiUxHWtsFL0RCv1uibG4mE+Fm8I8kJcf3WYa4jpeCCM2x3tpRAZOfoh9HR836HqbDTv1dylcoq+nee6BBnpfZ3nn3qyLTS81YjH7oPsFXfsehcUNPLWQwkAnr5eW5erpva0134sSKLJn/ygAIHIPB4znjuyB2cXyiQDb8odjDwIAtiXn8eMJrlNP/5DHL2+T1aTHLMppt36C13AG11Kngb/MeiUucJ+LBLW34bHTaYWiKJ5saM1heYvTsVoawJZvc9nbqb9GVaIR8qHrPeqeqYssPvkIR6DYlG0mHsmIWhoQ41XbKYvl7a0oPQBYlMCe9AnmkADQXC+w1lwaiAO0kq9s2zGFhgSAjVyk3u1flFwWsQYCOQkKy60camoJ28yrsBaGrYjkwMhtlojV3Rn4X6BVrSgrKJNv06hajwCLuyQ9nLiUqx1UIfImgMnHqXLEJT+C0wSjMwZz1ORRTXO0Tpxne5e3+jDwE25b3nLt+jo3cPyNGBaDnLI5t3GknR1UrfoRCrFeZ2foigxKkpnoSAhf/OJPAAD//p1HAAA7dp4DAPzoxG7Ej/CZ65jh8b6quMJ3+OCr8plLjEqQ3pBp1ikyszLJS7Gf1+t92aDQp6syFUW5A2xozaHtPKXo0lY/fDI3swHKu9RJSuZij8W5vyLztuM09IRkTf3SbqBNgm1qkvBzeXsrYCoqqRBcroF62AXf+JprMipt3BaZX9tJ4LlzHG63bXt/fgaX1WnbZhmuN1/fOV1wz+wCjWuJt1bPqurCfLsOsW6Vs+mmFc0lSXU5MRIXWi5oZ2iLjXAeHp+wWDggRrmEjJov8N7mB3zwi7u5611+ljokiM0YzN0jq01v0Bbjgq1Sx11IPD9DAJZ3ceTf+k3WZfq/EGPlniJ+OLkbAPDhPXyoXnyeKzZ73rPNtIK5Qf+KPgCAriM858XPsZ09z/N6tQjD+IFWWH9R1pKstdYAqOagKMoV2NCag0vv3Qi2FhSVujmk5zZTCqdPGuACJXH6DOeOk4+6nAGm6eUoySrA+Jhb6Wkx9ziHkOg5jrDdkiNwYZ9t2hpik5fldVgjXIDN+CSTu8bu4yodYyxSEVr9r5UX8nLiIdpior0MxR7ZRe3EpVa/pfrKorjMDsmsFbewMhC7JLJZCWfPbbEIL7Afo/KKgK4jEsz0IT+63uFx+X4JMhI7UWQGaJsW7XG7827IubcAHXKOYs/qjYHpo+ybkb/K0f6/uevHAID/35ufwuQp2nwW+vlQuWej2G2amkMrPT8/TQO4+Gn+3/9D7mwEnJbgQ+8bfAZdbgqX/Wnoj85h7EvbVq1dXmxo4bC0vbVazyUyNQ1ZlXdSjG0Fi5nH3FJY6rrFPRQSnT8JN11tHUdZZuop3ozEeABb/1Ci0R7j9coplk2OAH5JVuur8mFcazXP/e7FXY76jzqb+8Ylas557IK7uIy4ry17Xed2QsXf5rKi3PpjMb9PDKEitENLBr1vst9n76FwdnEk4XnT9N+7941E5+VdDhMGJYl+HHiB8Q3TDzHLSXy6jonP8X6FIowtCJyjMTG47MPSDkkAdB3v2bheXGboXz7wBgDgt04xbuEzdx3Fewtcxr38HfrT3cAz9GwWp/66NNYv0aFB9nHiokFSUs7lZcpQ7BejbFcF45JKL0nbZjNJ8vgX11YwADqtUBTlCmxozcGKFE5dqDU1hpyonvMPUu1LvxtE2zGWd2V6nuEcwFe1KHWJIVPEZO+P2SX5Ph+qMZYb+gmHv4U9HPHyAy1jkovyc9GXd4Lo7EpjaDYl+cPasqg32LAL49Q0IheoPZW3lbB1YG7N6uTccqEcP8ttBuc/J+/4OMMyfrfKcryBgiRkcW68pR0tY2J+K7UIX4VDcfosDxz7qB/pV9ieYE7S/X1etMIfRhDKyRRDEqbcKuU08NHHDwMA/sOLVCe7tnJK9p3Dd8O3LMmKJTnNwLPcd/y/bAMkn4NzM7vExNUE0HGcGlv7YdZ9cT+joXLDoWbiYveahGaEZK31/1qhmoOiKJ5saM2h/bjkcLg32HzDVVRWavpllWR8ut5MHhuUBKqbvk8JfeHT0aYrzKUSc+v6fXU0XWjZrdQYel8X49cOfzNpaSt91x3KMOtBeFYasfuSYCnJUTAWSwMANne8379XL67e4+DyT6RfZ0qn6Y8NInVa8mO8LWntPiUp5tt9CObcXFzC3mWVZvroMkafZhDU0m6Xpo/ti8z5mslmm+tsCrywabQSCjs7za1S7q1hqcpnYdMuupQ/O8Cksv/HuU+g72WWK4g2Ova02IVMFdERak3ueatKyv5gFvCVZQXqP+bOzi9xeWb21/ah3C7a8Yi8emGfuOUvWjTCa/vMqeagKIonG1pzmH2An4lRIH6BTRn+c87zMgfSAIDlTX4EZA449BxNvblhSv/UWSAlL1FZ2krJntvEc4YXDWyUI1Xb25It6iLVhNByG6Iz4gERW8M6SsLUfHP4yGQntvSvTKA41Jlp/u9WII7OMjY3cWL1/LGJCfbI8X/IMGP/NNAm2boW7qLtwOV1WN5u0UPjf9PVl9kltoftbc1cDyXREqKySCsyX8foJ6kduIxem5kxHqV2iwTXRqGavLUR1mlBm7fNYKlMr8PEAu0Cv/XeJ9m+ukFZFk4NfI/a0thn6b3oeTGAalyC5SSP6eRT8oKlYhBL23nO2rfkFQr/gC7lUBaIjvCcs1+gVtH+XXpjlreYZjDfWrGhhUMjJq9pv6uO2HkaIs/9Eh905zcfeG4Jkx+hzlmL8UHKDcr6ifN1XPi0ROKNynRkH9Xt5dk4QrLaUTygmP3/0BDmewlN99paJ9y4Gdw6gcixKEYD7I8OyXrsYhsyxQgWxtMAgNTJ1X8M3FqJjhfZ54Veg7wkx247x/vmXlqcGzLI7FyZCs4txW4EgewWcSGf4+fiTt6/Yq9F95tuPQLL18PcFyxYNPyrIxSiH+GUzO9rICqZdepT/JEGLlk1WRFXd34P4x1cFK2/YpuGxbl75TlblinPokXmLu6rDXCgSrzLA0NLttkfQXlninOddxxvIDe0thZJnVYoiuLJhtYcYpIqvpqyzfh5Z6TJbqeoti/5URdtObNDIiXPct/svQGERetOTMqaihepZfRPNDD5KYrt5KvyDoZnqCLXuloqq3O9rZbRazUJlIDA6xxx5oaosk5L0pj4qRBS5bW79vCfjvN6H6Nq7a+0jL2FXnnLuWgHNmjR9Y7kc6iLy07ycZRTvubbzZe28gRBmSaGlkzzvRXutQNjP8vzJI+EmhrfzWp3JUkd1xNuddR7I1R/nCvSuZHjMzWMSYrB/CbW0yfJijN+XzMQy0pobXycfZDZ3cD2P5ZkL/fzXvkl3f7i3baZ+yI+6nKKiNY7UoV74/ZaoZqDoiieGGtvU0KCqxDtG7Y7vvzVO10NRVmB/TCzb3Un8s1tbnVsa8XmxufIb371LWvtwcu3X1NzMMYMG2N+ZIw5bow5aoz5e7L9fzLGjBtjDsnfZy455jeMMWeMMSeNMZ9a3aYoinI7uB6bQw3AP7DWvm2MSQJ4yxjzjOz759baf3ppYWPMXgBfArAPwACAHxpjdllr15O3T1GuSTiw8pHNV0JNr9hPA9fUHKy1k9bat+X/LIDjAAavcsjnAPyRtbZsrT0P4AyAh1ajsopyO5k/34758+3N7zNnOmnkXYfu67XghgySxpgtAO4D8Jps+jvGmPeMMb9jjHG9OAhg9JLDxnB1YaIoyjrkul2ZxpgEgG8C+PvW2mVjzG8D+Edg2pN/BOCfAfjr8M4D/j6rpzHmVwH8KgAEk+3vO0BR7jTJc3QVzp1jhFXyTlbmDnBdmoMxJggKhj+w1v4JAFhrp621dWttA8C/RmvqMAZg+JLDhwBMXH5Oa+3XrbUHrbUH/bH4rbRBUZQ14JqagzHGAPi3AI5ba3/zku391lr3to8vADgi/38bwH8wxvwmaJDcCeD1q13D+oCqvF6wllipZFyeUl1ZyeX9tdZ43Y9L67DR7tft7r8b4U735fVMKz4E4FcAHDbGHJJt/x2AXzLG3AtOGUYA/OcAYK09aoz5BoBjoKfj19RToSgbj2sKB2vti/C2I/zFVY75GoCv3UK9FEW5w2j4tKIonqhwUBTFExUOiqJ4osJBURRPVDgoiuKJCgdFUTxR4aAoiicqHBRF8USFg6IonqhwUBTFExUOiqJ4osJBURRPVDgoiuKJCgdFUTxR4aAoiicqHBRF8USFg6IonqhwUBTFExUOiqJ4osJBURRPVDgoiuKJCgdFUTxR4aAoiicqHBRF8USFg6IonqhwUBTFExUOiqJ4osJBURRPVDgoiuKJCgdFUTxR4aAoiicqHBRF8USFg6IonqhwUBTFExUOiqJ4osJBURRPVDgoiuKJCgdFUTxR4aAoiicqHBRF8USFg6IonqhwUBTFExUOiqJ4osJBURRPVDgoiuKJCgdFUTwJ3OkKAIBpAMEC/w8WzJ2tzAZjPfTXeqjDzbKR677WqOagKIonKhwURfFEhYOiKJ6ocFAUxZN1YZBUlLXEyhDY+NASAKA3lcWFmQ4AQPyN2J2q1rpHNQdFUTz5qdAcSp0WABBeoNuqmuD2cm8d4R76UMszK0eQ4KIPkfn15+ZqO18DACwP89ZF5xuoxlhP62cZX4WfkUwDpsG2j3+U40DXWyw794BF5yH+X+jjZ9d7VQBAaLGC/HCU5Q5wn+uLSgqIT/Cc1YRcV4aY1Egdk49z25b/xHOd+wVWKn0sgHqY5Rry1LnjonMWlRSPi87y3At7ua/nbYtaRK4t1yv1yHHTFu0n2diJJ3ny0FKrr9z5s/tYZksyx+tbg/pSCKtN+0m2eX5fEADg461C76t5XPgMn6/UOW4zDX5WkwbLO/ilkeQBpsQ+i475ESixXGKsDgCYOdgaz2tJboOffebL87jInA+B4q235wMrHBq8PyjdW0B/J5+YaoMd6zciLAK11gHtSyuOr9b9zfJzh/k0xibvvLCYv4u3bPAneQDA9INx1OLcF15gu5yQqIcMit38Ep6D7GOZRrSOYg87qdDPhzM3x3PnHwwiOs1ywexKAVDaVIG5m09s8KUUAKDtHB/S7LAfiRGWm7ubP9b4RX4PLVuUOkQYbWZ5G+I1Ok4aLD7JH3Dv/+mEA4XT9MNA4gKPCxS4z1/i98wei6XHWPe2l7ivHm3do9xWXmfL8CwAYHy+DQDgOxVHanH17+XoJ9jXXe+wLnOfYD/5ynEkL3CbE8Slffz1xg5FEchxm1nm/QgtOYFsYaWamR08d4C3HUM/KmLuAPuowmY171Hfa2XMHQjfcnt0WqEoiicfWM0ht4cj0baehea2oJ8jyVyOQ+1cJY56zb/iOCNaxXD3YrN874FpAMB8vg8AEFpew4pfg/gk6zf5ONtgDeAX1TM+zVF06hHK/FpnDaEpjkZOpV+QUabnpQBgeS6nEblRLbwANIIyeok2ktnF47peDKIekXNG5LoXqa7PHEw1z7V0F/suMsn+zQ2apqobzMqYxFMj9c4EljcPsl0f4rZGhG2JzPiaKnhkkf/4ajzeV/UhcIKVcFMcX73VVzbcWNF3tXmOtMk10BoAYOAFNmj2XtF0LkakvhbBPPdFF1in8gT3xacqaATZnuwQ+2ppD8vUoj7ERIO7nMXdEdTYHFQTLJM6z+/nfskgdfTW26Oag6IonnzgNIdKkp99/YvNbcslSunldzsBoDnHC9aAcG3l8W6Od25XGFu2U2NwtonqLg59/vei8FfWpPrXxNlShr5Hjejiz3YgsCzz2W7K+u63+X3qMw0kZTSpi1HPjdZz9zUQnmd5I6Pt8DNUiUY/kULyIkevxT08zl9mmehCHbP38LEJZblt/KNUR/perSE3wNGv/T2e22ks/pIFpAq+ivQ/FQ6M/NJg838fbXpInRED6uEiIArA2S/RiJg4K3UqAtF5GZHlMzd4yXgX4IFWbmp4dqWWuNrM3sPzxyad7Ue0mZrFzIMs00ixs5PHWbbQF0I1yfLJEZbpfoOfy1uB3BDPkTrPMvP3iJ3mBG1KANDzDh/GShvvS/z06hhbVXNQFMWTD5zmYAMyjw5Wm9vmzzLgJTl37bmmcfPgkwGMGHoptmybAQBs7p0HAIy2DyI6fWc8Fz0vU2M4/g9ERTJlFH3iWRjjMB04ybqFRiKYf4yjSvptjibhJY6m4Tkf0mf5/9JWjhEzD/KchW1V+CtUUYzYJeqisSzcFUAj5ObBvE5sRuwS2wLwV/i/00YGvs++u/iFHiRGeb34GPcVe1vaTKDI45afoms5dIg2lYkPRVHq5nHt75qVbViqY+phVqyS5raEeEdqEWCbeCkmM/SqXOrmXAvi4yu9RX5pU6HXwHZR9QpM8R45DanjWBWVFA9wWuHiXUb2NVCN8/+WBsbvC3eZpq1pYY+4cbPyHORb57oVPnjC4TJd6Px4F5IjN6cgpU6wh5cH2PnpKO9Gra8CTN+6q+hmmHuQgi52WmI2UrbpO1/cJz/S3RIXMFBG6l3Ws/sd+sDm99PfHpuymH6Ix7kfVHyKP7DQT/yoxkRNv0fmExkRFnN+hDIrBaNTdevxOhJn+Eglxnmus3+1m/vCDfiLvA/lDrnuKI8rtxuEZGoUOEahkD5L6VLs9CG0xONKnBU2hUqxvwGT4D3p+QGFX5GXQ2Hnrc/7yu3SrkhrW3BZpkT595cvfYrTssoZCiMXbxIft0h+n/Wb/jg3xt4OyjX8yOxg+9pEWAdkGXluwDR1+8QY9/W8xTplh/0ILYlrt8zPYpev9T1464OXTisURfHkA6c5+HbnVnw3i7dunFmY5UiQ3sRRKpkuALgzmoMLJAovugCiBkxdplLUopHZxtvadiqEYq8c18P6pi7SuDp3IIjkCM+VH3IjONVbfxkobGK52HEOm05NjU82mhGLVYlqNFUxjL3sx/IWliuKcbTzPY54c/eYpkuy45hML6aolUwfjLaiPGXAW9rKutSiLWOc29n3Eo3Ni3e3IZxhWycfcxGWLNrV0/I3ly8yJPZ6NO3luyvNY9tkaur3tVyilTrrNXG+CwCQOtX6CZVHOC3b/ANqB2Mf4bMXWbTNUX3zH8sU7j5+Dr5QxPx+9vHSdm5zxtnQkkVQAr9KHWLgFS2mHgIKcm+3fLcsZejbrIdXZ8qrmoOiKJ58YDSHqiyNiIYptUs1Ni2QvXUpGpiTMWfTLZ/qlkldlECuAz753sDYR9nWLd/hCJLb7trsR1hcfJW4m7zyIzrdWs+w898xtvrclzhhT15ooPOYlJugvzK7lbaAQo+vabR12KDMgzf50PsmR9uJJ1mncjuvG14AanKP3Mh29i9Rm/GXLbL7ed+CUzLayjoYmJY7ryI2gNFUO7+3WYREc3Cu5YYYA42xTRd2YvT9Y6CzTVWoFCJ1kCrHtnC5WaZQ5X2fkLDr9lQBqQi1x+5N1F7yo+yzQBEY/JEEoT0kBkJRXoqdphnAVexkfXvfYIVr0QDiEyvX/Lg1GeVOg2IP9yXE2Fm/pJ3D3+MFLnyuTbaJBjll0FgFb6ZqDoqiePKB0RyKQxS3/TEGKl2Ypmk7vrA686/1wvhTHB3aD/N7/v+xhMiLnP/O7+VIuet3OdpPPJlEKOfmrBI4JO6vatI0vQaNCIeZbf/iOM/zc3tQi3DcmL+Hw9nydpb1lYGuw62FVgBgaq1zz9zP0dYt9EpMcMTM9/uQkgVaE0/zMzrC6w4+X8TUw9Hm+QE0VyryvDz/pu9x2MwN8jh/2TRdfM51mt/Ef7pCFYzIqJ66rA9rEaCwmc+LC3RzLJcimBvnSBwbYVvi0mcVxHDxQbpah7qpOSxHxMtSNJg7wJ/T0HM0Gpz+FbYpft6PjNhg3MK3UjvbUE0BxQHWJTIt2q4kWy51tVQ038hKj0R2bwXVJOuZFJtMQLwWS9uMrsq8lFQ/fxBuOuE/F7la8Q1L+3sSWdcvBrjvdcHK02/lbs7dyx90NdkSCtWki3SUVZkBIMywDZz/BZ6g/2Xq/dlhg7CsDEyf4TShICs44xMW1rdS4LqVm8mxRmtZdefKMvUQMCtGuIREB8YnKQAuPB1BdIrl2kb4Q0lMsuziztY0pprkcTMfkSCBqg+pE2y0i4VAWyu+xeS9IyILOyvYtonxFw0xcroVm/5jCaSynocBAGp5mWJ2v39fpU0iUx9m/ydPc3uxz8JXlpiQexmnsnyKU6PUGYPwIttQl6mAi/kI5A0qbWzX5fEOvT8OoCz33bl2CwMyrZi4cv1vBJ1WKIriyQdGc3Bk8lTlIh+w6YQjNidRjbvESLe5ioH/5IxyLjqRn7VIEOEM/89z0WNz7b+/AHSf5ijtk5WpUw9xVIxNWiztkgCbIs8dm5KgqGGD9JmVFkmnBhe7fGg/wXlBuYND3Nw9srpzFvCJyzMveRYGnueB7YerOPNljqRpCeia/BlOIbp/GMbyVklQc7es6ZBkM8kLLXde8qxP2nnZYplLqErei4GB1kpdN/VIHuWwbRrvO8yT2Sy1A5d7AQA635OkK/WVa13ajwHZYZaxP5DprvzyjLVY3ixuaVlJ6QzFvgrgrziNQT6jLLu4x6AeXplzIzq1umO9ag6KonjygdAcGgEgKIEqtfoHW941A4mOyDyzN4zkSY6EJ/4ug3ASZ1rhPgv7V6aQS5/gZy1qUUmyr5y7zAUnAUDjPPct75RR7bDLtASUUyu1MqeNBIrAyOfFKPcMR/AFyVxVabeodK5Mazb+FOsbmbeoJ3jtzHbRgqZa6w3azrm1FBI8dY6LJBbua0dM7InONRmQlZg+01oF6ihupTbSH6jh/IQEMR2+QZ+fnL/REBvOJZrG0k5Z5SpmDzfK57ZYhOedNuDWW7BMaMk0Q7GzW/gZFsUmdaGBmYP832kTva+zDycf8zdzZ7h1GiU2CVv/ZBETT7XfWLs8+GD/khRFuWk+EJpDob+BLeLCnFpKrvr5633lFd8t3jco3TZczgCXk7DSbnH2lzlKRDtoZre+lu2gHly5cjI6LyPPo35kn6CPrv374nIb5/fp/7qMmtghur/BubXLkxDMApl9kpPgHMtUxaKeGDXNoKdpyV/QfkI0gp0+JE/zces+xP6c38cykcUG4iM8lwvyagRE1bEt+8bMfTze3MN5e2TeotgtNo25lf3UsKaZu8Lhj9Sb+4KXJ/K4Di5d6elc5ZeGZJe7eP49/yvLjH6emcMCEwZLe7kvUGS7+l/hcD/9UBB1WeXq7AvLu1jWNPwY+hHrWRbtbOwp9m9k1jRDqaNzotWJXeL0X21rrny9FT4QwmGtcOpyf09mxfbSiTSit786AFqrAZ2rsOOIba3KG6dva3knf5D1kA91WV05L2sl8pN8nDuONVA7x1aUuniuxX38Hn0+hmjOPbB8UHveoOAY+WwUwaWVCmdokd+rSWDT9/nDX94ckjq4tSBAcpTnmn5QIiPFFx9arjeFQVHWELi0aoGyxfIWWZW5hdOCrhfZhqWdQEqMo3mJorya0I68x/YVPxxEf7tEF+6UdG0XVkZaenJwCVVZWxE4+35XuUnz4MlPUSg0V3NaIDznXNCsbzUuS81TDXS/xWIzj8qUSFavBvK2mTynuRxbUuxFFixSIxQwi7vY1279izP83io6rVAUxRPVHK5ApQ2obOfQFpE0cROLVCUis3fOTdpM37aLcn324QZ6X2J9XDy9X/IBlDsbTVdb3084Ak0/zOOXt/iaRsqkRN8ZMbJlt9URmRaDYICPyPhHZI3EvGkmVnHjdN9rHMGWtgSwvIWVcMFXZYny85WBapznDIrrs/MotYwLPxtE2wlJSX8vzx0dl+jLOpA+w22lLtYlt0nWG1ywiM+IkdPH8vbglYd+pxVkX+hB7kEaNbfspEVzqptal3k92VzbUOyV0XontYzuZB4XzjIBkFdq+23/iuXPfol16ntR3I8hg5kd3Lb3H08CAM59hb7NzncNYrPsvz3/ewYAcOr/zanxciXSjD4NH+O506dbEadL26gqZLdzW8d7LJu5uwhM3/qbvK6pORhjIsaY140x7xpjjhpj/qFs7zDGPGOMOS2f7Zcc8xvGmDPGmJPGmE/dci0VRbntXI/mUAbwlLU2Z4wJAnjRGPNdAH8JwLPW2n9ijPl1AL8O4L81xuwF8CUA+wAMAPihMWaXtbZ+pQvcKoGiQbm2OkqQyydQ2V7Eln7GF7vVffY4jXPXGyizFszfLSngJOVZfdmH5c2yTcJ+3Zu92o8DwYIYBOWlKC6JaWK8gXIby80+ylvjS3AEa38hgnBGUqinxaA5wXFk4PkcSt0ROSe3hRZk8cHWBBLjPEdN1mvkYzxPesTXtG2kJfhq4kmW6TzUMpTGRCszDckE1eVDdIZDfkJSvbs1B0u7LQoyJ0+fEs0jzzLpaAn+TmomjQscRV3a+kAJwAvUAsd7qTG41buNAJDby+Oc8dHZGS6c7VmRv+FyZg7yOv6CWwchoetxINHL9RbHv8potLZTPKaSMjj/BP9vf4sBWT3f5vGRxSpm72UfTX6Sfdb5KrUFf6m1ejOQ53U63+MDUA8lUV8Fo9g1NQdLXAaVoPxZAJ8D8Huy/fcAfF7+/xyAP7LWlq215wGcAfDQrVdVUZTbyXUNt8YYP4C3AOwA8C+tta8ZY3qttZMAYK2dNEaysQKDAF695PAx2bZmROYMsiVJripRKc2EnOUrHUXcev7SFhaMt3EU3HLJ6pvlQ+I6W6OXodwIO75OH9Xxf8ju9s0HERU3nstl0AyRLhtYn1sNyG1B8ULEx0vwVdlJkcmA7OOnNa33U7rQaPd+y4ufSjQzFTkN6txfkkQEBugQ16Vb8dnEtNyvDefBkMCgzC6LOffeTXHrRaZkhWIeGP241FOyPDk3nb9imvV0Ls3gu6zL7MNVbJIXGk0+wiG2dDYp/dKqVjUtLsYdbGhPW5YBVGi9/KhwMg0ASE1d/f67l8w023U3tajYhSAS3xQNpW3l/Rj89hisfwgAEFqW0HjJCDXV70f7EdbFeWgq6VYdWtmzWObip3mNSsoivgqLr65LOMiU4F5jTBrAnxpj9l+luFcP2vcVMuZXAfwqAASTtx7N5ehKyDskH5Lov/LVm9iZ5pN+abZqgIk+pqfTAIB45s4LBcfU03yQOl7i93KbQW7YrYOQh2Un21TLJZpvgnLTkJq8S3L23lhTUFST4kKriEsy0Wrv0HM0ys7vo7pu6gblDvkBSzZvt47F+oCJD7Fcxwl58askmSn0t9YA1JZFYZWPyLxBXh70tsOt9HAAIzLduzcW97t6thLBxGjfQ3B5ZUbs2mvtWHiE23rbRNDf3xL4TgC4VZmXcn6M6n3sOIVS7Dpz1bqXAAdKLvGrLF/PWISy8g4N30phNv7ZoeZK1FBGXpK8RfrwkK85dcgPuFW10t58692jHUclsbCst+l/pYHlzbf+jo4bcmVaazMAfgzgaQDTxph+AJDPGSk2BmD4ksOGALxPjllrv26tPWitPeiPxW+85oqirCnX1ByMMd0AqtbajDEmCuDjAP5nAN8G8BUA/0Q+vyWHfBvAfzDG/CZokNwJ4PU1qPsKnOo3LfHtvVdblO+BG0FcPojpc11InlnbNyTdDAsPcCjpekXcejuraDvKEcqpqoFXXL6x1kpENy1ITHBEL/T6m4bInpddoA2/j32mgfa3ecDUoxzC2yRVvL/sQ+YTEr00RwNcUAJ03ApQoJV8xa1UzOxquU5b7/ZkW3L9/uZbnqYlV8PAD9wbswymP8pyLjmMewN4LWaQH+aI3CnvtHD6X6AE1F7i3ObigKyHiLFSg5vmMT4qee4v02mjo0EkV+Yovm7cezBzfeKGlWlXNWGQ75eU/WPiHhWtLfN4CfWIC0aT+5DhccUe01w34VLtJ07IW78m6s1oy+RFqhOVlLw3NLY6mu71TCv6Afye2B18AL5hrf1zY8wrAL5hjPkbAC4C+CIAWGuPGmO+AeAYgBqAX1tLT4WiKGvDNYWDtfY9APd5bJ8H8LErHPM1AF+75drdADExFtUytMbNSrBPd8Lj7SOXcE7W8/uWOPomLnDEWv0VGqtD7Ly8XKbBkad7KAMjaeLcNveG7ErSNJOcLj/A4dqlqDc1NLMoBfMr076Fpn3NEasmIcDTD8uLV04BHd+TkU4GX/eeyuyQr2lxcslSy7ISM33MIjbLe5LdJCOehFOnc3Wc/6Kc/zDbVxDzdi0KtL8hLxfaLhqKGIYbQaD7TZZzBuhLcW1w99TNorPH+96XOm41CEg/Lu9g/XqkbnP3GtS6WBmfpHuqyPsxh/8ogEV5g3m3vLBm/oA7n0HxfhpKnQ4bnWFfZLYHmnYZa3iTcpIAOXFBw6cVRVlDjLXvcyTcdqJ9w3bHl796p6uhKD+VHPnNr75lrT14+XbVHBRF8USFg6IonqhwUBTFExUOiqJ4osJBURRPVDgoiuLJusgEZX2tt2TX4pe9MCV/5YCOy8uuF65WZ+XWcff9Wv18p5+Pjf4cqOagKIonKhwURfFEhYOiKJ6ocFAUxRMVDoqieKLCQVEUT1Q4KIriiQoHRVE8UeGgKIonKhwURfFEhYOiKJ6ocFAUxRMVDoqieKLCQVEUT1Q4KIriiQoHRVE8UeGgKIonKhwURfFEhYOiKJ6ocFAUxRMVDoqieKLCQVEUT1Q4KIriiQoHRVE8UeGgKIonKhwURfFEhYOiKJ6ocFAUxRMVDoqieKLCQVEUT1Q4KIriiQoHRVE8UeGgKIonKhwURfFEhYOiKJ6ocFAUxRMVDoqieKLCQVEUT1Q4KIriiQoHRVE8UeGgKIonKhwURfFEhYOiKJ6ocFAUxRMVDoqieKLCQVEUT1Q4KIriSeBOVwAATAMIFvh/sGCu+7gbKat8cLje+67Px62hmoOiKJ6ocFAUxRMVDoqieKLCQVEUT1Q4KIriiQoHRVE8WReuzJ9mOo5VAADz+0LwVbmtEeJn/0s57tsfR6GXbrmuozUAQLHdDwDIDxokRi2Pk7u5uI/fI7Mt2d92vgEAyA1yW25zHQPPc1+hi9vq0Vt0/Vmg2Mtrdx7m58I+nrP3jToK3axzbpjFK20sk7jog2H1UOzhtlqcGwIFH+oRbkNXGQBgfPyefDnWvHQlyc9ST6N5zivRfqqKWpT75/eyTtFZnrPYa9B+nOfIbpK+2s4+D08H0HGM+5a2cV9ouXXeQp+VOq/sx+ichZ9VR+KvjgMARl8flOuuX3erag6KoniimsMdZuphqgnVtgaiU5TVgSL3nf7bQQDAwH9qoBHkvuwQb1lkniNYucuiHuHoEx/jcZE5li1sqSL9Ls/hRubsPmoqbYdCmL2PI932358DAIw/3X1LbUldqCOY57UX97BOlgMz8r1+ZDfz/8tHy/CCRSXNbR1HWKeO758BAJz9+7vQ/wK35QajK85Z6gSiM9znr/D44PK1x7uLn/Kh7RRP0nWkDgCYuZ/HJc9bQKrXYNeh73nuK6WBmYPcOfQc1bzZe1mo0m4Rnea+SorHdYgGUuzwYf4+ah/Vb1JjqO2TGzLrR100xWqKbal0s+y2bdPNOo9MdbJOBd7/2PkgAqVrNvWWUM1BURRPVHO404h43vT9Ggo9vB3ZYW7c8h84El38pIENyLy7IfaEBX6NzPhQ2MxRLLzIUSwxxjKhTBD+Mv+ffIKfpsgRc+C7U5j8VB8A4PwXqTGEsrfWlMVd/qbWY8UuILVGqdM05+vLm9mubf87tYPFT2xHKcBthT62vfyXdwMAfBVg5iC3Oc3InbSaajTtKs7WEcxeew7fdtKPcof8f05sG3n2S6kLyG12JXnOvNTJVwVgxL4TNvLJkqYB5Id4rtgky09+mN/7fwJURgPSL3LdY7ze0r46urbwZsZDlffVtWF5nU29Cyu2Xwh3Inkocs223gqqOSiK4olqDneYnrc4v1zaGmx6K9yc2vo4akRmfajcmwcAVBc5VM18moUTyRKe6KUF/O3z+wEAuSEZhXeWsf3/khHcz+PqMuLNfLgX5TSv0/QGXMeoezV63yijmmLl/WV+Lm/jvrZzDSzcxbGo7zXWvfDgFrYp3rqum38XBjjqtp00brBuzuWz+2n6TxwNI/MAR9vUER5Yu47BNHN3Df6seE4GAlJf7kuO1lHo9Uu9uC13Dyf3yXciGHhRtJ9hHhcXLa3YbVDdRPuFqbOdifMsU26zKO2lSpV4hhXMbmKbN++aumI9xxfa0N++7Llv88A8Fg4NXruxt4AKhztMvo8PYjltkBwVI5Xhg3PhM3zI7r/vFEI+PnhjHWkAwM62WQDAyUwPwrIv+jgNi+Uazxmr+zD6cf6i2k7JqWVaUm43TTdceGGlinyzLOwNoybeRffZdprXW9zlQ3KE/1/4WV4vfYSPX/ehPM7+Io2Nne9wX/spMUIO+FDo5/9Rsc91vUBBUOwBwuNiNXSu0AH2RfKc/4r1jJ8PILTU6gcAzanK4m4/gjK9Kndw4+Cf8Rq5QWDiFymM0s/x/IaXg78CdL3ObdmtMvUQQWfqBoN/zHNMfojbfJtzV6zfhZOc7iXO+zGyi8Jky7aZK5ZfK3RaoSiKJ6o53GHSZ6jPljqDKCclsCbDff/ZX/4hAOCF+Z04MdsLACifpiYw2t4FAAhkApgI9wAAet7gcVv+1ggAYCqXxFxagp82cVQLz7NMfksdHe/wegtPsA63auDK7K+h90Vxo/bIue/h9X0lH8LLMrz7XcARv049GkeKtklYv8WldB4ro5qgSlPqEldfG0f77ndrqId5nYYYbIM5fq+mrlzPSnvL/dv/CjUB51IOZYDeNziqn/4ytZnxj/O4tuNA6nlucxpH5zFOkaILphlYFZsUN65MC9tGqpgRl2etm319Vw+1vGItiOUS21d4g/c0IUqFsUDylGgcU5xCpB+kBhENVmE/vMhyP2m/cmNvAdUcFEXxRDWHO8zM/RytA3mLrBjvIjszAICpchsAYLYQx8A/5a069wWW6XyD3/P9LWNe/ueXAADdYQ49R0f7YQMcrYNigKuLctB23I/5hzjqDf4njk7Z4Vtry85/X8Hpv8nrpN/kaOiT4KTInMGCC1W+yM/kRTGW+oCe/2wEADDz77Y0twHA0pZQ093YdjwDAJh4in7IQpcf6XMcifN9HPn94g2sXqWeiQtoGmPHPsK2d79D48H0Iwb5QVoig+xOBPIS3JRsaSSpc1aO530wFhCvI+ppGpmHvstGjH7Cj+ikXFxcydO5BPsgXMHiBY78SQ9XsgteC7PpmB5j2S1bZ9CdoJH63AEaeFLv3aLR6DJUc1AUxRPVHO4wgTxHoJ43s9jzK3RJvnJ4JwDgTxbvBwAMfNePc1+gHG8u4Bnn6FQPB2EOMkCmXOXt7A3TDRF7L9ocgZ113rkWfXWD1FGOmnMHuC+8eGttyW6KoOdHHD6rcedhkACiCjD0HIfG7BaOdBWxsVgfcOHPtwIAOubZrrGPyb5AA+mj/H/ulzhqmrqzLwDVREDOxeualSYLTwJFoOtQXq5DLWF5s9hk5lqjtTtX52HqIWNPBZpBXu2n6N5c2klVLLxgEJmz0j7WafyjEhQ17kPoQzT2dEV4A/e204X5w7O7kDx7Zc/K5aSO857N98SamkMiLZXC6moOKhzuMEVZbXnmvwqibYGRig/uPwsAeOMUfzD5Pn/TJVjqZPms+Nlzwxbtfj6E/2L/HwAAfu3dXwYApM/UsbRF/PniV4+KR8z6gcQEVelC/yopkAaoy/OZuYt1ik7y+pEli7GPcemkE3CxaZaZfswiMsVyk485FyHb23bKh8UDrGdkmm128SDVOBCZ5Y90cSfV9I4T3Lm4O3jFaha7DUyDAqq0k8fv+LoYbociKIvBs9jDz9n7eK6OIxaBkqw23UOhUOvg9eoRPxoB1j0+LlJljP1qrEU2z/KLE5wqFoZ5zsZ4a2XpjZAbS6F7T/6mjr1ervlUGGMixpjXjTHvGmOOGmP+oWz/n4wx48aYQ/L3mUuO+Q1jzBljzEljzKfWsgGKoqwN16M5lAE8Za3NGWOCAF40xnxX9v1za+0/vbSwMWYvgC8B2AdgAMAPjTG7rLX11az4B4XotIyQD2ex+DY1h/P3cl/fM7w9tYhFKCcqKmOfML9PDIy9ZbRFOPr9zde+AgDwn6Sq3Ag04JeVezWJ9qtH5TMMZHbyHE5VvlXm7zaoRyXXggQh9b7Bdw7Uon4EShwtpz8k0wK3gtLYpirfCLeMlABQ+GgOkaOSrEEMfi73QqBgcf5zYjwUY97UI+yzq02Rut6rYO4eGjB7v8/Psad4wf6Xy/DVeY7MXl4wPM99HcdyGHsquaIOqHHfjj8qYfJDrEvbeU6Nph5ie8s9dewfYATXA3svAgD+8OQDALg25lYJBXi9khibV2u15jVrZokL5wrK39Vmdp8D8EfW2rK19jyAMwAeuuWaKopyW7kum4Mxxg/gLQA7APxLa+1rxphPA/g7xpi/CuBNAP/AWrsIYBDAq5ccPibbFA/cWofpsXbc9yQjgaYLHJ1KYmSrhw3KHWIwW6Bcjk3y829+6Ud4NUPbxFSEx4UmuC+zw4dqUlZoLonBjoMMAnU0RXzbWQ7b+YFbG8WGflRFRdZWLOzhtvEnOacOL1oEZYrc8S7rssSFl4iN+lEYlhDwMdE43qJPcuSXQ0AP93W+dcnqSMGFfredZxm/2AQyO6/8aBd6gggvslz7GzTClNt6pd5hhLKi2UifubDvyceS6DrCi7ugps3f5r7TfzOI2Gme/+LTkivibe6b7TTojVC1OVtgoFPwzeT72nKzpMTImZV7HSitTnap63oarLV1a+29AIYAPGSM2Q/gtwFsB3AvgEkA/0yKe9XsfZqGMeZXjTFvGmPerBfW1rCiKMqNc0PeCmttxhjzYwBPX2prMMb8awB/Ll/HAFwaTjMEYMLjXF8H8HUAiPYNX4cD6oNJREawffvOYLLACJsHuzkv/fY+hkV37Zhruik/sukEP1PHAQDPLO3Hv97Crv+F8i8CAKaTaQC0mmcld0Iow+tUEzIajjVQ6Ja8A5/8/7d3psFxXdl9/9/XezeABho7QJDgvogSR7tGkjXSjCxpVnnKsSLHVpzYqYkrrvKUVanYk0pVPqniJJVJPiXl8SdVpmaLpzySZuKRNdJQMxGpIUUtFBeRBEkQAAFiazTQ6H25+fA/t0FIj+ICUqDI8/vS3e/dd999r98759xzzz2X5oSb2XileJU65jewjtqt0hM9IT6BnEFJBhDcZKWmM/xc2FRHs2RmciMoMztZj5mro+0I25wRa6TtMD8zWwzaD9PqcQFI058Jyvku3M5KE5Dr4/fyVznJyflkSlsKaN5Px4zzY6Rv5WfzaYvF3uWzOEutEth10sPg90YBACf/mI9/gUYCurdM4U+7fgkA+O8TjwHgSBKwNIS6ErxLGb+9Ai4qHIwxnQAqIhhiAB4F8J+NMb3WWhf39XUAh+T7iwC+Z4z5NuiQ3Axg39Vv+o2Bm2L8zsvb0f0gZahL8PHMw78GAPzwhYfQehc9kX/3Dh1ZLx/5LMsGgfceZa9tcjc/jQwnztxhEZWUbE4oOJM5MVFCKcmT9/+DzCZcYYTk9G1RNMnM0qk+ido7J/MMDFBipjPEx5fPiFz7cg3n7pFkrh7bkjpKe7vvVwWkd9IEd9fiAhDCWYOqXOvigIuLuHg7K00GIZFdoUc4xyH0EhsXnYmizNFGVOMyK/N1SfEW9xozZl28SNsJXm+xCzj1L3kDSwPsEsWmKODypTAydQqcxQobnN4qzuCr4Dx0z8vV5lIsh14Az4vfwQPwI2vtT40x/9sY8xmwyzAM4F8DgLX2sDHmRwCOAKgC+DMdqVCUTx8XFQ7W2oMAbvfZ/szHHPMcgOdW1rSbg1iamid9dx3V79Ap9voAbV5PTNeOr53D2VO0UZtPSvDTWh5nUiUMn2L3o3tY0qZvFC0arsOruLTzrGtBTN3IfAS5PmqcYP7qBEF5VTQCiKKSbDU7yDYN/qyMkd+m1kxMUVcUO3gtM7eFGun18zJXZG4b+yDxtgBmJYIzvMB9OUkp1zxSb1xrWOZB5PuWO2D9CBSX0soFXuMNaZlhm8YfMmh/h+Vy/ctzPZg6MP0Uh2btmYSUl6TAOYPWY5L091xoWVsqi1H838wu1lmlNdF2nOfLrl15t6JSkwQ75atrQejcCkVRfNHw6VVmbosMUY4HMLuD26KScyE6J6nN/74brSE3ZMe+eHaTmwLoYWCQ/eZzs3Sudbwn1kgw0Mhm1HKKxYs7loKMPuzM8x9ounSSpyvIbAw16geAdhm2nN4VRZTNRDXKna7fHyxYVCVVXGORGAmKWlzjNeaf2B30EDa9xFDpcvNS4t16yKXUu/g1VJqW0s47C8zl5gvmLDIyxOoW1llYy9ek2GngHeW5w+LwdDM2g0WL1sMZlt/WCgBoHeJ/Vb9vDhNFOjLGM3Q6t39D4th/3nvR9vrRtGYpfdz4CQbPNc9fUVUXRC0HRVF8UcthlXFp5E3NotguvgLphroRqlDOYvIhesxLbVR5iW6qifx4ExZ+Ru0TdAlYZfhy4JUCJu+ms8ENq7Wc5L7FNUBuJ13lLkfiSim2BRojEm7yUXbQNH67Jf3KMnLiRgW6DhSQ2cR2uiX93LUXeoCKZLOKHOGoxZxYP9WWOuKSG6IiEdae5HPw8hduJ/0TLpMT+/7pbWLNZE1jRunAL8SHs56N6t1Twsyt9Ju4ZLcuV2YtAiwOpM6rH5h4kMf1/U039t1Pq67eyQYWIiuLfoqGqsiV6b9InFm538IPFQ6rzMJ6WenpSL3hJIuPi4k9L2Phg0H0SkJT719wqu+9ncMAgBdP3Nt4QN08isQEH+rhr0bRxJCJxmxJN4chfs4iMc6Ha/zrks357diKriXX62HNL/hWZtdxmNSdP7xYR1xWp3JTtb0Kr/PcfXFkN1P4mZIYs22SVXp/FJ5ksnbDhosyi7S2dRH1c5Qw3ft5/NzWiz/S1Sag7YTMSO1g3U4oVZst+l53dVFoth3jizxzW6TRBQuK8Gk9yXpmdwZQ6JVrkLkZXftkdme7QT3Ctu9Yy9H/wQSn2b+a7GgIk0shu4v/VUek1FgFq+kajQVqt0JRFF/Uclhlms9Qm87s8hpJVl3i1TO/KytW/wYoylqSf7XhVQDAv3/p9wEA9aY6QmJaz2+RgJwOt3amaVgMhTVUL56YoJk7KujaTc3oIgLtCq3TWhQ482XW5ZKsLmzmeYM5D4kR6dKsc8ONPK7SDHgJaueONyTvwXY2PLuxjqbT1GHjT1AzD7wg63GGkyj18ppnd0rSl130FMb2JS7YztSRKs4+Il046b8EemkKxPcmUE5K1OMM6564n3Wv+WUJk3ezXc5KG3tMZtUeBSrNgWXXld4p/8OIQWSK+44cWgsA6LyL3thSew3h+YvfeJdct7s7AwDIlcMIH1+ZpXcx1HJQFMUXtRxWmaoI/3oYaB6VsNs58S9UZJ5CGJi/m33NF2cYj7bxM1w48vipXtSiLJ88Rlmf2SWrYe33GkN8bv3Hnr1M3jAeiGHmdmpGFzB0OX1fP4IFpn0HgKAM9YXm2ab1f7+AsUfpH+jdS2uiIqncS60G4YzMZ8hzn/Uk5Xykjmpc6vgB6xx9VEKsD1nAcF/n27w/C9N0wHxcavrRxw3Cc84pKqt6/1Qct13AuSdk7dHTskpYjPdp/KEIun+z3JFYj4Tkei2Sx7it/FUmk+j7GxlyTQZQSskqWJLnYk8nZ9L2bZ7G2TB9By0fLHcMVxJAWZyxnbcwfD4quRuGT3ehZYVrm14MtRwURfFFLYdVpvZ4hp8jLRj+KjWHG8brflNyLN4DpGQJuKEujkkGPO5rOhFq5HhoJEaNuVGOEFpPUtO0HabGGnmcffrmYYv8gFgOc6IjVqgqTB0wVVnOboidck9mlo09mkSLhHdnNvCxaxqXocLNFnGZoFVKshFrX6EVdfoZoLBdks5KLsnOn/D4iYfrjbySC+t4f8rim3H3wo++3cDcVsmpWXDn5b7Od4tIHfGkTgkge1MmXv1FGqMtDFVf+7JL8LtUb9M4t4X/i/xXT7Oe6LTXWFNzVqy16Pu0KiZ3eOhfx6i3+S7eq/wZmj2dW2YQCy23VEanmWQ3dubqDD9/HMba1Z8tHesZsJv+4NnVbsZ1Tf2hDACgQzIOu/UULydzsXL9UQ8tLdhb20yn6JpOdkuGT3c1ykUmKAxcBOnHCb/L5dC3nz1grb3rw9u1W6Eoii/arfiUkF8U+7VJs2bdSHiVpdWssJ/O1DQkF8bqNKmBWg6KoviilsOnBDNFy2EqTkdWKKNyXbm26BOmKIovajl8SkjI0moY45ib/nHKtea6eMasB1RkZmG1efnQajB7bZJnXo98+NqvJat5X911Xkobqs32qrX1cs57Ner/uPN9kv/1hdpwMbRboSiKLyocFEXxRYWDoii+qHBQFMUXFQ6KoviiwkFRFF9UOCiK4osKB0VRfFHhoCiKLyocFEXxRYWDoii+qHBQFMUXFQ6KoviiwkFRFF9UOCiK4osKB0VRfFHhoCiKLyocFEXxRYWDoii+qHBQFMUXFQ6KoviiwkFRFF9UOCiK4osKB0VRfFHhoCiKLyocFEXxRYWDoii+qHBQFMUXFQ6KoviiwkFRFF9UOCiK4osKB0VRfFHhoCiKLyocFEXxRYWDoii+qHBQFMUXFQ6KoviiwkFRFF9UOCiK4osKB0VRfFHhoCiKLyocFEXxRYWDoii+qHBQFMUXFQ6KoviiwkFRFF9UOCiK4osKB0VRfAmudgMAwNSBUJ7fQ3mzuo1ZRW6Wa7+c67ya9+Ra398P1/9x5/s0/NdqOSiK4osKB0VRfFHhoCiKLyocFEXxRYWDoii+qHBQFMWX62Io85NiYWsVALBh0zkAwNl0EpG9zavZpGtKoGjRtW8BAJDd0AQAiM7yHsxvDCO7juUqzRYA0LWfvzNbDMrrSvIjDABoGaIesQEgPlUHAMzu5HBc04gU/WwJyX1RAECpjdui98wCALyXUih0LB++y69jW0KtRXjHEzyus8a253i+zz14CPMV1nk6kwIALOb5uzwbhamzzsRwgMdJsxfuKiL5m2ijzQCQHOb5Mpuu7LEvtlu0H+K9mryX20ILbGd0FohN875Uo2xTvoefpg5EZ3hcy2iF1/I0j2/bH0I4y32hPD+n7mSdwZxBNcZtTaMsHyy6tlz7oVC1HBRF8eWmshwiqcKy3/2peYx1iEaduf6DUi6XYsrgzJeTAACPCgvtFWqiegDwyrzmnj3cNr+BuqIesmjbEwEAWLktc3dIBZ6FKEt07Q4BALLrWCj16wjqIdYVlCCf+POtAICZ25asEROkhsUij68UQtjyAFVjMsL/6MhUDwCgVA/inf2bAACdB3hYykqbthlEZuU8orULHWxc4nAU+T4WDM+zTGbjyh735BAwv571N53hts53qcrT2yJYHJB9o2xLJCOXOQB4FbZhLkJLLJBmmfnNFk2jPC7+vpg9dZZpP1zFzG1sc2KSVo8NsJ5ie2BF13IpqOWgKIovN5Xl0J+a/8i2cjv7uNGZG+9WRDJLff/YNLVoNXKePpCvhU5+icyxzLqfzGDkyU4AQLGDGi71Fu9Podug0sRyFboJEM7ws9JkUOjivo73eFyxlXWve3AEo3OtbEOFWs+0lAEAwaEYTiysAQBEZ6RRYlwcCCaRGpF+/udpvaT20eKIzBoUO7gvOmeWtckGgMRZLNvmrCdRzJdNrs806ggUnX+Afo2msToi89wWEOusFuW1mJpFjYbYkn8hy30d79dw7j5usx4LVZt58ZmNwYYPJd/Be7awmb9jk1d2DZfDJb8RxpgAgLcAnLXWfsUYkwLwQwCDAIYBPGWtnZOy3wLwJwBqAP7cWvvyVW73VaPl2I0nFBzWA3r38ukae4RvRHSWD2V43iI5xIdw6l4+nP2/5HFTD3SgcAvN+8jxGACgIn7byBxgqnwRW87wTclslO5BExCb5r6cOOM8WsMYOrgGz33phwCAb736e9xXEmfetEHvHhbMdfO4Qhc/g3NA+0E6VePTfMsX+5eusfNdXsPcFr481bgIwYRFeGG5YVxu4adz6l0u9SBgavJDulupD3gPim0BTH6B33te4f0Ii7CwxjQEknNShrL8PXNrADH6xxFNs3x2O++FqYURWuS2uVtF6B6QbkXHlV3D5XA53YpvAjh63u+/AvCqtXYzgFflN4wxOwA8DeAWAE8A+J8iWBRF+RRxSWrTGLMGwJcBPAfgWdn8JICH5fvzAHYD+EvZ/gNrbQnAaWPMEIB7AOy9aq1WLonYbB3R47Q/u1qobicepOZZ2FZDx5v8+41o91KLOO62W7S8SYuhGue+7HZ2AZIHw2gdovo88zvc1/satXel2cPCZhmenKc+sOepn/9w4EkAQPdgmuVfYNelebSC+fXyKIqz0Q0/5nsthp+kyu/ex/M67V3ssAgWeIL8Wp63eYj1dBysY34Dy9Vo+SNwhRaDw6ss3Y/UB2zE/HpaCaUkkHyb3YKmEU4xPvkUf0dmAqjfsggASOym9dN2jPez0BlCnVWgJF2w2DDNjHyfReoQ9yU/WG5RmfrKruVSuFTL4X8A+Hdo9AQBAN3W2gkAkM8u2d4PYPS8cmOybRnGmG8YY94yxrxVy+cut92KolxjLmo5GGO+AmDKWnvAGPPwJdTpNyZoP7LB2u8A+A4AxHoGPrL/WnBqiMNjLggKWAqMuhF9D+kdBpX4AL/volw34lBLHgohfRu3RaeoIwrSj93wd3mkd1JFzu9gmabj1GbFdmBhC8v17KZ6n/gcy8RHPHhlcW7OLvc5lFJAYC1PHvBYfvJO7gwvBFAQX4P17LLP1mPAzB38nu/i+cpJ8Ufk6ecAgHU/ZZlcNz/TT+URepOOkkKS5wvEeFxs8sqHrQd/SmfByX/igsq4vfVkHdE0r2fs87x3cXGIrnkti1NRll+UwLP5rTQXopNew7KJTfGzaZTXsDhgEFlgnRHxUSwM8Dm1n8DjeimneADA14wxXwIQBdBijPkugEljTK+1dsIY0wtALg1jAAbOO34NgPGr2WhFUa49FxUO1tpvAfgWAIjl8G+ttX9ojPmvAP4IwF/L5wtyyIsAvmeM+TaAPgCbAey76i2/As63GG4GOt+pY2GQ2jYxwk/n/W4ercJzQ4qW2xakjz59ZwLZQWrb1vclqGiKfez01gCikzyuyGhmdO2lJi61Aj0v0To4dw/7284DH8oB+TIDstZ9kbpidoKWnPXqSJwVjf9bHF2JH6Y6XdgINI2w/kVROS7QKjxvGnbq2D+jhq1luSF8rAmJcV5Dyxm5vnUrC+sJZ4Hxh2iNuOHb0h30JZRuN0j9jH6aUjfvVc+v2Jbxh5pRSYlP5DhfueSwhKBvBzy6H1CVIdfUB9xQao0gtMi6pu7k/Yyf47V8EuHTKzFO/hrAj4wxfwJgBMDvAYC19rAx5kcAjgCoAvgza23twtUo14r0jkBjyMw5+KwRh+S6YGNboMQHzo3h5x7IIXKQT+r8NolXkIi8Yk+tIRxah3hAWkzk0KLFzK2RZedLnuLLPv5AFMUePgZDGfZf7vrtIwCA+qOmse3pvmMAgO/n7mMFoTq8Ers0vW+wrtHH+NvUlyI4+7/PNkzdKU7WGjC3XW6EeEXdi1WLXtmLZc3S+Yqd0t16k92FQMmimJLujgxhFiXGJLulgq432K7MZhFUcoPCWTQiOUMTPH7ivoi006LQyesqdLJM8uTy/+NaclnCwVq7GxyVgLV2FsAXLlDuOXBkQ1GUTyk3nhdOaWCqS0OJpVZqnp49DG6a2xpDSUzThc0SBfkeCweOx+FVxYknDkYXqVdNeIhPSqBRjPvajtOCmNsSQnYzrYNQhvuGv0ItX2svoTXFUamazKQ8MtMNAMi/045yisfNdtJiaemlybP3rufxF+OfAwD844adAABPukblJBAosK7ZnXyUKxJd2PEuUAtzX/oRmVMzEb3EO+dPsdMieQJybtZdbuXvYG6pi+PGAF2wlZcLIC9DkKFbGKVr32AXK5i3SB5nuYXH2UWpjdGhWQ9ZzNwms2GlK3X2CekqHdG5FYqirBJqOdzA5AcrjUCnfC+3zW2VcOgWg3ILtVGbOB1zfdRubcfrqMuTEZQcAy6HQy0SQIGxSyi3UHvVg/zseL+CfB8PrEm4cHiedT/68EGky7QK3vvJDjmOZVKnlhynv37hdgBLQ3V3HPwm1smMzX/z2dcAAH/7/oMAgMSmAmr/SF9FeEHyHozxuFDOIvrP6YCef7u3cc3Akm/lcomPozFHwgUuBcUoCRQBryq+G5ntOr+JvxNnvcb8jvjPGdDl7mEkvRTQFBX/ResQnZelFg9V/l3I9/I+hk6LPv8E1LpaDoqi+HLTWw43YvCTY/PzZZz+Gq8vnHHht9xXDwPd+9nPn97FMkFZWMjUgVwP9Uapndovs8PVWkOwg53p8gj7xj1vygSu20NISaYkp2EzMtrxykt3N7Rs+2Gq7npIJml1BxDOyOzKzPLcEigbpL/LMcy/fayd1xKhZu2I53FO/r5IVoK86jJ5qS+IhSMcKm0ed5mZuM9NDrtcsoNAoMRja3Heu0qd7WwZto2RjKnH6aCJHpNMVB7Qs4/XnOthg9d/jxFSp/+wv+HP6X+F/ogzXxN/RGEpg1RA7l12I8/bfPo6G61QPl2YPe+h9jTzmXVJopRCuzi4PIOxR2XoTRx8waJMe44bpD7gCxiboBPx+Df5tgfHIrAFCoVImuXPfp51h+ZtI2LRxUy0HmWZesgiIOP5sztok7vp1jBAz16+3Iv9fOidcCkNlBGZExv+NM9bkYHxE2sjwHr+2Pj1UwCAdw/yxMmjQN/rEjUpQqHzXYk87L8y4RCbNshu40s++GNuy3fxfmYHvEb3IHSGjXfzL6afKqAwIU5GeaeH/rgPAFDurKLvVbZn9AkKhe79SzM960Huc3M6ev+fxHx8ZELC1Ue7FYqi+KKWww3M5J/f3zDXS8nl++ohYOBl7jv3WW5rmPtBg/HfooprPk0H2tof0JKY3mVQlFwP1SLNZjcEmvl8Ad4wPWipw6xzTrojtr+Ivv9DCyAtw3Oxc0u5JdLbeT5nRjuHZHg81EiNtuHHYnZ/hRfT8lYU2XvYlqDM13BzMmpRID4h+wrU5LN/yqFC83rbx9+4CxCbsrCG13D2IUl400rrIHY22BjKDMrw6tnHxZSYjyC9QyI3F8Q6a+NxwYUAymJtRWddl4rnaDlTRbFNolhdZr2+T06fq+WgKIovajncwFTjQLlNHHUyBLawlftCi0C5WQKcxNcw8k+pzSJDUXS+w+PKCekPP0YNlnrPonWI45TTHHVshCPH3okjmpE5GNtY3qUza/tVCHNbJFmqTOh3mrIWXppf4MKuK608f9ceg9ldsq0p2Wg7wGHWrn+gVbD/Efoa2tbQuihMtqPYQctm5lYJqd5Li2Elrjw3NyXKlBRYWC8W1ohFuWW5L6PtZd7fmdsCiE+4iccSKr3I47Lr65j9HC22sPgquvfTSktvD6LrAL2V01+QezUqc1ZmNTW9oiirxE1lOZxNU/Ocn2j2Rs7nUOyso+cNapjk0QwAoB5qBQB4FYvIgoQ6L0qAzQFq2si8XUrBPiap7Jt5n+rhUCP9u1vMJufScVggu0ZGG6LcFplzmYvsRxK8RufcjMMA6kE3W1Q08zjrmf5iAbFDEgnk8jYerchxS/4Il85qbpYd+E27Cxi/n8eF5e/O98qwYPrKtG5my1JC4vgonxeXz8HlpwSA2IwsbiP5IxJnLUoyKSt5ivtmbpVrOWgambVKbbyGsw9LWPoRi7mtvFltb7B8dvCKmn5F3HhvhNKgHgSya6XrIE9vbFZSuiW8xmzKRho1l+m4Z+mBnXPdA0ldVmpdmpG45lUZ629hPeWkbUT0hRYltkBeyHJLCE1n3ZwMlnFDqf2/rKMiL1JWplW3yuzD+nAElYTELqzncSNfYpnBF8uYvEtmaIZY/v7NHNI8tmF7Qwh5NWlDF1++aDp00XvnR/MwkC/KHA6JLnXCIbse6H1DhN0f0RGa+DkFVfreCuIn2ZjEKPeNP8LuQX4xiFy/DA1L9u+oxGGYmm0kpHX/jYvC/CTQboWiKL7cFJaDC7ZZ47NuxY1MYsxDbq1bl4N6oPrkHABgfiSJ5AcyN0IUqYvCK3TahqPNraPpVgQrt1jUWsX8fYaf3mk+Rl7ZoCpavtwpiWbT3Fe+NY/c3ewORH9GK6ZzP9s0eQ9Q62fUZdNbNCuyA24NStuwGAZeodfyzBephefXe2h+kAnI/nLDbgDAf9zzOwCA2BrT0LYVcaq6tlwptZhBOUULJTbB9hUZtInUoTqyA5JQR9ZfjUm3KXomjJIcN3U3J1l07JM5KzPVxkpcqQ/Y4FO/61Lbe0iMSyCXzHtpOcnzlVIrupRLQi0HRVF8uSksB7N5cbWbsCr07CtgSjr4bkVtr8K/vHuPwextMpNRHIsu0CZ5AijKrMHEqKSAa1/aF3/DhfCy7owMj8YmAVNzGo7nyQ7y92DPNIbOsdKwWCpFWXW7HqwjOEzHR/Z2WU8z7bSnQUgCh079PnXZY7veBQC81r8VtySY9+E/ff8pAEDrDOsuJ9FYmcvNh3B5E1wOhsuleayK+S1LyW0BILdGApc2eqhIItuIWFnn7pMUe6NAQrI8ob48zVssDbQfoUW0sE5S2YvPYdszR3Hqf/HmerKQUGYbq3HJaK8lajkoiuLLDW85FDvsBX0NY7/pR8sNuLq2Y/jLUYTk0l3ftT7F/v7iAFDppMYyp6mlXRr5mXtqSL3N/nNmG49rOSnDnfk6ZndKeQlcik3z0+U5BIByjyzpJkveWWvQ1kJ1O3OnHB9jmURTETs6GS3VE+XSd+/Ocu3M7OYwXP6mNXEe/94MZx2FIxWMfZfBT+ZxHtf539ju0ccSaJaVsJ1fYHGtC2H+uLt2YUwVSIyx/kBZJqvl5PnxgHrYWSrcNPginTgzu+JoHeLGxT7JjCWrkE88EEA16YaC6aMY+Dl/ju/bhKis4FNuYvme12kanX2i88ou4jK44YXD+bh1K1xMw8qShl3/hOaXBF/6XlnH8ReS/CXkYd2PuX/qTiknz2hsLIi5+5eb99kNMnchEEAo6xavFedjs0y97reIj8viseMu6SzrHJ4bgHUjiB1sS1KExWBrGkemmTIu2ccX6swwH35T8RCURWcXDN/yNlmU0Q6YxrCoPUShd/IpiQitWhQ6lq8OFVph73L69iBqEUm3t5cO1FwfGxA/Z5E4K0lzZMbk6Se5LzoLDP8rERzDy7sXiVGDUk6GgiUqNJin0Kw0BRoLEVfFqXrsG7wHrit4LdFuhaIovhhrP7mgigsR6xmwm/7g2YsXVBTlqnPo288esNbe9eHtajkoiuKLCgdFUXxR4aAoii8qHBRF8UWFg6IovqhwUBTFl+siCMp6QEVSb1eblw+tBrOXF8H44eOvBpfbhpuVa3Hvr5Tz/7OPe6ZW+rzdyKjloCiKLyocFEXxRYWDoii+qHBQFMUXFQ6KoviiwkFRFF9UOCiK4osKB0VRfFHhoCiKLyocFEXxRYWDoii+qHBQFMUXFQ6KoviiwkFRFF9UOCiK4osKB0VRfFHhoCiKLyocFEXxRYWDoii+qHBQFMUXFQ6KoviiwkFRFF9UOCiK4osKB0VRfFHhoCiKLyocFEXxRYWDoii+qHBQFMUXFQ6KoviiwkFRFF9UOCiK4osKB0VRfFHhoCiKLyocFEXxRYWDoii+qHBQFMUXFQ6KoviiwkFRFF9UOCiK4osKB0VRfFHhoCiKLyocFEXxRYWDoii+qHBQFMUXFQ6KoviiwkFRFF9UOCiK4osKB0VRfDHW2tVuA4wx0wByAGZWuy1CB7Qtfmhb/Lle2nKl7Vhnre388MbrQjgAgDHmLWvtXavdDkDbciG0Lf5cL2252u3QboWiKL6ocFAUxZfrSTh8Z7UbcB7aFn+0Lf5cL225qu24bnwOiqJcX1xPloOiKNcRKhwURfFFhYOiKL6ocFAUxRcVDoqi+PL/AapMkOFD4VMgAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "model.eval()\n", + "with torch.no_grad():\n", + " for data in test_loader:\n", + " data, targets = get_batch(data, seq_len, digits_per_batch)\n", + " data = data.to(device)\n", + " targets = targets.to(device)\n", + " output = model.visualize(data)\n", + " \n", + " fig=plt.figure(figsize=(12,8), dpi= 100, facecolor='w', edgecolor='k')\n", + " matrix = torch.empty((2*data.shape[0], data.shape[1], data.shape[2]), device=data.device)\n", + " matrix[0::2] = data\n", + " matrix[1::2] = output.view(output.shape[0], 1, output.shape[1]).repeat_interleave(data.shape[1], dim=1) * 10\n", + " \n", + " plt.matshow((matrix[:matrix.shape[0]//8,:,:].view(-1, data.shape[-1]) * mnist_std + mnist_mean).cpu().numpy())\n", + " plt.show()\n", + " break" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 2.757269, + "end_time": "2021-01-23T23:15:29.601281", + "exception": false, + "start_time": "2021-01-23T23:15:26.844012", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "### Average pooling - one layer" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-23T23:15:35.168442Z", + "iopub.status.busy": "2021-01-23T23:15:35.167648Z", + "iopub.status.idle": "2021-01-24T04:34:31.974232Z", + "shell.execute_reply": "2021-01-24T04:34:31.974692Z" + }, + "papermill": { + "duration": 19139.616994, + "end_time": "2021-01-24T04:34:31.974849", + "exception": false, + "start_time": "2021-01-23T23:15:32.357855", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 121.97 | loss 61.86 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 121.59 | loss 25.64 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 121.81 | loss 18.93 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 121.93 | loss 16.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 119.12s | valid loss 19.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 123.15 | loss 14.00 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 123.05 | loss 12.89 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 123.40 | loss 11.29 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 123.53 | loss 10.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 120.38s | valid loss 15.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 126.36 | loss 9.60 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 123.18 | loss 9.16 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 126.58 | loss 9.05 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 123.54 | loss 8.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 122.30s | valid loss 23.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 123.81 | loss 7.48 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 126.84 | loss 7.32 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 124.01 | loss 6.87 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 124.06 | loss 7.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 121.53s | valid loss 8.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 124.46 | loss 5.76 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 123.82 | loss 7.40 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 123.73 | loss 6.22 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 124.27 | loss 5.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 121.12s | valid loss 9.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 124.38 | loss 5.98 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 124.02 | loss 5.81 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 123.98 | loss 5.94 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 123.57 | loss 6.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 121.03s | valid loss 9.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 124.29 | loss 5.25 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 124.78 | loss 5.15 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 123.95 | loss 5.01 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 124.33 | loss 4.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 121.33s | valid loss 6.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 125.00 | loss 4.58 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 124.14 | loss 4.65 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 124.14 | loss 4.82 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 125.96 | loss 4.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 121.75s | valid loss 8.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 125.86 | loss 4.48 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 124.99 | loss 5.13 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 125.80 | loss 3.59 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 124.17 | loss 4.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 122.06s | valid loss 7.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 124.42 | loss 4.26 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 124.07 | loss 4.23 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 124.29 | loss 4.76 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 124.85 | loss 4.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 121.54s | valid loss 9.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 126.86 | loss 4.44 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 124.16 | loss 3.55 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 127.01 | loss 4.18 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 124.17 | loss 4.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 122.88s | valid loss 5.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 124.51 | loss 3.40 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 125.43 | loss 3.86 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 124.38 | loss 3.52 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 124.36 | loss 4.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 121.70s | valid loss 7.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 124.82 | loss 3.36 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 124.06 | loss 4.25 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 124.87 | loss 3.55 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 124.28 | loss 3.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 121.53s | valid loss 6.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 124.48 | loss 3.25 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 124.52 | loss 3.82 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 124.67 | loss 4.21 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 124.27 | loss 2.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 121.56s | valid loss 7.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 124.67 | loss 3.25 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 124.46 | loss 4.03 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 124.48 | loss 3.08 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 124.54 | loss 2.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 121.53s | valid loss 5.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 124.97 | loss 3.16 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 124.25 | loss 3.03 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 124.14 | loss 3.19 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 124.64 | loss 3.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 121.53s | valid loss 8.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 125.21 | loss 3.15 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 124.34 | loss 2.61 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 124.22 | loss 3.31 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 124.49 | loss 3.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 122.16s | valid loss 5.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 124.34 | loss 2.69 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 124.15 | loss 2.56 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 124.28 | loss 3.02 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 124.17 | loss 1.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 121.23s | valid loss 6.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 124.52 | loss 1.99 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 123.98 | loss 2.57 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 126.81 | loss 2.74 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 123.52 | loss 2.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 121.66s | valid loss 6.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 124.04 | loss 2.08 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 124.17 | loss 2.26 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 123.63 | loss 2.78 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 127.03 | loss 2.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 121.60s | valid loss 6.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 127.39 | loss 2.46 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 124.00 | loss 2.06 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 126.70 | loss 2.36 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 124.03 | loss 2.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 122.23s | valid loss 6.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 124.22 | loss 2.43 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 124.28 | loss 2.31 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 123.48 | loss 2.10 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 124.16 | loss 1.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 121.09s | valid loss 6.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 124.03 | loss 1.82 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 124.37 | loss 1.73 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 126.96 | loss 1.97 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 124.81 | loss 2.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 122.28s | valid loss 7.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 124.37 | loss 2.36 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 124.51 | loss 1.80 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 124.28 | loss 1.56 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 127.23 | loss 2.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 121.90s | valid loss 6.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 123.86 | loss 1.34 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 123.70 | loss 1.83 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 124.74 | loss 2.17 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 123.08 | loss 1.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 121.03s | valid loss 7.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 123.95 | loss 1.31 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 123.68 | loss 2.10 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 123.84 | loss 2.17 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 124.07 | loss 1.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 120.88s | valid loss 7.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 123.93 | loss 1.18 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 123.73 | loss 1.54 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 123.96 | loss 1.66 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 124.38 | loss 1.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 121.67s | valid loss 5.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 124.53 | loss 1.46 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 127.49 | loss 1.23 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 124.04 | loss 1.59 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 126.45 | loss 1.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 122.41s | valid loss 6.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 124.12 | loss 0.94 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 123.99 | loss 1.98 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 124.26 | loss 1.42 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 124.07 | loss 1.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 121.25s | valid loss 7.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 124.71 | loss 1.19 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 127.67 | loss 1.46 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 123.89 | loss 1.43 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 124.55 | loss 1.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 122.13s | valid loss 6.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.35 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 125.34 | loss 58.22 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 124.94 | loss 23.55 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 125.23 | loss 18.34 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 125.51 | loss 16.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 122.87s | valid loss 34.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 125.23 | loss 12.37 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 128.18 | loss 12.25 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 124.98 | loss 10.81 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 128.51 | loss 11.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 123.34s | valid loss 20.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 125.11 | loss 9.30 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 125.10 | loss 9.09 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 125.01 | loss 8.33 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 125.17 | loss 8.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 122.13s | valid loss 9.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 125.22 | loss 7.40 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 125.53 | loss 7.33 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 124.87 | loss 7.27 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 125.51 | loss 6.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 122.28s | valid loss 11.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 125.35 | loss 6.36 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 124.84 | loss 6.44 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 125.43 | loss 6.60 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 124.82 | loss 6.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 122.07s | valid loss 6.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 125.30 | loss 5.40 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 125.18 | loss 5.03 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 125.35 | loss 5.83 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 124.75 | loss 5.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 122.11s | valid loss 8.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 125.36 | loss 5.20 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 124.65 | loss 5.27 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 124.89 | loss 6.08 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 126.06 | loss 5.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 122.27s | valid loss 8.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 125.09 | loss 4.55 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 125.12 | loss 5.27 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 125.05 | loss 4.41 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 125.02 | loss 5.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 121.98s | valid loss 6.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 125.29 | loss 4.43 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 124.93 | loss 4.20 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 125.26 | loss 4.38 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 124.71 | loss 4.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 122.03s | valid loss 6.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 125.40 | loss 4.28 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 124.96 | loss 4.49 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 124.61 | loss 4.36 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 124.53 | loss 3.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 121.91s | valid loss 7.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 124.76 | loss 3.45 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 124.52 | loss 3.90 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 125.12 | loss 4.24 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 125.17 | loss 4.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 121.77s | valid loss 7.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 125.24 | loss 3.37 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 124.76 | loss 4.38 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 125.28 | loss 3.85 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 124.24 | loss 3.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 121.88s | valid loss 8.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 124.71 | loss 3.32 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 124.36 | loss 3.77 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 125.01 | loss 3.01 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 124.48 | loss 3.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 121.64s | valid loss 7.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 125.07 | loss 3.14 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 124.75 | loss 3.97 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 124.63 | loss 3.18 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 124.73 | loss 3.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 121.81s | valid loss 8.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 125.43 | loss 3.91 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 124.42 | loss 2.98 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 124.79 | loss 3.49 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 124.51 | loss 3.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 121.78s | valid loss 7.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 124.89 | loss 2.36 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 124.30 | loss 2.97 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 124.83 | loss 2.96 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 124.78 | loss 3.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 121.76s | valid loss 6.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 124.31 | loss 2.54 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 124.38 | loss 2.81 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 124.37 | loss 3.16 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 124.53 | loss 3.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 121.46s | valid loss 6.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 124.54 | loss 2.87 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 124.17 | loss 2.78 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 124.22 | loss 2.42 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 124.54 | loss 2.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 121.54s | valid loss 7.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 125.03 | loss 2.43 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 124.21 | loss 2.36 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 125.81 | loss 2.94 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 124.65 | loss 2.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 122.16s | valid loss 6.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 124.98 | loss 2.75 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 124.58 | loss 1.67 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 123.98 | loss 2.48 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 124.62 | loss 2.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 121.59s | valid loss 7.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 123.97 | loss 2.60 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 124.70 | loss 2.23 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 124.32 | loss 1.60 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 124.45 | loss 1.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 121.41s | valid loss 7.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 124.47 | loss 2.15 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 124.01 | loss 1.79 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 124.07 | loss 2.47 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 124.47 | loss 2.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 121.35s | valid loss 6.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 125.07 | loss 2.21 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 124.49 | loss 2.10 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 123.73 | loss 1.81 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 123.96 | loss 2.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 121.36s | valid loss 7.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 124.54 | loss 2.30 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 123.73 | loss 2.25 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 124.20 | loss 1.49 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 123.88 | loss 2.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 121.13s | valid loss 5.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 124.04 | loss 1.88 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 124.21 | loss 1.98 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 124.01 | loss 1.74 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 124.82 | loss 1.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 121.49s | valid loss 7.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 124.69 | loss 1.99 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 124.69 | loss 1.97 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 124.00 | loss 1.54 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 123.91 | loss 1.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 121.28s | valid loss 7.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 124.47 | loss 1.43 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 124.13 | loss 2.06 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 124.47 | loss 2.50 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 124.71 | loss 1.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 121.53s | valid loss 6.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 124.10 | loss 1.70 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 124.73 | loss 1.67 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 124.30 | loss 1.74 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 126.78 | loss 1.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 121.94s | valid loss 6.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 127.82 | loss 1.70 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 123.93 | loss 1.61 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 124.14 | loss 1.22 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 124.25 | loss 1.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 121.94s | valid loss 6.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 124.27 | loss 1.28 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 124.60 | loss 1.72 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 123.96 | loss 1.76 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 124.69 | loss 1.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 121.40s | valid loss 6.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 31 | 200/ 938 batches | lr 0.00021 | ms/batch 126.63 | loss 1.23 |\n", + "| epoch 31 | 400/ 938 batches | lr 0.00021 | ms/batch 123.93 | loss 1.20 |\n", + "| epoch 31 | 600/ 938 batches | lr 0.00021 | ms/batch 123.48 | loss 1.45 |\n", + "| epoch 31 | 800/ 938 batches | lr 0.00021 | ms/batch 123.98 | loss 1.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 31 | time: 122.04s | valid loss 6.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 32 | 200/ 938 batches | lr 0.00020 | ms/batch 124.09 | loss 1.46 |\n", + "| epoch 32 | 400/ 938 batches | lr 0.00020 | ms/batch 126.86 | loss 1.55 |\n", + "| epoch 32 | 600/ 938 batches | lr 0.00020 | ms/batch 123.75 | loss 1.13 |\n", + "| epoch 32 | 800/ 938 batches | lr 0.00020 | ms/batch 126.69 | loss 1.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 32 | time: 122.08s | valid loss 7.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 33 | 200/ 938 batches | lr 0.00019 | ms/batch 126.80 | loss 0.92 |\n", + "| epoch 33 | 400/ 938 batches | lr 0.00019 | ms/batch 124.17 | loss 1.17 |\n", + "| epoch 33 | 600/ 938 batches | lr 0.00019 | ms/batch 123.58 | loss 1.63 |\n", + "| epoch 33 | 800/ 938 batches | lr 0.00019 | ms/batch 123.75 | loss 1.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 33 | time: 122.10s | valid loss 7.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 34 | 200/ 938 batches | lr 0.00018 | ms/batch 124.02 | loss 1.23 |\n", + "| epoch 34 | 400/ 938 batches | lr 0.00018 | ms/batch 126.85 | loss 0.70 |\n", + "| epoch 34 | 600/ 938 batches | lr 0.00018 | ms/batch 123.74 | loss 1.04 |\n", + "| epoch 34 | 800/ 938 batches | lr 0.00018 | ms/batch 124.42 | loss 1.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 34 | time: 121.70s | valid loss 8.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 35 | 200/ 938 batches | lr 0.00017 | ms/batch 126.78 | loss 1.52 |\n", + "| epoch 35 | 400/ 938 batches | lr 0.00017 | ms/batch 123.84 | loss 1.62 |\n", + "| epoch 35 | 600/ 938 batches | lr 0.00017 | ms/batch 123.56 | loss 0.71 |\n", + "| epoch 35 | 800/ 938 batches | lr 0.00017 | ms/batch 123.54 | loss 1.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 35 | time: 121.53s | valid loss 7.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 36 | 200/ 938 batches | lr 0.00017 | ms/batch 124.20 | loss 1.27 |\n", + "| epoch 36 | 400/ 938 batches | lr 0.00017 | ms/batch 124.43 | loss 1.21 |\n", + "| epoch 36 | 600/ 938 batches | lr 0.00017 | ms/batch 124.39 | loss 1.29 |\n", + "| epoch 36 | 800/ 938 batches | lr 0.00017 | ms/batch 124.76 | loss 1.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 36 | time: 121.59s | valid loss 7.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 37 | 200/ 938 batches | lr 0.00016 | ms/batch 124.97 | loss 1.10 |\n", + "| epoch 37 | 400/ 938 batches | lr 0.00016 | ms/batch 123.97 | loss 0.89 |\n", + "| epoch 37 | 600/ 938 batches | lr 0.00016 | ms/batch 124.43 | loss 0.86 |\n", + "| epoch 37 | 800/ 938 batches | lr 0.00016 | ms/batch 124.01 | loss 1.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 37 | time: 122.00s | valid loss 7.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.46 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 125.04 | loss 59.19 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 125.41 | loss 24.39 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 125.61 | loss 18.56 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 125.54 | loss 16.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 122.38s | valid loss 20.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 128.36 | loss 13.63 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 125.17 | loss 11.79 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 124.85 | loss 10.58 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 125.00 | loss 10.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 122.67s | valid loss 15.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 125.15 | loss 8.79 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 124.85 | loss 9.85 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 124.61 | loss 8.33 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 124.87 | loss 8.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 121.89s | valid loss 11.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 125.79 | loss 7.61 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 124.63 | loss 7.16 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 125.00 | loss 7.57 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 124.69 | loss 6.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 122.08s | valid loss 11.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 125.05 | loss 6.65 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 124.96 | loss 6.37 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 124.92 | loss 6.20 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 125.11 | loss 6.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 122.03s | valid loss 7.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 124.96 | loss 5.38 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 124.77 | loss 5.61 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 125.16 | loss 6.35 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 124.46 | loss 5.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 121.89s | valid loss 9.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 125.23 | loss 5.46 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 124.76 | loss 5.34 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 125.17 | loss 4.83 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 128.52 | loss 5.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 122.65s | valid loss 10.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 125.50 | loss 4.86 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 124.64 | loss 5.36 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 124.90 | loss 4.97 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 124.66 | loss 4.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 121.97s | valid loss 6.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 124.87 | loss 4.00 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 124.94 | loss 5.17 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 124.74 | loss 4.53 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 124.77 | loss 3.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 121.86s | valid loss 7.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 125.22 | loss 4.99 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 124.43 | loss 4.19 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 124.74 | loss 4.90 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 124.97 | loss 4.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 121.86s | valid loss 7.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 124.50 | loss 3.87 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 125.31 | loss 4.91 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 124.50 | loss 3.48 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 124.59 | loss 3.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 121.76s | valid loss 7.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 128.25 | loss 3.87 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 124.72 | loss 3.65 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 128.14 | loss 3.98 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 124.36 | loss 4.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 123.12s | valid loss 6.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 124.80 | loss 3.67 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 124.99 | loss 4.02 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 124.02 | loss 3.20 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 124.43 | loss 3.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 121.54s | valid loss 7.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 124.66 | loss 2.73 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 124.01 | loss 3.65 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 125.20 | loss 3.62 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 124.68 | loss 3.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 122.29s | valid loss 6.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 124.69 | loss 3.63 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 127.62 | loss 2.96 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 124.48 | loss 3.03 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 124.63 | loss 3.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 122.17s | valid loss 6.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 124.88 | loss 3.08 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 124.28 | loss 3.05 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 124.49 | loss 2.95 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 124.25 | loss 2.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 121.51s | valid loss 6.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 124.61 | loss 2.80 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 124.31 | loss 2.60 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 124.06 | loss 2.58 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 124.45 | loss 2.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 121.36s | valid loss 5.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 124.58 | loss 2.80 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 124.41 | loss 3.13 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 124.06 | loss 2.39 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 124.22 | loss 2.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 121.40s | valid loss 5.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 124.74 | loss 2.01 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 124.77 | loss 2.56 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 124.37 | loss 2.66 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 124.43 | loss 2.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 121.60s | valid loss 6.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 124.42 | loss 2.60 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 124.19 | loss 2.07 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 124.86 | loss 2.41 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 124.07 | loss 2.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 121.48s | valid loss 7.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 124.56 | loss 2.68 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 124.41 | loss 1.84 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 123.96 | loss 2.85 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 124.14 | loss 1.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 121.38s | valid loss 6.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 124.90 | loss 2.20 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 124.50 | loss 1.85 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 124.45 | loss 2.06 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 124.44 | loss 2.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 121.46s | valid loss 6.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 124.07 | loss 2.17 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 123.63 | loss 2.27 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 124.14 | loss 1.64 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 124.30 | loss 2.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 121.02s | valid loss 6.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 123.93 | loss 1.60 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 124.33 | loss 1.96 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 124.22 | loss 2.32 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 123.42 | loss 1.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 120.99s | valid loss 7.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 124.22 | loss 1.52 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 123.81 | loss 2.24 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 123.47 | loss 1.86 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 123.71 | loss 1.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 120.89s | valid loss 7.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 124.36 | loss 1.64 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 123.64 | loss 1.55 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 123.83 | loss 1.06 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 123.81 | loss 1.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 120.95s | valid loss 6.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 123.76 | loss 1.80 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 123.93 | loss 1.90 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 123.80 | loss 1.32 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 123.66 | loss 1.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 120.82s | valid loss 7.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 124.42 | loss 1.64 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 124.26 | loss 1.38 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 123.39 | loss 1.91 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 123.79 | loss 1.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 121.04s | valid loss 6.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 123.98 | loss 1.45 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 123.80 | loss 2.00 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 123.73 | loss 1.47 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 124.10 | loss 2.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 120.96s | valid loss 7.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 124.00 | loss 1.71 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 123.90 | loss 1.09 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 123.72 | loss 1.35 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 123.81 | loss 1.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 120.97s | valid loss 5.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.52 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 124.22 | loss 61.11 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 124.98 | loss 24.69 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 124.94 | loss 19.63 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 124.52 | loss 16.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 121.70s | valid loss 22.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 124.94 | loss 13.12 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 124.97 | loss 12.06 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 124.39 | loss 11.80 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 124.67 | loss 10.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 121.74s | valid loss 14.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 124.97 | loss 9.23 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 124.87 | loss 9.65 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 124.00 | loss 8.32 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 124.91 | loss 8.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 121.81s | valid loss 12.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 124.92 | loss 7.65 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 124.51 | loss 8.25 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 124.17 | loss 6.49 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 125.09 | loss 6.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 121.64s | valid loss 8.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 124.69 | loss 6.15 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 124.53 | loss 6.33 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 124.55 | loss 6.73 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 124.72 | loss 6.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 121.61s | valid loss 9.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 128.18 | loss 5.72 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 124.22 | loss 5.09 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 127.63 | loss 5.66 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 124.30 | loss 6.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 122.80s | valid loss 7.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 124.99 | loss 5.37 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 124.65 | loss 5.91 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 124.65 | loss 6.29 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 124.76 | loss 4.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 121.70s | valid loss 9.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 125.30 | loss 4.68 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 124.69 | loss 5.11 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 124.67 | loss 5.01 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 124.75 | loss 4.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 122.49s | valid loss 7.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 125.06 | loss 4.52 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 127.57 | loss 4.21 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 124.73 | loss 4.33 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 124.78 | loss 4.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 122.33s | valid loss 7.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 124.60 | loss 4.42 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 124.49 | loss 4.51 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 124.53 | loss 4.04 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 124.66 | loss 4.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 121.61s | valid loss 9.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 124.80 | loss 3.62 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 124.77 | loss 3.88 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 124.28 | loss 3.99 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 124.75 | loss 4.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 121.66s | valid loss 5.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 125.09 | loss 3.46 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 124.53 | loss 3.71 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 124.39 | loss 3.67 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 125.03 | loss 2.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 121.66s | valid loss 6.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 124.84 | loss 3.68 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 124.34 | loss 3.25 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 124.48 | loss 3.71 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 124.49 | loss 4.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 121.51s | valid loss 7.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 124.87 | loss 3.37 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 124.58 | loss 3.12 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 125.03 | loss 3.26 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 123.92 | loss 3.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 122.22s | valid loss 7.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 124.59 | loss 3.70 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 127.44 | loss 2.87 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 124.35 | loss 2.79 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 124.36 | loss 3.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 122.05s | valid loss 6.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 124.80 | loss 2.51 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 124.62 | loss 3.12 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 124.31 | loss 2.44 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 124.84 | loss 3.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 121.55s | valid loss 7.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 124.55 | loss 3.42 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 124.61 | loss 2.83 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 123.85 | loss 2.54 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 124.34 | loss 2.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 121.37s | valid loss 6.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 124.64 | loss 2.16 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 124.37 | loss 2.85 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 125.10 | loss 2.80 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 124.56 | loss 2.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 121.59s | valid loss 7.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 124.67 | loss 1.91 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 127.74 | loss 2.47 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 124.00 | loss 2.48 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 124.70 | loss 3.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 122.15s | valid loss 5.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 124.49 | loss 2.38 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 124.20 | loss 2.24 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 124.37 | loss 2.40 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 124.64 | loss 2.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 121.43s | valid loss 6.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 124.80 | loss 1.35 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 124.18 | loss 2.80 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 124.09 | loss 2.15 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 124.19 | loss 2.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 121.33s | valid loss 6.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 124.84 | loss 1.61 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 124.20 | loss 2.49 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 123.97 | loss 1.99 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 124.46 | loss 2.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 121.34s | valid loss 7.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 124.77 | loss 1.78 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 127.17 | loss 2.63 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 124.33 | loss 2.02 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 123.83 | loss 1.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 121.99s | valid loss 5.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 124.37 | loss 2.41 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 123.68 | loss 1.45 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 124.17 | loss 2.07 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 124.49 | loss 2.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 121.17s | valid loss 7.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 124.41 | loss 1.62 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 124.30 | loss 1.94 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 124.15 | loss 1.75 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 127.27 | loss 1.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 121.85s | valid loss 7.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 127.57 | loss 1.49 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 123.90 | loss 1.61 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 127.14 | loss 1.71 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 123.96 | loss 1.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 122.35s | valid loss 6.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 124.56 | loss 1.68 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 123.81 | loss 1.49 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 124.20 | loss 1.97 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 123.97 | loss 1.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 121.12s | valid loss 6.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 123.84 | loss 1.37 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 124.49 | loss 1.66 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 123.91 | loss 1.14 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 123.77 | loss 2.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 120.99s | valid loss 6.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 124.69 | loss 1.38 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 123.77 | loss 1.62 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 124.20 | loss 2.04 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 124.17 | loss 1.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 121.26s | valid loss 6.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 124.17 | loss 1.05 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 124.63 | loss 1.37 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 124.21 | loss 2.29 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 124.23 | loss 1.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 121.30s | valid loss 6.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.07 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 125.03 | loss 61.77 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 125.12 | loss 24.00 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 124.88 | loss 19.50 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 125.31 | loss 16.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 122.06s | valid loss 26.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 125.54 | loss 12.77 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 125.06 | loss 12.30 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 125.25 | loss 11.77 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 124.93 | loss 10.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 122.21s | valid loss 18.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 125.05 | loss 8.70 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 125.29 | loss 9.01 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 125.39 | loss 8.38 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 125.21 | loss 7.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 122.18s | valid loss 12.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 124.96 | loss 7.32 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 125.21 | loss 6.60 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 125.11 | loss 8.01 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 124.47 | loss 6.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 121.86s | valid loss 8.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 125.25 | loss 6.28 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 124.96 | loss 6.22 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 124.51 | loss 6.77 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 124.92 | loss 5.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 121.85s | valid loss 6.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 125.13 | loss 5.38 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 124.85 | loss 6.09 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 125.15 | loss 5.90 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 124.55 | loss 5.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 121.81s | valid loss 7.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 125.03 | loss 4.88 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 125.05 | loss 5.15 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 124.31 | loss 5.13 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 127.79 | loss 5.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 122.36s | valid loss 6.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 127.95 | loss 4.99 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 124.43 | loss 4.85 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 127.80 | loss 5.01 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 124.40 | loss 4.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 123.43s | valid loss 8.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 124.76 | loss 4.75 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 127.15 | loss 4.91 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 124.48 | loss 5.10 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 127.52 | loss 4.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 122.71s | valid loss 6.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 128.13 | loss 4.32 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 123.84 | loss 4.04 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 127.64 | loss 4.62 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 124.39 | loss 4.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 122.67s | valid loss 7.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 124.64 | loss 4.23 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 124.04 | loss 4.02 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 124.05 | loss 3.55 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 124.34 | loss 4.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 121.31s | valid loss 7.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 125.02 | loss 4.20 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 124.10 | loss 3.84 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 124.52 | loss 3.95 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 124.29 | loss 3.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 121.48s | valid loss 7.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 124.81 | loss 3.40 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 123.96 | loss 4.09 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 124.75 | loss 3.41 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 126.51 | loss 3.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 121.88s | valid loss 6.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 127.68 | loss 2.88 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 124.32 | loss 3.05 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 124.66 | loss 4.10 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 124.12 | loss 3.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 122.04s | valid loss 5.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 124.84 | loss 3.51 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 124.67 | loss 2.89 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 124.20 | loss 3.98 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 124.31 | loss 2.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 121.56s | valid loss 7.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 124.27 | loss 3.29 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 123.71 | loss 3.35 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 124.47 | loss 2.74 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 124.41 | loss 2.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 121.23s | valid loss 6.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 124.72 | loss 3.71 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 124.00 | loss 2.33 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 124.06 | loss 3.16 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 124.52 | loss 2.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 121.39s | valid loss 6.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 124.72 | loss 2.80 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 123.73 | loss 2.62 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 124.25 | loss 3.14 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 124.13 | loss 2.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 121.21s | valid loss 6.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 124.79 | loss 1.69 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 123.82 | loss 2.26 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 124.37 | loss 3.72 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 124.22 | loss 2.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 121.29s | valid loss 7.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 127.55 | loss 2.27 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 124.10 | loss 2.00 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 127.34 | loss 2.54 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 124.08 | loss 2.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 123.02s | valid loss 6.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 123.90 | loss 2.31 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 127.09 | loss 2.13 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 124.55 | loss 2.03 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 126.86 | loss 1.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 122.32s | valid loss 8.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 127.63 | loss 2.50 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 123.99 | loss 1.83 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 124.14 | loss 1.63 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 123.75 | loss 2.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 121.78s | valid loss 6.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 124.04 | loss 1.53 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 124.53 | loss 1.64 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 124.11 | loss 1.49 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 124.30 | loss 2.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 121.19s | valid loss 6.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 124.44 | loss 1.54 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 123.83 | loss 2.32 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 123.96 | loss 1.61 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 124.35 | loss 2.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 121.78s | valid loss 6.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 124.09 | loss 2.19 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 124.17 | loss 1.99 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 124.35 | loss 2.20 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 127.37 | loss 1.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 121.78s | valid loss 6.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 127.52 | loss 1.52 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 123.86 | loss 1.43 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 124.32 | loss 2.05 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 123.90 | loss 1.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 121.84s | valid loss 6.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 124.52 | loss 1.55 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 124.24 | loss 2.09 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 124.16 | loss 1.83 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 123.87 | loss 1.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 121.23s | valid loss 6.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 124.97 | loss 1.51 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 124.09 | loss 1.55 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 123.47 | loss 2.26 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 123.91 | loss 1.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 121.19s | valid loss 6.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 124.39 | loss 1.46 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 123.37 | loss 1.53 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 123.64 | loss 1.64 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 124.13 | loss 1.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 120.92s | valid loss 6.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 123.77 | loss 1.64 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 124.26 | loss 1.68 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 123.94 | loss 1.42 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 124.15 | loss 1.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 121.60s | valid loss 6.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.34 |\n", + "=========================================================================================\n", + "Result over 5 runs: 5.207828521728516\n" + ] + } + ], + "source": [ + "num_runs = 5\n", + "best_model = None\n", + "best_result = 1000000\n", + "results = []\n", + "\n", + "for i in range(num_runs):\n", + " features = nn.Sequential(nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(64),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(128),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(256),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=(0,1)),\n", + " nn.BatchNorm2d(512),\n", + " nn.ReLU()\n", + " )\n", + "\n", + " classifier = nn.Sequential(nn.Linear(512, 1028),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=1028, bias=True),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=10, bias=True))\n", + "\n", + " conv = nn.Sequential(nn.Conv1d(512, 512, kernel_size=64, stride=1, padding=32),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.AvgPool1d(kernel_size=64, stride=64))\n", + "\n", + " model = ConvMNIST(features, conv, classifier)\n", + " model = model.to(device)\n", + "\n", + " optimizer = torch.optim.Adam(model.parameters())\n", + " scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "\n", + " epochs = 30\n", + " model, result = train_model(model, epochs, train_loader, test_loader, seq_len, digits_per_batch, optimizer, scheduler)\n", + " \n", + " if result < best_result:\n", + " best_result = result\n", + " best_model = model\n", + " results.append(result)\n", + " \n", + "model = best_model\n", + "print(\"Result over {} runs: {}\".format(num_runs, torch.tensor(results).mean().item()))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 2.952406, + "end_time": "2021-01-24T04:34:37.888811", + "exception": false, + "start_time": "2021-01-24T04:34:34.936405", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "### Max pooling - one layer" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "execution": { + "iopub.execute_input": "2021-01-24T04:34:44.104704Z", + "iopub.status.busy": "2021-01-24T04:34:44.103953Z", + "iopub.status.idle": "2021-01-24T09:55:24.001434Z", + "shell.execute_reply": "2021-01-24T09:55:24.002080Z" + }, + "papermill": { + "duration": 19243.168951, + "end_time": "2021-01-24T09:55:24.002285", + "exception": false, + "start_time": "2021-01-24T04:34:40.833334", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 122.74 | loss 67.42 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 123.62 | loss 22.29 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 128.29 | loss 17.70 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 124.56 | loss 15.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 122.03s | valid loss 18.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 125.78 | loss 12.29 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 124.72 | loss 11.52 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 124.94 | loss 10.99 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 125.43 | loss 10.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 122.21s | valid loss 12.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 128.16 | loss 8.36 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 124.81 | loss 8.38 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 126.00 | loss 9.35 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 125.03 | loss 8.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 122.96s | valid loss 8.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 124.80 | loss 8.07 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 124.85 | loss 7.64 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 125.04 | loss 8.66 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 124.54 | loss 6.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 121.91s | valid loss 8.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 125.95 | loss 6.74 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 125.14 | loss 7.18 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 124.96 | loss 7.76 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 125.18 | loss 5.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 122.26s | valid loss 6.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 124.97 | loss 5.67 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 124.90 | loss 6.18 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 125.40 | loss 6.84 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 127.37 | loss 6.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 122.55s | valid loss 7.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 128.83 | loss 5.19 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 124.92 | loss 5.66 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 127.89 | loss 6.08 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 125.37 | loss 6.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 123.99s | valid loss 6.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 125.11 | loss 5.78 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 127.74 | loss 4.86 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 125.39 | loss 6.14 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 127.65 | loss 5.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 123.24s | valid loss 6.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 127.86 | loss 5.22 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 124.89 | loss 5.22 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 127.78 | loss 5.94 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 125.22 | loss 5.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 123.79s | valid loss 5.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 125.32 | loss 4.26 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 127.58 | loss 4.96 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 124.44 | loss 4.02 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 128.27 | loss 5.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 123.15s | valid loss 8.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 128.60 | loss 4.42 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 124.57 | loss 4.48 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 125.02 | loss 5.13 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 124.69 | loss 4.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 122.57s | valid loss 5.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 124.99 | loss 4.96 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 124.52 | loss 4.22 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 124.37 | loss 4.22 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 125.67 | loss 4.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 122.04s | valid loss 5.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 127.98 | loss 4.15 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 125.58 | loss 4.28 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 124.60 | loss 4.79 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 124.71 | loss 3.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 122.65s | valid loss 7.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 125.08 | loss 3.56 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 125.12 | loss 4.55 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 124.86 | loss 3.57 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 124.89 | loss 3.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 122.09s | valid loss 5.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 124.99 | loss 3.74 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 125.62 | loss 3.95 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 124.23 | loss 4.01 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 125.31 | loss 3.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 122.03s | valid loss 5.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 124.94 | loss 3.32 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 124.91 | loss 3.49 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 124.82 | loss 2.94 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 125.26 | loss 3.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 121.94s | valid loss 5.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 124.97 | loss 2.72 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 125.04 | loss 3.25 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 124.37 | loss 3.68 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 125.07 | loss 2.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 121.84s | valid loss 6.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 125.48 | loss 3.61 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 124.93 | loss 3.17 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 124.86 | loss 3.23 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 124.47 | loss 2.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 122.06s | valid loss 7.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 125.41 | loss 3.28 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 124.36 | loss 3.27 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 125.00 | loss 3.17 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 124.57 | loss 3.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 121.97s | valid loss 5.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 124.44 | loss 2.04 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 124.67 | loss 2.95 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 124.61 | loss 2.39 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 124.67 | loss 2.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 121.73s | valid loss 5.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 124.97 | loss 1.95 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 124.89 | loss 2.97 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 125.04 | loss 2.16 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 124.59 | loss 2.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 121.89s | valid loss 6.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 124.97 | loss 2.60 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 124.62 | loss 2.39 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 124.50 | loss 2.63 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 124.70 | loss 1.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 121.92s | valid loss 5.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 124.78 | loss 1.99 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 124.50 | loss 2.71 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 125.16 | loss 3.20 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 124.17 | loss 1.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 121.78s | valid loss 5.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 124.77 | loss 2.65 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 127.46 | loss 1.84 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 124.82 | loss 2.29 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 127.98 | loss 1.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 123.04s | valid loss 5.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 124.77 | loss 1.65 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 124.23 | loss 2.04 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 124.78 | loss 1.92 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 124.13 | loss 1.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 121.55s | valid loss 6.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 124.53 | loss 1.67 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 124.67 | loss 2.79 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 124.28 | loss 2.57 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 124.80 | loss 1.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 121.58s | valid loss 7.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 124.63 | loss 2.04 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 124.39 | loss 1.70 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 124.72 | loss 1.40 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 124.74 | loss 1.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 121.88s | valid loss 7.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 124.25 | loss 2.06 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 126.86 | loss 2.12 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 124.24 | loss 1.40 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 127.16 | loss 2.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 122.43s | valid loss 6.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 124.73 | loss 1.40 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 123.83 | loss 2.24 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 124.19 | loss 1.30 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 123.96 | loss 1.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 121.30s | valid loss 5.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 124.26 | loss 1.45 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 124.73 | loss 1.94 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 124.22 | loss 1.19 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 124.25 | loss 1.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 121.57s | valid loss 6.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 31 | 200/ 938 batches | lr 0.00021 | ms/batch 124.67 | loss 2.34 |\n", + "| epoch 31 | 400/ 938 batches | lr 0.00021 | ms/batch 124.43 | loss 1.89 |\n", + "| epoch 31 | 600/ 938 batches | lr 0.00021 | ms/batch 125.05 | loss 1.65 |\n", + "| epoch 31 | 800/ 938 batches | lr 0.00021 | ms/batch 124.32 | loss 1.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 31 | time: 121.64s | valid loss 5.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 32 | 200/ 938 batches | lr 0.00020 | ms/batch 125.20 | loss 1.97 |\n", + "| epoch 32 | 400/ 938 batches | lr 0.00020 | ms/batch 124.77 | loss 1.10 |\n", + "| epoch 32 | 600/ 938 batches | lr 0.00020 | ms/batch 124.65 | loss 0.90 |\n", + "| epoch 32 | 800/ 938 batches | lr 0.00020 | ms/batch 124.98 | loss 1.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 32 | time: 121.86s | valid loss 6.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 33 | 200/ 938 batches | lr 0.00019 | ms/batch 124.86 | loss 1.13 |\n", + "| epoch 33 | 400/ 938 batches | lr 0.00019 | ms/batch 124.55 | loss 1.39 |\n", + "| epoch 33 | 600/ 938 batches | lr 0.00019 | ms/batch 124.28 | loss 1.33 |\n", + "| epoch 33 | 800/ 938 batches | lr 0.00019 | ms/batch 124.62 | loss 1.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 33 | time: 121.66s | valid loss 7.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 34 | 200/ 938 batches | lr 0.00018 | ms/batch 124.91 | loss 1.13 |\n", + "| epoch 34 | 400/ 938 batches | lr 0.00018 | ms/batch 124.41 | loss 1.38 |\n", + "| epoch 34 | 600/ 938 batches | lr 0.00018 | ms/batch 124.21 | loss 0.96 |\n", + "| epoch 34 | 800/ 938 batches | lr 0.00018 | ms/batch 124.85 | loss 1.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 34 | time: 121.65s | valid loss 6.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 35 | 200/ 938 batches | lr 0.00017 | ms/batch 124.72 | loss 1.04 |\n", + "| epoch 35 | 400/ 938 batches | lr 0.00017 | ms/batch 124.23 | loss 1.09 |\n", + "| epoch 35 | 600/ 938 batches | lr 0.00017 | ms/batch 124.80 | loss 1.15 |\n", + "| epoch 35 | 800/ 938 batches | lr 0.00017 | ms/batch 124.48 | loss 1.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 35 | time: 121.63s | valid loss 7.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 36 | 200/ 938 batches | lr 0.00017 | ms/batch 125.02 | loss 1.36 |\n", + "| epoch 36 | 400/ 938 batches | lr 0.00017 | ms/batch 124.37 | loss 1.27 |\n", + "| epoch 36 | 600/ 938 batches | lr 0.00017 | ms/batch 124.56 | loss 1.71 |\n", + "| epoch 36 | 800/ 938 batches | lr 0.00017 | ms/batch 124.73 | loss 0.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 36 | time: 121.77s | valid loss 7.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 37 | 200/ 938 batches | lr 0.00016 | ms/batch 124.50 | loss 1.12 |\n", + "| epoch 37 | 400/ 938 batches | lr 0.00016 | ms/batch 124.58 | loss 1.09 |\n", + "| epoch 37 | 600/ 938 batches | lr 0.00016 | ms/batch 124.53 | loss 0.79 |\n", + "| epoch 37 | 800/ 938 batches | lr 0.00016 | ms/batch 124.25 | loss 0.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 37 | time: 121.54s | valid loss 8.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.38 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 125.46 | loss 56.35 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 126.14 | loss 21.82 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 125.38 | loss 16.07 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 125.52 | loss 15.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 122.63s | valid loss 18.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 125.96 | loss 12.41 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 125.98 | loss 11.67 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 125.53 | loss 10.89 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 125.65 | loss 10.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 122.77s | valid loss 15.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 125.91 | loss 8.71 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 125.36 | loss 8.23 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 125.37 | loss 8.87 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 125.66 | loss 8.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 122.68s | valid loss 10.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 125.58 | loss 8.17 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 126.03 | loss 7.20 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 125.31 | loss 7.29 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 125.31 | loss 6.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 122.63s | valid loss 7.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 125.37 | loss 6.45 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 125.14 | loss 6.03 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 125.86 | loss 6.15 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 125.49 | loss 7.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 122.53s | valid loss 7.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 125.32 | loss 5.94 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 125.15 | loss 6.62 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 125.18 | loss 5.65 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 125.35 | loss 6.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 122.32s | valid loss 8.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 125.63 | loss 6.43 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 125.70 | loss 5.72 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 125.12 | loss 4.89 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 125.66 | loss 5.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 122.60s | valid loss 6.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 125.59 | loss 5.36 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 125.14 | loss 6.01 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 125.68 | loss 5.79 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 125.11 | loss 5.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 122.47s | valid loss 7.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 126.01 | loss 4.90 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 125.49 | loss 5.36 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 125.71 | loss 5.56 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 125.37 | loss 4.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 122.65s | valid loss 9.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 125.71 | loss 4.79 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 125.52 | loss 5.56 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 125.72 | loss 5.17 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 125.25 | loss 5.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 122.58s | valid loss 6.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 125.19 | loss 4.61 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 126.27 | loss 4.66 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 125.40 | loss 4.00 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 126.04 | loss 4.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 122.68s | valid loss 6.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 125.95 | loss 3.89 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 125.27 | loss 5.16 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 125.26 | loss 3.90 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 125.57 | loss 4.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 122.44s | valid loss 6.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 125.53 | loss 3.30 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 124.99 | loss 4.28 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 125.54 | loss 4.07 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 124.86 | loss 4.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 122.30s | valid loss 6.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 125.36 | loss 3.39 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 124.66 | loss 4.01 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 124.92 | loss 4.47 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 124.90 | loss 4.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 122.17s | valid loss 6.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 125.31 | loss 3.84 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 125.41 | loss 2.89 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 125.05 | loss 2.88 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 125.08 | loss 3.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 122.37s | valid loss 4.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 125.14 | loss 2.93 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 125.08 | loss 2.91 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 125.09 | loss 3.61 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 125.67 | loss 3.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 122.23s | valid loss 7.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 125.55 | loss 3.21 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 124.98 | loss 3.38 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 124.75 | loss 3.79 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 124.79 | loss 3.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 122.16s | valid loss 5.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 125.18 | loss 2.97 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 124.70 | loss 2.69 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 124.92 | loss 2.92 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 124.83 | loss 3.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 121.93s | valid loss 5.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 125.08 | loss 2.34 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 124.83 | loss 3.35 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 124.63 | loss 2.81 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 124.69 | loss 2.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 121.90s | valid loss 6.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 125.35 | loss 2.64 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 125.23 | loss 2.27 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 124.69 | loss 3.19 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 124.74 | loss 2.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 122.02s | valid loss 5.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 125.09 | loss 1.97 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 124.91 | loss 2.33 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 124.13 | loss 2.83 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 125.10 | loss 2.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 121.89s | valid loss 6.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 125.01 | loss 2.44 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 124.73 | loss 2.61 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 125.07 | loss 1.95 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 125.03 | loss 2.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 121.99s | valid loss 6.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 125.69 | loss 2.44 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 125.26 | loss 2.54 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 124.55 | loss 1.96 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 124.71 | loss 2.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 122.11s | valid loss 6.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 125.02 | loss 2.03 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 124.80 | loss 2.29 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 124.90 | loss 2.35 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 124.76 | loss 2.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 122.03s | valid loss 7.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 125.44 | loss 1.99 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 124.76 | loss 2.11 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 125.06 | loss 1.84 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 125.04 | loss 1.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 122.11s | valid loss 6.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 125.79 | loss 1.61 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 124.70 | loss 1.44 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 128.39 | loss 2.64 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 125.15 | loss 2.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 123.14s | valid loss 5.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 124.66 | loss 1.70 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 125.18 | loss 2.01 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 125.22 | loss 1.97 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 124.86 | loss 1.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 122.08s | valid loss 8.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 124.69 | loss 1.89 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 124.10 | loss 1.31 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 125.32 | loss 2.07 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 124.47 | loss 2.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 122.30s | valid loss 5.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 124.94 | loss 1.12 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 125.47 | loss 1.87 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 124.74 | loss 1.30 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 124.57 | loss 2.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 122.02s | valid loss 7.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 125.06 | loss 1.35 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 125.51 | loss 1.57 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 124.67 | loss 1.09 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 125.15 | loss 2.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 122.17s | valid loss 6.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 4.76 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 126.33 | loss 64.78 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 125.97 | loss 22.06 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 125.56 | loss 17.85 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 126.23 | loss 15.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 122.99s | valid loss 20.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 126.00 | loss 12.24 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 125.75 | loss 11.95 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 125.87 | loss 10.79 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 125.56 | loss 10.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 122.84s | valid loss 19.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 126.09 | loss 9.42 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 125.82 | loss 8.20 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 125.57 | loss 8.31 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 126.16 | loss 8.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 122.85s | valid loss 8.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 126.40 | loss 7.69 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 125.84 | loss 6.91 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 125.81 | loss 8.09 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 126.15 | loss 7.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 123.07s | valid loss 8.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 126.47 | loss 7.22 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 125.31 | loss 7.40 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 125.85 | loss 6.84 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 125.91 | loss 6.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 122.93s | valid loss 6.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 126.10 | loss 6.45 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 126.52 | loss 6.00 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 125.97 | loss 6.30 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 125.62 | loss 6.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 123.08s | valid loss 7.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 126.35 | loss 5.86 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 125.81 | loss 5.27 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 125.88 | loss 6.01 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 126.10 | loss 5.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 122.92s | valid loss 6.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 125.76 | loss 5.66 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 126.23 | loss 5.08 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 125.39 | loss 5.09 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 125.34 | loss 5.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 122.61s | valid loss 9.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 126.17 | loss 5.66 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 125.25 | loss 4.53 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 126.09 | loss 5.90 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 125.38 | loss 5.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 122.74s | valid loss 5.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 129.03 | loss 4.13 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 125.54 | loss 5.01 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 125.38 | loss 4.72 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 125.79 | loss 4.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 123.23s | valid loss 5.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 126.14 | loss 4.31 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 126.05 | loss 4.42 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 125.16 | loss 5.15 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 126.16 | loss 5.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 122.85s | valid loss 6.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 126.40 | loss 3.95 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 125.13 | loss 4.93 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 125.40 | loss 4.46 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 125.28 | loss 4.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 122.61s | valid loss 6.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 125.73 | loss 3.90 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 125.27 | loss 4.55 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 125.34 | loss 4.42 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 125.34 | loss 3.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 122.44s | valid loss 6.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 128.59 | loss 3.01 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 125.49 | loss 3.67 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 125.32 | loss 5.13 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 125.12 | loss 3.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 122.93s | valid loss 5.84 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 125.61 | loss 4.07 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 128.31 | loss 4.12 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 125.19 | loss 2.75 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 125.51 | loss 3.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 123.02s | valid loss 6.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 125.71 | loss 3.40 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 125.12 | loss 3.94 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 125.69 | loss 3.57 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 125.26 | loss 3.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 122.38s | valid loss 5.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 125.47 | loss 2.60 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 125.81 | loss 3.89 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 125.21 | loss 2.70 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 125.36 | loss 3.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 122.49s | valid loss 6.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 125.75 | loss 2.82 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 125.60 | loss 2.79 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 125.29 | loss 4.14 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 125.25 | loss 3.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 122.63s | valid loss 7.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 125.66 | loss 2.57 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 125.23 | loss 2.22 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 125.68 | loss 3.14 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 125.17 | loss 3.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 122.42s | valid loss 4.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 125.36 | loss 2.10 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 125.53 | loss 2.33 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 125.21 | loss 2.70 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 125.10 | loss 3.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 122.33s | valid loss 5.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 125.47 | loss 2.80 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 125.05 | loss 2.46 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 124.98 | loss 2.49 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 125.02 | loss 2.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 122.20s | valid loss 8.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 125.42 | loss 3.05 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 124.91 | loss 2.35 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 125.53 | loss 3.33 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 125.05 | loss 2.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 122.31s | valid loss 6.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 124.86 | loss 2.11 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 125.59 | loss 1.50 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 124.84 | loss 2.72 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 125.13 | loss 2.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 122.20s | valid loss 5.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 125.27 | loss 1.80 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 125.18 | loss 2.61 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 125.11 | loss 2.03 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 125.45 | loss 3.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 122.25s | valid loss 7.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 124.80 | loss 1.80 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 125.65 | loss 2.68 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 124.59 | loss 1.83 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 125.10 | loss 2.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 122.10s | valid loss 6.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 125.53 | loss 1.41 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 124.95 | loss 1.39 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 125.08 | loss 2.04 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 124.90 | loss 2.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 122.20s | valid loss 7.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 125.90 | loss 1.88 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 125.43 | loss 1.47 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 124.81 | loss 2.26 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 125.72 | loss 1.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 122.38s | valid loss 6.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 125.46 | loss 1.59 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 125.00 | loss 1.66 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 124.84 | loss 1.95 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 125.56 | loss 1.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 122.19s | valid loss 7.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 125.20 | loss 2.02 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 124.86 | loss 1.49 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 125.48 | loss 1.41 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 125.31 | loss 1.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 122.21s | valid loss 6.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 126.24 | loss 2.00 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 123.96 | loss 1.48 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 128.36 | loss 1.21 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 124.60 | loss 2.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 122.61s | valid loss 7.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 4.78 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 125.61 | loss 60.94 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 126.47 | loss 22.75 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 125.35 | loss 16.56 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 125.79 | loss 15.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 122.76s | valid loss 12.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 128.77 | loss 12.18 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 125.42 | loss 11.50 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 125.36 | loss 9.85 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 125.54 | loss 10.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 122.98s | valid loss 9.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 125.48 | loss 10.04 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 125.98 | loss 8.24 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 126.13 | loss 8.75 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 125.30 | loss 7.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 122.70s | valid loss 8.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 125.63 | loss 7.33 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 125.45 | loss 7.77 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 125.75 | loss 7.10 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 124.83 | loss 7.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 122.46s | valid loss 7.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 125.53 | loss 6.92 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 125.25 | loss 6.09 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 124.74 | loss 6.71 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 125.14 | loss 6.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 122.30s | valid loss 6.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 125.63 | loss 6.23 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 125.08 | loss 6.36 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 125.59 | loss 5.65 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 124.81 | loss 5.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 122.31s | valid loss 8.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 125.52 | loss 5.26 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 125.53 | loss 5.26 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 125.31 | loss 6.83 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 125.23 | loss 6.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 122.37s | valid loss 7.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 126.56 | loss 5.83 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 125.10 | loss 5.36 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 126.40 | loss 4.98 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 126.38 | loss 5.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 122.92s | valid loss 6.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 125.31 | loss 4.79 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 125.41 | loss 4.79 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 125.06 | loss 5.63 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 125.35 | loss 5.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 122.40s | valid loss 8.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 125.74 | loss 5.58 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 125.49 | loss 4.44 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 125.76 | loss 5.20 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 125.10 | loss 4.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 123.12s | valid loss 6.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 126.17 | loss 4.62 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 125.72 | loss 3.43 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 125.83 | loss 5.83 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 126.58 | loss 4.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 123.01s | valid loss 6.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 128.19 | loss 4.05 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 124.70 | loss 4.37 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 128.46 | loss 4.16 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 125.00 | loss 4.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 123.29s | valid loss 4.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 125.52 | loss 4.00 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 128.56 | loss 4.40 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 125.49 | loss 3.46 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 128.44 | loss 4.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 123.76s | valid loss 5.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 129.05 | loss 4.07 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 124.68 | loss 3.54 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 128.44 | loss 3.94 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 125.19 | loss 5.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 123.60s | valid loss 6.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 125.22 | loss 4.03 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 125.05 | loss 3.80 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 124.67 | loss 3.07 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 125.37 | loss 3.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 122.13s | valid loss 5.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 125.51 | loss 3.07 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 125.44 | loss 2.95 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 125.13 | loss 2.90 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 125.16 | loss 4.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 122.29s | valid loss 6.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 125.61 | loss 3.74 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 125.51 | loss 3.94 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 125.29 | loss 2.43 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 125.26 | loss 3.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 122.46s | valid loss 6.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 125.29 | loss 3.03 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 125.01 | loss 2.97 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 124.99 | loss 2.57 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 125.13 | loss 2.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 122.16s | valid loss 4.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 126.12 | loss 2.15 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 124.84 | loss 3.28 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 124.99 | loss 3.80 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 125.03 | loss 3.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 122.24s | valid loss 5.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 125.02 | loss 1.95 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 125.03 | loss 3.54 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 124.86 | loss 2.46 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 124.87 | loss 2.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 122.04s | valid loss 5.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 125.45 | loss 2.73 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 124.97 | loss 2.51 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 124.83 | loss 2.75 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 125.37 | loss 2.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 122.10s | valid loss 6.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 125.02 | loss 1.92 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 124.86 | loss 2.55 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 125.06 | loss 2.50 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 124.75 | loss 2.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 122.00s | valid loss 6.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 125.22 | loss 2.71 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 124.66 | loss 2.38 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 124.79 | loss 2.79 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 124.80 | loss 2.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 122.03s | valid loss 6.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 124.88 | loss 1.91 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 124.43 | loss 2.05 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 124.86 | loss 2.36 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 124.63 | loss 1.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 121.91s | valid loss 6.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 124.77 | loss 2.23 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 124.75 | loss 2.01 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 125.17 | loss 2.35 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 124.65 | loss 2.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 121.88s | valid loss 6.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 125.21 | loss 1.36 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 124.90 | loss 1.97 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 124.86 | loss 2.21 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 124.92 | loss 1.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 122.03s | valid loss 6.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 125.27 | loss 1.33 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 124.68 | loss 2.45 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 124.78 | loss 2.02 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 124.83 | loss 2.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 122.00s | valid loss 7.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 125.09 | loss 1.42 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 125.23 | loss 1.50 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 124.88 | loss 1.68 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 124.72 | loss 1.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 121.96s | valid loss 5.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 125.68 | loss 1.28 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 124.66 | loss 2.32 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 124.82 | loss 1.62 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 124.53 | loss 1.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 121.99s | valid loss 6.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 125.40 | loss 1.74 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 124.66 | loss 1.30 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 124.87 | loss 1.94 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 124.41 | loss 1.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 121.88s | valid loss 6.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 4.63 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 125.38 | loss 60.78 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 128.28 | loss 21.16 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 125.49 | loss 17.34 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 125.60 | loss 14.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 123.16s | valid loss 19.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 125.88 | loss 12.52 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 125.83 | loss 10.87 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 125.70 | loss 10.69 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 125.90 | loss 9.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 122.87s | valid loss 15.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 126.08 | loss 9.69 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 126.12 | loss 8.64 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 125.73 | loss 8.86 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 126.10 | loss 8.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 123.00s | valid loss 8.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 125.89 | loss 7.22 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 125.66 | loss 7.78 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 126.02 | loss 7.21 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 126.20 | loss 7.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 122.91s | valid loss 7.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 125.91 | loss 7.55 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 125.34 | loss 7.23 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 126.13 | loss 6.46 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 125.62 | loss 6.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 122.78s | valid loss 6.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 125.80 | loss 5.67 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 125.23 | loss 6.01 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 126.52 | loss 6.65 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 125.38 | loss 6.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 122.63s | valid loss 7.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 125.81 | loss 5.07 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 125.38 | loss 6.13 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 126.10 | loss 6.04 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 125.59 | loss 5.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 122.67s | valid loss 11.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 125.88 | loss 5.36 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 125.32 | loss 4.98 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 125.52 | loss 5.98 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 125.12 | loss 5.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 122.55s | valid loss 7.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 125.72 | loss 5.11 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 125.42 | loss 4.77 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 125.53 | loss 5.04 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 125.10 | loss 4.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 122.49s | valid loss 6.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 125.51 | loss 4.69 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 124.96 | loss 4.50 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 125.46 | loss 5.14 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 125.16 | loss 4.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 122.35s | valid loss 5.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 125.69 | loss 3.93 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 125.46 | loss 5.06 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 125.49 | loss 4.57 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 125.31 | loss 4.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 122.45s | valid loss 5.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 125.54 | loss 4.80 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 124.97 | loss 4.07 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 125.08 | loss 4.72 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 125.00 | loss 3.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 122.13s | valid loss 5.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 125.19 | loss 3.38 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 125.07 | loss 3.89 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 125.09 | loss 4.78 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 128.64 | loss 4.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 122.76s | valid loss 6.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 129.05 | loss 3.39 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 124.26 | loss 3.29 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 128.07 | loss 4.15 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 125.20 | loss 4.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 123.24s | valid loss 5.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 125.63 | loss 3.87 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 125.26 | loss 3.21 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 124.90 | loss 4.24 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 125.07 | loss 3.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 122.22s | valid loss 5.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 125.27 | loss 3.06 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 125.32 | loss 3.85 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 124.99 | loss 3.74 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 124.97 | loss 3.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 122.07s | valid loss 5.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 125.35 | loss 3.37 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 125.40 | loss 2.67 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 124.48 | loss 3.79 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 124.93 | loss 3.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 122.04s | valid loss 6.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 125.47 | loss 2.64 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 124.86 | loss 2.99 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 124.47 | loss 4.25 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 125.02 | loss 2.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 122.06s | valid loss 4.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 125.29 | loss 2.61 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 124.69 | loss 2.06 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 124.42 | loss 3.70 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 124.81 | loss 3.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 121.76s | valid loss 5.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 124.76 | loss 2.31 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 125.13 | loss 3.45 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 125.02 | loss 3.55 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 124.68 | loss 2.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 122.17s | valid loss 6.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 124.59 | loss 2.40 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 125.00 | loss 2.92 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 124.55 | loss 2.52 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 124.70 | loss 2.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 121.73s | valid loss 8.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 125.81 | loss 2.13 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 124.97 | loss 3.17 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 126.26 | loss 2.28 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 124.88 | loss 1.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 122.55s | valid loss 7.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 126.24 | loss 1.92 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 124.79 | loss 2.60 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 125.42 | loss 2.29 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 125.68 | loss 2.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 122.63s | valid loss 7.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 125.57 | loss 1.94 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 125.29 | loss 2.32 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 125.34 | loss 2.27 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 124.86 | loss 1.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 122.40s | valid loss 7.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 124.94 | loss 1.51 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 125.15 | loss 2.28 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 125.44 | loss 1.73 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 125.89 | loss 2.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 122.33s | valid loss 5.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 125.50 | loss 1.69 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 125.38 | loss 1.57 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 124.94 | loss 2.44 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 125.00 | loss 2.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 122.29s | valid loss 6.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 125.51 | loss 1.11 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 125.36 | loss 2.36 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 125.50 | loss 1.97 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 125.32 | loss 1.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 122.54s | valid loss 6.92 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 124.82 | loss 1.50 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 125.32 | loss 1.73 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 125.49 | loss 1.23 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 124.88 | loss 1.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 122.35s | valid loss 5.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 124.99 | loss 1.32 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 124.95 | loss 1.64 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 124.89 | loss 1.92 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 124.87 | loss 2.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 122.01s | valid loss 5.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 125.25 | loss 1.48 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 124.92 | loss 1.66 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 125.20 | loss 1.54 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 125.06 | loss 1.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 122.13s | valid loss 4.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 4.72 |\n", + "=========================================================================================\n", + "Result over 5 runs: 4.923704624176025\n" + ] + } + ], + "source": [ + "num_runs = 5\n", + "best_model = None\n", + "best_result = 1000000\n", + "results = []\n", + "\n", + "for i in range(num_runs):\n", + " features = nn.Sequential(nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(64),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(128),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(256),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=(0,1)),\n", + " nn.BatchNorm2d(512),\n", + " nn.ReLU()\n", + " )\n", + "\n", + " classifier = nn.Sequential(nn.Linear(512, 1028),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=1028, bias=True),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=10, bias=True))\n", + "\n", + " conv = nn.Sequential(nn.Conv1d(512, 512, kernel_size=64, stride=1, padding=32),\n", + " nn.BatchNorm1d(512),\n", + " nn.ReLU(),\n", + " nn.MaxPool1d(kernel_size=64, stride=64))\n", + "\n", + " model = ConvMNIST(features, conv, classifier)\n", + " model = model.to(device)\n", + "\n", + " optimizer = torch.optim.Adam(model.parameters())\n", + " scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "\n", + " epochs = 30\n", + " model, result = train_model(model, epochs, train_loader, test_loader, seq_len, digits_per_batch, optimizer, scheduler)\n", + " \n", + " if result < best_result:\n", + " best_result = result\n", + " best_model = model\n", + " results.append(result)\n", + " \n", + "model = best_model\n", + "print(\"Result over {} runs: {}\".format(num_runs, torch.tensor(results).mean().item()))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Results\n", + "\n", + "Average of 5 runs\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DataAvg PoolMax PoolSmart PoolAvg Pool (1 layer)Max Pool (1 layer)
0No gaussian noise6.4991776.8995555.1899095.3860365.667505
1With gaussian noise5.9945756.2982296.1706126.2061005.279452
2Gaussian noise patches5.7780705.6687324.4731045.2078294.923705
\n", + "
" + ], + "text/plain": [ + " Data Avg Pool Max Pool Smart Pool \\\n", + "0 No gaussian noise 6.499177 6.899555 5.189909 \n", + "1 With gaussian noise 5.994575 6.298229 6.170612 \n", + "2 Gaussian noise patches 5.778070 5.668732 4.473104 \n", + "\n", + " Avg Pool (1 layer) Max Pool (1 layer) \n", + "0 5.386036 5.667505 \n", + "1 6.206100 5.279452 \n", + "2 5.207829 4.923705 " + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "pd.read_csv(\"mnist2digits_results.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zUFnk6G-JHnu", + "papermill": { + "duration": 0.034695, + "end_time": "2021-01-21T10:12:56.107344", + "exception": false, + "start_time": "2021-01-21T10:12:56.072649", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "# Experimental" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "id": "G7IV5OZuJHrM", + "papermill": { + "duration": 2.279969, + "end_time": "2021-01-21T10:12:58.457234", + "exception": false, + "start_time": "2021-01-21T10:12:56.177265", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "from PIL import Image\n", + "from torchvision import transforms, datasets\n", + "\n", + "import time\n", + "import math\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import copy\n", + "\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "id": "3SvcSw131eON", + "papermill": { + "duration": 0.124179, + "end_time": "2021-01-21T10:12:58.629076", + "exception": false, + "start_time": "2021-01-21T10:12:58.504897", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "class ConvMNIST(nn.Module):\n", + " def __init__(\n", + " self,\n", + " features,\n", + " conv,\n", + " classifier\n", + " ):\n", + " super().__init__()\n", + " self.features = features\n", + " self.conv = conv\n", + " self.classifier = classifier\n", + "\n", + " def forward(self, x):\n", + " x = x.unsqueeze(1)\n", + " #print('before features:', x.shape)\n", + " x = self.features(x)\n", + " x = x.squeeze(2)\n", + " #print('after features:', x.shape)\n", + " x = self.conv(x)\n", + " x = x.squeeze(2).transpose(1,2)\n", + " #print('after sq and tr', x.shape)\n", + " x = self.classifier(x)\n", + " #print('after classification:', x.shape)\n", + " x = x.view(x.shape[0] * x.shape[1], -1)\n", + " #print('after x view', x.shape)\n", + " return x\n" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "metadata": { + "id": "lpLifQs091Yv", + "papermill": { + "duration": 0.088504, + "end_time": "2021-01-21T10:12:58.764183", + "exception": false, + "start_time": "2021-01-21T10:12:58.675679", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "class Smartpool(nn.Module):\n", + " def __init__(\n", + " self,\n", + " factor,\n", + " search_perc,\n", + " entire_batch=False,\n", + " mlp2=False\n", + " ):\n", + " \"\"\"Smart pooling algorithm\n", + "\n", + " Args:\n", + " factor: factor by which the sequence's length will be reduced\n", + " search_perc: percentage of length of sequence after smartpooling to search for border. Ideally the border is located somewhere in +-search_perc\n", + " \"\"\"\n", + " super().__init__()\n", + "\n", + " self.search_perc = search_perc\n", + " self.factor = factor\n", + " self.entire_batch = entire_batch\n", + " self.register_buffer(\"filters\", torch.FloatTensor([[[[-1,1],[1,-1]]]]), persistent=False)\n", + " self.mlp = nn.Sequential(\n", + " nn.Linear(512, 2048),\n", + " nn.Dropout(0.1),\n", + " nn.GELU(),\n", + " nn.Linear(2048, 2048),\n", + " nn.Dropout(0.1),\n", + " nn.GELU(),\n", + " nn.Linear(2048, 1),\n", + " nn.Sigmoid())\n", + " \n", + " if mlp2 == True:\n", + " self.mlp2 = nn.Sequential(\n", + " nn.Linear(2, 256),\n", + " nn.Dropout(0.1),\n", + " nn.GELU(),\n", + " nn.Linear(256,512),\n", + " nn.Dropout(0.1),\n", + " nn.GELU(),\n", + " nn.Linear(512,256),\n", + " nn.Dropout(0.1),\n", + " nn.GELU(),\n", + " nn.Linear(256,1))\n", + " else:\n", + " self.mlp2 = None\n", + " \n", + " self.visualization = False\n", + "\n", + " def warp(self, X, new_lens):\n", + " #print(f'X shape = {X.shape}, new_lens shape = {new_lens.shape}')\n", + " new_lens_cs = new_lens.cumsum(1)\n", + " #print(f'new_lens_cs shape = {new_lens_cs.shape}')\n", + " #print(f'new_lens_cs ends = {new_lens_cs[:, -1]}')\n", + " # This really searches for the low boundary of each new pixel\n", + " #pixel_contributions = new_lens_cs.view(1, -1, 1) - torch.arange(torch.round(new_lens_cs[0, -1]).item(), device=X.device).view(1, 1, -1)\n", + " pixel_contributions = new_lens_cs.view(new_lens_cs.shape[0], -1, 1) - torch.arange(torch.round(new_lens_cs[0, -1]).item(), device=X.device).view(1, 1, -1)\n", + " #print(f'pixel shape = {pixel_contributions.shape}')\n", + " #print(f'pixel shape[0,:,:] = {pixel_contributions[0,:,:]}')\n", + " pixel_contributions = pixel_contributions.view(X.size(0), X.size(1), pixel_contributions.size(2))\n", + " #print(f'pixel shape2 = {pixel_contributions.shape}')\n", + " # Zero out the negative contributions, i.e. pixels which come before each row \n", + " pixel_contributions = torch.max(torch.tensor(0.0, device=X.device), pixel_contributions) \n", + " #print(f'pixel shape3 = {pixel_contributions.shape}')\n", + " \n", + " # # This contains the cumulated pixel lengths for all pixels in each \n", + " # pixel_contributions\n", + " \n", + " pixel_contributions = pixel_contributions.unsqueeze(1)\n", + " #print(f'pixel shape4 = {pixel_contributions.shape}')\n", + " interp_weights = F.conv2d(pixel_contributions, self.filters, padding=1)\n", + " #print(f'interp shape = {interp_weights.shape}')\n", + " interp_weights = interp_weights[:,:,:-1,1:] # Removing padding\n", + " #print(f'interp shape2 = {interp_weights.shape}')\n", + " interp_weights = interp_weights.squeeze(1)\n", + " #print(f'interp shape3 = {interp_weights.shape}')\n", + "\n", + " # # Each column corresponds to a new element. Its values are the \n", + " # # weights associated with the original data.\n", + " # interp_weights\n", + "\n", + " interp_weights = interp_weights.transpose(1, 2)\n", + " #print(f'interp shape4 = {interp_weights.shape}')\n", + " Xnew = interp_weights @ X\n", + " #print(f'Xnew shape = {Xnew.shape}')\n", + " return Xnew, interp_weights\n", + "\n", + " def nonzero_interval_length(self, x, dim):\n", + " nonz = (x > 0)\n", + " _, low = ((nonz.cumsum(dim) == 1) & nonz).max(dim, keepdim=True)\n", + " rev_cumsum = nonz.long().flip(dim).cumsum(dim).flip(dim)\n", + " _, high = ((rev_cumsum == 1) & nonz).max(dim, keepdim=True)\n", + " \n", + " return high - low + 1\n", + "\n", + " def forward(self, features):\n", + " #print('features shape', features.shape)\n", + " B,T,C = features.size()\n", + " #print(f'features shape: {features.shape}')\n", + "\n", + " padding_mask = torch.zeros(B,T, dtype=torch.bool, device=features.device)\n", + " padding_per_batch = (padding_mask > 0).sum(1)\n", + " total_T = padding_mask.numel() - padding_per_batch.sum() if self.entire_batch else (T - padding_per_batch).view(-1, 1)\n", + " #print(f'total_T shape: {total_T.shape}')\n", + " #### B*T ^^^\n", + "\n", + " \n", + " #print(f'features shape2: {features.shape}')\n", + " new_lens = self.mlp(features).view(B,T)\n", + " #print(f'new_lens shape: {new_lens.shape}')\n", + " new_lens = new_lens / new_lens.sum(1, keepdim=True) * (total_T / self.factor) # Reducing the original length T by some factor\n", + " #print(f'new_lens shape2: {new_lens.shape}')\n", + " \n", + " \n", + " # MLP test\n", + " #new_lens = self.mlp(features.view(B*T,C)).view(1,-1)\n", + " #new_lens = new_lens / new_lens.sum(1, keepdim=True) * (total_T / self.factor) # Reducing the original length T by some factor\n", + " \n", + " if self.visualization:\n", + " return new_lens\n", + " \n", + " features, interp_weights = self.warp(features, new_lens)\n", + " \n", + " if self.mlp2 is not None:\n", + " features = self.mlp2(features)\n", + "\n", + " return features\n", + " \n", + " def set_visualization(self, value):\n", + " self.visualization = value\n", + " \n", + "\n", + "class DoXTimes(nn.Module):\n", + " def __init__(self, model, classifier, features=None):\n", + " super().__init__()\n", + " self.model = model\n", + " self.classifier = classifier\n", + " self.features = features\n", + " \n", + " def forward(self, x):\n", + " #print('1', x.shape)\n", + " \n", + " #print('2', x.shape)\n", + " if self.features is not None:\n", + "\n", + " x = x.unsqueeze(1)\n", + " x = self.features(x)\n", + " x = x.squeeze(2)\n", + " \n", + " #print('3', x.shape)\n", + " x = x.transpose(1,2)\n", + " B = x.shape[0]\n", + " #x = torch.cat([self.model(x[i].unsqueeze(0)) for i in range(B)])\n", + " x = self.model(x)\n", + " #print('4', x.shape)\n", + " x = self.classifier(x)\n", + " x = x.view(B * x.shape[1], -1)\n", + " return x\n", + " \n", + " def visualize(self, x):\n", + " self.model.set_visualization(True)\n", + " if self.features is not None:\n", + " x = x.unsqueeze(1)\n", + " x = self.features(x)\n", + " x = x.squeeze(2)\n", + " \n", + " #print('3', x.shape)\n", + " x = x.transpose(1,2)\n", + " B = x.shape[0]\n", + " #x = torch.cat([self.model(x[i].unsqueeze(0)) for i in range(B)])\n", + " x = self.model(x)\n", + " \n", + " x = x.squeeze(1)\n", + " self.model.set_visualization(False)\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "papermill": { + "duration": 0.083492, + "end_time": "2021-01-21T10:12:58.894166", + "exception": false, + "start_time": "2021-01-21T10:12:58.810674", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "seq_len = 128\n", + "digits_per_batch = 2\n", + "divider = seq_len // digits_per_batch\n", + "\n", + "dataset_root = \".\"\n", + "mnist_mean = 0.1307\n", + "mnist_std = 0.3081\n", + "batch_size_train = 32\n", + "batch_size_test = 64\n", + "\n", + "train_loader = torch.utils.data.DataLoader(\n", + " datasets.MNIST(dataset_root, train=True, download=True,\n", + " transform=transforms.Compose([\n", + " transforms.ToTensor(),\n", + " transforms.Normalize(\n", + " (mnist_mean,), (mnist_std,))\n", + " ])),\n", + " batch_size=digits_per_batch * batch_size_train, shuffle=True)\n", + "\n", + "test_loader = torch.utils.data.DataLoader(\n", + " datasets.MNIST(dataset_root, train=False, download=True,\n", + " transform=transforms.Compose([\n", + " transforms.ToTensor(),\n", + " transforms.Normalize(\n", + " (mnist_mean,), (mnist_std,))\n", + " ])),\n", + " batch_size=digits_per_batch * batch_size_test, shuffle=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "id": "QEN8XMlE9CDm", + "papermill": { + "duration": 0.051266, + "end_time": "2021-01-21T10:12:58.992204", + "exception": false, + "start_time": "2021-01-21T10:12:58.940938", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def train(model, epoch, optimizer, scheduler, loader, seq_len, digits_per_batch):\n", + " model.train()\n", + " total_loss = 0.\n", + " start_time = time.time()\n", + "\n", + " for batch, data in enumerate(loader):\n", + " data, targets = get_batch(data, seq_len, digits_per_batch)\n", + " data = data.to(device)\n", + " targets = targets.to(device)\n", + " optimizer.zero_grad()\n", + " output = model(data)\n", + " \n", + " loss = F.cross_entropy(output, targets, reduction=\"sum\")\n", + " loss.backward()\n", + " torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)\n", + " optimizer.step()\n", + "\n", + " total_loss += loss.item()\n", + " log_interval = 200\n", + " if batch % log_interval == 0 and batch > 0:\n", + " cur_loss = total_loss / log_interval\n", + " elapsed = time.time() - start_time\n", + " print('| epoch {:3d} | {:5d}/{:5d} batches | '\n", + " 'lr {:02.5f} | ms/batch {:5.2f} | '\n", + " 'loss {:5.2f} |'.format(\n", + " epoch, batch, len(loader), scheduler.get_last_lr()[0],\n", + " elapsed * 1000 / log_interval,\n", + " cur_loss))\n", + " total_loss = 0\n", + " start_time = time.time()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "id": "AEyFhNHW9EJA", + "papermill": { + "duration": 0.054936, + "end_time": "2021-01-21T10:12:59.094630", + "exception": false, + "start_time": "2021-01-21T10:12:59.039694", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def evaluate(model, loader, seq_len, digits_per_batch):\n", + " model.eval()\n", + " total_loss = 0.\n", + " seed = torch.seed()\n", + " torch.manual_seed(0)\n", + " with torch.no_grad():\n", + " for data in loader:\n", + " data, targets = get_batch(data, seq_len, digits_per_batch)\n", + " data = data.to(device)\n", + " targets = targets.to(device)\n", + " output = model(data)\n", + " total_loss += F.cross_entropy(output, targets, reduction=\"sum\").item()\n", + " torch.manual_seed(seed)\n", + " return total_loss / len(loader)" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "id": "-TTTvnM0D9Ru", + "papermill": { + "duration": 0.052094, + "end_time": "2021-01-21T10:12:59.194702", + "exception": false, + "start_time": "2021-01-21T10:12:59.142608", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def train_model(model, epochs, train_loader, test_loader, seq_len, digits_per_batch, optimizer, scheduler):\n", + " best_val_loss = float(\"inf\")\n", + " best_model = None\n", + " patience_expansion = 1.5\n", + " \n", + " epoch = 1\n", + " while epoch <= epochs:\n", + " epoch_start_time = time.time()\n", + " train(model, epoch, optimizer, scheduler, train_loader, seq_len, digits_per_batch)\n", + " val_loss = evaluate(model, test_loader, seq_len, digits_per_batch)\n", + " print('-' * 89)\n", + " print('| end of epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} |'.format(\n", + " epoch, (time.time() - epoch_start_time),val_loss))\n", + " print('-' * 89)\n", + "\n", + " if val_loss < best_val_loss:\n", + " best_val_loss = val_loss\n", + " best_model = copy.deepcopy(model)\n", + " epochs = int(np.maximum(epochs, epoch * patience_expansion + 1))\n", + "\n", + " scheduler.step()\n", + " epoch += 1\n", + "\n", + "\n", + " test_loss = evaluate(best_model, test_loader, seq_len, digits_per_batch)\n", + " print('=' * 89)\n", + " print('| End of training | test loss {:5.2f} |'.format(\n", + " test_loss))\n", + " print('=' * 89)\n", + "\n", + " return best_model, test_loss" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.049243, + "end_time": "2021-01-21T10:12:59.271906", + "exception": false, + "start_time": "2021-01-21T10:12:59.222663", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "## No gaussian noise" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "id": "SSUEYv2oBYZG", + "papermill": { + "duration": 0.042199, + "end_time": "2021-01-21T10:12:59.341892", + "exception": false, + "start_time": "2021-01-21T10:12:59.299693", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "def get_batch(batch, seq_len, digits_per_batch):\n", + " batch_size = batch[0].shape[0] // digits_per_batch\n", + " width = batch[0].shape[-1]\n", + " data = (torch.zeros(batch_size, batch[0].shape[2], seq_len) - mnist_mean) / mnist_std\n", + " choices = torch.multinomial(torch.ones(batch_size, seq_len - (width - 1) * digits_per_batch), digits_per_batch)\n", + " choices = choices.sort()[0] + torch.arange(digits_per_batch) * (width - 1)\n", + "\n", + " a = batch[0][torch.arange(batch[0].shape[0]),:,:].view(-1)\n", + " b = torch.arange(batch_size).repeat_interleave(digits_per_batch * width * width)\n", + " c = torch.arange(width).repeat_interleave(width).repeat(digits_per_batch * batch_size)\n", + " d = (torch.arange(width).repeat(digits_per_batch * batch_size * width).view(digits_per_batch * batch_size, width, width) + choices.view(digits_per_batch * batch_size, 1, 1)).view(-1)\n", + " data[b,c,d] = a\n", + " batch[0] = data\n", + " \n", + " return data, batch[1]" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "papermill": { + "duration": 0.490095, + "end_time": "2021-01-21T10:12:59.880226", + "exception": false, + "start_time": "2021-01-21T10:12:59.390131", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAKoAAAOfCAYAAABYKVgBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAACdn0lEQVR4nOydd3yN1x/H3+eOTElEyJCEJJLYeyaUGh021UEVVapqlOqi89c9tNXaparaKqVq1mrV3nuThBghxMhA5r33+f1xI0RCEu7OeXt53ec5z3nO+Sb53LPP9whFUZBIbB2VtQ2QSIqDFKrELpBCldgFUqgSu0AKVWIXSKFK7AKrC1UI8bgQ4rgQIk4IMfo+3g8WQqwVQhwVQhwWQozIDS8nhPhHCBGb++ldwnTVQoi9QohlD5qeEKKsEOJPIcSxXDujHjC9V3N/1kNCiDlCCJeSpCeE+EkIkSSEOHRb2F3fF0KMyf37HBdCPFbM9Mbm/rwHhBALhRBli5teoSiKYrX/gBo4AYQBTsB+oEYJ0wgAGuReewAxQA3gK2B0bvho4MsSpjsK+B1Ylnt/3+kBs4CBuddOQNn7TQ8IBOIB19z7ecDzJUkPaAk0AA7dFlbo+7m/y/2AMxCa+/dSFyO9RwFN7vWXJUmvUJutLNQoYNVt92OAMQ+Y5mLgEeA4EHCbmI+XII0gYA3Q5jah3ld6gGeusMQd4febXiBwFigHaIBluaIoUXpAyB3CKvT9O/8mwCogqqj07njWHZhdkvTu/G/tqv/mL/0mCblh94UQIgSoD2wH/BRFSQTI/fQtQVLfAW8ChtvC7je9MOASMDO3KfGjEML9ftNTFOUc8DVwBkgEUhVFWf0A9t3kbu+b4m/0ArDiQdKztlBFIWH3NacrhCgDLABGKoqSdt8GCdEJSFIUZff9pnEHGozV4hRFUeoDNzBWrfdrnzfQFWO1WRFwF0I8ZwpD75ZlIWHF/hsJId4BdMDsB0nP2kJNAIJvuw8Czpc0ESGEFqNIZyuK8ldu8EUhREDu8wAgqZjJNQe6CCFOAXOBNkKI3x4gvQQgQVGU7bn3f2IU7v2m1w6IVxTlkqIoOcBfQPQDpHeTu71/338jIUQ/oBPQW8mt5+83PWsLdScQIYQIFUI4AT2BJSVJQAghgBnAUUVRvr3t0RKgX+51P4xt1yJRFGWMoihBiqKE5Nrzn6Iozz1AeheAs0KIqrlBbYEj95sexiq/mRDCLfdnbwscfYD0bnK395cAPYUQzkKIUCAC2FFUYkKIx4G3gC6KoqTfkU+J07NqZyr3S9YBY0/9BPDOfbzfAmPVcQDYl/u/A+CDsUMUm/tZ7j7Sfphbnan7Tg+oB+zKtXER4P2A6X0IHAMOAb9i7EEXOz1gDsb2bQ7GEm7Avd4H3sn9+xwH2hczvTiMbdGbf5OpxU2vsP/iVokskdgu1q76JZJiIYUqsQukUCV2gRSqxC6QQpXYBWYTaklXRQkhBpk4f5meDaRlsvTMNDZa4lVRwC4T2yDTs4G0TJWeuUrUJkCcoignFUXJxjgV2dVMeUlKARozpVvYCpmmt0fIrQ4GAahRN3TBDU9RzmSzDzI920irOOldI/myoigV7pWGuYRa5AoZRVGmAdMAPEU5paloayZTJLbOv8qfp4uKY66q3ySroiSSm5hLqA+8KsrcqKuGE7HTmcsvRRGx05nl5/aw7Nxulp/bw5UXo6xtnuVQqWl/OIUTv9eztiX3xCxVv6IoOiHEMIzbDNTAT4qiHDZHXvdL2G9n+SpgI9r3t5Cj6DGgJkfR83piS3zXJ6G3toEW4nqPRgwvO4UVfrWsbco9MVcbFUVRlgPLzZX+/XL5pSiiBu5hYuB2chQ1WqEGQIVAK9TsuRSEV0ycla20HGW3JnBOn150RCtjNqHaGlntG3NjaCrb6k8kR9GToxhLUIAcRY9WGO/1BsearNMEVuTIB4H4BSVTZqwn6rV78j1PrxlAOZWGE3uDqEKClawsGsf6q9yDa8EattSfg1ao8/6/ntiSToENaXPg2bwSNco/HnVkFWubaxLUnp6ofzcQ1/EHrqa6k1OmYLl0po8eV+GEKruwgRrboVQI9fRHUUS9uCe3JDX+H3KuGad6Gzc/6g0qDCjkKHq+CdhGUsuSbuC0TWLfqcnC8OX0O92G8CFncFlacMdH/crG4W7XS1KoVqdso0tMDNyeV5K2/HAEJxpnoo85AYBaZcgrUVWIwkeB7YzLg6JY2XMsWYqOw7/XQJ+cXCCOyt2d94OXAeAdk2NpE0uEQ7dRT38URdlGlxgdsTKvJB1xvnmBXv3tJapWqO9zw7btoAkO4rGXNxOicSN81SAiJ24pEEfl7k7y/ABqap3oFf8ILqv22vSP7ZBCzWrfmGvBGtq1v9m717M6w52vh/XBaeVOjOtlbiGE4jAlqtBoiP3KhyW+S+h/5mGqj4otdKgt5uPaxNSdTJI+naTPwnDW7bS4rSXBIYVq7N3Pyde7vyXS/GS1b8zbkXMcpkQ992oTjracSIaSzcHZtfBNKViaXhocxbzu3wMa2k19k6DlBePYGg4l1Kz2jTn9BMTUn5o3Ppq/JC3I6Sego1tqXom6OsMdjwSdJc02GZrKwYwe+AcADWe9SkghVb7a05MBryyjnpOGjsc7U3niIbuY3HAood4YmkpMbkkK3LMkBWMbNqbDxHzjqPeKb+ucGBBEzzKXAHCumYKhVX1U6/fmi3P0q2os9VoHgJsmm32fVzfGv6im0ifbwWCbsnUYoV5+KYpt9SfS7KNhbHt/onFM9H/D8Fm59a7veDa8nK9t+npiS7sVaU67hqx9fizgBsDexrM58UsG+7Iq5ovXwW0bxrXsML/KKrhtyLjTNw9huHHDQhaXDIcRKoqxBI0auCevRL1XW7OwtumOH+rjw92FbctkVNByyaDBV30rrIrGlSqaO4eljCKtvvF5VMfL5IX67dThkm67X1KHEerVxjq0Qp1/Dv8uvffkvyPYWu8HDCgO0TYF8JyzjTHrupFZIxC9i4qkBtq8Z2P7/cTjrrfm88OXDKbq8D0oOvv5eR1GqBjI1zYFCi1RCytJI5e+TMgixW6r/ZvoEi+gSbyABgj+2ximCalEyIBkwJm51yvwxY/PUHXcDrsSKTiQUKt/m8ybDR7i+4pb89qcWz+ciOpDQfiyl4jrdLME3VOgJHUEkd6NnIreZCrG9sAHS5+iytgtdjn65jhTqIqCAZFXUt7+ebjjpELD7b2XXxxS3kmnnpOG95LqUfW7s0W/YKM4jFD1MSf4b0V9Htr/DP9mlM3Xm7/z8/XEltSeMYzuQU0cWqRqT0+GVFkPwB/ro9ElnLOyRfePwwgVoPL7W/HqEMe44b0JX/ZSoSVo0/8N5VTvQCq/b5+9+5KgT0vjo22dOaNLJ+hfQ9Ev2DA24R9V7kIt3fyr/LlbUZRG94rjUCWqxHGRQpXYBVKoErtAClViF0ihSuwCKVTJ/aNS476hAsvP7eHCyGizZuUwU6gSy3PujabsrTKBJH2G2TcHyhJVcl9oQirx/gDj8abRy0bhvNy8M3xSqJL74vz3bnR3v8rqDHeqv3+i6BceEClUSYlJfa4Z6xrOBGD48n7oL18xe54O10ZVV6hAzOgq1GoUz4LwFXnhDXb2xr/bUSta5hgIrRPPv70UL5UroUsGEfnK9qJfMgEOI1S1ny/Hvgxmb7uJlBHOGFDYmQVaoae6FlY2mM6AyH553lEk90fMuPoM8NrK5kxB9Q9OWWwHq90LVVWrGiee9WbaMz/Q3CWHsVfq8c/FapzfUZHwifGg0ZA63Ym/a80mPaIczlKo941oVItFnb4HtAwZPwz/i5bzB2DXQk1/oikvfraAXh4XOaPLoMH3b1Lx6+04GU4TwmlubrZo6WegjHDGd8xJUv+2qsl2TcxwJ6prtezOgsCfLOsPwK6F6j3iNL09kuge1xHdQDcqxhb8hovGtRlVfirgyhMV9jCTypY31AFQmtdjfsupgJp3B7yIOm1Pke+YErvt9V8YEc3C8OVU39CfrFYX0MeeLDTeqc5l8Fa5ohYqbhicLWyl4xD7vJZ6ThoiF79cwBmwJbDbEvXx57dgQCFk4t09mmU/3pheXddjQAHFwCebOxHJLgta6RgYHqrP4kcmkGzQEz7n1gyUurwP8cOqklk5C+/tTvj/fhh9WppZbLBboTZ0jwdAE3MuX1tJaV4PnauaMy/o+bHpDJq7GH+xWYqOwBXqQlKS3At1hQpU/+4g1bVaqv0xkvCN2wAQDWsSPXM3i31WGyM+Ck/16kBGKynUQqm9+hLzdjfC2TOLdqExfBUwDWehwYDC2xcbsTRF8IX/TuZdD8J9gWXG/ByJS53CWeo/iemplai8PIez70Xz2rN/8Zj7JvZk+dL20JPkGFRsqDOPTysvYhTmOfrIbtuoo7f2AOAz3z3EtP+Bg81/ZlzFLWiFmqXpntSbOJyDTdQsXW082fLLg49Z01y7pdxzxi3W6QZn/jdtBvsHT6Cv5znaznqDKQ0a4vpYPBdO+wDQa98LZrPDbkvUiH57aN/6Rc4PzUZRjO1Uw7EyVNyUg9OqXQRhdLSQ45uDCoEQ1t/EaG8o0XWZX3UqauHKCO84shQdrQ/2xPNVNSFHt2JQqYmZ3IS4zlOZfc2PADPO/NmtUAHUa/cQvPbecZ5usAsDSp6YJcUnw88FN+GEXjFwXckievJrBH22BT1GX6zx33gREzWF6amVWNI9CjDf+Vx2LdSiUHl40NbD2Ph32uxhZWvsD491MazOcOdx13Q+uxRFpQkHEd7eJLxQnT+Gf0241pm2h56kzKta9MdjzGqLQwuVKsHUdloBuBK46pJdeFa2JfTJyXz6zvNU+GICn/juZu1eF5yEnuYu/5KlqKjx6zDC/7cXfWam2W1xaKEa9h1hY2Yg3d2vYjh5xtrm2CUef2zjnXMv0nryVt7wOQLAB0n12TOoLmE7t2Ip/ysOLVRVrWpEu2xChRtKVpa1zbFbVJv2sb6OK+tpeFvoQcvaYNHcLIzOxxU/tSvd4zpY2xTJA+LQQs3yNnpdjl0TZmVLJA+KQwv1/NPZALhdkGOo9o5DCzW4QjK7s8Bv3hFrmyJ5QBy6M+X0yGk+oCGQam1TJA+IQ5eoEsdBClViF0ihSuwCKVSJXSCFKrELHEKoMTMb0vtYAmpPT2ubAhj3Et14simn/qjDr2c3s/zcHlad38eVF82z+r00YP9CVanRuubwjEciLTZdROXmZlVzDC3q4bZQsPb7yRxqMRNvlQuHc7JJ0t9g/rtjUaLronJ3t6qN9ojdC1VdNYyDD80A4DWfQ6R0rWM1W7I6NCZ60k7+CFvNzLRgIle9RPO3hzEmqhtbMv0I0rjy1q+/oVtSzmo22it2P+Df/s8d1jYBoXXizJuNmPvit1TXavn8Sk1Wv9+SyEVG23TAq2t70bHjVFq6ZHPE/wArQ+qiOyWXHhYXuxfq4LIn89ZEJuiycLlq+VOTE4c2Yv+QCYCWx492x2UguJ669QUSDWsS1/GHPDuX9W8Jpyy7TM7eseuqX1W3er77Xof647TK8g4mfDufRYVg0NmH0bQ7g+7UGYSzM0lDomlz8AZ/L/kVtVAx55ofXdv1hB1SpCXFrkvUuF5l865P67Jxn1D2rnHNhb51A+ZGTsCAC8e/q4kH27jaP4pHXtnMh74TADAA318N45/nmmE4KhfI3A/2K9RmdZj61LS823XpEVY5KVrnpsZL5QLAxaZwqUs9jrUyHrv+WmIzNk9rhDoLvGdtBaRI7xe7FerZR8rQwiUTUHFel8Uv73bGHct7QnH59wCNdj7Hrsa/cezpSQBMTQ1h3N+diHhvL+UzHf8Ua0tgt0L9rO8vedexOd5Wc9ejZGWRcaQsNL4VtrRPS6rsttzGt9KAXXamdG0aUtfpAipUaIWakbNetJotMdMbc7TfJFSIvP8ZFeWAvqmxS6Emj7xORY0zBgxkKTlo0i1vg7qsFxdGRhPX4QfWZDjT/M0hvHmhEQYU0irbbUVls9jdb1TtU47R1Vbm3Uftfo6AbyznS/4mF37xZ1fDCezL1vNd+yfx0SXy0mcbARfcLkpXF6bG7krUUy9Xo6v75bx73Xofi9twfmENNjX4laM5ObzXpCP62JMors6oMW4idD8vfQiYGrsT6lNPrc93b+nSVO3ny8/1fkYr1Dz5+6voL10C4GyH8pRTGX+dYvM+i9pUGrAroWqCgwh1vpR332Db8xa3IW5EFeo5aRh09mFC3zYOPanqVufjl37BU+VC1YVDLG5TacCuhHr1oSB6eZzLu6/0oeXbgp0f244BhS1ragGgRNXlyblr6eiWyu5sPdUmJ1vcptKA3XWmblJvywtUOmTZmR5NSCU6lDUeVGVQw8nf63Gs1U8YUFh4oxzfvd8LjyPbLGpTacFuhZqZ5gwGy5aoulNnmJDQjpbhyznSZ2JuqKDfqXYkvROKxwbLT+GWFuxKqF7Hr7E105n116tR490ELL+gDzIfTaH6Z0N54ZG1lNNc56v/OlFtcjLqI5Y/e6k0IRTF+n6ZPEU5paloa20zJFbiX+XP3YqiNLpXHLvqTElKL1KoErtAClViF0ihSuwCKVSJXSCFKrELpFAldyVpaDTdj1xi1fl9xM+pa1Vb7GrA35HQBPiDq3FToHLtRt4qLGtzYWQ0rw7+k67up3BT7USFihwFDrf8iZpz+xPa84BV7JJCtRQqNaoaERgOHSPh7Wi+euEnHnW9AcAvaYF8++sTBH2x3eLTwgA0qU38qypWRE8iQL0DrVADzgWijW/0B+OoXvB9CyCrfgugCa3MqY+b0Gy2sTSqsDeHj2M6MeeaHwB9Pc+xb+gELg5ranHbLoyMZujsBRxu+ROVNK5ohZrVGe5U+28gj77wEt2adObRF16yuF13IktUC9B88TGW+Cxk9MWG6No2wHnFTpxXwB/BzZjSIogrdQTvdp9PWoQePwvadWFkNFtf/y63BIUlN7wZs6A3ERNPE35uL2D0m+Vc0fK7KO5ECtXMqGtWpX/ZGYAbn/ntYu+PO3jvmRdgx0F0ZxPwnJOA5xyYO+MhnD+33C7FmJ8a8WfrWyKt+tcQqn99ntDTW62y2KcopFDNyNX+UfzvnZmUV7vmhT039xVCdxR0SqGPi6eqrx8ZFrLt37bfsTEjjDcHdEabmknVw/vRFXJKtNrPl5wvUixk1d2RbVQz4nZJz5szXqBLxz4A7MgShH9zl3Ptm9Xhx7C/LGbbyz2HMq9zCzT/7UbZfRjDXY4yz4moyPJqiyxm192QQjUx6ogwkvsZXaC7LNtB0OdbSPgAritZ9Fk6BP3lKwVfalaHN2fPpvHqERazU2zdjz72ZJHxkqsZa4OL+gzemjTA3GbdlSKFKoT4SQiRJIQ4dFtYOSHEP0KI2NxP79uejRFCxAkhjgshHjOX4bZIwphoZqz5hf8+HYehRT0MD9XnnZP7iA6Mp8OoV4kYUfg2lU4z1rMytQ7VX4u1sMX3JvW5Zvz4zncAjDrTFf9xlvefcJPilKg/A4/fETYaWKMoSgSwJvceIUQNoCdQM/edyULkttYdGZWahLejOTx8Mr5qNxbfCORET2c+nvUj0y48zJmWCmXmFRSp2tOTNgdvkJjtxeG2ZdGn2NZRmL4vnqKmk4aL+gwSJkRY1ZYihaooygbg6h3BXYFZudezgG63hc9VFCVLUZR4IA5oYhpTbRdVnarsGzqBHEWPAYWnyyQxt+NEeq0ZzKXolELbf4aH6jNyz1ZalznC6snN0Sfbzu5VdXkfrgyM4t1KSwFjaerxh3U3Ld5vr99PUZREAEVREoUQvrnhgcDtP1FCbpjDcuXFKJ4buSLv/rI+gxdPPoXSSyEyseBmP5WLCyfer89b3RcyPqEdSi8Fn0TbcE2prlCBnKqBXHozg+0NJwIqFlwvz/XnPYFC2tYWxNTDU6KQsEI3ZQkhBgGDAFyw7pE7D8K1djcYWvYEAAuul+fT6b2o+HXhbTnRuDbqsZf5L2wsbbcOIeQZ68ybF0ZyvyjqD93HxMAZ+cLfW9iTsDjrf5HuV6gXhRABuaVpAJCUG54ABN8WLwg4X1gCiqJMA6aBcXPffdphdY60+BkDsCbDjV8fb0nF+MJFeuLrZnzZ5Xeu6MvQfvybhNxFzNYgpU8Uv330NaEalwLPPGteQR1ZBX3MCStYdov7FeoSoB/wRe7n4tvCfxdCfAtUBCIA65+vY0bmXq/A02WM39MbNXxRVSlPhq8WgCt1BK1aG0vN+DWC8aN6UubQBSqesh2Rqst68c77swoVKcC2BnOIWZ3NsMGvWMX1/E2KFKoQYg7wMFBeCJEAfIBRoPOEEAOAM8BTAIqiHBZCzMPorF4HDFUUxaF9MH79/TNkDv+Lvp7nqDVlHOmKyPdH/yUtkK/nPkHoh0Zx2tr0ZPo8b9q7Xcu7v27IouGiV3E9r6b1E7v5n/9/RGpdqPDuSRK8muG9+SyG1DQM167dI1XTI/f1mwB1zaqceP+OZXGx7oT9mYrqShq6swnWMawIMjs14edJ3xKkMQ7qpyvZRE96jaDPb5X459+MZsnQr/LiANT5YTiVPjJdrVCcff1yrt8E6A8fJ+SZguGG3P+2yuXamjwBfnmlJou/a03QT/kFWPGrLQz79UmqLUtijO9GOhx4noqbLO//VQq1FOO/PYv4wZmEalyYO7sNgT8VXkrqEi9wqCH0pjneWGf2TAq1FKP5bzfDKzcHIBDb6eAVhlyUIrELpFAldoEUqsQukEKV2AVSqBK7QApVYhdIoUrsAinU20h8LZpl53YTM9Xh13rbHVKot6OAAQPHOk/i2jPNrG2N5DakUO/CpfqFrQGXWAspVEmJEBoNKg8PaFKbpGHRaIKDLJKvnOu/DbeLBq4ZsvFQOdGvw1o2ji58MXFpQl3eB6HRcP7JKnh1OU8L3xN8WGEHYDw8ue2Jl3C2wDJGKdTb8Jq9jf0fedLCJZNeXrvYSAtrm2QV1GW9uN6qKgnd9axr8z2B6lt72mZf86XFgadI2e7Hqhe+wvXsNYssZZRCvYMZF1rSImQ1AWonLr0cRYUp1t/YZilOfRJFTnAWnzVbyFNl1uaGunHFkMF/6UF8PrkXAZvT8Nx1iAuTfWi3eSihh/ZbxDYp1DvYdTYYQkAr1GT6lK4O1ZH+k/KuF9zwZvSWJ/Ha4Yzf9lSU3YfxZwsKoAkKZHGH8Yx4aZjFbJNCvYPg8inWNsFqvJ3UgNUzovHdcQ31sdNEpO0GCu53j//Omye2DKbKql0Ws00K9Q4S1wdhJe/fVmdfffDNXUB9tx2Z6d2b8n7teczq8ZhFt9nI4ak7EDpQoUIr1BicrL/x0ZbQBAXS89PlfDm+F4ZDxyyatxTqHVSecpgEXQY5ip7xz/5obXOsSmanJqQ+1wxVnWoA3PhJy5LEuvjPsPyR77Lqv4NTw2tSUfOvtc2wKqq61bnUuCy+vU/zU9g8/NQa0hU93qrdjDjfnLVvNsD9vILfPwnoTp+1iE1SqHfgclkhR9HjLEpvZWPYfxSf/aD/EYbQgphpjYnr+AM69EyouIXkF9cAUP4jdwBO5Fyn79G++dKo4X2RM01vmMwmKdQ7qDBlKxdH66ik0VrbFJvBP/gqi26UZcKIZ8gqq8ZzzjaEszNn3mhIlo8BdYDxkAx/72ukZLiQmaklblw5nDDdqIAUquSeKM3rMavGJJ5/8zU8VmzLOyZNycoi+JOCW6xdC4SYhtJbv92FpCHR+KmN39/3Y7pa2RrrE9tfw+NLRlndka8U6h1k+ZB39tJHkYuLiO3YGB6qz7J2Ewj/vfATUyyJFOodlD+g45ohG4DP4ztY2RrrEjQ2jp57ByC2WmY+/17INuoduC7ewQujeqBCwbnThcLdZZcCUp9rRrTrBi6O9rvrLJUlkUIthKxWF6xtgtVJriaYta4lEcet2za9iaz6JXaBFKqkUKp8H4ffdmtbcQtZ9UsKRX/pEp6/X7K2GXnIElViF0ihSuwChxCqJjiILkeusOzcbjK6Si8njohDCFX9m45BXqcwYOBqNdnsdkTs/q8aO6Epx8Mno0KwO0tF4FrLnn8ksQx2L9TjT0zGgIHdWSreHfAi6h2WX33uyKjc3Djzayj7mv3CgDOt2XWuEvujZuU9350FH9Z92OwHpNlt1a8u68WNlWGoEKhQMezTYajXSpGamgsv1GNPs58xYGBGpbXsjZqJ4bZ/DZ3h2vwKqH3KmdUOuxXq+T41WVP7DwwoVP1zKD4zSo+jCEuhNK9H+wGb8oWd1mXzVFxnmn06gprrXwRgTe0/SG0baVZb7LLqj/mxETHtJ2AAujXtQkSCbcxHOxIpfaLY8MX4vPsmY0cQ+PNh9CmpwAV8uUB6xShoZRl77E6opz+MJqb9BFQIqi4YRkRC8ef5NMFBNnsuqa1R9tettDIMJ7m6wPuogv/sLQVWUencFLRCzep0F7wOJ5t1lZXdCfXnPhMwYODhg89Q7d2jxf7lXBkQxZi3ZjMrMRp9bzW6hHNmtdMR8Jq9Da+7PLv6QhTrnxxLjuLK1POPoD983Ky22FUb9cqAKBo7CwafbUOZ993Rp6UV+Y66RiQ3Voax/aNJdHFPZmH4ck4OrGwBax0TtU85zr8ZzewPvqaC2pnD2Tqynnc3e752JdRHh23GgMK2FbVhx8GiX2hSm/eXzcnrdBn7qQo/9p1ofmMdiJQ+UVx8JZpWBzLosvE4u0Z8T2WNEwDdVwxHd/KU2W2wm6pf16YhH/pO44Ok+oTNOI2uiPjqsl6kfXSDxs4CAyp6xHXk5PIw9r8ykSbOpXXdfsk59UkU2/p9g5tKiwoVhjs8Tv3b8VueefENfKabd9TFbkrU+O4aDBhYMrdFsdqXtw9ftTzwNFmtLhD45Za8klVSPGq2jKOMyjnPH5eqwD/wOWg6RxN3w25K1NgnpmAo5vdKExzE7tETMaAicsVLRA685Qjh5lSrpHA0/n4oZdzQx8UDELskgkYtypN2xR2vfU558d4b/hud3K9wWucJ2w6Y3y6z52AibpaElWcXXe0f+cA/L36lxUZRaoKDUP+mw4BC/1nDqWTj59NbA01gRTzmZdKt/DpmVjV2OCt+vQW+Bv874h59sSKd3K9YzDa7KVr2ZhtQoeJ073v32HVtGhLT/gdUCJp9PgL3jce5ODyav7cvY0H43/SI60jYjNMWstq+SJjkxayQVXQvk2RtUwpgN0LtN2sEBgx06bmJzM53X3N6paZzXu8+NVJP+82n2D76e3IUfV5bVY6hFk6o99VixUvv3pSh3sZ1FZ+e6mhOk/Kwm6q/0vJrXHwhi49992GYupcavQagAJVnqI2+uwVcqeHMiy8vzW3kC473MC7/G3i2DedfCcGzOENapRiVMNZa/U+3BVIKPFf7lCPutaoc7jcRcKb/6bboP/dFhfldT9qNUNlxkB7/e4MtH0/EgIEjrWZgwICqlXHI5ObQya0hFBWTUqowfs1jVJtyFY5KkRaFQTH+7jYfiCSSHYBRnPj6oCvrxqM/rmdh2VUYgB9Twzj7WSQu/+6wiG32I1Sg3E9b6fRTwxK9E8F2m/D0YU983no+4595BgDRP4n3wpfQwPkq3ioXDMD7SY050KcaLocsI1KwM6FKzMv+2GAIh+5lkujx7aQ7xpud+CCpPit/aI7//OMYrljWh78UqiSPaq8cod3jwwh+LYZZIbfcw9fd/ALu/5bBb94RKqRstUoNJRTF+tOJnqKc0lS0tbYZEivxr/LnbkVRGt0rjt0MT0lKN1KoErtAClViF0ihSuwCKVSJXSCFKrELpFAldoEUailA7VOOpMXVuLSkKuqq4dY2576QM1MOjrq8DwHLslga/DsA5/5JZ9ATg1F2HbKyZSXDIUrUCyOiOfF1Mypu8yDh7Whrm2NTJPSryrTgDQCsy9QSqHYjvquHla0qOTZfop76JIodz39Lg/mv4rsTLnXMQn9NizZZDaE32NniB9zEblQI1ELF1OdOsvCzCtY22ybIadeQX4aNA7SEr34R7UUnjvaZRMVm561tWomxeaEqaigjnIl5ejIZT2XjLLSoELfFcGZdppYUvTvd3FOsZabNoa4RyWfTp1DbSUv4qkFUfekAxyfWBeDq34H4Y1/bcWxeqLfjKpz4Oa0i8/u0I6mRBwFLjb9sJTsHpz9VdAtfYWULbQN1eR88p1+mnpOGwQkPUfWlA6j9fZnzyFRicrIIXJZod2t0bV6o4V8do8v0J/LulRvpKBcPUWEXebtRNSGVaOlj7BxM/rErAaV8h2l64zCWhvzAmgxnzvX2RcmJx1DOg8bOgjUZXnlboe0JmxeqPjkZkpPvGed6LX9GeC8CQHvd+ssWrY1TWg7VNz5PlS90GOKOABDT3+jubMiO3lRhnxWtuz9sXqglpdHAfZyaZm0rrIvYvI/QzeRbnz+gzVoA/BY6W8eoB8Qhhqdup36ZM9Y2weZQ1anGIO89xORk4rX/srXNuS/srkRN7d2MzHIqMsvDE12MbrsrOf+d9/xJjxjO7PUB4GKWJ0fG16LsoRQMByy7x8eWUJy1eKtc2ZTpiT7mhLXNuS/sTqiXG8DxnrfcRqqFCr1yq5LzVrnyse8+0gyZ7Mzyou3YDdT4eSgh5nePZLOceLIMAD+dbwHY5xHvdidU58sqrhgymHK1MeezvHjT7x9CNG4ATE4JZcqcjvgc0aPONOCSlMHHldypcvSy3Q3HmAqViwujuywkJieTjHf8UUmhWoagz7fw1KFXcd8Sh/7KVcbtbMv3Fbfyc1pFVnaoQ/DpW0NTCuC2i1IrUgB9g6o877mFudeDUW3aZ21z7hu7EyqAy9IdBcQ393xjVKfN71rG3jjbzui2/JLO/ub3b8chev1J+nSyvwuwthk2SWaAcVrk+y2PILRORcS2XRxCqP+78AguSy3nXsZeEBoNH7ReBEBchx+If79k7pBsCbus+iV3R+XuzsU+dXDtdpFa5RLp42HsPBlQ0KSLIt62XaRQHYwPD62jodNG1EJFuiGbfdnQf38/svd7U/lz+10DIYXqYHx2phPnrnmRctiHiM+PoE9JJYCj1jbrgZFCdTAyWl2kHBcph2MNy9m9UDecC0Os88a/lC/tc3TsXqgB3ey/WpMUjUMMT0kcHylUO8CwJphrPZtZ2wyrUqRQhRDBQoi1QoijQojDQogRueHlhBD/CCFicz+9b3tnjBAiTghxXAjxmDl/gNLA4OD1aPpftLYZVqU4JaoOeE1RlOpAM2CoEKIGMBpYoyhKBLAm957cZz2BmsDjwGQhhNocxktKD0UKVVGUREVR9uReXwOOAoFAV2BWbrRZQLfc667AXEVRshRFiQfigLufYCaRFIMS9fqFECFAfWA74KcoSiIYxSyE8M2NFghsu+21hNywO9MaBAwCcMGtxIbfD4ZW9Ulq4ApARtPrKAYVfuXS0M/yxXPOtiLelliTYgtVCFEGWACMVBQlTYi7zhsX9qDA1lBFUaYB08B42ERx7bhfNKGVOdJHzeHHvwNAK9TkKMYh8ZjPFPr7vIrvRNsdi1WJ0r27tli9fiGEFqNIZyuK8ldu8EUhREDu8wDg5kmvCUDwba8HAVb1IWNoVZ8+Kzdw+PHJhT6P1Ao2j/6Oq/2jLGxZ0agrVMBfk4JBsd8FJaagOL1+AcwAjiqK8u1tj5YA/XKv+wGLbwvvKYRwFkKEAhGAVdfgnXjaiS7uRfea5/1vLNmP3fMUGYtjqOxHDW2mtc2wOsUpUZsDfYA2Qoh9uf87AF8AjwghYoFHcu9RFOUwMA84AqwEhiqKYtVp5+PdCi9J7yRI44rr6PMkP287JaveXYunyoVzR/2sbYpVKbKNqijKJgpvdwIUeoqZoiifAp8+gF0m4+yftdCKfQXC627tR6UvQNl1iPTuTflv4mS0Qs3iyKVUrz0U74JJWZUyp0r33Izdz/XfC0Or+nxX7zdyFH1exwlgV7YTwU8eyuvhaTIMxOsyCdW45ItnC5zs5szmLBWBPx1yqNVQJcVhv6bp3ZuSMExHC5cb+cL/vF6Rj/v1zxfmtHInXXcMtqR5xUJonSgXcZVpFx5Gn5ZmbXOsisMK9fxDgt1RMwqE//JyF1Qb9+YLy368MYubTLWUacVGFRnKtvpz2Xoy1NqmWB2HFWphPHKoJ077Crpc1LmqCFJrrWDRvbnSsBwA4eN1RcR0fByyjaquWZVjz0wCjEsMtELNguvlKfP4yULbeYnRKrRCjdbKSxLSn2hKYrTAKeQ6AO1DtxOTk4nmYgqlXaoOKVSgQKfo0xm9qHjHLgChdUIVGcq33WblxU81ZKJNtVxFI5ydiZ1Rg3nNf6Ce0x5O6dLJVlSsT4/gRa+zgAudV+xhSeua6C8mFZmeo+KwQr0T7+MFyyRVZCh/rvo17/6Ty3WYt6AVlT6y3FTq8fF12NPqex758DXcLutx/+cwhhs3uPBqNG1GjqXb9DfYMfhbfv+lCa6leMFkqWijNt/XE/d/D+cLOzE2isGLluYLm7O8pUVFCqBN1uCm0uIVn43roh1GkY6MZv8bk5mX2pDgT7Yw/mpdgsqkWNQuW6NUCPVauguGG8ZhqpgfGnNmfm2OPjuRdq4peXHqbBhE6JitFrct/LPDjE+uxguTFxEzoxHem8ux443vaX24K1sfCwFg+s6H+DVkDZqwEIvbZysIRbH+qhxPUU5pKgqd5LovNEGBNPk7nrfLHwSMnamwv14iKCKJNbX+zAu72S7td6odV5rf+5wAc2JoUQ/3TxNZEL6Cr69WZdq+FkQMPIKSlQWAumo4KfXK4/GHYy5F/Ff5c7eiKPdcZOGQQgWI+7YZB58eD+QX5U1uhh3NgRf298Nf7ma1GsURqsNW/YEbDNTfPPCecY7mwKuvDpMitQMcVqiui3YQ9sIJHt7fu9Dn1eYP5b1He+K6SHoBtAccenjKcOMG19JdAKi1cDgecbcG9MO/31KqF3nYGw4tVIBKTx2kEw2JYLu1TZE8AA5b9UscCylUiV0ghSqxC6RQJXaBFKoDIrROxPzQmGXndhM7vinqquHWNumBkUJ1QNQV/TjWaTIGDBzvMZnRK/5E7edb9Is2jBSqA3Lx0aB8902dczj+VpiVrDENUqgmRImuS/XdGoSzs9Vs0FQO5qd3xhUIn9nN9vaElQQpVBOS4efCWP/tHB9fx2o2ZIVVoJxKx78ZHjw2aAhNdhU+hWxvOPzMlDVwKWc9FzzqtXsYUKkFAM7sJPWZBqhQocZQxJu2jSxRTYjH4ctc1Gfwff251jYlj1qVz2PAwOwr0dY25YGQQjUh+pgTpBrUBGpsx1nEgvC/AVi9oZ51DXlApFBNzLP7XqCmkysZ3azvZFsdYd89/duRQjUxvh7X0Su20R48/ZR/3nXZI/btX1UK1cQkXC1rbRPyyPJW0Ao1KlT4rT5rbXMeCClUE2NLHakh7VeRo+jpd6od+qRL1jbngZBCNSGaoECqOSUTk5OJ29/7LJ9/5WBULsYdDSoXF9xUxl2ssckV8na02itSqCZEH1AOD6GiokZAvaoWzTtmemMWbVmE9xpXkv+OwOc/F/p7nQLgcqKXRW0xB3LA34QoOw9yXi/yri2BysODzL/Ksb3ad4ALM0NWF4wkFFQeHhiuXbOITeZACtUMhGrU6No2RLNmt1nzUVeowLFvgjleYzrgggrBnZWkVqiJaf8DtM8flqPoGX2hMftfq4d63R6z2mkKZNVvBnLQo84wv6PIi93DOdr2Bwy5/2pPHkb97X3z7g0Y+DnNN9+9AQM5ih4DBj7z305KhPUW0JQEWaKagTLCmUxfZ1wtmGfL0a+Q0SaLnY2nAU5cM2Tz1MARuO44wfwyTYh9ORiDs9ErjiJAKKDKElSZvd8uVgFIoZqYZdfqoPLYbxHHFn7rkqjW5GWOtZ/Chi/Go0KFIVekbb57g4BVub4LkpMJfTuh0DTsQaQgq36Ts/lqFQx3Pe3ItOhjTlD15QNUWzo0L+y3tGCaLBlFwDe2e1zm/SBLVBMTm1SBCmEG1DWroj983Oz5KTnZRA7eQZfBjfPCHNHZhixRzYCPypXrEfY/dmlLSKGamJBPdExNrWyVmSlHRlb9Jsaw/yhLavgA2dY2xaGQJarELpBCldgFUqgSu0AKVWIXSKFK7AIpVIldIIVqI+jaNMRvq2epPvTsXkih2gAnvmnG+J8mMr3SGtIjylvbHJtECtUG0HvoidQ6WduMIlGHh9LpcDLJf0ega9PQonlLodoQ53VZqLNsc+Gdumo4rRYeYnDZk/xRayY5HuqiXzIhUqg2xJMHXsBWt4WUmZHMqHLHAHj8tze4FmjZ2Xc5129l1FXDiek41egowkadmVx/uhk/V/qaw9lahh3vRY8Om3m3wi5mDquCGgP6Qsq7b/7pSNjCbEz1xZNCtQGM+5jAYP3zkwsl8JVYyqtd+fJSU9wfP8nOVg3598c4BnmdQoXAQH7D43KyGO/fGvW6AyazQQrVhvCYYntrWK8904zplb4GXFnzRxMqsgXV+r1M7vUEX1V2BwF36BTnFB0hJt6BK4VqZXQ+7nnXbrFXbO581hv+KgLUriTqMyh/ICcvXNl1CPddlrNDdqasTMo76dY2oViMOt0Np5U7rZa/FKoVyWrfmM315qJChVZYdrinpLiodQgrjvVKoVoR17UHefxo9zynELbMzMpruPhnGKc/tI6LdSlUK2LIzORk3C1nu+mRPla0pnAqHMjk4YNPoUKwq9HvHB44keXn9pD9T2Wy2jcuOgETIYVqZTRpt6r8a4NTrWhJ4ajX7sGjcwKPPzuAqH3PYEDBgMLqGn/x57TvMKwJJqOr+d3Ay16/lfGuednaJhSJkpONav1efLa70DXoSeJ7B+BUP5ldjX9jZbXFnBmfzrD9z6I7dcZsNkih2gA3O1O2OjN1E0NmJsTFU+nDeIRGQ+d6/Xh81iaGlj2BojFvZ1BW/TbAzc5UuU9crG1KsVF0OpRdh5hyqCUACV0CzJqfFKoVUZrXY2KN3wFYm+GC5txVK1tUMtRVwznS4mcAgpYkmjUvKVQrku2ppW7u0OSQxQPQnS3c454toq4RSbdFW1ALFYezdYgc8/qDlUK1Iu7HL/HZ5XoAVHltm3WNKQHqmlXp+OdW+nueJT7nOiOGD0d32rzHA0mhWhHdyVNsq6ulS6DlxiMfBKHRkPB2NB3nb2GQ1ymSDZm0++t1XJaZ3xes7PVLio0qJJgBz67ME+nTg0YSvsIyNYEUqqTY6OPiWVXLk1UY90s5Y7lFKrLql9gFUqgSu0AKVWIXSKFK7AIpVIldIIUqsQukUCV2gRSqNVCpUfuU4/wb0VxdFsnyc3tIGBMNKtveN2VNpFAtTNqzzTj3ZzUWH/iHta+MZXrNXzGg0KvXf6i9bW9ff1Goa1YlbUUV3j+5h7PvmG8/lZyZshAnZtdH66RDq02lXXAsjb4ejttFo0O0DWMnMWNTS6plH7WylSVApUbfqi6vTf+V1q6ZqBB88fzPTPo0EqF1InFoIxAQMGEHiu7BV1ZJoZqZGz2akv3CVfbVncL2LHcGLXmRo910+GM8qzTt2WYAfN52Pr+4NYZr16xpbrGJ/6QJh/tNJF6XySeXmzD7cGOqjDcABzjzZiP2D5mACkHHjX1h16EHzk8K1UxoAvy53qgSEW8cYVrwOoada82ZF0MI378NlYcHOY0icDp0lqs1jftP/DWpCJV9tMQyujZhTq/vidfl0P/1UXguP0SVG/vynn/bfwYAn1+pgfrcZUyxUlUK1UzEDwhj/8sTAFh4oxxnWqsw3LhVtYt3L9GiwhkW+6wEYHVaLZTMLKvYWhJEw5qsnTyVJH02Pd54HY/52/IdpZ7ZuQlNnbcAzvzzdktcEk2zBFAK1cRo/P2IGxrGyr5fAa4suF6en3t3RLlxq/oTTlqqe13gDZ8jeWELYuoRkmw673fmQDSuzd+LZqEWKrq98wZl/9ia9yyrY2MuNtZy6MWJgAsRC18mYpnpTrmWQjUhmgB//BdeZ1HwRMCVFmOGUX59AsrpWyI98U0zvuz8O13ck/PCpqaEEfa5Dtv0NZ0fAwooBpKa60muHkXfTmvx1txgsNcPuXv+jVRabtqfRgrVhBz5JIiZgd8DLjR/exjl/tiDLstYnZ9fWINJdX8nRLOJALVrvve+X9Ge8MO26Wn6dlSp6RzI1lPPSUNM5yl3PL2117vJrt5U3HvWJG3TvNQVxfreYz1FOaWpaGttMx4ITWhlft84Fzdh3K13WZ9B5m2/2iCNKyoEQ841Z2Lgprzw2dcCmN8hGt3JUxa2+P7IadeQHuNXcznHg9+XtQIgZPF1khp5sOOdCXQ+3gWl3QUwFN+X1r/Kn7sVRWl0rziyRDURSkoa711oybgAY7vMV+1GhpLNJb2xXHkqrgMpX1XCecUeGv71HHsbz2bRjbKM/64HFU5uvVfSNoX23925x7xDCLfsTn+tDgDnVlSmouGcyfOVQjUR+uRk4jr6UnvQsLwwrzgDnnNu7im6iLPqMobmddjd+EcMwIxzLagw1X5EejcyOzfh+EPT2JFloOLYLWbJo0ihCiFcgA2Ac278PxVF+UAIUQ74AwgBTgFPK4qSnPvOGGAAoAdeURRllVmstzH0F5MI/jjprs8ND9Vh+e8/ct2QxZvnH4GXXO8a155IjtSgVwy8H98NMH1pCsWb688C2iiKUheoBzwuhGgGjAbWKIoSAazJvUcIUQPoCdQEHgcmC2HjXmotQMKYaFb+PoN0JZvuR5/lTNMb6GNOWNssk6Bumevh5U1vs+VRpFAVI9dzb7W5/xWgKzArN3wW0C33uiswV1GULEVR4oE4wPx+CW0UTWhlTn8Uxfah36JDT5NZo3B+9JS1zTIZon5NPqm5iNcuNIF9x8yWT7Hm7IQQaiHEPiAJ+EdRlO2An6IoiQC5n7650QOB291mJOSG3ZnmICHELiHErhxsf0bmftCEhXDknQocHDARZ6Glxn+DCHnH/tukt3N8kDuPut5g5YnqJll8cjeKJVRFUfSKotQDgoAmQoha94hemPPEAmNgiqJMUxSlkaIojbQ4F8tYe0ITFkLE/ARi2v/AL2mBRO/tRbX3rljbLJPTsdF+tmapqfz0QbPmU6JVEIqipADrMLY9LwohAgByP2/2IhKA4NteCwLOP6ih9oQmtDJH3i7PWH/jUNVnu9tT/skzZnV0aw1uPNmULwLWM3BHX7PnVaRQhRAVhBBlc69dgXbAMWAJ0C83Wj9gce71EqCnEMJZCBEKRADmd05kQ1yv5UdM+x8AmJAcQdWRZ4xOcB2M8510OAsthnNuZs+rOOOoAcCs3J67CpinKMoyIcRWYJ4QYgBwBngKQFGUw0KIecARQAcMVRQbP/LDTExNCWNN59roL5+2tilmoVJFY1MmaI35/7xFClVRlANA/ULCrwCFznsqivIp8OkDW2enuCzdQaelN8+zd0yR3uRoTg6u52+YfUGNnJmS3DfOj57iDZphrDzNi30sKZeUeqRQJXaBFKrELpBCldgFUqgSu0AKVWIXOIxQhbMzKg8PrgyMImZaY4bGxrDk3E6urwyTfp0cAIcYR73xZFMGfrKQ3h6JGJciGFGhZl3t+VAbGqUPx//7W6vPhbMzQq3GkJ5ueYMdBHXVcGJeLA+AvpyOuMemAbAvW8fboaZd2Wn3Qr0wIpr5o8YSqrl1jmhMTjZDY3pxJrEcaq2BI61m8PdrX9E68A0qbtDjsmwH1zvVIy1ETcA35tk64SiceT8avYtx8ZtTtTQ2Np6e90wltuZtZgTyZqfqOKk5+240wZ+Y7ndr10LNat84n0jrThmO7+4c3I9cwPn0KSI4Rc6jjaAV+KldOdJ7Ii0ODcN+jsa1LglvR7Pvpe9R5WshFr0kc2+2gcpf7zHptKpdCzVnxJU8kbY++BSVv92HIT391n5ylRrljUv53rnW5RplfwHPdbF4ubiYdO+5o+F9XM/gs22YFrwuL2zB9fK8s6Nb3n2l39S4jznHwohlACToMhg55nU8Mk17UJrdClXl4cGIsDUA7MgSeLzuVKC9GTOxITE18jtK8J8my9Pi4r5gOxfXeNGtXI9bgRmZhCfuzbtNfC2aP6r8Cbn1VIcZb1JprumbU3Yr1At9atPdfR0Az/37EpGH8p8iFzO5CXFdp+arfmqsH0D4hkMYgLSHI3LbqKVqTXeJ0aekQkpqoc/Ovx7Nile+wltl3E27It2D0AlHMceiP7sdntLdtlbXZ8et75toVIuK2zw41nVSvvh7sw1EvnXZIRcwW4NLS6qy4pWv8Mt1TxSvy+Tjz/qhT04u4s37w25L1NtJDxDoh0bj0SWRP2v8gLfKhTu/gy9+NwL/s7eqpDLL9pHZp4GFLXUMNCGV6F9lW55I43Ky6LhhOBEzzbdx0W6FWubcrUp9/+AJtz0p2AZ9N6kh/hPyu0C80aEe2V6F7UOUFEV8nyAGl12Yd//kD68T8bl5h/nstur3nLONRl8PZ22GCyoEKgSphkxaHniaprufRYVALVRcN2Sx7+U6BZx2KXb7k1uXCyOj+XvgV3n3nY51JeTnk2bP125LVAD/cVv4bkYUH3SqAYD3siN4ZSXQbW9Cnh/PhmuGE7Ftt5UtdQw0QYG8NngeQZpb7dLsr/whcZf58zZ7DmZGn5aG5+/GMTs9EDupKQO8jNXQR5drU/2dxELHSr02n8Jrr6scRy0BulmCXh4X8+57HehP+VXmFyk4gFBv58qAKHZ3+QZwJkGXwa4OldGdK3z4yXAjHZGTY1kD7RiNvx/9gjfm3Y+9UgP/l29Y7IvuMC010bAmr785lzIq4xRf28Wv3VWkANcerUHC89UsZZ5dc6NHU6r9fYnu7kZnaAm6DOZNb4suwTye+wrDYUrUmFec6VHmct6973bZozcVZYef4Qt/44RKoj6DJ8a+id9Eyy7mcYgS9fwb0exuMzHvPnLlS3j9fu8TOTyPXMV/6w1zm2b3JD8fxdSw+Xn3HXa9hK+FRQoOUKIm94tiwytf51X5g8+2ovro0+iLOJtAfzS2UG9ukvxM+eD7vIH9VEMmgV9ZZwG63ZeoqeHkiXRSShVOj45Ef+lSEW9JioPKxQX1bY4YH945CLZZ5ywsuxdq0Lpsjubk8F1yJEuHt0W9zvaPwbEXTo1uQE2nW5VukBWdNNm9UDVrdvNGSDP+reWB5j85sG9KArZkU2P2MOJ1mUQuG4z6gnkWnBQHec6UxOoU55wpuy9RJaUDKVSJXVBqhKpr25D3T+4xbvaT2B2lRqjJI67TzBnin7G2JbaL2tOTuHHNeDk2jqGxMZz4ppm1TcrD7gf8i4uPu3Hjn/+aUvMjlwhN5WAaLTnJ0vKTidr3DCkHytOnw3q2e/uZbXtJieyztgGWQO3tzcuV1gJQ5pxjnmn1oBx5x58l5RcTvnIQVV/ah7culu1f2YZIobRU/b4+dHZLY1sWOJ21jV+8LaF/uAH7Ooyn2m9DqfrSvryDzWxFpFBKhHrsHU8A3op5Et3JU9Y1xsbQhFSi79QlzEipSeR38WY9fe9BcHihqmtWZVHLyQA4fVvOytbYHkdHBdDbI4nZ4x9Dl3jB2ubcFYcXamagBzW1RkdeLnvirWyNbaGpHMxfncfTK/4Ryk+z7TNaHV6oZ9toARhxPgrDtetFxC5dJHYIoraTlt07I6xtSpE4tFA1/n481349WYqOfV/WQ8mSPf6bqNzdafj8AbZlQdWPjgMgtE5kdG2CuryPla0riEMLNbFbGO+WP8TRHHD/894r/ksbKu+yTA3ayC+Xm6NPTkZTOZhz88JZO3kql2eVQxMWYm0T8+HQQr3JsrR61jbB5rjWMBCANXFVATj5dVn2NfmNBTe8+afuL3ywZj761rbj8sihherVw7gLdfGUVla2xPZIjjDO9fgtciFxVDSHomfROaYTs9o+xMNfvEa6wZlPZkxHE1rZypYacVihakIq8UPk72zLgoDViXnhKnd31J6eGB6qj9rT04oWWpf0QAMbMzWoswxsHPUNH12ujXjWgO5sAr4Tt/DFk73QI/j0v3k20WZ12CnUmMGBVNG4kqlkc2mCFjD2bGfVnEWk1gVYR5aio+aKoVQbcbjUHTpRqVYiD7noGP/dBP68FsrWwY0Qifvznit7DzN4yjD2jZzI+V5V8Ztg3bMOHFKoKg8PRnYxuuquqXVia70/bnvqwjl9On5qZ5yFhrgOP1Cv/HNU7G7+E5JtiQsbA6EGxOT48udTDyMO7S8QJ2jCHuYP9KFs5/MwoZBELIhDCjXjoWoM9loPwM4shWc3DKLSH7daOS6XM8n2ckLvqmbQ1wtY02g6zzUfiti8z0oWWx5dGeMWpNFbniTiUOF7zQyZmaTo3Qp9ZmkcUqjXA2/9WP1mDyPivfyzLgqgzf3//eA29Kz3B9me2mKc9+E4zHhyCmqhAnH3PXNC64SbKotMncbqvxuH7EyltTG2Nx892o2Q9+9+OoemcjCLa89kX7YOtxNXLWWeTVBBlc62TD0rH55ASp+oAs/VkVW4uqgyvT2ScJ5i/TUSDlmi3iQt04Vy99hlG/9cML5qN56P7QIxJyxomfV5vXUvFBcnUsfp2fLFJL59K4LZPzyGS7KBiy0MrO/wLVqg1rQ3qbxiB9beqyy3S5dyhLMz8e814NnO63naaxepBmc+Ot2ZhGUhBP5zFcOBY2a3oTjbpaVQJVZH7uuXOAxSqBK7QApVYhdIoUrsAilUiV0ghSqxC6RQJXaBFKrELnDoKVRHIWloNCn1cojpMDUvTCvUDDvXlFVrGhA54fQ9z9RyBOTMlA2jqlONY6+UYd/jE3AR+csUFSoMGE/YfutCFFsmNMb7Z9PvzVe5uZHZsiZneuuY2XwmzZ2NeTbb25PyvRIxXLv2wHnIKVQ75+oLUWz6eDy7stQM3N0PgD5Vd/Dr8SYIobCh6Q94qIzONa7qs/g8qTVxfULQH419oHxVbm5kPFyT013hrYeWM8DrTME4CKrOHUKV1+6+Oq24SKHaMSe/jKJLu+1suxSCmF4Bz3WxnOtTjcyo64R+pUfZfRhRvyYXo7z4462xVNYYBfvxpQbs6VXtgcR6YVF1djX+jb3ZBnZmhLExOYKEa2W5scSfKr1imBO2CoCYnGxGhRRcIlhS5Fy/HfNdj5lsuxRCmcdP4nI1hwszK7Djje+p4nuZwXMXEzO1CRyKxXfyFoY9O5THDj8FwHsV9nB80INtxruR7szebAMLUhqzvEN9kptfxf3xk/hO3sKhxADAWKJ2+m/4A/+cxUWWqDbKK3HH+PiD5zFoBFP+9z21nPKfM6hCRafuz8OOgwDktGvIhB8nEq7VsCrdi0kRkQ9sg7pmVfSHjV5U1N7exEwM4fDD01ChokdcR3RPZKO/8uALzmWJase8M/4F1NkK0/73XT6Rbsp0YUpKBFNTwjj5ZJm8cO2/u+nzxShO5uRQzzmJCyOiH9iGmyLVVA5Gs8iZow//iCpXMmlZLhx/L5L07k0RuU7ozIksUW2Uits8yNJrmBmyGoB0Qw4zU2ux+vlolF2H7vrejZVhrKn9B6d12Yxs0PmBS7zsxxvz9ZRJ1HG6+xmokStfosanl+/b96wsUe2YJ8vvzBNppqKj4V+jWFXL854iBUjPNnovrKxxIu71qg9sxyNfbcgT6ZIb3tTc8AK1ZwwjcunLzEwLBiDu8WmkTlahLuv1wPndDTngbwc0mP8qEa8WbxiowidOsNB0ea97qRm/vdYYn9/d8dgQR+jlW/v//9IG8dWsRzne6ifW1p5Po/7D8R9nHkcVUqg2yNl3o3nUdTegYsH18kT+nJo7tF88brYjTXHOu9i6n+Anjdf6O54pOdlUee4gfx8vQ3u3a6RV1eH/4FkWiqz6bQxdm4a4N7uMIfff1FFPYth/tNjvX34vK+9di2wdNegx5Mrow4f/AtXd27IPghSqjXHmUSc21Z8NQNfj3XBdc7BE77tqLX9YhCq3vA9xuoymcpCZ8pDYLNeynTFkZhYrrqZyMIY1wfxU/VfAOErgGWdO64xkdmpCK9crAOzPqIwu/rRZ8pFCtWHGV51LSt+ipyg1lYOJezGIZdX+yptKHXu5GT4zzHuAhLqsF+HvHsFNGPOcsLSD2fKSnSkbw+2CIFPR4SI01HISTP7wewaWGYl3XDbqLD2q9XsBUIeHkhlmnCr1ej+eA6Hj89LYlaVm3zMRgHm9v5wcVYPFwRNRIfjgUl0ipiRgroaHFKqN4f/dFuqHjWR393G4qbTUchJse+d7AC7qs3j2SF8Angjax1Dv4wXeX5HuzZsL+hAac/+laXK/KLK8Bf7f3X2oKfW5ZqzoNxZw5Yohg62jmqA5XbhXQFMgZ6ZslHp7YXT5LbiptIU+v3096k1SDdl0e+M1POY+2NK7pMXVmFX3Z3pNG0XlRZfRH4nJe6YJDuLEN95siJqKj8oVAwo1Zg8j7M37/2IUZ2ZKlqg2yr760HD8qyzt8h0AFdXKXUWboMtixtVoVk9ujs9c07RLa2qd2Dd0AskvZzLqbEcMikAlFJ7z/Ye2rumocCXNkEmDZSOp9uGBEo3z3g+yRLUTzo2OZvfw7/Pub5ao7yc15lCniibdiqJyceHYxFp82XI+3d2vokJgQEGVO4NgQCFRn0G7394g9O0H/2LIhdOSB0Lt58vpAeFk10pnYO3NrL5YnZAyV9lwMhzv1a4m2/oihSqxC+TqKYnDIIUqsQukUCV2gRSqxC6QQpXYBVKoFkATFsLpj6JYfm4Py8/tIbV3M2ubZHfImSkzc/KrKP7XdR5Pl0ky++yNI1PsElUIoRZC7BVCLMu9LyeE+EcIEZv76X1b3DFCiDghxHEhxGPmMLxQGxvV4uczm0jrZf0SS+3tTcKYaI71nsTTZZLyPQsfeozkfg/uYaQ0UZKqfwRw+56I0cAaRVEigDW59wghagA9gZrA48BkIYR59ifcQaafK75qN374/DtUdatbIstCUaLq8vz2PewbVvhJtzMrr8G3/ylUbrZxzqg9UCyhCiGCgI7Aj7cFdwVm5V7PArrdFj5XUZQsRVHigTigiUmsLSY1tU5kBJYpOqIZEI1rM+rXuXR3v7Wf/kC2njUZ+UW5MGIZws3V0ubZLcVto34HvAl43BbmpyhKIoCiKIlCCN/c8EDg9nVmCblhZsflQjqJ+nQC1JYvqdJ6NSOjvIpRQ+fR1tV4FuuBbD1D330Fl6t6tOk6/hfkzKWGcOyZScaX/MrD5SsWt9UeKVKoQohOQJKiKLuFEA8XI83CNukWWFAghBgEDAJwwTTCiu3jYRWRArz10W90dEvNu9+Q6cTnA/rhte7Wd9YLQDSDZ4z3CZ+qCOhmUTPtluJU/c2BLkKIU8BcoI0Q4jfgohAiACD382aPIQEIvu39IKDAGjRFUaYpitJIUZRGpjqAXCmXbZJ0SoLKxYVzf9XkUddbVf2C6+X5ol8f1Ov2FIhf9o899IjraEELHYMihaooyhhFUYIURQnB2En6T1GU54AlQL/caP2AxbnXS4CeQghnIUQoEAHsMLnlNoLw8GBv01/Q3tZf/PGl7qg27Ss0vpKTTZbOWJEFeFwzqxscR+JBBvy/AB4RQsQCj+TeoyjKYWAecARYCQxVFOVOJxtm4a+HpgBwxZCB89UsS2TJ6YER+e5bH3wK7bYjxXp3WbXFpDd/cPeQpYESDfgrirIOWJd7fQUodBGpoiifAp8+oG0lRiWMTeGuh/riue2ARfJ0anY1b+X7GV06hp99MWSevOc7apUBFQK1UHHuIQ1hf1vCUvvGIadQvV0yLJKPvnUDQr2vYEDBgMJbZ7sWa2Od3qDCgEKOoqd/h/8sYKn945BCHRC4ySL5PDJ+I39UWQnAh5fqcf15T4vkWxpxSKFW0V5CExZi0Ty3XwlBHxdv0TxLEw4p1HLqHPRe7pbN9IPiHfCgCalEHe9zZjbG8XDI1VM7Myui7D1s2UxF0c5I1RUqcPaJIBb5mdDTbinBoUrUt091B6CbewrZjze2aN69p/9N/BfGFVGGVvVR17zlllxdsyqxk5pSfkkWu1/Lv1BlxprWFrXTXnGoEjVlQiXI9RWmWOAr+O+QFoTOSKK7+1V6eyTy9HPfc7qnDg/VJgzADYMKJ2FAKzYRoC64ACV6by8ifrtmEX+79o5DCdVry2l2Zik0djaBT/BioNq4l7f+e4bunY0TDVqhJlx724pGNXleRm5nwfXynMzypUKfS+iTky1iq73jUEJVynlRQZ0BJlrkUhyqjTpEvdPDyaiWyfF20ws8VwsVhtyJue6xnYhfFUrlxTcdj0mRFheHEqr+8HEOZvsTokkjOUJrtoMPbseQnk7Q50b3jJ1oWETsRIJILHBog6RoHKozBfDJl31od6Q7mRVky8+RcKgSFcDnx63wI4RgHl/yEuvgcCWqxDGRQpXYBVKoErtAClViF0ihSuwCKVSJXSCFKrELHG4cFYxbmON+qkbswz/z9Mm2pLaQTh7uhtrPl4z6lfOFOa3eAwbbmj9zSKFWXKdhSfAMchQwKJZZoGKP5DzaCJ8PTrA05AeuGDLQKwq+ajcGnW3JhV4+ZjuA935wKKGqXFyInVGdZcFGF1mHs3UcuehPMJetbJntofbz5eVJc+nolkrLg0/hNUyB5FRixkRyoOd4OkUMQWtDQnWoNqqhXiRHH77lx63f968S/OQhK1pkuxz9MITObmkABHukEDfQn2utIqjy+jZavTeCpPpOVrYwPw4l1JjnXfKuq80biv/47Va0xnZR16yK9qo6b53s7JB/OdxnIi9+tgCAvq8vJ6zDvX0TWBqHqvorh91ymOu7A5vrEFiTKy9GcbWWQrNGx3kvcCaRWhfWZLiwKLkhR1P8+LXqbOb0fBQ4wpMehxlc9mQxli1aDocR6qWXo9hecyIAqzPc8YqRWzwAkoZEE9ozlgVh43AVThhQ6BH3BFcmheC1+ij6lFRc3C7Rv/FwVPv25r0377rvPVK1PA4hVLVPOZLrGkvPHVmCSY2aoqRYeBeqjaFE1eXhH7bxms/3rEj3pvHWF1GOliFs0gkMVy5TRnchbwG3IT0d1XqjSPWtG+Cm2sr/dncmjH1Ws/9OHEKoVzpUZc6jxtL0xT19qZwZZ2WLrMull6NY+fbX+KhcqbFpAOEjLlLpwkGAIncXxHdxooxwxnuVbXnDtvvOlKFFPQa8vZiGzvDMicfxme2OITPT2mZZjfQnmjL0lYW4CTV1Jg0jpOdBdBcuFutdVd3qHHl6Av1Ot8H7l6J9aFkSuxfq9Xeu0d/zLACnfg/HbWHp7elrQirR/n/rCNQmEz1uFEGfbYHinh6uUhP/jgYNauLHVS3+exbCrqv+jK5N2FDHuFW51qb+hExzWH/BxSJ2UCBLfBbRaNezBHyzpUTvXn2+CYebT6LVwSfxWLTX5jqidluiqlxcyBl8aw5fOele6oejtFXTuK5k4TOuZNvFbzzZlI/f/oml6Z54Pn0ZJcfyLuaLwm6FKioHsaHOPACmpoQRMa3AMQGljsNRs0k16Cns7IC7kfpcM5743z+4qbJ4c0Ef9Glp5jPwAbBLoWpCKlFvbkze/fSfOtrUAgpr8cGlmqgAlYdHkXHBWJI+PXo1XT0OMHDuy4SO3mpeAx8AuxOq0GhIbB/Ih757uW7IotfJxwiaLufzAeYcbYif2hVlseddD7FQ+/lytX8UMT824tdvvqGR20me/PJNQt+2XZGCHXamzr/SJM8j3l/Xw8h4Sm2z1ZWlqfJZNjP/CObvqktZt1vLiysHEv57FurrWcT098K/WhLTqs2mmtZ4XNKv18KY/WJHfDeWrONlDYRiA8MQnqKc0lQUem5FPkTDmoz6Yx6tXTNJNmTy5PBXcV1Uunv6d6KuWZUyP1xmTug/d43TOaYTZ1aGUGnKIZv4kv+r/LlbUZRG94pjVyVq7EgnWrsaB/PnX6smRVoI+sPHSXtYQ9tHXiKpkZZHu+3AV3uNmYei0B51I2TCYfRpFwk0nLcrH1h2JVTXw67QBv5O92Jp75ZA6Z7PvxuKTofzip0Er4CjH8NRXPPm7e1JnLdjV1W/xDEpTtVvd71+SelEClViF0ihSuwCKVSJXSCFKrELpFAlxSZuXDNWnd9HevemFs9bClVSbE48MxWAjZN+sHjedjXgb26EszNCrUapWYWYPm60anKE9TtqAPBP129o989IfDdo8Z6z2ybXbJoTYym6z2r5l3qhqtzdoUowx4Z48nnr+TxV5gpw2zHqwetzL9yIaz8N2kN41GAiXy5d07fnW1rXh1epFKq6rBfn+9YktW42r0etYrDXRmubZPOEv7oNnrFe/qVKqELrROxPNRlUbxOvl1trbXPsitur/ip/DCYcy+5SLVVCvdyvITFtJhUZ74ohg40ZAcxKjObIOX+Ot/op33NNqvoubzousuq3IFfrGO75/IohgyGnupI4vgpeh65yuUl5Wg85UCBexJfH7HYV0v3SvNmRvOuKGyy/kKlUCbUw/slw5ZeL0Rz7tRradPCKS+d8X4VJX/1GbSdtgfgdj3eG65esYKl1+aXyhrxra/hOKFVCdb2gZnOWimsGF/683JhDP9SiwrbL6I/G4lc3hZNva9nS4ufc2AVFCvBsxe3MqdoO5dAxi9ltC/Q93TKfWC1NqVuPKhrXRmTkYLhNaOqaVRm99A+aO9+7aXCTudcr8GutKqVqLHXV+X15149VrGfStOV61EJQdh7MJ1IAnZdLsUUK0LPMJc4Pv+fvVWJiSp1Q74fh56OZfc2XNMMt52vXKxdf2JIHp1S1Ue+G5lA81Tc+z9GHfua1C004muLPld+D8UjIAcBl01FOVWzJV0/4s3n4N5QRzla2uPQhhQro09IIH3SKxxoMxOngKbiSgA8Jec8NALEnCfzyJKdfFtQsvJ8lMSOy6s9FuLvhlJCM/srVu8ZJGhJNVa1xsL/sMXl+lSWRQsXo2OKlDRuoOe8U13o2K/BcVbc6MT82YsrrE9Cg5vvkcHx/3G0FS0svsuoHLjT3oqPbdTq67SZ57CZGjWjPnsRg2occoblHLCGa7bmD/8ZSdMLGdkTmlK7VU9ZGChXQPnrrZD9vlSszK62DSvliAKAWKj67XJUanySgs6iFEln1A9f2+BQrnl4xsKlHTXTnSp8v1r6nW+b7tDRSqECVmef5Oa3iPeN8eaU6XZp3Qx8XbyGrbIuLUWk8VrEeF6Os41RNChXQxZ9mfp1KNPhqGP3PPMzSdM98zyP+HcjCcW3QnTpjc4cwlBZK3Vy/xPaQc/0Sh0EKVWIXSKFK7AIpVIldIIVqRoSzM1f7R1nbDIdACtWEpD3bjEExJ7m+MoyFCTv468RGtn0yCe/N5axtmt0jhWoCUp9rxvWVYWwYO4lu7ilsqP0nb1+MpuPRp4icP4RfQ/4h4e1oi9iSNCyaVef38VrcYRLGWCZPSyCF+gDktGtI6vJw1n85gQ21/yTZkEm7I91pPXQwx5qpcHrkNJFvH2B3Fmwf8q1ljFIgR9HT2jWT2YPGQbM6lsnXzMhFKfeJukYkn02fQj0nDUn6LJqve4VqX97A6fBx4HTe6cyG9HR6LxpKzDOTLW6jnzqHlEh3ylrWqYlZkEK9TwzuzkxNas1/sZFEfpFBxKE9d3VK4RFvnYprT3Z5yv5i20dHFhcp1PtE2XmQM00hnL3ca5ufcHbm+cHLLWaX4ZFki+VlSWQb1cxkta7D8LInqbOtj0XyUzZ6511f0nneI6Z9IYVqRlJ7N+PzyVPJUnRUHpBQ9Asm4KuXZ+Rdfz/+SYvkaQmkUM1Eau9m/Pn51zR0hq7He6BPSbW4De+O/M3ieZoL2UY1MeqIMI6NqMCGLl8ToHaj3+k2qNqetYotk888jIYzVsnb1EihPgDqGpEoWjUiW4fh5Bni323AhF4/0tY1Cx3OVF3/ApGvWE6kSUOjaeq8BTA6yLj6VxC+UqilizP/i0bvpJBTTk/d6qc5siWMX56ZSGNnQbIhg5mpdRjlbXQSvDtbz4DJI6gydotF/ahmlocyKsf04iKFeg9UHh6cHlmbR7vtYIn/92gwOp84ocug/Z7XGDJ2GDvfnYS3ypVR3rGA0Rnwq2+OouL8LRa3t0MXBxjZvwtSqPfg+KQIYttOJM2QSdfjTxG3J5jAdQaEQWHJ5HFU0xpLryxFx5IbfqTo3XjO8wQjPp3LzJgOGPYftai97/pu4ma1vzrDHb/tqVh/o5FpkL3+e1De5xqLbpTlqT7DUNqcI3zudc49rOKN8b/miXRzloqm341kZtXKLKxRgajvR9HDPZmuc6zn9BZg2L99UXYftqoNpkSWqPdgXPU/WHu9Bhcbu5AxqB77Wk7FVTihQ8/SdG9e/a8X1SZdo+KBW9V80KR9tGz7JKtq/c70QV0oP806U5idG+3luFVyNg9SqPdgyoU2zKr8H2NGGg9aWJpenjG/9sVvZw7OK3YSyc4C06eG9HTKvpiNdpuaKn1jSJ1mebsBNk9vRHkcY54fpFDvyfn3q1B/WAAAOXu9qfzFbiplFd1J0p1NoP+pR/kkeAnDaW5uMwvF5apjORoullCFEKeAa4Ae0CmK0kgIUQ74AwgBTgFPK4qSnBt/DDAgN/4riqKsMrnlFkD7724C/r11X5KOSfLLfgzxGY6aPSa3qzik+6kpY5WczUNJOlOtFUWpd5ujgNHAGkVRIoA1ufcIIWoAPYGawOPAZCFEqTtBzHDgGOq1lhXpQztfzLtu1Ge/RfM2Nw/S6+8KzMq9ngV0uy18rqIoWYqixANxQJMHyEdSTPwmuhCXkwXAka9rWdka01LcNqoCrBZCKMAPiqJMA/wURUkEUBQlUQjhmxs3EPIdlJmQG5YPIcQgYBCAC273ab7kdjT/7WZkiHGfVBksf2iZOSmuUJsrinI+V4z/CCHudRpYYT7DCzTvcsU+DYy+p4pph6SUUqyqX1GU87mfScBCjFX5RSFEAEDuZ1Ju9AQg+LbXg4DS51BUYlKKFKoQwl0I4XHzGngUOAQsAfrlRusHLM69XgL0FEI4CyFCgQhA+hGXPBDFqfr9gIVCiJvxf1cUZaUQYicwTwgxADgDPAWgKMphIcQ84AigA4YqilLaDmOWmBjpH1VidaR/VInDIIXqIGhCK7Ps3G6S/44gcVQ0mtDK1jbJpMi5fgfCgIGN9X6HesBr0GhHP7SrvfCfd/yeJxI+KOoKFbjcIZxLUXoqhyXhrNaR+HclAr413eJxWaI6CIaLl2g4fkS+sD1NfmXru+OJmVDpLm89GCoXFzK6NaHe6ou8/s7vDIjeQHnX65y56k1Uz72o3Ew3kSOF6iio1ZRrk1joo+XNJyHq1zRpdsLZGe2qsvzw3XcsWNacnxvVZWMdF649dJnQMdcJc72MUj3UZPnJqt9BUJUvxz+15gHQYm9vfIdmYPBw59gwT9zOaAg+tMu0+bm6UEaTRdffXyP0va35NjHGDvRnabkFzGvYjvImOjJWCtWBUKEi1ZBJuU9c0J2OASBysPGZqQchU+f6ELfBj9C3Cy7O9qlziT6n2uI397DJduHKqt9BONs9EAMGHpr+Bmw7YPb8KnskEzqm8B0EPSvvYuvBCPRppjvlTwrVQbgWaTxGuNwx86/sP/N+NFsPhxf6TGlej0FeMSbPUwpVUiL0rRvweZ9fqD7mRKHPUyJccRYafHaadq28FKqkRJxt68ziK/XvOi6b2S2FdZlafLdcLvT5/SKF6igIBRUqctwKWw5sGW70aMq2RrMY+M8A9EdjTZq2FKqjoAgMGPDvb95j2vXOsCsxuEC4xt+PH775DjeVE5X+Nn2+UqgOgutZy4w0tnroINmHvQqEH307hM0ZVdArBq4HmH4vpxSqg6Bzt9xyTaea+Z0Sn/kgmp87/cDCp1qSpE/H7ZLpRx6kUO2A1N7NiJly7428OeWMQ+uH9oWY1ZZdF4KpUu4KJ7+I4uIr0VTZ6cK7z/7B0KlD0Hm74qZSU2bVQZPnK2em7IAbT6bxZc2/mVmvI4Z9RwqNE99lGjmKioDN5rXF9U8v/vfxz6QEufKQi46usR051a8tFY9sIeHtaPqd6IYh/aLJ85VCtQOuX3Knq/tlMueuZk7PRwuINbV3M3KU3RjueZCQafD6bRvvrO+B3q8s7we64f7PIQzpxsUwWd7ma35IodoB1cec4OHgnqyrMxenP1Yw+dTD6H7yAyC5uooRTy8uIgXTojubAGcTcN1Fvq/Gow/v5Xx6wY6WKZBCtQP0V67i/Ry0nf0Ma2r/QY9af2L49s7SU8WhbAX3hAyr2Kjx92N4hYWMOd3dPOmbJVWJydFfuYrHUzm06jqcpEezOdr2h7xnb19oyuHUANS99YhEK/mccnXBQ2XgwvRQvDB9G1XuQpWYBE1oZaIWH2djHZcSvyt3oUoshi7+9H2JtLhIoUrsAilUiV0ghSqxC6RQJXaBFKrELpBCNRNK83okvhaN2s+36MiSIpFCNROf/DadvaMm0nfDTi6+Em1tc+weKVQz0WvzIACeKnOFBa99RaXt7iQ/H2Vlq+wXOYVqJtRnbg1+h2jcmBq0kVeHZRO7tzqGA8fAGjOCzepw4slb/qBUgRkcbTkTtVAR9u8LRPS1zplYxUEK1UxUXpXJvl466jnd+hWPC9gOy7dTfUN/It5OQRd/2iK2GFrU43+//ERdp204C23+Z4BB0XO07Q80GzIC38mWP769OMiq30yo1u9ldN+X2J1d0KnN0ZYzSRhnmSOLDC3q8fEvM2jirBQQ6e2oUNm0GmzYNPtHtXEv71drQfjiwQWe7Wr8G2ffMX8nS7M3ljEnngBg9rUAqv03MO//moxbX5Zf0gLx/2mf2e25X2TVb2aUrCyqjTpA53FPcqKvH0demASACkG1x2K58al58zfcuIFrDzXdyveA9AzCL+wFIKtjYxo9fBUwtqUzFS2G9HTzGvMAyBLVAhgyM9HHniT0w508/uwAMpRsAGp4XkATWNHs+evT0tCdPIXuwq11ojoXFV6qWx2+qTM7m92OB0EK1YKoqoRwqr1LXlvxwwr7SYku6MzBEmgHX8i7npYaQvBP9zqM0frIqt8CqNzc0DWsSu/pS+ntkcTNUzjVovByQu3pifDyNO5NMgPn34jm76pfAa4APFHmKNfXGUvXH/9uR9jowt1JWhMpVDOjcnHh2De1iOsytcAzvWLAoBV5vu4N6ekIjYaEXwL5utafDP1rIG7nBO4XDXjM3Vbg/ZKS0bUJSQ00rO7/FQFq17zw8mpXnvbcS9fxbxKy3TbbqXIrihkxtKrPpz9Po6FT0S5ushQddea+QuV651ldfVGB5x0CG9y3HWfej6Z15z285/cv5W8T6E1mXwvgt8GdUK+zzoB/cbaiyBLVjFxs5FoskQI4Cw3He00u9NmriU2BnPu24/u+02ntmsnNqv4m8bpMekx6gwp7stCuM5GzfTMhO1M2wN3aqgBphky2TbxnYVMkw+cMZN71/Ku44nWZ9PzidSp+tQXtv7YtUpAlqk2gV27t0R9yrjnZBg0HZ9TC40wO6mwD3msfrHMT8t5WZq3sTMRv06nvZPxSvJ/QmQpTbK/TdDekUM1I0LIk+j/5MF8GLsdXfWsWaGpqZQzKrVI0NsOXHd81BKDsH3tQcjLwwbQiEh9dzhMpwInk8pTDfKf5mRopVDOiPx7HxSh44tnXSfe7JZKACTtQdLrbYubghbFXb46urdqnHEMr/5t3f1Gfgee3HmbIyXxIoVoAz9+34XnbvaXHWeKHV+NR13/y7ttPfJOK/9nmKqm7ITtTDo7Kw4MpfW6N4aYaMvHfbh3/VA+CFKqDc2ZYbZq73Braar+/P6r1e61o0f0hhVqKyFH0eH9WcMDfHpBCLUU0mTASscVK3v4eEDmFKrE60pufxGGQQpXYBVKoErtAClViF0ihSuwCKVRJoWgC/In/PIpTf9QhYqczmrAQq9ojhWom1BUqEDuxKe+f3MOq8/vw2uSDrm1Da5tVLNQ1q/Le5r+Z9vQPqNUGKrte5om/t6GOrGI1m6RQzYAmsCLVV14htvsU6jvp2Jap5+tKixk3YxK6NrYv1mMvlSXF4Mbo9wcR/OQh1vaoz/Oe5/H7JQnh7GwVm6RQTYyqTjWe/W87X/nv4lhOFg1mjOS9fgPp338ElTUK51ta5w9dXFR1qrGu2zcMXdIfr9nGpYf643E8deIxfgxeD7UjrGOXVXJ1UNTVI+j/5wp6lrkEQDWtM4cHTmT27xNxOXmZa4aCfqhsjZg3XZl2NYqIN3flCz+0ORyAuGess45VCtVEnPimGVNXzaSHezJndOm0O9KdZ04+SqI+HW+VCyeeDyRA7Ua5wzYsViFoGnqKVQnV71jYDZVXZgJgcDP/wcCFIRdOm4CEMdEc7TkBFW5E7XuGCoNu4HTuNNeAp1b0w9Mpi1ptYhhyrjmeK49Y4Azo+0Pl5savIWuovWIY5YjJ90ybdJ1EfToeMcXbVWty26ySqwNx9p1o9g2bgArBt8kReHc+ie7c+bznbl+XJfmXYP4IW83WufUxXLtmRWvvTUqX2gAE/3u9wDO9lysBajfSat3/tu0HQZaoD0Bar2asfOkrdme5supaLeYueJhKhvxbPFyOJfLC5O08deIxKo7fYfFtKCXh0dEbWXDDG7HrSAE700KNmxOrTbxulRpBCvUB8B50hkC1Gx0mDcN3bzaBOZn5nivN6/H2rz8BkPmiF4rukjXMLDbdPfdwOLtigfYpQK2RB40XsZbxkn0nUqgmQHsDtKvz95JvenrORs0Xj/dAHxNnJescA9lGfQBObKmMAYUvR04nrVczUBk7GvqHG/DOrF/Ykh7Bx/37o485YWVLi8eq67UI0V5CXaFCvnCViwuNPeMBEJUDrWGaFOqDEPLOViJXvERb1yw2fT2ZU3NqkvZsMxb8Nol5V5vw78OhdrWR7peYJjRzBn2VgHzhyT3qMcAzgXP6dJT4s1axTVb9D0jki7tp+eTL9PlwGUda/AwtYGOmO6c6eKC/bNtt0uKSEmksz1qtHklk5q4iYpsHWaI+KIpCmfnbGbva6Fpchx4/9XWSuoRb2TDToAkL4dNevwFQeaGwmh1SqCZA7VOOkY+soN2R7jwyZCjpiobJ744nq2Nja5tWIrJOehYIS6vnRzf3FJINGbjFWs9XlRTqA6L29qbF2nMkZHvj+lwGrot38PYzA6ih1XN5QHpeB8seCJ9nHOjP8L91CMXFp4xDbo3/fcWqnUIp1AfkSudqvOVzlGXzo9FfTDIG7jhI7ZXD2N/0V9TlylrVvpKg7DpEr/hHiBh9BHVZL2J+asTRljM5o0un6sTMohMwI1KoD0hqp+sczskm+Jv8znBVLja8+ORuKAq79oczLXgD0RsuEv/4j6iFirbrX0HZfdiqpsle/wOSfcGNmlonkl5oQMCqRAAMHq4MqGNf3vJuEjlyLxH6lznwxPec0el45q3XqbrssNUX0khPKQ+IJjgI9zkZzAn9p8CzqH3PUK5rfKFTkpJbyMMmLIDubAKpLaADBU8t8SbWpheh2BOyjSqxC6RQJXaBFKrELpBCldgFUqgSu0AKVWIXSKFK7AIpVIldIIVaSinU4ZlKzamPo5h8ehOrzu+jzcEbnPi9HqJxbRDWW4sKDjozpa5ZlZ4L1tDL42K+cK1Qk6MUXCxSf3tfKn6tsdsTQ+6HO5fsGVrUI2LcMZZXnMQHl6JYHG/c4/9SnY2MWhRL1NtD8Z5lvUN+HUaohhb1ONfKjaxqGaxtNQE/tXOBhRQ5ChgKWV6xu+nPXJqbZTx68Sv7XExy36jUnPqwCX/1+ZZMRU2Nn0dQ5etjVEw+AsC6gAiur3ThUlM93rOsZ6bDCHXkz3Np53rTC0nJPeZVUDtjcJjfRvGJ/7QJB/uOp098J9KG+xOydyu31zm6xAv8urcZjeucINVqVhZTqEKIssCPQC2MZ86+ABwH/gBCgFPA04qiJOfGHwMMAPTAK4qirDKx3QWI0F4BnPg3w4PP4jrke3Z9pT8Bm9IKvJPh70bfsUt4zvMsmzJdCPnpBKVpnZPKw4O3ui2k8c6+BHQ7ClwpNJ7aSY+7JptUlRqs5JGwuGXI98BKRVGeFEI4AW7A28AaRVG+EEKMBkYDbwkhagA9gZpAReBfIUSkohTSODQh3aa9Qf9eq1j8v3aU+XN7vmdlOFnoKibdyjCe8zRu/5158SF0Fy4WEstxSepVi+c91/PtDu+7xjn7bjQ7HvoaT5ULnSs8fmsXg4UpstcvhPAEWgIzABRFyVYUJQXoCtxstcwCuuVedwXmKoqSpShKPBAHNDGt2QUJ/nQL67rUxvPfY0XG1QQFcuHVaKZVm50XdmhuDXOaZ5Nk+hh78hX23d3xmd5ZwVPlwtJ0T5R0651KXZzhqTDgEjBTCLFXCPGjEMId8FMUJREg99M3N34gcLuXgoTcsHwIIQYJIXYJIXblkPVAP8RNdCdPoU+5d0tK4+9H7NBK7Hj9e8K1Gi7qs3jxTFsqrrJOSWFNvGONlVxOmcI3IGoCK0LkDQDe+KuPVT0RFkeoGqABMEVRlPrADYzV/N0obMCtQM2rKMo0RVEaKYrSSHsfnZ/7xWdhBgf7jgfgqj6L/jHPcjEqDf3x0ucbymvzaZL06Vx96gYqd/e8cJWbG1cGRPHsf9uNTjUA/23W3YxSHKEmAAmKotxs+P2JUbgXhRABALmfSbfFD77t/SDgPFZG5e5OSt8ovg1aAUC6IYd2U99E0+6MlS2zHrrEC7T58U0ORc9CWVqWE98048Q3zai/+TrbP5qU5+J90Y2yeKwtukllTooUqqIoF4CzQoiquUFtgSPAEqBfblg/YHHu9RKgpxDCWQgRCkQAO0xqdQlRubsTuk7Pps8n4qFyAqDT4ecI+qyUjZkWQuUvd9P4k6G0LB/HC4+sZXGPcZRRZ9FgZ2+a7OkJwOiFvYtsUpmb4vb6hwOzc3v8J4H+GEU+TwgxADgDPAWgKMphIcQ8jGLWAUPN3eO/F2qfcoStvMG4iltyZ6bg26vVKPtidqkairobSlYWFaZsZf0UVwA2h/VCd/IU/hwlfk5dANwTrDt9CsUUqqIo+4DCdgkWunVUUZRPgU/v3yzToPH3w3dROt9U3IQB48xUo7HDqTh9P4YbCdY2zybRnTyVd/1RgyUABP593upfaodelHJqUnmmBv+Xd//t1Wq5Ir1hRavsA3XVcOo6n6P6xufRn7H+l9ohJw01lYOJ7xvMokZjAWObtNqyIdT4JFGWpMUkZlB5IrUuqI6XsQm/BA4p1NO9gtk7+HtuivSLy3Wp8eUldGelSItLjUanAPDda32RggMK9cz82vzZ+Btu/mgfX2rAnqcj0Z+0D/fktsCNJ5uysMoE3ktqhOuS3UW/YAEcpo2q8fcjaWg0i5tMJVxrFOmHSQ3Z80xVu/GhbyskthA4Cw3zVzW32iKUO3GYEjV+QBX2DjFW91f1WXx6sS0nnwsulTNOpkJlGxoFHEiot9M3theqtmcxroeR3A+Hc7IJn37O6sNSN3GYqv8m312tgdOL9uPl2RYJH7mNN0KaoTtlO9PL0u2kxOoUx+2kw5WoEsdEClViF0ihSuwCKVSJXWD3Qs1aHcJrcYeN3jzs6EwnScmwe6EmbahIW9cs/l40i+s97tlxlNgxdi/U0F/PMjklFIC3PvsVdc2qRbwhsUfsfmZKd/osU+Z0ZMjLE+nodp0va5SljBXO7spq35jTXQVlA4yOLvqFb2N42ZMAqIUKvWJALVQ02fsU3h1jLW+gnWP3QgXw3X1rX3qZwedgvmXz99vqyeTgCbgKp3zhVwwZvJ7QPu9+eqU1POR/gkOWNc8hcAihWpu9F4Jol/wcyXsqAFBlzlXIykbo9HnTkEp0XXLm2dAqDztDCtUEVOxu9HznlbsIpjA5JldzK1DiSoqP3XemAE73sP56haJIjTR+Xs1xv3dESaE4hFCrV7G6f4siKVv7MgDbVtS2siX2iUMI1eZRqWnmd8raVtg1UqgW4OKwpowL2M73yeGEjpN9/vvB7oWqrlCB2mVtu+pXch2NbEsJRZ9W0KGwpGjsXqjZtYL5zHePtc24J2XaXwDg0Co5a3a/2L1QL9d2sbYJ9ySrQ2N+qf4LAH47sq1sjf1i90K9FmZdv51FkdRAS4jGjf5nHsbpn73WNsdusXuhhi7OX0r9GvkHSnRdK1lTEK/mxnMBdi2vZTN75O0RuxeqZsthep9ql3fvo3LF4GQb61I1wUFMrDaHLEVHyB+l6yALU2P3QlWyskjLvtVOferEY2g228YQUOzLwdRz0tB05/PSW8sDYvdCBbj+XVDe9Y0cZ5Qc63da1J6eDO6yigwlm7K/eFjbHLvHIYTqunQ3kfOGoBYq4ncEF/2CBYh9pyYjvONosm0gbgu3F/2C5J44xuopg57wV7fx2Kv1CMV6B8veTmD9RADEHk8rW+IYOESJaosMD/mPv9PLUHnacWub4hA4Rolqgyy7Wpf1u2oQcVlW+6ZACtVMnG92jQikSE2FrPoldoEUqsQukEKV2AVSqBK7QArVgmiCgzjxez3U1SPMk4EQXB4UxWOH0lh+bg/Lz+1h1fl9VN+tIXZiUzQB/qjc3MyTt5mRHqctiPN6f65kuOH19GWzrPTXhFRi0eaF94zT59QjJH4ejssyq56jnA/pcdqGSOvVjAXhf5M1389s21GOfexTIGzslRr8ne6Vd/9ryD98Nn4qqno1zGKDuZBCtRDjPp3EpJQq+MwwzxSvunoEWx+emHd/NCeHxl8MZ1NLf6Z1epxq84bmPWvirHD+fwpqT/uZ3pVCtQAZXZvQ2FmwbFgbs+VxbIwH3qpbyx2f2DIYvwlb0Kekoj8eR8Qbu2j12i2x7mr8G0pooNnsMTVSqGZGExxEi/9tY3JKKE67zXPu1aWXo1jUcnLefbM9vYh4Kf/6V0Wnw3PBHh453CMv7NiQMmaxxxxIoZqZzAg/PvTdy7dbHzFL21RVtzrfvv4D1bXavDD96vIYrl0rEFfJySYt0znv/ukmO1F52MdaWSlUMxPfXYMBAzX+l2jytIWzM8decae5yy23mw13PoffhC13fef6jVvNg098d6Nyt4/hKilUM6IJDiL2iSmoUKFLOGfy9LNb1Sbm8R/y7rOUHAK+0N7jDYgYdZFvr1bLuz/3VBWT22UOpFDNyOlnK2FAoeqfQ4uObAIyFT1sO3DPOLrEC2y+ekuc2V73iGxDSKGaCXVZL8q2vsDf6V5UH3vWpGlfHB4NQHJEyf2txo1rxieVF90KECYyysxIoZqJ831qsqb2H+y4Hmbyaj/wr1MAuHTKvwW71eQ37vme2tubjg/tztfxqrw0xaS2mQspVDNRpuMFVKhYPbG5ydPWnSvoFO6zy7WpNO7uPrhULi44LXLim4BteWF1fhiO4YB9bJWRQjUHTWqztvZ8s85E3cmsfc0wZGYW+iyrQ2MqrHVifvjyvLAJyRGETTthN95bpFDNQOLbOlQIxq95zGJ5urhnF3pyodrbm7OPqplRaW1eWLIhk1kzHkd3wX68t0ihmhhNcBAf1PibnVkK1aZctVi++6NmofbMP9OkrlABwwI3jj01KV/443tfwH/c3cdabRG5uc/EHH0zkC7uydSfMJzAo+YVw4WEclDn1r3TYhcujzeOkZ7vmkOfutt5t3z+4ao+px4hYMh1dGa1zPRIoZqSJrU5/sRkJqVUofJvp80uhmqTbrCjraCJs3FN8fzw5TD+7vEHnGnNtadd0J0z/eSDuZFVv4lQl/Wi08wNaIWaZcPamGUm6k4M+47w6aM9mJoSVmTcav8N5HIX50JHDOwBKVRTUdGPPdcqUf+zIWZbJVUY+rh4/nrtUZ458XiBZ7+kBdLk8+F0a9iB8L770F+6ZDG7TI3ciiKxOnIrisRhkEKV2AVSqBK7QApVYhdIoUrsAilUiV0ghSqxC+QUqqREZHZuQnp5NZeb6Si7X4vvZMssbpFCtSNuPNmUtd9PpkdcR7JaXbBo3mpPT+LersnBPuPRYFxO+Oi8Fy2Wv6z67YhrwWoMGPio0mI0IZUslq+hVX0G7d7D0T6T8kQK8NesiYw9tY0Li6qjjjTvblYpVDvhWs9mzB3xNQDVnVQktrecO572k9bT2a2g84wywpmaWif2NJ7Nd//8wtl3os3m0EJW/XZC4iM6wnI35eUoetSF7zoxOWffiWZA2W8Ao4eVx491RTPKuED7RogHic3V1Gx6kgXhKzg4ZCJDujbnVBPT2yGFaofUWTyCiJnm34uV3r0pW1/+hjLCmVO6dN5L6Iy2Rxr6FOP2b9f9ELYYsst60brVYNTDLvJT5GzajXud8Fe3FZF6yZBVvx0gGtVi5sM/5d1XG3PUIvmefyKbMsKZwQkP8VK/V0h5wQd9SmqBePqUVFwX78DpkdN0nvwmy7p/i6hf06S2SKHaOELrRMdZG/P8S312uR5KtmUOJa5bOYEMJZu9P9ZBvW4P+uNFr7MN/vsqwRoVJ980bWUthWrj3OhUn16eR/Luf93U4q7bok2JOiKM70IW0mpvX3ymF7+ZYTh0jHcvtmBes2kmtUcK1YZR1amG574LHMo29qQ/TGpItXdjLJJ3SkNfAtVu3Nhb0N16UaxY3hgvVQ6asBCT2SOFaqOceyuajEAPLrWsSJRLFgAx133RJydbJP/kHjcA8N9a8i2KAVt0hGrLcOIL0w1VyV6/DXJpcBTlD+bguvYgtTYqqFChFWrOTo+gLJctaotb7GVK6kvlYlMtesVgUjtkiWqDVJi6FeflO9E3rMY3FTdhwECOYlnXOy7rPMhQsrnc3L/E7xrUxn146n2mK1GlUG2Yy3Vc866rLhqCz9+Wc2jmO2kLA093wKnXRVTu7sV+T+XuzlOdNwHgdcJ0paoUqo2i9vOl6+D1AFzUZxE5ah/6K5ZzEQRwemIkP1X/ldiPahf/pYjKfFhhP4n6dLyOppjMFilUG+XoF8GMLr8fgJ5vvo6SlWVxGzznbOOxf0ew5elvuPBqdLFK1rieRhfWl/VaDAeOmcwWKVQbZVarGQCsSvfCe3OC1eyo/losvWN6suf1iVxb4EdKn6hC46lcXLj0chQbnh0LQM+dA01qh+z12yBKVF0C1ZsAZz6L7YDXWct5XrkTfUoq2h7w+IKurK/9J9c/zyKq64uIPZ64n1dQZyukhahwbnaFnQ0nAW78k+FK2NALJR4tuBdSqDZIbF9ngjTOrEr3otwrepP+we8HfUoq2u4K1ccMZftz33Aw6hcopGDVoWfA6Ue4/EoQyqVDJrVBVv02SPBKSDVkU9MpidiPvGzizFJ9WhqhY7bybPsXCF8xiMa7e9F4dy/mXq8AwEeXa9NyzCtcik5B2WVakYL0PWWzOK/3Z374Ulrt74XPi+l264WvOJjE95QQoqoQYt9t/9OEECOFEOWEEP8IIWJzP71ve2eMECJOCHFcCGE5/+AORFarC3QJbIxXhziHFmlxKVKoiqIcVxSlnqIo9YCGQDqwEBgNrFEUJQJYk3uPEKIG0BOoCTwOTBZCFHQuL5GUgJK2UdsCJxRFOQ10BWblhs8CuuVedwXmKoqSpShKPBAHmGFzgqQ0UVKh9gTm5F77KYqSCJD76ZsbHgjcflRdQm5YPoQQg4QQu4QQu3Kw/GC2xL4otlCFEE5AF2B+UVELCSvQY1MUZZqiKI0URWmkxbmQVySSW5SkRG0P7FEU5ebhRBeFEAEAuZ9JueEJQPBt7wUBsjcgeSBKItRe3Kr2AZYA/XKv+wGLbwvvKYRwFkKEAhHAjgc1VFK6KdbMlBDCDXgEeOm24C+AeUKIAcAZ4CkARVEOCyHmAUcAHTBUUSy8mFJSAE1wECcGVqL5YweYFrwBAwoqBAYU9mYbeHar0T2PNsaNihuycN4Viz6toNMJayEH/G0I0agWBq0a7cVUdCdP5XuW0bUJaydPpfXQwbguKlkFdfKrKL7oNpsu7skYMKBCVeTnpJQq/PZte4uc5SoPm7AjrvaPYtKCqSz980eOflCuwPPyr8UTk5OJ64WS70Ad2WkZXdyTUSFQoSrW59CyJ5j87nh0bRqa4sd7YKRQbYRyM7fSbs1IAJSM/C2yy4OieC1oFSuu14JtBwp5+94YFGNJaUAp0Wd9JxUf/TjdFD/eAyNXT9kImsrBLG4zkaPZgurjU/JWTKkrVOCDN2bRyFnP9vT7S3vWtx1IHLaZQ2kVObnceMpf5dmn0VU0ltznWuff21S29QXW1p6PilvHV1obKVQbQBPgT8RfiVTVqqm2ejCRR3bnPRNuLjzmZnSjM3nFY1Sh5G1Gnxlb2T1DBVwgEKNfVR1A7jGYgXc0edVLI9mxzChSA6bdTXq/yKrfymgC/Il5NZSx/tvJUfQErNTme35sZGBuu1FF2KIMi9ikPxLD36n18tqs596Ktki+90IK1coc+SiYI70nYsDAI2+OxOOP/F7w+rbZgAEDv6QFIjbvs5hdS0/Vymur2gJSqNZCpSb+8yj2tR9PlpJDnZ9fwXPuznxRlKi69C1rrJc/Wd/F+Jq7O6jMvxhNCCWvRLUFbMOKUkj8p0042Hc8LkJD35OdCXlnKxhuzYuoy3px7f3rVNQ4czTbQPXxKQAk9a5D/CdmXozWpDbLGky3qRJVdqasgKpeDVY8O5abXpxPp3qT+XZEvjgVHj7PuppzAfBS5dBu3i6O3gigumYnH/ptpIl+FCHv3d9gvLpGJMde9kblc2vVmu8SFzzmGpsdJ0aqCVC7okIAKgLXXruvfEyJFKoVOD7SlUqaW15QdjSYi6FB/mEgrVCToxgrvCCNK8O9Y9GWO5nr2seJ8o0vUlI0wUEc+cCfOW1+oL7zHTNSrQx8+JZxcP9Drxm5JalxhoodB+/7ZzUVUqhWQOOSk1el/pYWzOd72uO2xxXnqwr0uMIvtX4mXOuMAQO1fnuFgK0Fl0qU3X2ekvrZOzGwEjHtJ+TO8avySsybnx/77rttDcCtGaoVTZ63uljlXL8VULm5ofIxDrYr128UcCX52KE0hnvHclmfwcCop02zZ6pJbT6c+3NeSToppQrLhrXhSk0XUiONX4TjPSYXOvd/UZ9Fj/+9QbmfzDPvL+f6bRRDejq6swnoziYUEOm1Z5rRvoxxu3H0qldNtrHvXGsPGjsbe/E1Zg9jRc2yqNfuwXfiFkIX6Wjd5PBd5/4D1W5s+XgisROaWm3rthSqDaHy8KDNmM2EabXMvhZA9TdM6136Zi8+7M1bJePJr6L49MdpTA5em/d8Z5ZCq1FDqTV9GIPOPpwXfvyJyaTOK29Sm4qLFKoN4b1SwwcV9qFCxUf/dC/0BJL7pUvPTXklpCY4CJrUpsuRKxzpPTGvpFUh2JulYtinw/D4YxuV/reF882usTfbkOdMeF3tonYimQcpVBtBXTWc9wL/zpuFihi+3aTpz1/dPK9kfGHNBqb/OYVBXqfyrZbamaXw7oAXC6xBHTNgMDuyBDmK3jgKYAVkr98G0FQO5tK3gjCtlmuGbCZOegJfTHtqs1cM7M02Lt3r5p6CgVvjpBf1GbTfPYiK3Y+gZk9B+/7bzUc9c3cdWan3L4VqA+RULMfGej8CEP3b64RONP3R4j4ztjImfjCVPothWvA6DBh4+OAzuH5ZFs31HCruLEKAVh6ekkK1IS7ps6i0ynw+DjT/7eZ8M+iEcWC/DCeBQvay2yBSqDaA2LqfLoGNAQqteiWyMyWxE6RQJXaBFKrELpBCldgFUqgSu0AKVWIXSKHeJ+oKFdAE+LMgYRurzu8jZsY9V6nZPKc+jmL6mU0sP7cHv62enPyi8POkrIUUaglReXiQ/Vgjvti5lEW7/sZZaDmanU74z/brB07l4cGHT88lQO2KAYWZldaxutdYbjzZ1Nqm5SGFWgLUVcOpsFrwzLgVpBu0jL7QGAMGwrXOnG3nWnQCNsqZnyvRo4zxePUD2XpmX/MlSOPK9K/Hce2ZZla2zogUajERjWtz4iM3ZlRaS22Xs/SbPYxDDQ1kKsYNIW8/Mw9NUAEP8DZPWq9mzGloPM6ywbjhjOn9Ir/XDqP25ucJ1zrTZsxmK1toRAq1GAiNBtVXVzjUYia7s+DtVwbn7QCdcLU+AL08LnJyYGVrmnlf1BhxiOpaLT3iOhI0eR9iy36UnGyqvHqFRw73YP6yFtY2EZBCLRZPHDzP4silHM7WMXrYYFyW3XLWtLVjOFX/fZGZacGsHzAWw0P1rWhpydmaEALA8Qu+GNJveWHTnTuP86OnCHnX/P5Ri4MU6j1QouuSsSqU/p5n6RHXkWd/ehXn5fm9mejOJhDRbw9zh7fHW+VCQhv7aqtWnOjE0Zwcfm/yI5rAitY2565Iod6DS/XcWVPrTxJ0GZxcHkbwx3dfJ6rOMW5/fr3nX3bVVlWv28OC1IbUcVKD0y0HbWpPT64MjCJpSDSVtrujrhpuPSORQr0rSvN61OtrXCz8wqCRBH5578XM6nQdCboM+nqew+Bj/UN2S8Lvh41LDPUzdAhno/eWY59XZ83/vuWDV39h09910R+33lHsIIV6V04NU5gWvI7q6wbi/F/RXp6VnQfpMulNAE48U9bM1pkWz3XG5srSqktwWV0W9w0VqFLtPBOu1mfU5meo9JHpdxyUFCnUwmhWh8VRU1id4U7Vt5JQcrKL9VrQqmQ2ZDphCLGMH1NTUX7aVhp+OxwVggXhK/gr/B/O/xvMxjouRDy/u+gELIBc4X8H6vI+HH/GjTdO9eDI6QAiEoq/4v5auCctXYonaltD0+oKhtxNKS0PPEHQ59YvRW9Hlqh3oAT5cezpScRsC6HaiBMletfj7/18drm2mSwzHzntGjK7zsy8+7o+tnfQohTqXTBolZI7gIgMIco91jwGmQnRsCa6N64SrnXm26vVWJPhxlcB60h71jamTm8ihXoHJ94yutNRVcxA5eFR9Au55DzaiNcXzqe1ayYRX9rHadmGh+ozbsF0KpZJpUu1h/mvtjvjwqvz1/Ugln75Dam9bUesUqh3UGGhK01dTjGv2TRC/8sh+fkoVHWq3TV++hNNiZ3UlHemzKSlSzZ1pwxHxJ+zoMX3T3ZZLeFaZ04kl8dwLddZb5PaTDvVEi+VCxm+tiMP2Zm6g7I7Ezmt86atazrfV9wMn25mbYYLgzf1pUbIeS7MDsHtyQs0rXAKgMWrVbzZegn+6us0+3AUIfMO29QZosWhZUAcS+fUpW/N7Tzq8RMNndQ24hD9FtI/aiFoQitzfEhFurbZzrDyGwjSFJwWnZkWzFd7HyW8/1HU5X1Q3F3Rx5Ss82VtVC4uVN2sY6x/fj9XKgSPH+uKplvKrZLWjBTHP6oU6j1QubsT914ddGX1xHSekhfeYEcfnFZ4UX6abSzYeBDOvRWNV+sLJCaVhVQtzRrEcPr7qngs3ouSZZm2thSqxC6QHqclDoMUqsQukEKV2AVSqBK7QApVYhdIoUrsglIj1KSh0Sw/t4fLSyNLNIcvsQ0cXqgqFxdifmhMxxc3YkDhraqrUHnZ11YRSSmY64//JYKY5lMBSFey+fqzZ/FOsP8ZpdKGw5aoKhcXTs+rzcHmPwOQbMik46HeeP8sRWpqVC4uJLwdjc9mb/ofP82ZD6LJ7NzEpHk4ZImqqRyM1+/XWRLyc17Yo2PfwG+8bW2vsGua1UH3SQoAp5PKcbTVzVOrFXoMmkBcThYjLw2GbUVvjCwODidU0agW8aNhf8gi5lzz45N9HQic6UTFHUexX397tkeGvwvLq83DWWihujHs57SK6FHxVJk4wrXOpIW64bnNNPk5lFCVqLq8+utcHnHN4Oe0QKaO7U5o7tHdUqSmxXXRDuo/PZi5UdN4dsdAAmc4oV29CwD1UQN9PU27eNxhhKqOrELkxMO0dU1nXaYTP7/dlXILZXvUnIQ9u4+3aUIIpqne74XDdKbiPnLPWwA87JeXcFto2kNvJcVDVa8GD7vF5Z6zasJ0TZqalYifU5cVUZMBqDt1OKGT7WsnqCOR6edGJY0rM9KC8Pprr8nStfuqP2FMNEdbTkQtylB7+7MEf7xFtketiObNiwD88l5n3LNMV6vZdYmq/n97Zx4f07n/8fczM1kHESGLJEQiiS0tIhGpolpU7VW9LYpWm6qdVq/e3+3torooLaW4abXaotprF65LtZZrCWLfsthlIfask8zM+f0x6ZCbhCCZMyc5by+vM3POmTNf8/r4Ps/5nuf5PPXq4Rh1DTMSZwqz8ZrlLHdI1Zr0idEsC/2VxVk+1NqcVKHXVrRQJR8P5oQtAaDb4klot6gL3srJ9JHf4Coc+e6vfTFdv16h11auUDVaLk8108rRSOimGALfsw8zr+qKNjiQrq6FJBUWUONYZoVfX5F91Jz+beG1THaH/QxoOfzU13RYPtR63LDLA7/NWbDniHxBViOEgyNnPtZjksz02jKa4JSKTxqKE6quUUMe+3s8H3ne/jGchAPx4Uus7zXhgi8GBfPHk0GYLl2WI8xqhdazLoejFwKCJp/dqpSbWcU1/Rld6/N87T2km/Ksznm5UgGdJoyi6yuvE7p8JHlSAePdk6i14raDskrlkdY3AIDlOe6QeqlSvkNxQvXanEGmqSYOwJbLwQC4CkeaTDxKvoeOxr8YiL3RDICr+XoZI60+6HtlADDt84GVZmekuKbflHKGt+cNZ/Srq9jYbIV1/3z/rfD5VsBiSbM21w3H17QYbeT2UV3J6xvJ1rD5gMDjm8p7ZK04oQL4zNjJ6sVhrKzbAYDEEW7ofbPYH/kTAK9e6MiujS1olFZxT0ZUSqJ1d8fzzdOYkUio5HygSKECGDMuQYalPxQ82rKvJ+FFR7NoyC67c6Sratx8KoTVgXMxSIW8smAi/lTeeF/F9VFV7Ie0Jy2+ZetzvfCfWrmD0lWhqjwwTeZlEXszgIWPV/5y6Ypt+lXkx3zoBGuaeQCVX6tWM6qKIlCFqqIIVKGqKAJVqCqKQBWqiiJQhaqiCFShyojO24voQwVciwuROxS7RxWqjJybW5f36h1nd6ulXHg3Wu5w7Bq14C8DOn8/Trzty+HIWZgkR66a86h7xCh3WFZ03l5IddworKcny9+JzNYgeRTQwOcavesfZnO3JhhTbbsCtSpUGTj/YgOSn50DOJJuyuX5t9+i5qoKMmkqB7qABmR28sXvlZRSj3ete5TO+iSCSlmxEODIcl/SbLyerypUG5P2VjR/jPkcsIhgaNJAai61nUgBrnTwZdfUr4vtMyORKxUAUEM4WePLlizj9yJ3vM7R9t+jQdC9zmEW0MimMatCtSG6hv4MHroJd41FBE+f7IPTX7JtbpjhmGUmWzJQQzhx1pjL6qxHmLuhK0Fv7Ubj6sqFsS0B0OaB9yzLqKhGTom89HsXFgf8ZuNoLahCtREFT0cwavZierhmA3DWmEvBTB80V/fYPBbXlfH0yR+LyUWDS0Y+YuchgrBkdXNuLr6flhyyd/7tcNYFzAFg2eU2wFVbhqwK1RboAhpQ//2TVpGeMuYx8MNJeKyVz23Q6d97y32ucHJi4SuzAC0A2f21lRRV2ajlKRuQ/y1867/V+r7f/El4LFCOJealXxsR7mgR59BznTFl2jabQjmFKoSYIIQ4JoQ4KoT4WQjhLISoI4TYJIRILtq633H+O0KIFCFEohCiW+WFb/9kjI/m+5AlaIXlp+6R2IuAhadljqr8XH2tHatbfQvAbgOkvd8YzLa3obunUIUQvsBYoI0kSS2w5P8XgMnAZkmSgoHNRe8RQjQrOt4ceBqYK4SwfVthB+T2a8vaidPw1bpiksz0SOyFdpAJY3qG3KGVC52/H5v+MQNfrStmJF5aO9LqKm1rytv06wAXIYQOcAXSgD7AD0XHfwD6Fr3uAyyVJMkgSdIZIAWo2CUyFEJmSy2+WlcAPrnaTFEiBUiJ8aeWxuKQGLZjGMHjbFtGu5N7ClWSpFRgOnAeSAduSpK0EfCSJCm96Jx0wLPoI77AhTsucbFoXzGEEDFCiH1CiH2FVL2595lvtGPjy9MAS3P/2+THFSXSm4OiOPDyLAAWZ3kS+F6+rPGUp+l3x5IlGwH1Ab0QYvDdPlLKPqnEDkmKlSSpjSRJbRyoWrY7mSPaseadz63ZtHCqN07ry3+XLTfCyYmoiftwEjqyJQM/jOyD6YS8Lt7lafqfAs5IkpQpSVIhsAKIBi4JIXwAirZ/zvC6CPjf8Xk/LF2FakH2gLZs+L/p+BSJ9MUzXXA+mS5zVPdH8ietmOFtqe92OTQE3e/yW3qWR6jngSghhKsQQgBPAieANcCfXo9DgdVFr9cALwghnIQQjYBgwPZVbZnwGXsKj6InT4POPkXOAAebD+B4GLR1PVj57EwAjhUWUG945XhJ3S/3LPhLkhQvhFgG7AeMwAEgFqgB/CqEGI5FzAOKzj8mhPgVOF50/ihJkqqFrf6NIe1YHDAdcGHouc7celaH6ZJy+qUApl9caO7gCMDggy/jk3FC5ogsCEkq0X20ObVEHamteFLuMB6ab87/19ov7dX1RUzHEmWO6P7I7xnJqnkzcde6MuRcBy53MiDZwGTuN2lZgiRJbe52jvpkqhKYkN4WLl+TO4z7IzKMd2b+QC2NMybJTPJXzWwi0vKiPuuvQIacHEy4x3lSutXEdLXifewrDY2Wa/8w0MUlD4DG614nZKl9LSinCrUCcep6lqMAKCubCq2Whm63Y3Y74gB20CW8E7XpV0EqLODSl0EAjLj4OPV/tr++tZpRVQBwXRHPMytaAzlFf+0LNaOqKAJVqCqKQBWqiiJQhaqiCFSh2gmmJ1rzyH5BXGoCcakJSO0eRVvbTe6w7IYqI9QpZ/bit7sG61P3E5eawOlp7eQOqdxom4fyzOwtfOS1B3PRn2lLYrnevancodkNVUOokWG0ctQw1/8PzEiYMTOgyw65oyoXhV3b4LUgjVHuxWuXzR11rJ42g7MfKec/XGWibKFGhpE8uy1xKxeiQaBBY91O8TzI+fft23gs64Uobo65Raz/llKP9zw8jIA4+6tpyoFiharz9yNm0WoSn51b1FhaMmmbT0bz9Y0gzEgUBufJHeZdqbl0N609U0s9dqzASJ2/O8LuwzaOyj5RrFBvfuNIb/11awa9ZMqj/btj8Zyzk8sFtdCUOiPGvrjwbjRz/bZZ/w1tPx3HlMzWlpZB2NezdrlRpFCvDm/HH2H/KpZJ+70/iTrf7ULn70e4/ixmJBySSnejswc0ej1u0Zes/4Z1uW54785iyX+jMWOmqYMDaZ3Vu/4/UaRQu47eUSyTtlg0ljrfWZxHEj+rS2/9ddbluhG44JzMkZaORq8naWoYWx5ZCsAeg2D65EFIe4/gtfN2S/DLmOnovL3kCtOuUN6glMgwRnjMw4yLNZMGFomUyDAWRS3AjJl3j/am/sXj8sZaBmcnPcrxAZapyP2Te1PYKR09lvGfpkG37XKGHB1GnYwkWWK0NxSZUQHW5brR/t3bmVRb242cD3OIcBKMuNCZ+v3sU6TXXm7H+qL5/gDZn/tZX2tq1qSr70k5wrJ7lJdR9xwhpv8baNOvUefibaOxW0s92Bz2C3sNGi68E4yW/TIGWTamftfwK3JyDtkYQ8i62/P9pdCGvFdvC6DBQWi5flNPHXnCtDuUJ1RA2nsEI5DXJxLzG1eQgG1hyzCjYfTU0Xj8Yb9OefHhSzBjBsD5THHjjeSBNazHThQYCJpdLSbvlgtFChUsIm3//m4+8DxQdOesIXTZKILt2M4xr08kYDFz6JPYl4DpB4tkCVovT0JbnQcg02Tg9XETcdldbewQ7okiharz9+Mvn/ybGLezRfVSyxOpwFUFcodWJlovTxzG3J7jfyPfBbdcS7FfW9uNthsvMrnuIQB67H8N79WqSO9EkTdT2kVGYtzOFqujmkvaW9kVZ19rzPqmy63vjcvrWV+LOu600Vs8U/sn98ZvQq7N47N3FCdUbW03utQ9UeLZvgZBrpej3OGViccxU7F4c+tb6qU6H28yH/ehq0sOGjRcz3fBeMY+679yojihGlo3JqZ2ijWTdjjyHJ2ODMCMxOVe9mOYcCcavR5jzJViLYA2D67EtCM47go7Pp6DGTMzr4fgONND7nDtEsX1UQ1vX0dTVL557PAAajx9mtTJ0Tg8ouVEp2/pSbjcIZYgu2sLVjSfAXfYa+6bMOuOMyz5YsXHXai1QT6zXHtGcRn1L/4JmDHz2OFncXv+CgANF52jUDJZSjuRYTJHWBLXlfG0X/YW398MKPOc5kvHUHuVOlKqLBQn1A1PhxH+1Thq/EOP6ZbFEtF4MZUnjgxAg4acD3Mwdg5H5+93jyvZlsYTd7PknR6lHov6cDQhU05gzlVvosqiyrj5aZuF8H9xS4l0kvj6RhArJ3fFea1a4lEC5XHzU1wftSxMx5P4MLC19b1z9fEOrhYorulXqZ6oQlVRBKpQVRSBKlQVRaAKVUURqEKtLCLD6H38KnGpCfjtrsGpJS3trrarJFShVhLh/zxkHeE1338rxzp+wyubtyEi7O/JmRJQhVqJ/O8Ir776G/RYuE3NrA+AKtRK5M6xsn9uY2qncG5gA/mC0mgpeDqCpNgIVl7cw3/SDjI86Qzn34/G+KT9Dej5kyrzCNXuiAzj9HM1GNBlB75O162zEcxIxN4MYF1UI+tYBVuhqVmTpLmNOdn5WwCyzQayJDM+WstkwwQDTHm8l82XxKxWj1Dtjj1HCNwDCWg4WKsRhbu0jKp9CjNmwp3Pss43Cmws1JPTm5DUeT4APU/24dYCP+rsSudWK28efecgX9bfiSHYG60drt2qCtUW+HkT5fKHdX7XmputbL6sePbzUWzvblmntdVXY/D7Yh+1Ci9iBFzPnCO+djuYstOmMd0Pah/VBqQ9WZdWTrf7qnJwuY3AS+tC1P4X8ZuZgFRYfCLk1Q6W9wXvXEfTookcId4VVaiVTF6fSBImzyl29/9zQqRs8eQc8CixxqlwcKT/IxbDjs0tlpEyxF2O0O6KKtRKROfvR/v3d5e4+9cn234SYqeOltkDBe4lM/rVl8L52Guf9X2DDfY37Vzto1Y0kWHcCtKT2SufCS03l/AekCs3bD8XCH7b2NP3CyI1E6mVqKVGqgmnN9JZFzodcLaeq8tWhVp1iQwj/W9G4lrPw0frUuTdoinqk1q2TbfE4JDkQuDicxhtHJ7XTy78GObLkFqpJPada91vKZk53+WT9oHa9FcA5z6I5oOlCzkQsRgfrUupngMaNHQMSqb+dgPGi6XboVcmzmv3sDSmOz1P9rHu25inp8miUQBMzogAYP6NQNhnf06IasH/IUlf1ZSEiEXFMujdtnsMgnFTR+FhZx5ZaSubcTByEf1SnsHQMePeH6hAylPwVzPqQ2DsHE5CxKIyM2hp20gnifgPvyb5q7b288xfo6WNzwXMSJxZGyh3NKWiCvUh+d9n+eXdnug/h1c2b+P0Z+24OlzetaSMnVoS678Fg1SIz077XC5IFepDcrcM+qcrdt+oPoR/NY7YmwHFjvevcYvjg+ewb8o8TE+0vveXVRKXx1iWOep2dCBi5yHZ4rgbqlAfAqeMLPYYRJmZ88uJA6nz3S6MFy7i+9lO1kU1InT5SObeaIQZs9XdpVCSz7BXODgyKnQrAA6z7Nf3Si1PPQSm40l8duEZVjZez5110v4pPTAN0uJ8sbi3gOnWLYLHxhOHOz8MH0tBrxv0bHiMNUvb0zDZ9iUrgNNTwhnutotsswGHrEIZIigfakZ9SEyDtNYMudcg0XHiKAwdM+5ZgvJYsAufvidIaKXB97OdspSscvu15beBnwPQef8raP570OYxlBc1oz4kxoupxDV3Jw5LHbImynHj6zdlE75aV1JNufhMKsSeVwxQM2o1xdyxFTG1LYX9jusmYko6JXNEd0fNqNUUzdYDjLnYBUeNkaYzMu06m4Iq1GpNWlRW0aszssZRHtSmX0URqEJVUQSqUFUUgSpUFUWg3kypPBwaLaJVExJf1Rfb3ezjNDAaMd+8VSFrE6hCVXlgRKvmpL1nJiHix5IHe1s2bRMG4v1mIabk0w/1XWrTr/JApL8ZzbK1C0iIWATAeWMekzMiaD1rDO3+Mdo6YyA+fAlhv55G07LZQ31f1R/hr9Gi0btyeWALHPpk0tEnhY8996MVGoK3DCPk7UxZnrMrmZzn2rLiyy9w1zhz05xP54RX8ZzlgvaP/dZzhE5HTu9wbgRp2T9+NnsMgsG/xxA66kiJ6drVeoS/0OlI/Ws0wfE64k5u5fd3v+D5BvtZmfgITbcPY/DZThzoMJ8Tk/zlDlVxpD0ucNc4M/N6CE/OmIRP3xPFRAogGY24rojHb/4RQtaPINJJIqn7P9G6136g76ySfVRdQ39OfV6b+OgZxN5oQejSkTRaZUCz/QCNsMxvzwSGbe+J71b5WxSl4btVIvz8GPx+TMY78+42QOasLDTZWgBmXw9Gys17oO+sckI1dI8gZtYvdHbJIPyXiQS9uZugMkY05b3giGtqvI0jVD6uK+JxhXuOD9D5eNNifQbL680EHNj48mNIt4480HdWqaZfo9fT8dOdNHG8RJcP3yTozbsPubO1vWJ1QdfQn6TYCNr95ywfeSaQUigIWf0GmvOXHvyaFRif7Fx79hEmenxB+5lvUv+b+3emkx5ricM59ebqQdDWqoWpWQBJw53Y2e1L6mpd2JbvSOiqkTT58jIhKXseaoRWlRJq7rM3SSnU4v/9yXL9KBq9nryOzbjawoHgnsksCoyl06FBuJe+tq5KGYjw5pz7GxyM+p5dBi3R/5mA5zYdHvuvEXwsvkKGEFYpoXrOdqHlTzpG7N7N5B+HUed4yZ8oq4EWJPDve4a/N4gjwmk7S7ProdcYcBI6cnbUwx3bepcqmYJubfhoXiyRThI78h34YNRwQjbsBe7dh70fqpRQdVsOEvnBKEKHnuTIiDkljm/P1/Hpue4kXfTi/NpGfLBpEOLSNaTsHDw2OeDvs5GA707JMslOiWSMi+aH8V/iIMyELh9Pk6+v4pi4t1K+q2oW/IVA4+pacr/JhDk/v8Tu1L9Gc2DsbEJWv0HISHVV6nshIsIY8OMmBte6QO+T/dANMT/UjWn19fCXJMw55XP8MDwTweihqwEI+peaS++G0OkwRbVg8g8/0doxiz6JAxDdMjAaK/93q1LlqftF51uf8CkJDK51irAdw0o8XVEpjiYogBHfraCDcwGxNx6FSbVJnNMaXWBApX931cyo5UDodCTPqMsa7zgarx6rNvn3wNy+JS5T0+mtvw7AxDonmbjmJAApTxt4ZsN4Gv9ciG5fUrlbs/uh2go1dUIkJx6fQ/+U7jSdXL5yVnVFin6U8QuX0tWldAE2dnAiqdc86AUjL3bgUr43eUYH8ubWL/V8/fL7fxpYLYWa81xbVo2extpcb/ImecIDPtarLvztp594zPm23c+6XDcmLX8J/98KKKypY/xnP1sz7Vy/bbc/+FXp1+u5/P5XCKx2QjU90ZqfZszgSIEn8wb1gz2qSO9FJxczhXcUh9471ovAyRYjYgfgu/0d+bZ2DZKHuCF8LFWVQx1icRCWwSib81yJTevAL0Eb+PjKgy1aXDXLU2Wg861Pu3+fZnydg/R4YyzOa9V+aXnQeXsVey/l52O6cbPcn5EKC5FyctHUdsOck4s5K6vYudW3PFUGidO8WOMRR+P1YwhRRVpujBn3P5iktM+YM0rWsMtLtSlPZYyL5kSnbxmTFk3o6MNyh6Nyn1QboQ58dROb81w5O6xhiakQKvZPtWn6fw/T8ztNgUS5Q1F5AKpNRlVRNnZx1y+EyARygCsVeNm66vXs4lrluV5DSZLq3e0CdiFUACHEvnuVKNTr2eZ69hib2vSrKAJVqCqKwJ6EGqtez26uZ3ex2U0fVUXlbthTRlVRKRNVqCqKQBWqiiJQhaqiCFShqiiC/wd67FPC7S2GTQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "for batch in train_loader:\n", + " data, targets = get_batch(batch, seq_len, digits_per_batch)\n", + "\n", + " plt.matshow((data.view(-1, data.shape[-1]) * mnist_std + mnist_mean).numpy())\n", + " plt.show()\n", + " break" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "papermill": { + "duration": 0.227794, + "end_time": "2021-01-21T13:19:14.482416", + "exception": false, + "start_time": "2021-01-21T13:19:14.254622", + "status": "completed" + }, + "tags": [] + }, + "source": [ + "### Smart pooling" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true + }, + "papermill": { + "duration": 22679.934735, + "end_time": "2021-01-21T19:37:14.644374", + "exception": false, + "start_time": "2021-01-21T13:19:14.709639", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 75.56 | loss 39.50 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 75.31 | loss 20.90 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 75.27 | loss 12.93 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 75.40 | loss 7.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 76.23s | valid loss 10.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 78.01 | loss 7.56 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 77.74 | loss 6.07 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 77.78 | loss 6.11 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 77.88 | loss 5.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 78.30s | valid loss 10.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 78.22 | loss 5.10 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 77.94 | loss 5.38 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 77.84 | loss 5.44 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 77.86 | loss 5.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 78.37s | valid loss 7.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 78.18 | loss 4.94 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 77.93 | loss 4.94 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 77.96 | loss 5.27 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 76.64 | loss 4.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 78.05s | valid loss 6.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 76.66 | loss 4.00 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 76.44 | loss 17.22 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 76.37 | loss 9.15 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 76.46 | loss 11.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 77.09s | valid loss 41.23 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 76.87 | loss 22.53 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 76.52 | loss 20.27 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 76.57 | loss 19.28 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 76.54 | loss 18.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 77.21s | valid loss 30.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 76.91 | loss 17.91 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 76.47 | loss 16.88 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 76.47 | loss 16.31 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 76.71 | loss 16.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 77.24s | valid loss 27.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 76.87 | loss 16.39 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 76.54 | loss 16.07 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 76.51 | loss 15.09 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 76.51 | loss 16.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 77.21s | valid loss 25.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 76.94 | loss 15.31 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 76.52 | loss 11.74 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 76.55 | loss 2.76 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 76.46 | loss 3.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 77.19s | valid loss 5.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 76.76 | loss 2.82 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 76.52 | loss 3.86 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 76.37 | loss 2.81 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 76.29 | loss 3.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 77.08s | valid loss 5.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 77.06 | loss 2.27 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 76.30 | loss 2.30 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 76.37 | loss 1.66 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 76.53 | loss 3.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 77.14s | valid loss 43.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 76.67 | loss 1.65 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 76.86 | loss 2.73 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 76.29 | loss 2.18 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 76.93 | loss 2.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 77.29s | valid loss 6.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 76.59 | loss 1.64 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 76.68 | loss 1.82 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 76.81 | loss 1.13 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 76.64 | loss 2.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 77.23s | valid loss 5.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 76.66 | loss 1.35 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 77.40 | loss 1.39 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 76.56 | loss 2.14 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 76.57 | loss 2.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 77.36s | valid loss 5.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 77.24 | loss 1.91 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 76.29 | loss 1.27 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 77.30 | loss 0.98 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 76.73 | loss 1.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 77.55s | valid loss 5.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 76.63 | loss 0.61 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 76.61 | loss 1.56 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 76.56 | loss 1.43 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 76.46 | loss 1.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 77.24s | valid loss 5.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 76.95 | loss 0.95 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 76.65 | loss 1.38 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 77.10 | loss 1.15 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 77.10 | loss 1.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 77.53s | valid loss 6.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 76.89 | loss 0.74 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 76.66 | loss 1.23 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 76.84 | loss 1.11 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 77.04 | loss 1.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 77.52s | valid loss 6.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 77.18 | loss 0.77 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 76.68 | loss 0.68 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 76.81 | loss 0.61 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 77.22 | loss 1.00 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 77.53s | valid loss 6.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 77.56 | loss 0.91 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 76.93 | loss 0.35 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 77.50 | loss 0.39 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 77.01 | loss 1.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 77.75s | valid loss 8.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 77.41 | loss 0.60 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 77.11 | loss 0.92 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 77.32 | loss 0.80 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 76.87 | loss 0.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 77.84s | valid loss 9.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 78.02 | loss 0.54 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 76.86 | loss 0.32 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 76.91 | loss 0.47 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 77.19 | loss 0.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 77.74s | valid loss 8.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 77.53 | loss 0.31 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 77.42 | loss 0.48 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 76.31 | loss 0.42 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 77.14 | loss 1.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 77.66s | valid loss 9.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 77.86 | loss 0.45 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 77.30 | loss 0.33 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 77.12 | loss 0.25 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 77.33 | loss 0.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 77.91s | valid loss 7.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 77.18 | loss 0.18 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 76.66 | loss 0.62 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 77.01 | loss 0.67 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 76.67 | loss 0.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 77.56s | valid loss 9.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 77.23 | loss 1.30 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 77.08 | loss 0.31 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 76.65 | loss 0.33 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 77.33 | loss 0.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 77.65s | valid loss 8.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 76.96 | loss 0.25 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 77.28 | loss 0.65 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 77.39 | loss 0.17 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 76.94 | loss 0.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 77.77s | valid loss 10.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 77.58 | loss 0.21 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 76.76 | loss 0.60 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 76.86 | loss 0.33 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 76.65 | loss 0.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 77.59s | valid loss 8.77 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 76.99 | loss 0.11 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 77.18 | loss 0.30 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 76.85 | loss 0.25 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 77.36 | loss 0.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 77.76s | valid loss 10.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 77.50 | loss 0.25 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 76.65 | loss 0.37 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 76.93 | loss 0.23 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 76.97 | loss 0.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 77.56s | valid loss 8.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.06 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 78.52 | loss 39.96 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 78.02 | loss 12.16 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 78.45 | loss 9.87 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 77.73 | loss 7.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 78.81s | valid loss 24.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 79.04 | loss 7.42 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 78.35 | loss 6.94 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 78.28 | loss 5.92 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 78.14 | loss 5.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 78.90s | valid loss 11.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 78.22 | loss 5.17 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 77.86 | loss 5.84 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 78.24 | loss 5.05 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 78.20 | loss 5.15 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 78.69s | valid loss 6.65 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 78.36 | loss 3.89 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 77.88 | loss 5.10 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 78.17 | loss 4.31 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 78.00 | loss 4.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 78.56s | valid loss 9.01 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 78.92 | loss 3.40 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 78.23 | loss 3.86 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 78.05 | loss 3.97 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 78.09 | loss 3.94 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 78.85s | valid loss 5.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 77.89 | loss 3.37 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 77.20 | loss 3.58 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 77.52 | loss 3.24 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 77.56 | loss 3.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 78.16s | valid loss 9.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 77.95 | loss 2.75 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 77.51 | loss 3.50 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 77.36 | loss 2.69 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 77.67 | loss 2.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 78.20s | valid loss 8.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 77.88 | loss 2.66 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 77.57 | loss 2.69 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 77.33 | loss 3.12 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 77.82 | loss 2.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 78.29s | valid loss 6.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 78.09 | loss 2.29 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 77.23 | loss 2.01 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 77.23 | loss 2.44 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 77.41 | loss 2.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 78.05s | valid loss 6.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 77.83 | loss 2.24 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 77.85 | loss 2.33 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 77.61 | loss 1.78 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 77.35 | loss 2.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 78.23s | valid loss 7.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 77.62 | loss 2.24 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 76.94 | loss 1.81 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 77.43 | loss 2.05 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 76.94 | loss 1.32 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 77.81s | valid loss 9.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 77.46 | loss 1.82 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 77.25 | loss 1.44 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 77.02 | loss 1.60 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 77.24 | loss 2.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 77.81s | valid loss 8.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 77.01 | loss 0.94 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 77.25 | loss 1.84 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 77.21 | loss 1.89 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 76.84 | loss 1.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 77.72s | valid loss 6.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 77.26 | loss 1.07 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 76.90 | loss 1.06 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 76.94 | loss 1.20 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 76.78 | loss 1.85 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 77.57s | valid loss 8.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 77.48 | loss 1.34 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 77.36 | loss 1.54 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 77.20 | loss 1.61 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 76.99 | loss 1.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 77.82s | valid loss 7.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 77.63 | loss 0.89 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 76.91 | loss 0.84 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 77.15 | loss 0.87 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 77.62 | loss 0.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 78.11s | valid loss 8.18 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 77.58 | loss 0.91 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 76.41 | loss 0.69 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 77.08 | loss 0.69 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 77.77 | loss 1.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 77.85s | valid loss 8.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 77.39 | loss 0.78 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 77.06 | loss 0.54 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 77.58 | loss 1.13 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 76.99 | loss 0.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 77.75s | valid loss 7.66 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 77.68 | loss 0.46 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 76.71 | loss 0.41 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 76.70 | loss 1.19 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 77.40 | loss 0.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 77.76s | valid loss 10.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 78.04 | loss 0.39 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 76.96 | loss 0.38 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 77.17 | loss 0.97 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 77.42 | loss 0.83 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 77.88s | valid loss 7.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 77.31 | loss 0.29 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 77.00 | loss 0.87 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 76.80 | loss 0.55 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 77.52 | loss 0.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 77.69s | valid loss 8.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 77.29 | loss 0.40 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 77.09 | loss 0.87 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 76.59 | loss 0.62 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 76.71 | loss 0.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 77.52s | valid loss 9.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 77.18 | loss 0.14 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 77.58 | loss 0.35 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 76.77 | loss 0.20 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 77.17 | loss 0.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 77.76s | valid loss 9.43 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 77.41 | loss 0.18 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 76.90 | loss 0.19 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 76.84 | loss 0.34 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 76.85 | loss 0.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 77.57s | valid loss 12.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 77.50 | loss 0.24 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 76.91 | loss 0.39 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 77.02 | loss 0.55 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 77.01 | loss 0.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 77.64s | valid loss 8.80 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 77.15 | loss 0.05 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 76.93 | loss 0.06 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 76.88 | loss 0.20 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 76.89 | loss 0.47 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 77.53s | valid loss 10.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 77.47 | loss 0.45 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 76.73 | loss 0.25 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 77.11 | loss 0.10 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 77.52 | loss 0.28 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 77.81s | valid loss 9.45 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 77.92 | loss 0.25 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 77.23 | loss 0.27 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 77.19 | loss 0.26 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 77.32 | loss 0.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 77.89s | valid loss 9.58 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 77.12 | loss 0.14 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 76.74 | loss 0.27 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 77.03 | loss 0.24 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 76.55 | loss 0.36 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 77.49s | valid loss 10.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 77.42 | loss 0.04 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 77.43 | loss 0.03 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 77.62 | loss 0.15 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 77.07 | loss 0.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 77.88s | valid loss 9.34 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.83 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 79.12 | loss 34.70 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 78.57 | loss 16.57 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 78.39 | loss 27.04 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 78.38 | loss 23.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 79.13s | valid loss 28.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 78.52 | loss 7.19 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 78.14 | loss 6.51 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 78.14 | loss 6.63 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 78.31 | loss 5.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 78.89s | valid loss 83.38 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 78.34 | loss 5.00 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 77.95 | loss 11.94 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 78.25 | loss 5.15 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 78.18 | loss 4.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 78.64s | valid loss 8.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 78.63 | loss 5.44 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 78.07 | loss 5.28 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 78.36 | loss 4.96 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 78.77 | loss 5.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 79.02s | valid loss 7.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 78.84 | loss 4.69 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 78.17 | loss 4.09 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 78.64 | loss 3.48 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 78.63 | loss 4.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 79.04s | valid loss 7.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 78.16 | loss 3.70 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 78.07 | loss 3.68 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 78.51 | loss 4.49 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 77.93 | loss 4.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 78.67s | valid loss 5.24 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 78.15 | loss 3.19 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 78.16 | loss 3.26 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 77.63 | loss 4.02 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 78.38 | loss 4.12 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 78.62s | valid loss 6.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 78.25 | loss 2.59 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 77.76 | loss 3.69 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 78.41 | loss 2.68 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 77.91 | loss 3.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 78.57s | valid loss 8.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 77.93 | loss 3.47 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 78.17 | loss 2.58 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 77.81 | loss 2.90 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 78.09 | loss 2.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 78.49s | valid loss 5.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 78.15 | loss 1.88 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 77.93 | loss 3.55 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 78.25 | loss 2.20 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 78.19 | loss 3.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 78.61s | valid loss 5.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 78.33 | loss 2.50 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 77.86 | loss 2.38 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 78.34 | loss 2.62 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 78.33 | loss 23.54 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 78.72s | valid loss 6.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 77.66 | loss 1.68 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 78.06 | loss 1.41 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 77.92 | loss 1.86 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 78.02 | loss 2.42 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 78.38s | valid loss 5.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 77.82 | loss 0.95 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 77.06 | loss 1.76 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 77.09 | loss 1.78 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 77.07 | loss 2.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 77.88s | valid loss 7.57 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 77.69 | loss 1.57 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 77.23 | loss 1.37 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 77.32 | loss 1.50 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 77.36 | loss 1.09 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 78.02s | valid loss 6.41 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 77.47 | loss 1.67 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 77.46 | loss 0.91 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 77.39 | loss 0.96 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 77.11 | loss 2.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 77.90s | valid loss 6.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 77.76 | loss 1.22 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 77.77 | loss 1.09 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 77.89 | loss 1.47 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 77.66 | loss 1.37 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 78.38s | valid loss 5.74 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 77.70 | loss 0.84 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 77.93 | loss 0.77 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 78.02 | loss 0.66 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 77.07 | loss 1.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 78.26s | valid loss 7.29 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 77.73 | loss 0.83 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 77.51 | loss 0.77 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 77.55 | loss 0.94 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 78.06 | loss 18.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 78.39s | valid loss 7.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 78.21 | loss 0.49 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 77.70 | loss 0.57 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 77.67 | loss 1.06 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 77.82 | loss 0.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 78.48s | valid loss 6.81 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 77.84 | loss 0.90 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 77.74 | loss 0.84 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 77.49 | loss 0.72 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 77.72 | loss 0.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 78.30s | valid loss 7.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 77.77 | loss 0.26 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 77.69 | loss 0.78 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 77.89 | loss 11.26 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 77.91 | loss 0.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 78.26s | valid loss 6.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 77.90 | loss 0.44 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 77.23 | loss 0.67 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 77.61 | loss 0.31 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 77.46 | loss 0.87 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 78.12s | valid loss 7.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 77.89 | loss 0.55 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 77.61 | loss 0.29 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 77.30 | loss 0.28 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 77.42 | loss 0.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 78.00s | valid loss 8.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 77.37 | loss 0.35 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 77.28 | loss 0.32 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 77.33 | loss 0.40 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 77.26 | loss 0.31 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 77.88s | valid loss 7.44 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 77.86 | loss 0.57 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 77.31 | loss 0.13 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 77.40 | loss 0.18 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 76.71 | loss 0.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 77.93s | valid loss 8.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 77.30 | loss 0.28 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 77.16 | loss 0.46 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 77.16 | loss 0.29 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 77.36 | loss 0.60 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 77.83s | valid loss 9.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 78.09 | loss 0.61 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 77.02 | loss 0.21 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 77.08 | loss 0.27 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 76.98 | loss 0.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 77.91s | valid loss 9.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 77.05 | loss 0.62 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 76.63 | loss 0.22 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 76.84 | loss 0.17 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 76.68 | loss 0.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 77.38s | valid loss 9.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 77.40 | loss 0.36 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 76.89 | loss 0.25 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 76.95 | loss 0.36 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 76.68 | loss 0.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 77.59s | valid loss 9.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 77.18 | loss 0.27 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 76.92 | loss 0.06 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 77.06 | loss 0.34 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 76.63 | loss 0.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 77.61s | valid loss 9.20 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 5.24 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 78.85 | loss 42.17 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 78.35 | loss 11.76 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 78.80 | loss 9.30 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 78.62 | loss 7.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 79.28s | valid loss 68.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 79.13 | loss 6.14 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 78.66 | loss 6.17 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 78.78 | loss 5.94 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 78.32 | loss 5.73 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 79.22s | valid loss 9.55 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 78.61 | loss 5.30 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 78.49 | loss 5.68 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 78.52 | loss 6.23 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 78.83 | loss 19.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 79.03s | valid loss 5.70 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 78.68 | loss 8.28 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 78.37 | loss 5.59 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 78.35 | loss 13.15 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 78.05 | loss 4.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 78.87s | valid loss 10.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 78.50 | loss 4.28 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 77.89 | loss 5.04 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 78.21 | loss 12.02 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 77.96 | loss 4.67 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 78.68s | valid loss 7.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 78.39 | loss 3.80 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 78.02 | loss 6.14 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 77.48 | loss 3.97 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 78.12 | loss 3.39 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 78.56s | valid loss 7.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 78.59 | loss 4.23 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 78.44 | loss 13.79 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 77.83 | loss 21.64 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 78.29 | loss 20.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 78.87s | valid loss 33.02 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 78.64 | loss 18.16 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 78.57 | loss 9.95 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 77.79 | loss 2.96 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 78.01 | loss 3.91 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 78.76s | valid loss 5.22 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 78.44 | loss 2.51 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 77.77 | loss 4.46 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 77.78 | loss 4.50 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 77.91 | loss 2.64 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 78.48s | valid loss 4.90 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 77.91 | loss 1.82 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 77.86 | loss 3.74 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 77.98 | loss 2.24 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 77.56 | loss 3.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 78.34s | valid loss 6.89 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 78.36 | loss 1.66 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 77.64 | loss 2.43 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 77.45 | loss 2.87 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 77.59 | loss 2.07 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 78.32s | valid loss 5.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 78.25 | loss 1.99 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 77.50 | loss 1.93 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 77.30 | loss 1.82 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 77.79 | loss 2.51 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 78.44s | valid loss 6.97 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 78.18 | loss 1.91 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 77.16 | loss 0.93 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 77.69 | loss 1.98 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 77.67 | loss 2.06 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 78.20s | valid loss 6.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 77.84 | loss 1.25 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 77.38 | loss 1.48 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 77.66 | loss 2.36 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 77.53 | loss 1.76 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 78.30s | valid loss 6.62 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 15 | 200/ 938 batches | lr 0.00049 | ms/batch 78.63 | loss 2.16 |\n", + "| epoch 15 | 400/ 938 batches | lr 0.00049 | ms/batch 77.79 | loss 1.38 |\n", + "| epoch 15 | 600/ 938 batches | lr 0.00049 | ms/batch 77.80 | loss 1.75 |\n", + "| epoch 15 | 800/ 938 batches | lr 0.00049 | ms/batch 77.64 | loss 1.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 15 | time: 78.47s | valid loss 8.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 16 | 200/ 938 batches | lr 0.00046 | ms/batch 78.33 | loss 1.32 |\n", + "| epoch 16 | 400/ 938 batches | lr 0.00046 | ms/batch 77.32 | loss 1.37 |\n", + "| epoch 16 | 600/ 938 batches | lr 0.00046 | ms/batch 77.46 | loss 1.08 |\n", + "| epoch 16 | 800/ 938 batches | lr 0.00046 | ms/batch 77.47 | loss 1.10 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 16 | time: 78.19s | valid loss 6.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 17 | 200/ 938 batches | lr 0.00044 | ms/batch 78.34 | loss 0.79 |\n", + "| epoch 17 | 400/ 938 batches | lr 0.00044 | ms/batch 77.63 | loss 1.50 |\n", + "| epoch 17 | 600/ 938 batches | lr 0.00044 | ms/batch 77.13 | loss 0.71 |\n", + "| epoch 17 | 800/ 938 batches | lr 0.00044 | ms/batch 77.47 | loss 1.26 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 17 | time: 78.20s | valid loss 6.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 18 | 200/ 938 batches | lr 0.00042 | ms/batch 77.83 | loss 0.87 |\n", + "| epoch 18 | 400/ 938 batches | lr 0.00042 | ms/batch 78.05 | loss 1.33 |\n", + "| epoch 18 | 600/ 938 batches | lr 0.00042 | ms/batch 77.31 | loss 0.65 |\n", + "| epoch 18 | 800/ 938 batches | lr 0.00042 | ms/batch 77.55 | loss 1.21 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 18 | time: 78.25s | valid loss 6.96 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 19 | 200/ 938 batches | lr 0.00040 | ms/batch 77.73 | loss 0.92 |\n", + "| epoch 19 | 400/ 938 batches | lr 0.00040 | ms/batch 77.72 | loss 1.46 |\n", + "| epoch 19 | 600/ 938 batches | lr 0.00040 | ms/batch 77.32 | loss 0.95 |\n", + "| epoch 19 | 800/ 938 batches | lr 0.00040 | ms/batch 77.80 | loss 0.49 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 19 | time: 78.15s | valid loss 7.98 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 20 | 200/ 938 batches | lr 0.00038 | ms/batch 78.06 | loss 0.41 |\n", + "| epoch 20 | 400/ 938 batches | lr 0.00038 | ms/batch 77.30 | loss 1.69 |\n", + "| epoch 20 | 600/ 938 batches | lr 0.00038 | ms/batch 77.45 | loss 0.75 |\n", + "| epoch 20 | 800/ 938 batches | lr 0.00038 | ms/batch 77.40 | loss 0.68 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 20 | time: 78.13s | valid loss 6.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 21 | 200/ 938 batches | lr 0.00036 | ms/batch 77.58 | loss 0.58 |\n", + "| epoch 21 | 400/ 938 batches | lr 0.00036 | ms/batch 77.22 | loss 0.48 |\n", + "| epoch 21 | 600/ 938 batches | lr 0.00036 | ms/batch 77.27 | loss 0.82 |\n", + "| epoch 21 | 800/ 938 batches | lr 0.00036 | ms/batch 77.94 | loss 0.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 21 | time: 78.06s | valid loss 9.99 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 22 | 200/ 938 batches | lr 0.00034 | ms/batch 78.05 | loss 1.17 |\n", + "| epoch 22 | 400/ 938 batches | lr 0.00034 | ms/batch 77.55 | loss 0.59 |\n", + "| epoch 22 | 600/ 938 batches | lr 0.00034 | ms/batch 77.12 | loss 0.41 |\n", + "| epoch 22 | 800/ 938 batches | lr 0.00034 | ms/batch 77.04 | loss 0.48 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 22 | time: 77.96s | valid loss 9.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 23 | 200/ 938 batches | lr 0.00032 | ms/batch 77.58 | loss 0.58 |\n", + "| epoch 23 | 400/ 938 batches | lr 0.00032 | ms/batch 77.34 | loss 0.66 |\n", + "| epoch 23 | 600/ 938 batches | lr 0.00032 | ms/batch 77.22 | loss 0.45 |\n", + "| epoch 23 | 800/ 938 batches | lr 0.00032 | ms/batch 77.36 | loss 0.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 23 | time: 78.01s | valid loss 8.25 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 24 | 200/ 938 batches | lr 0.00031 | ms/batch 77.87 | loss 0.39 |\n", + "| epoch 24 | 400/ 938 batches | lr 0.00031 | ms/batch 76.91 | loss 0.69 |\n", + "| epoch 24 | 600/ 938 batches | lr 0.00031 | ms/batch 77.49 | loss 0.57 |\n", + "| epoch 24 | 800/ 938 batches | lr 0.00031 | ms/batch 77.28 | loss 0.30 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 24 | time: 78.04s | valid loss 7.59 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 25 | 200/ 938 batches | lr 0.00029 | ms/batch 77.92 | loss 0.31 |\n", + "| epoch 25 | 400/ 938 batches | lr 0.00029 | ms/batch 77.11 | loss 0.49 |\n", + "| epoch 25 | 600/ 938 batches | lr 0.00029 | ms/batch 77.35 | loss 0.41 |\n", + "| epoch 25 | 800/ 938 batches | lr 0.00029 | ms/batch 77.56 | loss 0.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 25 | time: 77.95s | valid loss 9.16 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 26 | 200/ 938 batches | lr 0.00028 | ms/batch 78.17 | loss 0.15 |\n", + "| epoch 26 | 400/ 938 batches | lr 0.00028 | ms/batch 77.61 | loss 0.19 |\n", + "| epoch 26 | 600/ 938 batches | lr 0.00028 | ms/batch 77.56 | loss 0.21 |\n", + "| epoch 26 | 800/ 938 batches | lr 0.00028 | ms/batch 77.16 | loss 0.46 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 26 | time: 78.08s | valid loss 9.78 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 27 | 200/ 938 batches | lr 0.00026 | ms/batch 77.42 | loss 0.62 |\n", + "| epoch 27 | 400/ 938 batches | lr 0.00026 | ms/batch 76.93 | loss 0.34 |\n", + "| epoch 27 | 600/ 938 batches | lr 0.00026 | ms/batch 76.72 | loss 0.21 |\n", + "| epoch 27 | 800/ 938 batches | lr 0.00026 | ms/batch 76.88 | loss 0.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 27 | time: 77.67s | valid loss 9.61 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 28 | 200/ 938 batches | lr 0.00025 | ms/batch 77.58 | loss 0.17 |\n", + "| epoch 28 | 400/ 938 batches | lr 0.00025 | ms/batch 76.89 | loss 0.13 |\n", + "| epoch 28 | 600/ 938 batches | lr 0.00025 | ms/batch 77.03 | loss 0.18 |\n", + "| epoch 28 | 800/ 938 batches | lr 0.00025 | ms/batch 77.09 | loss 0.08 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 28 | time: 77.66s | valid loss 10.69 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 29 | 200/ 938 batches | lr 0.00024 | ms/batch 77.34 | loss 0.47 |\n", + "| epoch 29 | 400/ 938 batches | lr 0.00024 | ms/batch 76.89 | loss 0.09 |\n", + "| epoch 29 | 600/ 938 batches | lr 0.00024 | ms/batch 77.06 | loss 0.41 |\n", + "| epoch 29 | 800/ 938 batches | lr 0.00024 | ms/batch 77.28 | loss 0.50 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 29 | time: 77.70s | valid loss 9.93 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 30 | 200/ 938 batches | lr 0.00023 | ms/batch 77.79 | loss 0.30 |\n", + "| epoch 30 | 400/ 938 batches | lr 0.00023 | ms/batch 76.79 | loss 0.08 |\n", + "| epoch 30 | 600/ 938 batches | lr 0.00023 | ms/batch 77.19 | loss 0.04 |\n", + "| epoch 30 | 800/ 938 batches | lr 0.00023 | ms/batch 77.11 | loss 0.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 30 | time: 77.87s | valid loss 9.88 |\n", + "-----------------------------------------------------------------------------------------\n", + "=========================================================================================\n", + "| End of training | test loss 4.90 |\n", + "=========================================================================================\n", + "| epoch 1 | 200/ 938 batches | lr 0.00100 | ms/batch 78.58 | loss 55.95 |\n", + "| epoch 1 | 400/ 938 batches | lr 0.00100 | ms/batch 78.51 | loss 21.86 |\n", + "| epoch 1 | 600/ 938 batches | lr 0.00100 | ms/batch 78.68 | loss 9.01 |\n", + "| epoch 1 | 800/ 938 batches | lr 0.00100 | ms/batch 78.89 | loss 14.14 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 1 | time: 79.15s | valid loss 9.13 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 2 | 200/ 938 batches | lr 0.00095 | ms/batch 79.35 | loss 7.41 |\n", + "| epoch 2 | 400/ 938 batches | lr 0.00095 | ms/batch 78.36 | loss 10.07 |\n", + "| epoch 2 | 600/ 938 batches | lr 0.00095 | ms/batch 78.17 | loss 8.61 |\n", + "| epoch 2 | 800/ 938 batches | lr 0.00095 | ms/batch 78.44 | loss 13.05 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 2 | time: 78.90s | valid loss 9.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 3 | 200/ 938 batches | lr 0.00090 | ms/batch 78.70 | loss 6.10 |\n", + "| epoch 3 | 400/ 938 batches | lr 0.00090 | ms/batch 78.10 | loss 6.09 |\n", + "| epoch 3 | 600/ 938 batches | lr 0.00090 | ms/batch 78.25 | loss 4.45 |\n", + "| epoch 3 | 800/ 938 batches | lr 0.00090 | ms/batch 78.29 | loss 5.75 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 3 | time: 78.78s | valid loss 49.52 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 4 | 200/ 938 batches | lr 0.00086 | ms/batch 78.27 | loss 13.12 |\n", + "| epoch 4 | 400/ 938 batches | lr 0.00086 | ms/batch 77.72 | loss 4.84 |\n", + "| epoch 4 | 600/ 938 batches | lr 0.00086 | ms/batch 78.26 | loss 4.84 |\n", + "| epoch 4 | 800/ 938 batches | lr 0.00086 | ms/batch 77.71 | loss 12.17 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 4 | time: 78.52s | valid loss 5.71 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 5 | 200/ 938 batches | lr 0.00081 | ms/batch 78.67 | loss 4.25 |\n", + "| epoch 5 | 400/ 938 batches | lr 0.00081 | ms/batch 78.17 | loss 3.99 |\n", + "| epoch 5 | 600/ 938 batches | lr 0.00081 | ms/batch 78.56 | loss 4.10 |\n", + "| epoch 5 | 800/ 938 batches | lr 0.00081 | ms/batch 77.99 | loss 4.72 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 5 | time: 78.79s | valid loss 9.56 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 6 | 200/ 938 batches | lr 0.00077 | ms/batch 77.96 | loss 4.15 |\n", + "| epoch 6 | 400/ 938 batches | lr 0.00077 | ms/batch 78.16 | loss 9.85 |\n", + "| epoch 6 | 600/ 938 batches | lr 0.00077 | ms/batch 77.77 | loss 4.22 |\n", + "| epoch 6 | 800/ 938 batches | lr 0.00077 | ms/batch 78.15 | loss 4.03 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 6 | time: 78.47s | valid loss 49.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 7 | 200/ 938 batches | lr 0.00074 | ms/batch 77.86 | loss 23.10 |\n", + "| epoch 7 | 400/ 938 batches | lr 0.00074 | ms/batch 77.91 | loss 16.75 |\n", + "| epoch 7 | 600/ 938 batches | lr 0.00074 | ms/batch 77.95 | loss 3.85 |\n", + "| epoch 7 | 800/ 938 batches | lr 0.00074 | ms/batch 77.71 | loss 3.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 7 | time: 78.44s | valid loss 5.53 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 8 | 200/ 938 batches | lr 0.00070 | ms/batch 78.17 | loss 3.24 |\n", + "| epoch 8 | 400/ 938 batches | lr 0.00070 | ms/batch 77.73 | loss 3.40 |\n", + "| epoch 8 | 600/ 938 batches | lr 0.00070 | ms/batch 78.01 | loss 2.85 |\n", + "| epoch 8 | 800/ 938 batches | lr 0.00070 | ms/batch 77.53 | loss 4.33 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 8 | time: 78.38s | valid loss 5.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 9 | 200/ 938 batches | lr 0.00066 | ms/batch 78.37 | loss 2.09 |\n", + "| epoch 9 | 400/ 938 batches | lr 0.00066 | ms/batch 78.02 | loss 2.66 |\n", + "| epoch 9 | 600/ 938 batches | lr 0.00066 | ms/batch 77.85 | loss 2.62 |\n", + "| epoch 9 | 800/ 938 batches | lr 0.00066 | ms/batch 77.36 | loss 3.35 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 9 | time: 78.45s | valid loss 7.40 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 10 | 200/ 938 batches | lr 0.00063 | ms/batch 77.88 | loss 2.55 |\n", + "| epoch 10 | 400/ 938 batches | lr 0.00063 | ms/batch 77.50 | loss 2.74 |\n", + "| epoch 10 | 600/ 938 batches | lr 0.00063 | ms/batch 77.88 | loss 3.16 |\n", + "| epoch 10 | 800/ 938 batches | lr 0.00063 | ms/batch 78.24 | loss 2.82 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 10 | time: 78.48s | valid loss 5.04 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 11 | 200/ 938 batches | lr 0.00060 | ms/batch 78.71 | loss 1.61 |\n", + "| epoch 11 | 400/ 938 batches | lr 0.00060 | ms/batch 77.70 | loss 2.81 |\n", + "| epoch 11 | 600/ 938 batches | lr 0.00060 | ms/batch 78.08 | loss 2.76 |\n", + "| epoch 11 | 800/ 938 batches | lr 0.00060 | ms/batch 77.58 | loss 2.63 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 11 | time: 78.52s | valid loss 5.86 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 12 | 200/ 938 batches | lr 0.00057 | ms/batch 78.12 | loss 2.21 |\n", + "| epoch 12 | 400/ 938 batches | lr 0.00057 | ms/batch 77.18 | loss 1.66 |\n", + "| epoch 12 | 600/ 938 batches | lr 0.00057 | ms/batch 78.08 | loss 2.00 |\n", + "| epoch 12 | 800/ 938 batches | lr 0.00057 | ms/batch 78.07 | loss 1.95 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 12 | time: 78.41s | valid loss 5.11 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 13 | 200/ 938 batches | lr 0.00054 | ms/batch 78.13 | loss 0.98 |\n", + "| epoch 13 | 400/ 938 batches | lr 0.00054 | ms/batch 77.97 | loss 2.55 |\n", + "| epoch 13 | 600/ 938 batches | lr 0.00054 | ms/batch 77.84 | loss 1.84 |\n", + "| epoch 13 | 800/ 938 batches | lr 0.00054 | ms/batch 77.29 | loss 2.27 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 13 | time: 78.30s | valid loss 9.19 |\n", + "-----------------------------------------------------------------------------------------\n", + "| epoch 14 | 200/ 938 batches | lr 0.00051 | ms/batch 78.07 | loss 67.85 |\n", + "| epoch 14 | 400/ 938 batches | lr 0.00051 | ms/batch 77.86 | loss 1.49 |\n", + "| epoch 14 | 600/ 938 batches | lr 0.00051 | ms/batch 78.11 | loss 3.83 |\n", + "| epoch 14 | 800/ 938 batches | lr 0.00051 | ms/batch 78.08 | loss 1.79 |\n", + "-----------------------------------------------------------------------------------------\n", + "| end of epoch 14 | time: 78.48s | valid loss 5.93 |\n", + "-----------------------------------------------------------------------------------------\n" + ] + } + ], + "source": [ + "num_runs = 5\n", + "best_model = None\n", + "best_result = 1000000\n", + "results = []\n", + "\n", + "for i in range(num_runs):\n", + " features = nn.Sequential(nn.Conv2d(1, 64, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(64),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(128),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=(1,1)),\n", + " nn.BatchNorm2d(256),\n", + " nn.ReLU(),\n", + " nn.AvgPool2d(kernel_size=(2,1), stride=(2,1), padding=(0,0)),\n", + " \n", + " nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=(0,1)),\n", + " nn.BatchNorm2d(512),\n", + " nn.ReLU()\n", + " )\n", + "\n", + " classifier = nn.Sequential(nn.Linear(512, 1028),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=1028, bias=True),\n", + " nn.ReLU(inplace=True),\n", + " nn.Dropout(p=0.5, inplace=False),\n", + " nn.Linear(in_features=1028, out_features=10, bias=True))\n", + "\n", + " model = DoXTimes(Smartpool(divider, 0.3, entire_batch=False), classifier, features=features)\n", + " model = model.to(device)\n", + "\n", + " optimizer = torch.optim.Adam(model.parameters())\n", + " scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)\n", + "\n", + " epochs = 30\n", + " model, result = train_model(model, epochs, train_loader, test_loader, seq_len, digits_per_batch, optimizer, scheduler)\n", + " \n", + " if result < best_result:\n", + " best_result = result\n", + " best_model = model\n", + " results.append(result)\n", + " \n", + "model = best_model\n", + "print(\"Result over {} runs: {}\".format(num_runs, torch.tensor(results).mean().item()))\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": { + "papermill": { + "duration": 0.755122, + "end_time": "2021-01-21T19:37:15.861895", + "exception": false, + "start_time": "2021-01-21T19:37:15.106773", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQcAAAMwCAYAAAAtS6WbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABGqklEQVR4nO3dd3wUdf7H8dcnhYTee5cOFhBEEQuKZztFvBNPz4KKx/lTT72zot7ZTk+v2MuJinp2BbGfDcVyIghKkY6CVGnSpASSfH9/fCchgW9ISHazm+T99OEju7Mzs59slvd+ZuY7s+acQ0RkVymJLkBEkpPCQUSCFA4iEqRwEJEghYOIBCkcRCQoKcLBzI43s7lmtsDMrivH521tZh+b2Wwzm2lml0fTG5jZB2Y2P/pZvxxrSjWzb8zsrUTWYmb1zGy0mc2JXp9+Cazlj9Hf51sze8HMMsurFjMbZWarzOzbAtOKfG4zGxG9j+ea2XHlUMs/or/RdDMba2b1YlVLwsPBzFKBh4ATgO7AmWbWvZyePhu40jnXDTgEuCR67uuAcc65TsC46H55uRyYXeB+omq5D3jXOdcVOCCqqdxrMbOWwGVAH+fcvkAqcEY51vIUcPwu04LPHb13zgB6RMs8HL2/41nLB8C+zrn9gXnAiJjV4pxL6P9AP+C9AvdHACMSVMvrwC+AuUDzaFpzYG45PX8r/JvtaOCtaFq51wLUARYCtsv0RNTSElgCNADSgLeAY8uzFqAd8G1xr8Ou713gPaBfPGvZ5bFTgediVUvCOwd2/vHzLI2mlSszawf0AiYCTZ1zKwCin03KqYx7gWuA3ALTElHLPsBq4MloE+dxM6uZiFqcc8uAfwKLgRXABufc+4mopYCinjvR7+ULgP/GqpZkCAcLTCvXMd1mVgsYA1zhnNtYns9doIaTgFXOuSmJeP5dpAEHAo8453oBmynfTat80fb8KUB7oAVQ08zOTkQtJZCw97KZ3YDfTH4uVrUkQzgsBVoXuN8KWF5eT25m6fhgeM4592o0eaWZNY8ebw6sKodS+gODzGwR8CJwtJk9m6BalgJLnXMTo/uj8WGRiFqOARY651Y753YArwKHJqiWPEU9d0Ley2Y2FDgJOMtF2xCxqCUZwuEroJOZtTezavidKG+UxxObmQFPALOdc3cXeOgNYGh0eyh+X0RcOedGOOdaOefa4V+Dj5xzZyeolh+BJWbWJZo0EJiViFrwmxOHmFmN6O81EL9zNBG15Cnqud8AzjCzDDNrD3QCJsWzEDM7HrgWGOSc27JLjWWrJd47lEq4k+VE/J7W74AbyvF5D8O3WtOBqdH/JwIN8TsG50c/G5Tz6zGAnTskE1IL0BOYHL02rwH1E1jLLcAc4FvgGSCjvGoBXsDv69iB/zQetqfnBm6I3sdzgRPKoZYF+H0Lee/ff8eqFotWIiJSSDJsVohIElI4iEiQwkFEghQOIhKkcBCRoLiFw96eaWlmw+NVy95SLWGqJSxZaol1HXEJh1KeaZkUL3BEtYSplrBkqSX5wwHoCyxwzn3vnNuOHw58SpyeS0TiIC1O6w2dEXZwUTNXswyXSQ3qWIOkGJGlWsJUS1iy1FLaOjaxbo1zrvGu0+MVDsWeERZtHw0H/0sdZifGqRQR2ZMP3egfQtPjtVlR7BlhzrmRzrk+zrk+6WTEqQwRKa14hUPCzrQUkdiIy2aFcy7bzC7FX5oqFRjlnJsZj+cSCVn0137kZPot2cY9VgMw4YAx+Y93+Oh8AGpPqg5A0/u/KOcKk1+89jngnHsHeCde6xeR+IpbOIgkwrq3OwHwbc8Hd3tsR4Fd4nOOehyA5/o0B+DlD44EIGf2/DhXWHFo+LSIBKlzkEohr2P4X88Xd3vs3+v3AeDuCb8AoF3b1bzf3V8u9KzaKwC4/bxGAOxzrTqHPAoHqdCyB/YG4KMDHoqmpANw77rOfPybPn7Scn/9187rJgOQkpnJHRP3A+D6RjP8eupnl1PFFYc2K0QkqEp2Dmt/1w+ANucsYM6qpgBsz/KfOC1f8D9rLP2Z3KmzElOglNjPLasBkBJ9zt27rjMA4wftR873c4PLLLilF883+Fd0zw/Aa/WuPid3pVdERIKqZOdwzdXPA/Drmuugwy4PDvA/FmVv4b7VR5Vq/ZNWtQWg5r/qApA2Lhm+xKpyqvefCQCcNtl/CZat819Ylr1iUZHLXHjih9RK0ZD94qhzEJGgKtk53H/9GQD8Zf8U6s/2I2PWdfMnklbbfz0Af9/3Ve5p7r8N7u0ttQD4ZY2fd1vXVrcdgIlZNQEYkLkDouU6/ub3AHQeF4/fQgrKmTWv2HkW3e73NQ2r908gE4ArVxwCQO0PZ/v1xKe8CqlKhkPN0ROjnzun1dllngeaDeCv/dv5xz5ZAMDfB3TcbV1pW/0XYtec7o+XN/x0DPtVi3ZqLkqPYdVSWuvP8aHwv3P/CUDdlEwmZKUCMPWvvQCovjGu31pXIWmzQkSCqmTnUBLZP66k5piVwM5Ws+botUXOv/JC/+nUo1oa//zJf/9suye/9+uKX5lSAmsO9JuOdVMy86cNHX8hAJ1fU8dQFHUOIhKkzqGM0tr6C149eL0/CzDdUnnlvmMAaLhiQsLqEtj+gT+kPKFr3oAn3zkcMGEo3a78DtAOyD1R5yAiQeocymjOH1sCcFCGPxQ6c/tWGszaksiSBEjbpx23dXwFgPrRvoYpWf6xtrflkLNuXaJKqzAUDqWU9cuDAPj6tHuiKX7E3f9dfjnVv9BOrkTr8PIyelUr3BifOe4iADpP+yoRJVU42qwQkSB1DqW0+ASfq7XMdwxnLvQXEqnx7jQS/u0mVdi6of6Q8i1N/0VeNzd0kd9B3O0aP5hNOyFLRp2DiASpcyiFlNq1OefwzwHYmLsNgFV3+EuRZWRpezYR0lq2AODwy/zQ+IJnXU6Y5Ye9d16nv83eUOcgIkHqHEph/s09eKvRwwCcMv/XAGS8o0+lRJp9vR+M9lqzN/OnHTVjCKB9DaWlcNgLG872p/dO/839fJe9A4Cf72oFQAYrElaXwJRBhQ8pA9S92J8xm60xDaWizQoRCVLnUAJ5O7uu+PNLAGRYGmdMOweAxv/V5kSy2tHUX6YvfXvLIufJWb0GAJflh09aRgapjRsVnqdxPQDmX1ltt+Vdjh8Z2/UPC8jZuLHMNScTdQ4iEqTOYQ8szb88B7y1FIAhtfz1HJ7b1ISmf/a5mpuY0qQE3h49qth5Dv3mTADWrPTXAqvfeBMTez+/18/V/cZL2eeaynUWrjoHEQlS57AnB/grOt3W5JlCkx+6Ywj1plWuT4mK7pRZZwEwbt/RxcxZ2Be9XijysS3RxYN3uJ394YnTzwNgw9TC+yVafl75rvelcChCavfODH/x9ULTuo+6BIB2z3yZiJJkD6oftxCAHndcCoALvLNrd/0JILjZ0OOz8/1yi2vmT9tndHS18Ukz8qfVZ36hn5WZNitEJEidQxHmXFyfk2sUPjTVarxvM3E67zJZtb+++M29k+i9+3JMj0c5FZo6BxEJUuewi20n9wVg3Mn/AmokthiRBFLnICJB6hx2sby//5q0Nmk7u4bnNjUBIH2j3+egPQ5SFSgc9uBva7sDMOG4dgC4FTP2MLdI5aLNChEJMpcEh+XqWAN3sA1MdBkiVdKHbvQU51yfXaercxCRoKTY5+Dq1mDbEf4QYo1FfuDRlnZ1Ct3fW3nLJ7vS/n5VSUX5W+5Jov/Oe3wN3wyfj6LOQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkaC0sixsZouATUAOkO2c62NmDYCXgHbAIuB059y6spUpIuUtFp3DUc65ns65PtH964BxzrlOwLjovohUMPHYrDgFeDq6/TQwOA7PISJxVtZwcMD7ZjbFzIZH05o651YARD+blPE5RCQByrTPAejvnFtuZk2AD8xsTkkXjMJkOEBG9XplLENEYq1MnYNzbnn0cxUwFugLrDSz5gDRz1VFLDvSOdfHOdcnvVrNspQhInFQ6nAws5pmVjvvNnAs8C3wBjA0mm0o8HpZixSR8leWzYqmwFgzy1vP8865d83sK+BlMxsGLAaGlL1MESlvpQ4H59z3wAGB6WuBgWUpSkQSTyMkRSRI4SAiQQoHEQlSOIhIUFkHQcWEbdhC5luTAMiNpmV+S6H7eytv+WRX2t+vKqkof8s9SfTfuTSvoToHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEJcWhzMrGevcA4O03ngFgv39fCkDr275IWE3JLLVeXeY+uA8Ac456HIAbV/UGYMZZncmZNS9htVVlCoc4WHVQHQCyyQGgxnKXyHKSXm77VswY8CgAO6KX6q9NpgBwwKmH0lrhkBDarBCRIHUOcbBuf98xLM3OAqDhExMSWU7SSmvdCoD2IxckuBIJUecgIkHqHGLM9e/JZyfdDcCRn/4BgI58k8iSks7ivxwKQO/jZwHw9+afFTlvrUNXs+TPfv5G07MBqP76pDhXKKDOQUSKoM4hxn7qXp3mqTUAaDk6PcHVJKfpv38AgB0up9h5xx/wXP7FCMdubg7AqE2DAUj7aEpc6hNP4RBjAy+ewGub6wFQa/xcAIr/J1A1pI/3/7jTLbXYeb/Z7k9yXrSjMafW/AmA02v5bzk4/ZmRAJzUsnc8ypSINitEJEidQ4yk9ugCwB1NXuCJjf4QXc76DYksKalsHdyX85u/AuzcnAhtVuw77iIAGo/LACBjQw4jBvjPsBlD7i8079IRh9Lqbxp1Gi/qHEQkSJ1DjCz7RcP821M2tY1ubU1MMUkkr6P6690j6VNte97UQvOM3dycGz/+NQDdrvFft5qzcWP+413mdwZg0qBMAPpmbAPgv//3d47NvAaAdnf4nZMuKysOv0XVpM5BRILUOcTIxu478m9PfbAnAPXQsOncav4ttrNr2OmCH44HYNNvqtN5qR/YFDqyk3dW5sVP+f0Rk39/LwDNU6vz9TB/+9ev+q9nddNmx6r0Kk/hUEZZJxwEwOvH+mP3t67pTYMx04HEX3E4WV2/sg8AGy/0m2I5S+eXaLl2Y9YA8OfBhwBwZ7Ov4lCd5NFmhYgEqXMoo6VH+5dw/2p+Z9nQRfvRZPOcRJaUlAoOfJp+YN71LUrWMeTz3+hOWkrubutcfov/2WxwaSuUXalzEJEgdQ5l1HhfP6Q3x/lPs7TX6yeynKQz9//8eSYlOY+iOIt+5fdRjG48KVpnav56W9zk59F+nthR5yAiQeocSimtvR/o9M8ufkjwYxtaA9BglA5fFnTj4W+Wafm01q3Y1LsFAP8+/+HdHp+U5ff12PbsMj2P7E7hUErzf+/fsIf4UwD43ddHAdCaSvCtr0lk1i3NmHnsg8HHxvzciEeuGgJA5mxdACbWtFkhIkHqHEopt/W2Qve3rs9MUCWVU961H/7WfEyR8zy17FAy31THEC/qHEQkSJ1DKT188LOF7rf8b/FXN6qKUm33AUsbf3tIoXluufUJjqpeuBPLm98fqgy/tu7oZTGsVHalzkFEgtQ5lMK2k/tyWGbetq5ewj2586XTADg9OnsS4NN/PAQUHhi1o4hvDNzT1aI68XWMqpQQvbNLYfEgR4b5l+7WNfsBUOv16GIjCasqOe3zkj+TctLZmfkXadkbk7IyGfnjkQCsu7gZAF0X+m/I0oV740ubFSISpM5hL6TW8d+efW3/d/KnPf/fIwDYJ1sjI0PyLtTylz9dyJKT/c7JeSc8WuLlLx51Ea1vz7uI7LpYlyd7oM5BRILUOeyF3OjipbO2tOCYZf5qRp3umAlo+7c41V+fROfX/e0jzrwEgPTzVgLwbo+XOPbbMwDIfaoJAM5fuoF2U1frtU0QdQ4iEmTOJX7/eh1r4A62gYkuQ6RK+tCNnuKc67PrdHUOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJSopBULn1a7J54MEA1J67AYBNXeoWul/e8p6/vCTq94yF8n6tSqsiv8Zltce/0ejRwcnqHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQVGw5mNsrMVpnZtwWmNTCzD8xsfvSzfoHHRpjZAjOba2bHxatwEYmvknQOTwHH7zLtOmCcc64TMC66j5l1B84AekTLPGxmqTGrVkTKTbHh4Jz7FPhpl8mnAE9Ht58GBheY/qJzLss5txBYAPSNTakiUp5Ku8+hqXNuBUD0s0k0vSWwpMB8S6NpIlLBxHqHpAWmueCMZsPNbLKZTd6R9XOMyxCRsiptOKw0s+YA0c9V0fSlQOsC87UClodW4Jwb6Zzr45zrk55Rq5RliEi8lDYc3gCGRreHAq8XmH6GmWWYWXugEzCpbCWKSCKkFTeDmb0ADAAamdlS4CbgTuBlMxsGLAaGADjnZprZy8AsIBu4xDmXE6faRSSOig0H59yZRTw0sIj5bwduL0tRIpJ4GiEpIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkaBiT9kuDynrNlNzzEQAcqNpNaML4eeGF4m7mt8WP08sJer3jIXyfq1KqyK/xmVVmr+ROgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKh0oo56gDyTnqQP5v/oJSLb/pN4eQ2qUjqV06xrgyKa315/Zj/bn9eGfZ17yz7GuW3HAolpaGpcVvHKPCQUSCkmL4tMTWD8dlANAgtXTfXv7jL7ez4xz/udHgpJiVJaWQ1rIFALf8ZVSh6TMufpCT7j8SALdpU1yeW52DiASpc6hELL0aAEcfPbVM66n9TSanD/sEgI/rtQIgZ/2GMq1TSmfVcW0B+EX1rYWm95n8W5r8PDeuz61wqEQ2nXogAPe3fACAbq9dSicm7vV6suo7Lqs/B4Dxtbv5iQqHcpdSowa/+MP/go9lvlQPnIvv88d17SJSYalzqARc/54APHTXfQA8u9G3ol1vnEdOKdbX79gKcoGGSi7r0G7c1uSxQtO2uu0A1Hn+y7g/vzoHEQlS51AJrBuxBYBWadkA/OkPvwQgfd2UvVpPWvNmADzZ5l12OH1uJNqiU3f/53navF9Ht5bF/fn1DhCRIHUOFdza3/Xjlf3+AcB/NuwPQPqHe9cx5Jl1a2sAdrgchi46BoCcVatjUKWUxvF9p+Xf3pi7DYCcW5oAkFIOnYPCoYJLGbyGFml+ROQTzx8PQCu+2Kt1pPboAsCzAx8FIMvtYPHdnQGombX3h0KlbLJOPAiAB1o8mj9tSY5v8lM++abc6tBmhYgEqXOooFIbNwbgxs5v509rdcfedQx55lxcD4A+Gf7A50Pruud/VYCUv5UHpe82bfCblwOUalBbaalzEJEgdQ4VlNXIBOC4Ghvo+9W5ADRjdqnW1ajdT4XuP7ewD42YV7YCpdTSe63Lvz1nRxYAXR9YA1CqQW2lpc5BRILUOVRQuT+tB+C21Qfy2w6TAfi0eQcAslf8WKJ1pLX1hy7/1/PFaIr/rNj6ZSNQ51Dutp3UF4CvDnoompLKnO1NAciZ912516NwqKByowt8vL+sK5/1fB6AFW/VBeCzR/sVudz67v5MvlrtNnBIi0V+Xbt8i6TF92Q/KcLWRqkApJGaP23ElFMBaM+04DLxpM0KEQlS51DB1b8lkyNvPhOAsfs+BcBdN00ocv7JWf5TKYcU+lTbHk21QvO0eWBGlf5G6kTZNnh9oftzdmTR6onE/RNV5yAiQeocKrpJM6h7or95zoDLAFjfKaPI2Rs+trOrWPZqDwCmHPxUoXly43TBUglL7ex3JH910H/ypgDwzs/7lvo8mVhQ5yAiQeocKpHU8V8D0HB8yebfuqi2v3Fw4emuf0/sf1NjVJUUZ+VR/kzLgkcpAB756JhyHS69K4VDVRbth0zZpYFUMJSvbQ0K7xCest2Pg+z296VkJ6KgiDYrRCRInUNVFg122nUQlJSvxkctL3T/jQ3+KwZyVq9JRDn51DmISJA6hyosN7Nwx7A6JytBlVRdlpHBoBbTC037aUdNAFzWtkSUlE+dg4gEqXOowp49/t8AzN7uO4gzn7oGgDZ7eQ1KKYOcHB6b3R+Ayw9dAMAni/2gqFbMTFhZoHCo0m5dOAiAzQ+3BKDNGIVCeXPZ2bQf8TMAPe4YCkDKtNqJLCmfNitEJEidQ1U2cCkANVma4EKqtpwFCwFoe3qCC9mFOgcRCVI4iEiQwkFEghQOIhKUFDskc+vXZPNAf95w7bkbANjUpW6h+1VR3muQ7Kry36ikyvtvuevfZI/PP3p0cLI6BxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJKjYcDCzUWa2ysy+LTDtZjNbZmZTo/9PLPDYCDNbYGZzzey4eBUuIvFVks7hKeD4wPR7nHM9o//fATCz7sAZQI9omYfNLDVWxYpI+Sk2HJxznwI/lXB9pwAvOueynHMLgQVA3zLUJyIJUpZ9Dpea2fRos6N+NK0lsKTAPEujaSJSwZQ2HB4BOgA9gRXAv6LpFpjXhVZgZsPNbLKZTd6R9XMpyxCReClVODjnVjrncpxzucBj7Nx0WAq0LjBrK2B5EesY6Zzr45zrk55RqzRliEgclSoczKx5gbunAnlHMt4AzjCzDDNrD3QCJpWtRBFJhGIvTW9mLwADgEZmthS4CRhgZj3xmwyLgN8DOOdmmtnLwCwgG7jEOZcTl8pFJK6KDQfn3JmByU/sYf7bgdvLUpSIJJ5GSIpIkMJBRIIUDiISpHAQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQkq9kttykPKus3UHDMRgNxoWs3oC/Zyw4tUCXmvQbKryn+jkirvv+Wuf5PSPL86BxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQUkxCCre0tq2BqDJS+sB+GRKdwC6PryenJlzS7XO1MaNAVh7QkcA6r/0NQAuK6sspYokDXUOIhJU6TuHtGZNuXX8GAC6pPtBpUevbQZAzsz5pVpnauPGnPW57xQOyRwLwCUzfu8f/GZmWcpNeqmNGjL3njYADOjkX79lR+4A1DVVNpU2HNJatQSg7ktb2L9aKgBdPrwIgE5Dvy7Tumf/tR2n13oXgAPvvQaAFt98UaZ1JrtVlx4KwE2X/4df1ni/0GODG50MQPay5eVel8SPNitEJKjSdg7r+vudkK+1eyh/WrcbVwGQXcp1un4HALDgpEc5csYQAFqPmgNATinXmexSO3cA4PEr7wWgZ7W03c74W/FIbQCa/74Z2St+LMfqJJ7UOYhIUKXrHPIOW64+ZVv+tD7//AMAzZaUbr9AXsdw43NP50/7+W2/U7Pm2u9Ltc6KYvZ19QHy99uETOz9PADzJmznV8/8CYB9bv8GgNxt24pcTpKbOgcRCap0ncOS+2oBML/vUwDcuKonLZ/0hxdLu19g2YCaAPTP8Fvb+34xlDYPVO6jE6ndOwPw4cB7oynVAbhrbTcmr/eHMl/q8G6hZTqnV+Oxsx7x8406BYDchT/Ev1iJi0oXDs4ZADucj4KJa9uRunXVXq8npXZt5t7uR1K+NuhuAHJJB6DNkBmxKDWprenbEIB2aTUAGL7kCACWHvIzKTW3AND7Ir+5dtXvXgbgrNqrOCLTL//mmMUAzPql3/zSjsqKR5sVIhJU6TqHXb3T9TWGjT8KgMWbmgOw/YlmRc7/4+EOgBMPnsobLR6OpvqOof/UMwCoT+lGVlYkORn+Zy7+9Zj+6H4ANGACuZs3A9D8X37T6uWTDwLgzNpvgfObXiuz/OFNt02jJisqdQ4iElTpOocmD/gdZx+P9Bu/R1XfxhNtPgYgBb8/IvduV+Ty+fOwc54XNjUFoOH1adFjlV/tX68odH/Dcb5baPDk7vP+pe0b0a2dnzWffdMVgM7rJsWlPok/dQ4iElTpOoe0j6YAcN9hRwNw26HtWHqs7wIWnPxvACZl+e7g7Pcv2m35Tv/x28hvvzIqf9rfZx0HQMtplfuMy4I2jfH7Z+jhf5zX3X/p0KcH9WV1L3+42J30EwD7pvvuYPaOHfRIrwbA2BMeAODaQ37nV/Dl9PIoW2LInCu6xS4vdayBO9gGJroMAFL29+3wW/99nr+u2ReASSftA0D2kqUJq6u8pTX3O23/PsGfkt45+kefghXa5AL4zXfHA7D1ssac+sJ4AM6vswSAbp8MA6DDb6fGu2QppQ/d6CnOuT67TtdmhYgEVbrNirJafJM/hyAXx/u3+4E/tZZ8mciSEiJv0NLwq68A4Ml/+oFgndNr5h+u7Pi+32Toeqk/MzV38yzu/Mhf22HY4GikZJ9XAXj8gF+SO212+RQvMaHOQUSC1DlE1gzvB8D0Q/z1HxZlb6X66u2JLCkp1HrF74g8H3+25U+nb2HbBj9CqtvV3wGQEw2KAuhy3SwABnb6FQAf9PCX6LvpphRa/qp8apbYUOcgIkHqHCJbfvFzofunTb2QJh+X7VqTlUleB1HrlZ3TQme55m7aBMDGsf5IT96h0Lv2H8PDzQcAOgmrolA4RB7t/QwAK3L8GYcN762RyHIqvMaP+rEPB5/wW8BfEObyq9oB0OFKhUNFoM0KEQmq8p3D0hH+kuv9M/wmxJdZvmNI1SZF2eT6jY6G//Kv55pntjL7DL+z9+TnzwXATak6I04rInUOIhJU5TuHs84cB+w8C3PY5PMAaMsMUhs28DM18VdFypld+a/jEGspn/gLzQ54+mpmXeA7h023bwWgzhB/zYe8nZiSXNQ5iEhQle8cdpWb4/Ny1aWH8ssLPwPgte/9GYoaxFN6HUcu4Zkh/mSuT/cbDcDxB1wAQMrnUxNVluyBwmEXs4/wVzPJPcLR41P/5u14sx8BWFm/1ao8ZC9ZysunHgnAOR++BMCaq/13WjT5PGFlyR5os0JEgqp85/DeDf7TbNYIv+kwYaK/nkPX+5bT4ce5AOToW5tiIm+H7m++PxaAN3s9DsCwQy7WxWCSkDoHEQmq8p1D5pt+mO/qN/39jvhrN5T2m7ileFtO9YeNJ37RAoB1XWpSv+pdMiPpqXMQkaAq3zlI+ctZsxaAkZ39tTnrMyGR5UgR1DmISJDCQUSCkmKzwtWtwbYj+wJQY+FGALa0r1PofkF5j1U0od8lkSrC65hsr1lFtce/9Rujg5PVOYhIkMJBRIIUDiISpHAQkaBiw8HMWpvZx2Y228xmmtnl0fQGZvaBmc2PftYvsMwIM1tgZnPN7Lh4/gIiEh8l6RyygSudc92AQ4BLzKw7cB0wzjnXCRgX3Sd67Az8RcmPBx42s9R4FC8i8VNsODjnVjjnvo5ubwJmAy2BU4Cno9meBgZHt08BXnTOZTnnFgILgL4xrltE4myv9jmYWTugFzARaOqcWwE+QIAm0WwtgSUFFlsaTRORCqTE4WBmtYAxwBXOuT2NTLHANBdY33Azm2xmk3ds3xxYREQSqUThYGbp+GB4zjn3ajR5pZk1jx5vDqyKpi8FWhdYvBWwfNd1OudGOuf6OOf6pFerWdr6RSROSnK0woAngNnOubsLPPQGMDS6PRR4vcD0M8wsw8zaA52ASbErWUTKQ0nOregPnAPMMLOp0bTrgTuBl81sGLAYGALgnJtpZi8Ds/BHOi5xzunarCIVTLHh4Jz7nPB+BICBRSxzO3B7GeoSkQTTCEkRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSC0hJdAIBt2ELmm5MAyI2mZX5LofsF5T1W0YR+l0SqCK9jsr1mFVVp/tbqHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEpQUhzIlsdKaNQVge6cWuz2WPm8ZAHNH7ANAvVlGg9nbAEj57JtyqlASQZ2DiASpc6iiNpx9CGtP9B3Adb3eBeDcOu/sNt8TG9oA8KvaYwGoPyQz/7GTWvaOd5mSQAqHSi7lgG4AzPlDTQA+O/ZeABqnfkVKCRrHYXUXR7cy9zifVD7arBCRIHUOldzm9rUBmHfCI9GU6iVa7t/r/Q7I5344qMh56rKgTLXFU0rP7mxr5rulRYMNgNP6fgXADpfKx8/0BaD5JxsAcN/MTECVyU2dg4gEqXOoBNJatQRg9rWtAGj6hf+krPPCl6RkOQDm7dgOwJLsegC0TlvPed8OBWDd7IZ+ua/8vPW+WIL7+WcA6q5P3u6gINe/JwDfX+LvP9/vMXpXSy16gav9WcBbr/Kvy8j13QF4eNqRdBo2G4DcbdviU2wFoc5BRILMOZfoGqhjDdzBNjDRZVRIqfXqcvAnqwG4vtEMAPpffykA9Z+esHO+Hl0AyJk519/v1omcud/7B3NzyqvcmMo9rCeLLva33+7/EAAd0nbuU/lgq799/azBAKxfXA+Abwc/wJ9XHgLA35tNBmD8tnQANudmMGLUeQC0+tsX8Sw/aXzoRk9xzvXZdbrCoYJKyfSHFre/1YR3u/kxCF1e9f9Sut7g2+KcjRsTU1ycff98TwCeC2w6nLnwFwB8Nac9XS+PNg82by40T9MJdVh1WVsAOj8yB4Abm44H4LOtzRlUcx0Agw85BYDsJUtj/0skkaLCQZsVIhKkHZIVTGr9+gDMua0zAHO7PcyULP9Y11v9ZkJl6xhSavpDkvNv3Q+A2Uf6TYgUUvkq2uF61ut+T2SXW3y30Hn95CIvMbdf7WV8kNYegMn/8KM8G949EYDBNdcDFutfoUJS5yAiQeocKpjlZ/vh0HNPfQCANzbX54mT/HZ2zurvElZXPK0f5DuGj4b8E4AUagAwbmsGd17sD8d2fP9LAEK7Vi3Nv81TunQA4PHXGvCP/zwNwH7VVkVz+XWmWgr7TfwtAC1XVc7Xs6TUOYhIkDqHCmbTwVsL3b9v4UCqz6vcn3AuOiCxzRXeF7Aptzo/HlwNgK2/8sOhO3Zakf/4hm3+iM6Qtl8DcEm9ZwCYvL0a/TPy9kjUKLTO/23LpeVf/fO4rKzY/RIVkA5lVjC3LfTnB/Sq5pu+dbnb6PfKlQC0f8OP9ksd/3VCaouXlNr+/JCtY/xIzme7PgtA09TqpJtPjhxXePdjlssmw4r/7MuONkQGTD8DgAaX5JD9/aKY1F1R6FCmiOwVdQ4VzHvLpwKww+2+6y1v2r7jLgKg7leZ/NzK/33rRIMhG03fOSBozf7+EGHT8X6nXE4F2TxJbdoEgNk3t+PY3n5U6LwNftoPyxr5earlMKjLdGDnKMiQzh8PA6DLlf5yeDkrVxU5b2WlzkFE9oo6hwpm3qP++grzTvp3zNY5KcvvgLtiVrTdfdK8mK07kZaP9WdaTu37bKHpi7K3MPiBawBoea8/O9NlZ5dvcUlEnYOI7BV1DhVM3oCe7QMOAODcB9+kRoo/5HZSDX92Zt4e/L2VGw047vH8ZXS4ekIxcye3hXf04+tz7wGgulUr9FifOy6lyUNV44zLkiiqc9A4hwomr/1N/3AKAC903fldE/ef5jcLctL9ZsKhV03izmZflXjdeRecbXXAimLmTF7Lrz4UgPfO+jvVrfAYhvvWdQSg2ZNTizzvQnbSZoWIBKlzqERqjp5Y6P6bB/TjznN857DF+QFSvT/9PwDaPp7Kmsu2ADD5oMI77CqiHcf6rvi1S/8OQJu0nV3D4mz/e75xrd90zdhS8m6qKlPnICJB6hwqsTbvZcE5/naNaKfc7COfAOCctr/gnXbvRXMW/oxY/GMDOrGonKqMjUUn+Z2w7Qp0DCtyfMdw7hV+eHmNtyfuvqAUSZ2DiASpc6jE0ifP55CvzwTgywNfKPTYM+0+IO+zIcvtAOCkaBBU18u+C14XIRmlNmwAwDe/ujeakpH/2IDP/YV2O4xVx1AaCodKLHfTJpr9wV9W7uRRgwC4vt3bAPTLyGHMz/48hBve+Q0AHf9Y9AVTklFq/fpcMfEzAGpZRqHH7lrbjU6/mw+gw5alpM0KEQlS51DJZS+KviX7aP/jssv85es3HbSVrjeuAaDjD18morQyWzOoK8fW+BiAnF0G+r5zywBqbtbmRFmocxCRIHUOVUzT+/05BU2Bin4e4q+v+nC3K0B1fNNfy6LzGHUNZVVs52Bmrc3sYzObbWYzzezyaPrNZrbMzKZG/59YYJkRZrbAzOaa2XHx/AVEJD5K0jlkA1c65742s9rAFDP7IHrsHufcPwvObGbdgTOAHkAL4EMz6+xc4NJFImVwQPXFpJr/fPtym397df+7v5JTRe+KkkGx4eCcWwGsiG5vMrPZQMs9LHIK8KJzLgtYaGYLgL5AxT4HWJLOFc8NY87vHgbgglF/AKD19zoVO1b2aoekmbUDegF5G3SXmtl0MxtlZvWjaS2BJQUWW8qew0REklCJd0iaWS1gDHCFc26jmT0C3Aa46Oe/gAsIf9HgbleUMbPhwHCAzF2+O0CkJNre9AXH3dQTgNaoY4i1EnUOZpaOD4bnnHOvAjjnVjrncpxzucBj+E0H8J1C6wKLtwKW77pO59xI51wf51yfdDJ2fVhEEqzYzsHMDHgCmO2cu7vA9ObR/giAU4Fvo9tvAM+b2d34HZKdgEl7eg5XtwZZh/sLp1b/YRMAW9vWLnS/Ksn73Suyvf27VdTfuaK8P/f4+r41Oji5JJsV/fEn/s4ws6nRtOuBM82sJ36TYRHwewDn3EwzexmYhd9pfImOVIhUPCU5WvE54f0I7+xhmduB28tQl4gkmIZPi0iQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJCgt0QUA2IYtZLz9FQC50bSMbyl0vyrJ+90rsr39u1XU37mivD9L8/qqcxCRIIWDiAQpHEQkSOEgIkFJsUNSpKpKqVGD3l9sAuCmxlMBOHbWrwCo9osfElUWoM5BRIqgzkESbtvJfan+368BcH26A7BwUE0ADj96Bp99tF+h+ZtPyAEg881J5VhlbKXUqAHAvJFdeK3xSGDnYdEl05oD0IHEdg7mnEtoAQB1rIE72AYmugwpJ6mNGgKQ81J1AJ7q9CIrc9IBqJuyA4A2aTWKXH5VzhYAludU4/d3XA5Aw8cmxK3eeFh0ez8App93P2d9fwIAa29vD0C1d78q11o+dKOnOOf67DpdmxUiEqTNCil38+5rA8Dcrk9EU2rQJNXfenh9ZwC+3uTnWbq5Xv5yqeYb77e7vAlAk1R46cZ/AHDR7EsBSPl8ahwrj53tTbLzb0//rBMA7d9Nru5HnYOIBKlzkHLj+h0AwEuHPhpN8W+/d7fW4M6rhwJQe+Ya/9DqnwBIWbdk5/Ipvr3o/K+LAZh1+gN0SK8FwNYbNwJQ97ymAGT/uDJOv0VspNfaDsCm3O20+SArwdWEqXMQkSB1DlJudtStBkDPav5tl4s/Unb1kxfQeuwXAOTsaQW5/tGOf/wSgG7VLmX6KfcB8Ml+owHof4zvKuo+m5ydQ2pHf0Ri5hGjALh8+UBSP/46kSUVSeEg5SYn0wrd3/+L8wBoc/sXpVpfp0sm8tYxfkzAkFprAVg/aDMAdZ8tZZFxNvfmejFZT9YJB7GpdeF/vo2n+E0rN2VmTJ5DmxUiEqTOQcpNlxGFP9FSp9Qu8zpv+GowAEOO8odFL+nxKQBvUb/M646Hew5+qdD9/z1/IM0ovnP67rleANx38AsA7Fftc5qmZhSaZ8EOf3j0lNF/pMNVX5a5VnUOIhKkzkHKRcr+XRlQ7wMA5u3YBkCj6TvKvN76n2T6G0eVeVVxlVqnDgA1U/xhy/e3+nNHmt2ze9dg6X7H7faj9ueGR54E4IjMKQCkmz+cOykrg3PnDAHgT+3fB2BQTT+s/OHBT3DvqFMByJk1r9Q1q3MQkSB1DlIu5g+txxm1VgNw2PRzAKjzTvmeYJRIC6/YF4DDMscB0P3jcwHoyDf58+Qd5px7iR/INev0B/IfG7fVD/a6+L3zAOh63xoy5n0HwEP4IecPjGsNwFtdX+VvbeoCUG1W6WtWOEi5+OMJb+dvTlR7qGE09bvEFVTObP+Nhe6nf1d9t3nyDnPOOeohwJ/CnXfG5sZrWgLQacJEIDweZMH3zfyNrmWvF7RZISJFUOcg5ebRtUcAkPlWxb1IS2l1bVL0iE3r3QOAsYc9Ek3x17boMX44nYbN9vNsm1bi5/rLqoPIHD8DKNv3aqhzEJEgdQ4SV6n1/I6x2ilLE1xJYrWqsR6AlLzPY9t5BbZ5l/nBTN3SfcfQ+6uzAehw1jd79cmfd6bn5uwMcrdtK1vBqHMQkSKoc5C4WjrMb0+fVftjvt7cLubrzzpxQ6H7W3Krxfw5YiHX+c/h3LxewO08Ca150/WFHuve2O+fWFfCde96pucR00+nTgyOBCkcpMLKPro3L/Z6MLrnW/Oxd/kLFdel7OcWlJd6w/wmwMTP/GbFg238ZfD63XUVne/3V6DOXra8yOW7veTnWZmzFYDM+xoQi8PE2qwQkSB1DlLhZB/dG4CfLt9M13TfMVy8rD8A9V6Kvv8iMaUFpXZszxF1Pyry8byu4K5jBgNwwJjvAfj27Pu5+Eh/0siKXzYAIGetv3ze+nP6cdgVfkDUX5r+D4DeL14FQId3Y9M1Fds5mFmmmU0ys2lmNtPMbommNzCzD8xsfvSzfoFlRpjZAjOba2bHxaRSESlXJekcsoCjnXM/m1k68LmZ/Rf4FTDOOXenmV0HXAdca2bdgTOAHkAL4EMz6+yc2+MVwKRyqrPI/9kXZW8p87oszb9d1//Rf7fk5ANf5IOtfhjyvD/7HZ/Vdkwu8/PEWs6Chbz4Y18ATu3wLgBtD1sM+LM1czb6odXZ3y8CYEov/5l9xDmX0WD6egCskT+DdeGD/vyJmUc8mL+PIb9jiME1HAoqtnNw3s/R3fTofwecAjwdTX8aGBzdPgV40TmX5ZxbCCwA+sayaBGJvxLtczCzVGAK0BF4yDk30cyaOudWADjnVphZk2j2llBoV/HSaJpUQTXH+O3id2/rRodMf1bm/Fb+DMXspcuKXT73sJ4s9NeM5dfdpgJwR5MX8x+/4yp/Sfvq7yX3kOxtF/rrOdw9xp8V9VbX1wG4fFx/Jv3bfzVereXZhZZZfVAuB13m9z/8q8XnwM5BVCM3tOOpf54EQIdR8fkynBKFQ7RJ0NPM6gFjzWzfPcxugWm77R8ys+HAcIBMiv5eRKk8Lq63EICVb/l/KJN/alPsMne2H5l/teo8U7b7TZVzJg2jw0dzgGKuWp0EcqLTqz89xW/+1H/bXwj3nhafwa2fFZo3LwByA+Mj9/38fAA6/mkNDZbF9xuy9upQpnNuPTAeOB5YaWbNAaKfq6LZlgKtCyzWCtjtIK1zbqRzro9zrk86Gbs+LCIJVuy3bJtZY2CHc269mVUH3gfuAo4E1hbYIdnAOXeNmfUAnsfvZ2gBjAM67WmHpL5lu/L76YJ+nHi5v/jrXxrN2Ktls6O+YJo/dYCzX7oMgPbXJdd3S+6N1KZ+K3zx+R3Z3N7vbHzv+HsBOO69K/xMBf5pdnncD5RyX+3da1cSRX3Ldkk2K5oDT0f7HVKAl51zb5nZBOBlMxsGLAaGADjnZprZy8AsIBu4REcqRCqeYjuH8qDOoWrIOwfgqNemA/Cn+vOLXabrJxdQbYbfJ9Xqb6X78hvZs6I6Bw2fFpEgDZ+WcpOzwB+t+HBf/2U2H3Jgscvsw9R4liR7oM5BRIIUDiISpHAQkSCFg4gEKRxEJEjhICJBSXEoM7deDbYO8Gd11/zOn9u+uUOdQvcleeX9rSqKeL2nKtrrkO+10cHJ6hxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkaC0RBcAkLJ+C9VfnwRAbjSt+rcUui/JK+9vVVHE6z1V0V6H4qhzEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISlBSDoEQE0po1BWBD/3YALPuFY+GgkQDscDkA9J96BgCrl9Sn+50/ApC9aHFc6lHnICJB6hxEEsgyMvj+lgMBePC0xwE4svqW/Md3OP/5nRsN+v6s5/P+gZ7Qs+EFALQZEp/a1DmISJA6hzLKGeBTP+0vKwF4s8sbpFsqsPt2YsMb0rFFywBYe3J3ABq85s/Wyd20qfyKlqSx+OrezDjnviIfP/+HgQA80faD3R6beugoAAZxUFxqUziUgmVksGlQTwBu+pv/A+W1grnADkd0u3AreOCfz+OAZr5Ze73dgwAcVO8PADR94IvyKF2ShOt3AACjLnigyHn2f/Iy2t/2NQBd77kEgDmnPBT/4iLarBCRIHUOpZA1YD8+uvfBQtM+3loLgL/89QLSt7hCj21s6zO42ha45irfaWzIzQag1oqceJcrSSSvY3B//QmA3hk7ry8x9ucmAIw6bxAA7SZOwuX690eXP04D4ITX/g+A2/49kj4Z/rFjvvWbpB/uWzumtapzEJEgdQ57IS/1//bIo/nTzvzuRAA23tQagPofT9htubod2wPQ85Xv6FbN53HX1/8IQOfRE+NXsCSdVQfVBOCrrr6DTLdUNuRuB+Cml/2O63YTdn8PuawsP//7kwE4+72LmHmy716vbvAdAI+9MBSA9mdOi0mt6hxEJEidw15Yd8NWwG8nnjjnVwCkXlXH//zm6yKXW9/bD4u9qcnL+dNavx+vKiWZpRyzFth5JGuHg/O/j/Yx/Hn3jqEonf9vEg8c1gOAPzWYA8BZ3b8C4AuqxaRWhUMJLHxxfwBm9noSgKXZW0m5oT4A7pvpRS5nGRkAdLxiFgAppOQft67+2qS41SvJJ61lCwCu7PLhbo99/0onAJqyeq/WOer1YwD40/lzylhdWLGbFWaWaWaTzGyamc00s1ui6Teb2TIzmxr9f2KBZUaY2QIzm2tmx8WlchGJq5J0DlnA0c65n80sHfjczP4bPXaPc+6fBWc2s+7AGUAPoAXwoZl1ds5V2GN253bPu2y+bwV/yK4DXxbfMcy91+/AfL3NQ9Hy8MM/ugBQA+2IrErWHdYGgNNqvV5o+vAlA2j5it+hmF3G59i3+lIAJu1zNNnfLyrj2krQOTjv5+huevS/28MipwAvOueynHMLgQVA3zJXKiLlqkT7HMwsFZgCdAQecs5NNLMTgEvN7FxgMnClc24d0BL4ssDiS6NpVUJqjy7M/kNdAOacXHio68dba1H7i4UAVNg2Skpl9YEWnP7dnd2o/mNs9j+dVNPv7Ly7TzNqlUfnAOCcy3HO9QRaAX3NbF/gEaAD0BNYAfwrmj30KuzWaZjZcDObbGaTd5BVitJFJJ726miFc269mY0Hji+4r8HMHgPeiu4uBVoXWKwVsDywrpHASIA61mBPmykJN2ZhTwCubjgDgF4Zmzl8+rbgvH1rvMpR1f1ju37t2pXTTqPVypnxKlOSWE4N/25I2eXzOBZHrXaeBVzmVRVSbDiYWWNgRxQM1YFjgLvMrLlzbkU026lA3jcFvgE8b2Z343dIdgIq9HG7Zmf706wHvXYqAG91fT0/KEIOv9afaZl7pm/z8s7KbPJYjXiWKUls//0XATt3asdS3qUBYr3uknQOzYGno/0OKcDLzrm3zOwZM+uJ32RYBPwewDk308xeBmbhd8BeUpGPVIhUVcWGg3NuOtArMP2cPSxzO3B72UpLHvkXYhnofx596sWs6l24Paw/2/d0dZ/7ktXP+H0oc3q+CMATG9oBUGPmijIfrhIpyg/Z/hyN6qu3x2R9OrdCRII0fLoUaoydSLuxRT8+52h/odC8bcCH5h4JQIsls+Jem1QdFw4ufILOKU9eDUCbj2NzVTF1DiISpM4hxlJ7dMGPF9u5Ddj0/swEViTJYPNf/IlXk5/0hx3zruK0+JX9aDOk6CNfe3JQdT+gblKWH1rU7h/+Og6xOmahcIix72/aebrskG8uBKDZx0Wfzi1VQ8on3wBwyb2XAvDVtf7Csh8c/AjnHXUZAKl78T5Z+OL+9M/0H0KHfnMmAA02z4tZvaDNChEpgjqHGMm7hNwbBz8M+M0IG1c/gRVJMmo+3l9Yts/RZwMw+aBnWTrAv1/aflz88pt/fTAALx98PxOy/Nm/Df4an81WdQ4iEqTOIUbyLhzaPi0z/xBm2rakPmVEEiB3ur9qU8sb/HU9xo5twBvn/QOA4xv9CYBOl+y81of19peCW9nPn+n76JX+27G6VUuh65vDAej8ZXzOTlDnICJB6hxiZFsj3yXkksu9P/nvwWz4WMkvGCpVS87MuQA8ffxRPDrSd5rvnnQ3AC8f3huAF58/mseH+6MavTIKH6A8ftZpdH3ED+eP/alcnsIhRs4evHNvUt6FP9uhcJA9y/5+ERlnNgbgol6XA5B+7Y8ATPnDfXR985JC87d/1UdBxsfTyd0Rm3MoiqLNChEJUucQI7teEEakpHJW+0vSp78fXZo+OmViEAfRuYhLoZTHrm51DiISpM4hRty4BgBc3+pgmk7WtW2k4lPnICJB6hxipOn9/hz6b++H6hX7kpkiQJKEQ269mmw52o8ZrzV/AwA/d6pb6H685T1fMiqv10DKJpnfQ3v06ujgZG1WiEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJEjhICJBCgcRCUpLdAEAKes3U2PsRAByo2k1vqXQ/XjLe75kVF6vgZRNMr+HSkOdg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEgpLiUGY8LbjnEL77zb8BOPeHIwBY2W9jIkuSOMg+ujcAC0/1b+krB77D8LqLAEjBAMjFAXDTql68uWhfAFr8LdWvYNKMcqy2YlDnICJBlb5z6H/IrPzb/2n7KQCHn/p7gPyBV1IxLLv2UAA2d9oOwJm9J+U/dkuTkQDkRkPGUkjJv91t/HAAmryRAUDtl76kBTvfFxKmzkFEgip955DXLRS0/Ai/DdpxbHlXI2Ux7bIHgZ37DlbmbOXhtb6b6Pxf3w3WnF8NgMw1joZPTACgA9+Ud6mVQonDwcxSgcnAMufcSWbWAHgJaAcsAk53zq2L5h0BDANygMucc+/FuO4y6fjHLxNdgpTCETNOA+Cj/V4C4OG1hzKll29+OzM5YXVVVnuzWXE5MLvA/euAcc65TsC46D5m1h04A+gBHA88HAWLiFQgJQoHM2sF/BJ4vMDkU4Cno9tPA4MLTH/ROZflnFsILAD6xqRaqdLq/W479X63nbc2N+StzQ0ZXG8Kqd06kdqtU6JLq5RK2jncC1xD4bOHmzrnVgBEP5tE01sCSwrMtzSaVoiZDTezyWY2eQdZe1u3iMRZseFgZicBq5xzU0q4TgtMc7tNcG6kc66Pc65POhklXPXe6/DSRbtNW3DPISy455C4PafER/aSpWQvWcp1Y8/iurFn0ataCtub1WZ7s9qJLq1SKskOyf7AIDM7EcgE6pjZs8BKM2vunFthZs2BVdH8S4HWBZZvBSyPZdEiEn/FhoNzbgQwAsDMBgBXOefONrN/AEOBO6Ofr0eLvAE8b2Z3Ay2ATsAkEiRv6LRUIlFvmoKxtkcmAA2s926zZUyeD0DORg2XL42yjHO4E3jZzIYBi4EhAM65mWb2MjALyAYucc7llLlSqfLSWrcC4M7BzwF+vMOXI+4D/IhIP23nCMkBM4YAkPVKD4D8cQ9SMnsVDs658cD46PZaYGAR890O3F7G2kQkgSr9CEmp+PI6hhPfmwbAoJrrgMJnV7ov6xVaZtAZn/OnfT4EYPCt6wHIvdXvFz/+nOHa5CgBnVshIkHqHCTp/dyzBQDD6/p93kdMPx2AOid8V+TZlVPuSmFaq8MBuPHCtgAccry/ZsO7z4zkofUdAPjv+X4eXc9hd+ocRCTInNttfFK5q2MN3MEW3LdZZu8tn7rbtONa9IzLc0nFsOzaQxl0xucADK7nx/aNGOYHy6V9VNKxfpXHh270FOdcn12na7NCqpyWd33BtOf8OL3m720A4NbHHwPg8tsv0SHPiDYrRCRInYNUSdlLlwHwyvXHAbDiZn+Nj4dvvJ+hrS8HoM3NXySmuCShzkFEgtQ5SJVW/XV/2s+0KTv3QUz9nR+SPejmgxJWVzJQ5yAiQZU+HPK+yKYgXc9BdpW9dBnZS5dx/7SjSIn+q+q0WSEC0Hc/AJ455In80ZNVneJRRIIqbeew5dSDAfhP20cTXInEyg+3+O+oyFzj7zd9oOyHGlO7dwZg462bAWiVtpV3z4vOt6Bqn2+hzkFEgipt59D+mtnFzyQVwtph/QCYceEDAHQbfyEATR8o2fJ514P44bdt8qftc+L3AFzf+gUAvtzq9zOcevPVNPhKw6dBnYOIFKHSdQ572tdw+CX++xQ7jtXX4VVE6dEXp80e4L9b6ZuFufx2wu+And+HcMQ+CwCYu74JH+/3CgApfA3s/I7NFIyH17cH4MyP/Hui+80rAGiwVF1DnkoXDrvq8NJF+d+NWYOJCa5GSiPvLMlDN/vTqledvPNLkJ7u9wQAfTP8P/y8w5C5WP7mR+5a/+W6+4zdkb9ctSk+RDpv9N+xmR236isubVaISFClv9iLiOxZURd7UecgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJGgpBgElVu/JpuP8SMba8/xlwrf1LVuofsF5T0WT6Hnlfgpj79pLFW098ceX99XRgcnq3MQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISpHAQkSCFg4gEKRxEJEjhICJBCgcRCVI4iEiQwkFEghQOIhKkcBCRIIWDiAQpHEQkSOEgIkEKBxEJUjiISJDCQUSCFA4iEqRwEJEghYOIBCkcRCRI4SAiQQoHEQlSOIhIkMJBRIIUDiISZM65RNeAma0GNgNrEl1LpBGqJUS1hCVLLaWto61zrvGuE5MiHADMbLJzrk+i6wDVUhTVEpYstcS6Dm1WiEiQwkFEgpIpHEYmuoACVEuYaglLllpiWkfS7HMQkeSSTJ2DiCQRhYOIBCkcRCRI4SAiQQoHEQn6f+O5iMJOz0t5AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "model.eval()\n", + "with torch.no_grad():\n", + " for data in test_loader:\n", + " data, targets = get_batch(data, seq_len, digits_per_batch)\n", + " data = data.to(device)\n", + " targets = targets.to(device)\n", + " output = model.visualize(data)\n", + " \n", + " fig=plt.figure(figsize=(12,8), dpi= 100, facecolor='w', edgecolor='k')\n", + " matrix = torch.empty((2*data.shape[0], data.shape[1], data.shape[2]), device=data.device)\n", + " matrix[0::2] = data\n", + " matrix[1::2] = output.view(output.shape[0], 1, output.shape[1]).repeat_interleave(data.shape[1], dim=1) * 10\n", + " \n", + " plt.matshow((matrix[:matrix.shape[0]//8,:,:].view(-1, data.shape[-1]) * mnist_std + mnist_mean).cpu().numpy())\n", + " plt.show()\n", + " break" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [], + "name": "Smartpooling toy task.ipynb", + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.8" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/uwr_related/experiments/jdzikowski/mnist2digits_results.csv b/uwr_related/experiments/jdzikowski/mnist2digits_results.csv new file mode 100644 index 0000000000..8653ff2ed2 --- /dev/null +++ b/uwr_related/experiments/jdzikowski/mnist2digits_results.csv @@ -0,0 +1,4 @@ +Data, Avg Pool, Max Pool, Smart Pool, Avg Pool (1 layer), Max Pool (1 layer) +No gaussian noise, 6.4991774559021, 6.899555206298828, 5.1899094581604, 5.386036396026611, 5.667505264282227 +With gaussian noise, 5.994574546813965, 6.298228740692139, 6.170612335205078, 6.206099987030029, 5.279451847076416 +Gaussian noise patches, 5.778070449829102, 5.668732166290283, 4.4731035232543945, 5.207828521728516, 4.923704624176025 \ No newline at end of file diff --git a/uwr_related/experiments/jdzikowski/scrib.sh b/uwr_related/experiments/jdzikowski/scrib.sh new file mode 100755 index 0000000000..c35cd806d7 --- /dev/null +++ b/uwr_related/experiments/jdzikowski/scrib.sh @@ -0,0 +1,30 @@ +# [!] needs to be run from fairseq main folder +RUN="smartpooling4" +RUNDIR="/pio/scratch/1/i273233/runs" +mkdir -p $RUNDIR/$RUN + +python train.py --distributed-world-size 1 --update-freq 2 \ + /pio/scratch/1/i283340/MGR/NewSetup/DistSup/data `#path to Scribblelens data folder` \ + --vocab-path ./fairseq/data/handwriting/tasman.alphabet.plus.space.mode5.json `#alphabet file` \ + --save-dir $RUNDIR/$RUN --num-workers 0 \ + `#--restore-file $RUNDIR/$RUN/before.pt` \ + --keep-last-epochs 3 \ + --tensorboard-logdir $RUNDIR/$RUN --log-format simple \ + --task scribblelens --criterion wav2vec --arch wav2vec2_scribblelens \ + --valid-subset test --pad-to-multiples-of 4 `#--max-sample-size 256` \ + --log-keys '["prob_perplexity","code_perplexity","temp"]' --quantize-targets --extractor-mode default \ + --conv-feature-layers '[(64, (3, 3), (1, 2), (1, 1)), (128, (5, 5), (2, 2), (2, 2)), (256, (3,3), (1, 1), (1, 1)), (256, (3,3), (1, 2), (1, 1)), (512, (3,3), (1, 1), (1, 1)), (512, (3,3), (1, 2), (1, 1)), (512, (3,2), (2, 1), (1, 0))]' \ + --final-dim 256 \ + --latent-vars 320 --latent-groups 2 --latent-temp '(2,0.5,0.999995)' --infonce \ + --optimizer adam --adam-betas '(0.9,0.98)' --adam-eps 1e-06 --lr-scheduler polynomial_decay \ + --total-num-update 400000 --lr 0.0005 --warmup-updates 32000 \ + --mask-length 10 --mask-prob 0.65 --mask-selection static --mask-other 0 \ + --encoder-layerdrop 0.05 --dropout-input 0.1 --dropout-features 0.1 --feature-grad-mult 0.1 \ + --loss-weights '[0.1, 10]' --conv-pos 128 --conv-pos-groups 16 \ + --num-negatives 100 --cross-sample-negatives 0 \ + `#--max-sample-size 250000 --min-sample-size 32000` \ + --dropout 0.1 --attention-dropout 0.1 --weight-decay 0.01 --max-tokens 10000 --max-update 400000 \ + --skip-invalid-size-inputs-valid-test --ddp-backend no_c10d \ + --labels `#can be removed for no labels` \ + --enable-padding `# crashes without that, needs to make all lines same-size` \ + --smartpooling --smartpooling-factor 3.0 --smartpooling-search-perc 0.3 \ No newline at end of file