From 254356005e87f005f2e41537a8bc8e4e247c8cda Mon Sep 17 00:00:00 2001 From: lvyufeng Date: Tue, 16 Apr 2024 16:00:16 +0800 Subject: [PATCH] update to ms2.2 --- 5-1.Transformer/Transformer.ipynb | 267 +++++------ 5-1.Transformer/Transformer_pytorch.ipynb | 486 ++++++++++++++----- 5-2.BERT/BERT.ipynb | 63 ++- 5-2.BERT/BERT_pytorch.ipynb | 538 +++++++++++----------- 4 files changed, 780 insertions(+), 574 deletions(-) diff --git a/5-1.Transformer/Transformer.ipynb b/5-1.Transformer/Transformer.ipynb index a7f811b..b98ce69 100644 --- a/5-1.Transformer/Transformer.ipynb +++ b/5-1.Transformer/Transformer.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "saving-region", "metadata": {}, "outputs": [], @@ -11,7 +11,7 @@ "import numpy as np\n", "import mindspore.nn as nn\n", "import mindspore.ops as ops\n", - "from mindspore import Tensor, ms_function\n", + "from mindspore import Tensor\n", "import matplotlib.pyplot as plt\n", "from layers import Dense, Embedding, Conv1d\n", "# S: Symbol that shows starting of decoding input\n", @@ -21,7 +21,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "burning-dining", "metadata": {}, "outputs": [], @@ -35,7 +35,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "resistant-giant", "metadata": {}, "outputs": [], @@ -54,7 +54,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "id": "confirmed-console", "metadata": {}, "outputs": [], @@ -63,27 +63,27 @@ " batch_size, len_q = seq_q.shape\n", " batch_size, len_k = seq_k.shape\n", " \n", - " pad_attn_mask = ops.equal(seq_k, 0)\n", - " pad_attn_mask = pad_attn_mask.expand_dims(1) # batch_size x 1 x len_k(=len_q), one is masking\n", - "\n", - " return ops.BroadcastTo((batch_size, len_q, len_k))(pad_attn_mask) # batch_size x len_q x len_k" + " pad_attn_mask = seq_k.eq(0).unsqueeze(1) # batch_size x 1 x len_k(=len_q), one is masking\n", + " return pad_attn_mask.broadcast_to((batch_size, len_q, len_k)) # batch_size x len_q x len_k" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "id": "parallel-magic", "metadata": {}, "outputs": [], "source": [ - "def get_attn_subsequent_mask(subsequent_mask):\n", - " subsequent_mask = subsequent_mask.expand_dims(0)\n", + "def get_attn_subsequent_mask(seq):\n", + " attn_shape = [seq.shape[0], seq.shape[1], seq.shape[1]]\n", + " subsequent_mask = np.triu(np.ones(attn_shape), k=1)\n", + " subsequent_mask = Tensor.from_numpy(subsequent_mask).to(mindspore.uint8)\n", " return subsequent_mask" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "id": "capable-target", "metadata": {}, "outputs": [], @@ -91,61 +91,57 @@ "class ScaledDotProductAttention(nn.Cell):\n", " def __init__(self, d_k):\n", " super().__init__()\n", - " self.scale = Tensor(d_k, mindspore.float32)\n", " self.softmax = nn.Softmax(axis=-1)\n", + " self.d_k = Tensor(d_k)\n", " \n", " def construct(self, Q, K, V, attn_mask):\n", - " K = K.transpose((0, 1, 3, 2))\n", - " scores = ops.matmul(Q, K) / ops.sqrt(self.scale) # scores : [batch_size x n_heads x len_q(=len_k) x len_k(=len_q)]\n", + " scores = ops.matmul(Q, K.swapaxes(-1, -2)) / ops.sqrt(self.d_k)# scores : [batch_size x n_heads x len_q(=len_k) x len_k(=len_q)]\n", " scores = scores.masked_fill(attn_mask, -1e9) # Fills elements of self tensor with value where mask is one.\n", - " attn = self.softmax(scores)\n", + " attn = ops.softmax(scores)\n", " context = ops.matmul(attn, V)\n", " return context, attn" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "id": "interim-selection", "metadata": {}, "outputs": [], "source": [ "class MultiHeadAttention(nn.Cell):\n", - " def __init__(self, d_model, d_k, n_heads):\n", + " def __init__(self, d_model, d_k, d_v, n_heads):\n", " super().__init__()\n", " self.d_k = d_k\n", + " self.d_v = d_v\n", " self.n_heads = n_heads\n", " self.W_Q = Dense(d_model, d_k * n_heads)\n", " self.W_K = Dense(d_model, d_k * n_heads)\n", - " self.W_V = Dense(d_model, d_k * n_heads)\n", - " self.linear = Dense(n_heads * d_k, d_model)\n", + " self.W_V = Dense(d_model, d_v * n_heads)\n", + " self.linear = Dense(n_heads * d_v, d_model)\n", " self.layer_norm = nn.LayerNorm((d_model, ), epsilon=1e-5)\n", " self.attention = ScaledDotProductAttention(d_k)\n", " \n", " def construct(self, Q, K, V, attn_mask):\n", " # q: [batch_size x len_q x d_model], k: [batch_size x len_k x d_model], v: [batch_size x len_k x d_model]\n", " residual, batch_size = Q, Q.shape[0]\n", - " q_s = self.W_Q(Q).view((batch_size, -1, self.n_heads, self.d_k)) \n", - " k_s = self.W_K(K).view((batch_size, -1, self.n_heads, self.d_k)) \n", - " v_s = self.W_V(V).view((batch_size, -1, self.n_heads, self.d_k)) \n", " # (B, S, D) -proj-> (B, S, D) -split-> (B, S, H, W) -trans-> (B, H, S, W)\n", - " q_s = q_s.transpose((0, 2, 1, 3)) # q_s: [batch_size x n_heads x len_q x d_k]\n", - " k_s = k_s.transpose((0, 2, 1, 3)) # k_s: [batch_size x n_heads x len_k x d_k]\n", - " v_s = v_s.transpose((0, 2, 1, 3)) # v_s: [batch_size x n_heads x len_k x d_v]\n", + " q_s = self.W_Q(Q).view(batch_size, -1, self.n_heads, self.d_k).swapaxes(1,2) # q_s: [batch_size x n_heads x len_q x d_k]\n", + " k_s = self.W_K(K).view(batch_size, -1, self.n_heads, self.d_k).swapaxes(1,2) # k_s: [batch_size x n_heads x len_k x d_k]\n", + " v_s = self.W_V(V).view(batch_size, -1, self.n_heads, self.d_v).swapaxes(1,2) # v_s: [batch_size x n_heads x len_k x d_v]\n", + "\n", + " attn_mask = attn_mask.unsqueeze(1).tile((1, n_heads, 1, 1)) # attn_mask : [batch_size x n_heads x len_q x len_k]\n", "\n", - " attn_mask = attn_mask.expand_dims(1)\n", - " attn_mask = ops.tile(attn_mask, (1, self.n_heads, 1, 1)) # attn_mask : [batch_size x n_heads x len_q x len_k]\n", - " \n", " # context: [batch_size x n_heads x len_q x d_v], attn: [batch_size x n_heads x len_q(=len_k) x len_k(=len_q)]\n", " context, attn = self.attention(q_s, k_s, v_s, attn_mask)\n", - " context = context.transpose((0, 2, 1, 3)).view((batch_size, -1, self.n_heads * self.d_k)) # context: [batch_size x len_q x n_heads * d_v]\n", - " output = self.linear(context) \n", + " context = context.swapaxes(1, 2).view(batch_size, -1, n_heads * d_v) # context: [batch_size x len_q x n_heads * d_v]\n", + " output = self.linear(context)\n", " return self.layer_norm(output + residual), attn # output: [batch_size x len_q x d_model]" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "id": "official-fetish", "metadata": {}, "outputs": [], @@ -160,25 +156,22 @@ " \n", " def construct(self, inputs):\n", " residual = inputs # inputs : [batch_size, len_q, d_model]\n", - " output = inputs.transpose((0, 2, 1))\n", - " output = self.conv1(output)\n", - " output = self.relu(output)\n", - " output = self.conv2(output)\n", - " output = output.transpose((0, 2, 1))\n", + " output = self.relu(self.conv1(inputs.swapaxes(1, 2)))\n", + " output = self.conv2(output).swapaxes(1, 2)\n", " return self.layer_norm(output + residual)" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "id": "incredible-packet", "metadata": {}, "outputs": [], "source": [ "class EncoderLayer(nn.Cell):\n", - " def __init__(self, d_model, d_k, n_heads, d_ff):\n", + " def __init__(self, d_model, d_k, d_v, n_heads, d_ff):\n", " super().__init__()\n", - " self.enc_self_attn = MultiHeadAttention(d_model, d_k, n_heads)\n", + " self.enc_self_attn = MultiHeadAttention(d_model, d_k, d_v, n_heads)\n", " self.pos_ffn = PoswiseFeedForward(d_ff, d_model)\n", " \n", " def construct(self, enc_inputs, enc_self_attn_mask):\n", @@ -189,16 +182,16 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "id": "greater-samba", "metadata": {}, "outputs": [], "source": [ "class DecoderLayer(nn.Cell):\n", - " def __init__(self, d_model, d_k, n_heads, d_ff):\n", + " def __init__(self, d_model, d_k, d_v, n_heads, d_ff):\n", " super().__init__()\n", - " self.dec_self_attn = MultiHeadAttention(d_model, d_k, n_heads)\n", - " self.dec_enc_attn = MultiHeadAttention(d_model, d_k, n_heads)\n", + " self.dec_self_attn = MultiHeadAttention(d_model, d_k, d_v, n_heads)\n", + " self.dec_enc_attn = MultiHeadAttention(d_model, d_k, d_v, n_heads)\n", " self.pos_ffn = PoswiseFeedForward(d_ff, d_model)\n", " \n", " def construct(self, dec_inputs, enc_outputs, dec_self_attn_mask, dec_enc_attn_mask):\n", @@ -210,17 +203,17 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 11, "id": "atmospheric-armstrong", "metadata": {}, "outputs": [], "source": [ "class Encoder(nn.Cell):\n", - " def __init__(self, src_vocab_size, d_model, d_k, n_heads, d_ff, n_layers, src_len):\n", + " def __init__(self, src_vocab_size, d_model, d_k, d_v, n_heads, d_ff, n_layers, src_len):\n", " super().__init__()\n", " self.src_emb = Embedding(src_vocab_size, d_model)\n", " self.pos_emb = Embedding.from_pretrained_embedding(get_sinusoid_encoding_table(src_len+1, d_model), freeze=True)\n", - " self.layers = nn.CellList([EncoderLayer(d_model, d_k, n_heads, d_ff) for _ in range(n_layers)])\n", + " self.layers = nn.CellList([EncoderLayer(d_model, d_k, d_v, n_heads, d_ff) for _ in range(n_layers)])\n", " # temp positional indexes\n", " self.pos = Tensor([[1, 2, 3, 4, 0]])\n", " \n", @@ -237,29 +230,23 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 12, "id": "seeing-germany", "metadata": {}, "outputs": [], "source": [ "class Decoder(nn.Cell):\n", - " def __init__(self, tgt_vocab_size, d_model, d_k, n_heads, d_ff, n_layers, tgt_len):\n", + " def __init__(self, tgt_vocab_size, d_model, d_k, d_v, n_heads, d_ff, n_layers, tgt_len):\n", " super().__init__()\n", " self.tgt_emb = Embedding(tgt_vocab_size, d_model)\n", " self.pos_emb = Embedding.from_pretrained_embedding(get_sinusoid_encoding_table(tgt_len+1, d_model), freeze=True)\n", - " self.layers = nn.CellList([DecoderLayer(d_model, d_k, n_heads, d_ff) for _ in range(n_layers)])\n", - " \n", - " # temp positional indexes\n", - " self.pos = Tensor([[5, 1, 2, 3, 4]])\n", - " \n", - " ones = np.ones(shape=(tgt_len, tgt_len))\n", - " self.subsequent_mask = Tensor(np.triu(ones, k=1), dtype=mindspore.float32)\n", - " \n", + " self.layers = nn.CellList([DecoderLayer(d_model, d_k, d_v, n_heads, d_ff) for _ in range(n_layers)])\n", + "\n", " def construct(self, dec_inputs, enc_inputs, enc_outputs):\n", " # dec_inputs : [batch_size x target_len]\n", - " dec_outputs = self.tgt_emb(dec_inputs) + self.pos_emb(self.pos)\n", + " dec_outputs = self.tgt_emb(dec_inputs) + self.pos_emb(Tensor([[5,1,2,3,4]]))\n", " dec_self_attn_pad_mask = get_attn_pad_mask(dec_inputs, dec_inputs)\n", - " dec_self_attn_subsequent_mask = get_attn_subsequent_mask(self.subsequent_mask)\n", + " dec_self_attn_subsequent_mask = get_attn_subsequent_mask(dec_inputs)\n", " dec_self_attn_mask = ops.gt((dec_self_attn_pad_mask + dec_self_attn_subsequent_mask), 0)\n", " \n", " dec_enc_attn_mask = get_attn_pad_mask(dec_inputs, enc_inputs)\n", @@ -274,16 +261,16 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 13, "id": "presidential-bailey", "metadata": {}, "outputs": [], "source": [ "class Transformer(nn.Cell):\n", - " def __init__(self, d_model, d_k, n_heads, d_ff, n_layers, src_vocab_size, tgt_vocab_size, src_len, tgt_len):\n", + " def __init__(self, d_model, d_k, d_v, n_heads, d_ff, n_layers, src_vocab_size, tgt_vocab_size, src_len, tgt_len):\n", " super(Transformer, self).__init__()\n", - " self.encoder = Encoder(src_vocab_size, d_model, d_k, n_heads, d_ff, n_layers, src_len)\n", - " self.decoder = Decoder(tgt_vocab_size, d_model, d_k, n_heads, d_ff, n_layers, tgt_len)\n", + " self.encoder = Encoder(src_vocab_size, d_model, d_k, d_v, n_heads, d_ff, n_layers, src_len)\n", + " self.decoder = Decoder(tgt_vocab_size, d_model, d_k, d_v, n_heads, d_ff, n_layers, tgt_len)\n", " self.projection = Dense(d_model, tgt_vocab_size, has_bias=False)\n", "\n", " def construct(self, enc_inputs, dec_inputs):\n", @@ -295,7 +282,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 14, "id": "hairy-internet", "metadata": {}, "outputs": [], @@ -316,24 +303,24 @@ "\n", "d_model = 512 # Embedding Size\n", "d_ff = 2048 # FeedForward dimension\n", - "d_k = 64 # dimension of K(=Q), V\n", + "d_k = d_v = 64 # dimension of K(=Q), V\n", "n_layers = 6 # number of Encoder of Decoder Layer\n", "n_heads = 8 # number of heads in Multi-Head Attention" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 15, "id": "pacific-pollution", "metadata": {}, "outputs": [], "source": [ - "model = Transformer(d_model, d_k, n_heads, d_ff, n_layers, src_vocab_size, tgt_vocab_size, src_len, tgt_len)" + "model = Transformer(d_model, d_k, d_v, n_heads, d_ff, n_layers, src_vocab_size, tgt_vocab_size, src_len, tgt_len)" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 16, "id": "activated-ordinary", "metadata": {}, "outputs": [], @@ -346,7 +333,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 17, "id": "0f7e451f", "metadata": {}, "outputs": [], @@ -360,7 +347,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 18, "id": "223f4a3d", "metadata": {}, "outputs": [], @@ -370,12 +357,12 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "id": "3c96e055", "metadata": {}, "outputs": [], "source": [ - "@ms_function\n", + "@mindspore.jit\n", "def train_step(enc_inputs, dec_inputs, target_batch):\n", " loss, grads = grad_fn(enc_inputs, dec_inputs, target_batch)\n", " optimizer(grads)\n", @@ -384,34 +371,50 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 20, "id": "human-reverse", "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[ERROR] CORE(1265263,7f03d442f4c0,python):2024-04-16-15:48:30.934.784 [mindspore/core/utils/file_utils.cc:253] GetRealPath] Get realpath failed, path[/tmp/ipykernel_1265263/3030654789.py]\n", + "[ERROR] CORE(1265263,7f03d442f4c0,python):2024-04-16-15:48:31.085.333 [mindspore/core/utils/file_utils.cc:253] GetRealPath] Get realpath failed, path[/tmp/ipykernel_1265263/1296245474.py]\n", + "[ERROR] CORE(1265263,7f03d442f4c0,python):2024-04-16-15:48:31.085.953 [mindspore/core/utils/file_utils.cc:253] GetRealPath] Get realpath failed, path[/tmp/ipykernel_1265263/1296245474.py]\n", + "[ERROR] CORE(1265263,7f03d442f4c0,python):2024-04-16-15:48:31.086.786 [mindspore/core/utils/file_utils.cc:253] GetRealPath] Get realpath failed, path[/tmp/ipykernel_1265263/1296245474.py]\n", + "[ERROR] CORE(1265263,7f03d442f4c0,python):2024-04-16-15:48:35.478.043 [mindspore/core/utils/file_utils.cc:253] GetRealPath] Get realpath failed, path[/tmp/ipykernel_1265263/2926503423.py]\n", + "[ERROR] CORE(1265263,7f03d442f4c0,python):2024-04-16-15:48:35.478.088 [mindspore/core/utils/file_utils.cc:253] GetRealPath] Get realpath failed, path[/tmp/ipykernel_1265263/2926503423.py]\n", + "[ERROR] CORE(1265263,7f03d442f4c0,python):2024-04-16-15:48:35.663.493 [mindspore/core/utils/file_utils.cc:253] GetRealPath] Get realpath failed, path[/tmp/ipykernel_1265263/3030654789.py]\n", + "[ERROR] CORE(1265263,7f03d442f4c0,python):2024-04-16-15:48:35.663.514 [mindspore/core/utils/file_utils.cc:253] GetRealPath] Get realpath failed, path[/tmp/ipykernel_1265263/1296245474.py]\n", + "[ERROR] CORE(1265263,7f03d442f4c0,python):2024-04-16-15:48:35.665.780 [mindspore/core/utils/file_utils.cc:253] GetRealPath] Get realpath failed, path[/tmp/ipykernel_1265263/1296245474.py]\n", + "[ERROR] CORE(1265263,7f03d442f4c0,python):2024-04-16-15:48:35.667.354 [mindspore/core/utils/file_utils.cc:253] GetRealPath] Get realpath failed, path[/tmp/ipykernel_1265263/1296245474.py]\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "Epoch: 0001 cost = 2.596976\n", - "Epoch: 0002 cost = 1.083525\n", - "Epoch: 0003 cost = 0.804770\n", - "Epoch: 0004 cost = 0.334796\n", - "Epoch: 0005 cost = 0.180342\n", - "Epoch: 0006 cost = 0.042456\n", - "Epoch: 0007 cost = 0.009277\n", - "Epoch: 0008 cost = 0.003621\n", - "Epoch: 0009 cost = 0.002399\n", - "Epoch: 0010 cost = 0.002193\n", - "Epoch: 0011 cost = 0.002270\n", - "Epoch: 0012 cost = 0.002314\n", - "Epoch: 0013 cost = 0.002193\n", - "Epoch: 0014 cost = 0.001918\n", - "Epoch: 0015 cost = 0.001599\n", - "Epoch: 0016 cost = 0.001310\n", - "Epoch: 0017 cost = 0.001075\n", - "Epoch: 0018 cost = 0.000890\n", - "Epoch: 0019 cost = 0.000741\n", - "Epoch: 0020 cost = 0.000619\n" + "Epoch: 0001 cost = 2.367968\n", + "Epoch: 0002 cost = 0.938194\n", + "Epoch: 0003 cost = 0.512835\n", + "Epoch: 0004 cost = 0.167661\n", + "Epoch: 0005 cost = 0.048837\n", + "Epoch: 0006 cost = 0.010902\n", + "Epoch: 0007 cost = 0.003625\n", + "Epoch: 0008 cost = 0.001862\n", + "Epoch: 0009 cost = 0.001336\n", + "Epoch: 0010 cost = 0.001196\n", + "Epoch: 0011 cost = 0.001198\n", + "Epoch: 0012 cost = 0.001234\n", + "Epoch: 0013 cost = 0.001247\n", + "Epoch: 0014 cost = 0.001202\n", + "Epoch: 0015 cost = 0.001100\n", + "Epoch: 0016 cost = 0.000960\n", + "Epoch: 0017 cost = 0.000809\n", + "Epoch: 0018 cost = 0.000667\n", + "Epoch: 0019 cost = 0.000544\n", + "Epoch: 0020 cost = 0.000444\n" ] } ], @@ -427,7 +430,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 21, "id": "rubber-difficulty", "metadata": {}, "outputs": [ @@ -443,12 +446,12 @@ "# Test\n", "predict, enc_self_attns, dec_self_attns, dec_enc_attns = model(enc_inputs, dec_inputs)\n", "predict = predict.asnumpy().argmax(1)\n", - "print(sentences[0], '->', [number_dict[n.item()] for n in predict.squeeze()])\n" + "print(sentences[0], '->', [number_dict[n.item()] for n in predict.squeeze()])" ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 22, "id": "postal-picking", "metadata": {}, "outputs": [], @@ -466,72 +469,10 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "id": "dried-depth", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "first head of last state enc_self_attns\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/lvyufeng/miniconda3/envs/ms1.9/lib/python3.7/site-packages/ipykernel_launcher.py:7: UserWarning: FixedFormatter should only be used together with FixedLocator\n", - " import sys\n", - "/home/lvyufeng/miniconda3/envs/ms1.9/lib/python3.7/site-packages/ipykernel_launcher.py:8: UserWarning: FixedFormatter should only be used together with FixedLocator\n", - " \n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAArUAAALUCAYAAAAVCMq+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAsUklEQVR4nO3de5TXdZ348dd3GB0JmIFE0GBkNGXdYy1KIqSuMtiKt7xkua1tUp7cLqaY1RaV/rysUm21Xjra1VulZzt6bEUtdV0I5KKUQIUXkjIx4yKXGW8MMHx/fxhzJBAFZ74fXt/v43HO93C+3+9nvryGzxnmOZ95fz+fUrlcLgcAACRWV/QAAADwZolaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIr77oAYCet27duvjf//3fePzxx+PFF1+MCy+8MCIi1q5dG+3t7TFw4MCoq/MzLgB5lcrlcrnoIYCec+edd8a//du/xYoVK6JcLkepVIrOzs6IiHj44Yfj3e9+d/zoRz+KM844o+BJAWDHOTQDVWzmzJnx/ve/PxoaGuKqq67aIlwPPfTQ2G+//eL2228vaEIA6B6WH0AVu+yyy6J///7x61//OgYOHBgrV67cYptDDjkkHnrooQKmA4Du40gtVLGHHnooTj755Bg4cOBrbtPc3BxLly6t4FQA0P1ELVSxjo6OaGxs3OY2a9as8SYxANLznQyq2L777htz587d5jazZ8+OAw44oEITAUDPELVQxU477bSYOXNm3HDDDVt9/hvf+Eb87ne/i3/+53+u8GQA0L2c0guq2AsvvBBjxoyJxx57LMaNGxcdHR0xc+bM+OxnPxuzZ8+OWbNmxUEHHRSzZs2KhoaGoscFgB0maqHKrV69Oj796U/HT3/6067z00ZElEqlOP300+Paa6+NAQMGFDghALx5ohZqxMqVK2Pu3LmxatWqaGxsjFGjRsXgwYOLHgsAuoWoBQAgPW8UgyrWq1evuOyyy7a5zeWXXx719a7DAkBuohaqWLlcjjfyyxi/sAEgO1ELNW7FihXRu3fvoscAgDfF7xyhytx8882b3Z8/f/4Wj0VEdHZ2xpIlS+Lmm2+Od7zjHZUaDwB6hDeKQZWpq6uLUqn0uttt+tLv3bt33H777XHsscf29GgA0GNELVSZm266KSJeidazzjorTjnllDj55JO32K5Xr17x1re+Nd797nc7Ty0A6YlaqGIf/ehH49RTT42TTjqp6FEAoEeJWgAA0vNGMagBGzZsiCeeeCLWrFmz2aVyX+3II4+s8FQA0H1ELVSxcrkcF110UVxzzTXx/PPPb3Pb14pdAMhA1EIVu+yyy+Lyyy+P/v37x5lnnhlDhw519TAAqpI1tVDFWlpaolQqxa9+9avYfffdix4HAHqMK4pBFVu6dGmccsopghaAqidqoYrts88+0d7eXvQYANDjRC1UsU9+8pNx1113xfLly4seBQB6lDW1UEWefvrpze6Xy+X43Oc+F/PmzYuLLrooRo4cGY2NjVv92L333rsSIwJAjxC1UEXq6uqiVCpt8Xi5XN7q45uUSqXYsGFDT44GAD3KuX2gipx55pnbjFcAqFaO1AIAkJ43igEAkJ6ohSr26KOPxtVXXx0rVqzY6vPLly+Pq6++Oh577LEKTwYA3cvyA6hiZ555ZjzwwAOxZMmSqKvb8mfYzs7OaGlpife85z1xww03FDAhAHQPR2qhis2YMSOOPvrorQZtRESvXr3i6KOPjunTp1d4MgDoXqIWqtjSpUujubl5m9sMGTIk/vKXv1RoIuDN2nfffeOcc84pegzY6YhaqGJ9+vR53auJLV++PHbbbbcKTQS8Wc8999xrXkQFapmohSo2cuTI+NnPfhZr1qzZ6vOrV6+OO+64I0aOHFnZwYAd9g//8A+xaNGioseAnY6ohSp2zjnnxMqVK6O1tXWLdbO//OUvo7W1NVavXh2f/vSnC5oQ2F5f+MIXYsqUKTF16tSiR4GdirMfQJX77Gc/G//1X/8VpVIpGhoaYs8994ylS5dGR0dHlMvl+PznPx9f+9rXih4TeINuvvnm+OlPfxr33ntvnHLKKTFq1KgYPHjwVq8meOaZZxYwIRRD1EINuOuuu+Laa6+NuXPnRltbW/Tv3z8OPfTQOOecc+K4444rejxgO9TV1UWpVIq//fb96qgtl8tRKpWis7Oz0uNBYUQtACRy0003veFtJ0yY0IOTwM5F1AIAkF590QMAPe+pp56Kn/zkJzF//vxob2+PxsbGOOigg+JDH/pQtLS0FD0eALxpjtRClbvqqqvi3//932PDhg1brMHbZZdd4utf/3pMnDixoOnobi+++GKsWbPmNddS7r333hWeiJ5yxx13xK233hqPP/54vPTSS/Hkk09GRMTjjz8ed955Z3zoQx+KIUOGFDwlVI6ohSp21113xUknnRQDBw6Mz3zmM9Ha2hp77bVXLF26NKZOnRrf+ta3YuXKlXHnnXfGCSecUPS4vAk//OEP45vf/GY88cQTr7lNqVSKDRs2VHAqesLGjRvjX/7lX+K2226LiIjevXvHyy+/3PWDzLJly2Lo0KFx6aWXxqRJk4ocFSpK1NagRx99NL797W/H3LlzX/OITqlUisWLFxcwHd1p3Lhx8Zvf/Cbmz58fQ4cO3eL5JUuWxMEHHxwjRoyIBx54oIAJ6Q7XXXddnHPOOVFfXx+HH354DB06NOrrt7667IYbbqjwdHS3b37zm/H5z38+PvGJT8RXv/rV+Na3vhWXXXbZZv+Xjxs3LtatWxcPPvhggZNCZVlTW2N++ctfxrHHHhsdHR1RX18fgwcP3uo3Pz/rVIdHHnkkPvShD201aCMimpub4/TTT49bbrmlwpPRna688soYOHBgPPjggzF8+PCix6GH3XjjjTFq1Ki49tprIyK2en7a/fbbL+6+++5KjwaFErU15otf/GJs2LAhfvCDH8SECROiV69eRY9ED1q3bl306dNnm9v07ds31q1bV6GJ6Al/+tOf4mMf+5igrRFPPvlknHPOOdvcZvfdd4+VK1dWaCLYObhMbo1ZsGBBfPCDH4yzzjpL0NaA4cOHx5QpU15zHeWGDRvirrvuEkPJ7bXXXk6yX0N69+4dbW1t29zmT3/6U/Tv378yA8FOQtTWmD59+sSgQYOKHoMKOfPMM+OJJ56I8ePHx69//evNnvvVr34Vxx13XDzxxBNO0J7chAkT4uc//3m8+OKLRY9CBRx88MFx7733xtq1a7f6/KpVq+IXv/hFjBkzpsKTQbFEbY05/vjjY8aMGUWPQYVMnDgxTjrppJg6dWoceuih0a9fv3j7298e/fr1i9GjR8cDDzwQJ510klN6JfeVr3wlRo0aFf/0T/8U06dPjxdeeKHokehB5513XjzzzDNx2mmnxTPPPLPZc4sXL45TTz012tra4rzzzitoQiiGsx/UmOXLl8cRRxwRxx57bHz1q1+Nt7zlLUWPRAXcfPPNcdNNN2128YWDDz44JkyYEB/+8IeLHo83adNSonK5vNU3DW3ilF7VY9KkSfG1r30tSqVS9OnTJ1588cWudbTlcjkuvPDCuOSSS4oeEypK1Fa5cePGbfHYmjVrYsGCBdGnT58YPnx4NDY2brFNqVRyiidIYuzYsduM2VebOnVqD09Dpdx///3x7W9/Ox566KFYtWpVNDY2xujRo+O8886L8ePHFz0eVJyorXJ1dTu2wqRUKnnjSRXp7OyMZ555Jp599tlYv379Vrc58sgjKzwVAHQfUQtVbOPGjXHFFVfEVVddFatWrdrmtn6IASAz56mFKjZp0qT4z//8zxg0aFB89KMfjb322us1rzQFAJk5UltjOjs748UXX4y+fftudWnCpuf79OnjPLZVYM8994wBAwbE3Llzo2/fvkWPQzc566yzolQqxRVXXBGDBw+Os8466w19XKlUih/+8Ic9PB3dra6uLurq6uLRRx+N4cOHR11d3RtaQ+2NgdQaUVtjLrroovj6178eS5YsiT322GOL51esWBF77713TJo0KS666KICJqQ79e3bNz7xiU/EN77xjaJHoRttiprHHnusK3LeCGvlc9r0RsAf/ehHMXToUG8MhNcgamvMyJEjY6+99trmNcHf+973xrPPPrvFyfrJ57DDDouWlpa45ZZbih6FbvSnP/0pIiKGDBkS9fX1XfffiGHDhvXUWACFsriuxvzhD3+I1tbWbW7zd3/3dzFz5swKTURP+vKXvxwf+MAH4pFHHomRI0cWPQ7d5G/DVKjWrpUrV8aCBQuira0tmpqaYsSIEbH77rsXPRYUQtTWmPXr17/urypLpdJrXn6RXE444YS48cYb47jjjouTTjopRowYsdXzEke8ckld8tqwYUNcc801ceutt8bjjz8eL730Utd6yvnz58f3vve9OP/882P48OEFT0p3eOqpp2LixIlx9913x6t/4VoqleLEE0+MK6+8MlpaWoobkDdl9uzZ8eUvfznmzp0bpVIpRo8eHZdffnkceuihRY+2U7P8oMaMGDEi6uvrt7m04F3velesXbs2Fi5cWMHJ6AkdHR3xsY99LG655Zaub3x/uxZv01WorLXM6+WXX45jjjkmZs2aFQMHDoxddtkl/vKXv3Tt07a2tthzzz3js5/9bPzHf/xHwdPyZi1evDgOP/zwWL58eey///5x+OGHx+DBg2PZsmUxa9asWLRoUQwaNChmzZoV++67b9Hjsp1++9vfxujRo7c4uNS7d+94+OGH48ADDyxosp3fjp2Zn7Te9773xfz58+Oiiy7aImI6OzvjwgsvjPnz58cHPvCBgiakO11wwQXxk5/8JN75znfGpZdeGt///vfj+uuv3+x2ww03xPXXX1/0qLwJV1xxRcycOTMmT54cS5cujY997GObPd/U1BRHHXVU3HvvvQVNSHf6whe+ECtWrIjvfOc78fjjj8f1118fkydPjuuvvz4ee+yxuO6662LFihXxhS98oehR2QFf/epXY+3atfHlL385li5dGkuXLo0LL7wwXn755fja175W9Hg7NUdqa8wLL7wQo0aNikWLFsXb3/72aG1tjSFDhsSf//znmDp1aixevDj+/u//PubMmeMUUFVg0KBBMWzYsJg9e7bz01ax4cOHR3Nzc9elrS+55JK49NJLN/vB9VOf+lTcfvvtsWzZsqLGpJsMGDAgxo4dG3fcccdrbnPyySfH9OnTY/Xq1RWcjO6w9957R0tLS0yfPn2zx4866qh46qmntuuNobXGd7ka07dv35g+fXp88pOfjDvuuCOefPLJrufq6uri/e9/f1x77bWCtkqsXbs2WltbBW2Ve/rpp+PUU0/d5jb9+vWLtra2Ck1ET+rs7HzdX0G/4x3vcDqvpJYtWxYf/OAHt3h89OjR8dBDDxUwUR6+09WgPfbYI2677bZYtmxZ/OpXv4q2trbo379/HHLIITFo0KCix6Mbvetd79rsBxeqU79+/WL58uXb3Gbx4sVbPTc1+YwcOfJ13/OwcOHCOOSQQyo0Ed1p/fr1Wz2w1KdPn1i/fn0BE+UhamvY4MGD44QTTih6DHrQFVdcEUcffXTcddddceKJJxY9Dj1kzJgxMWXKlFizZk30799/i+eXLFkS99xzz+sezSWHyy+/PI4++uj4wQ9+sMX66YiI733ve3Hvvfd2LUeBWiFqq5zLada2+++/P8aOHRsnn3xyjBs37jVP6VUqleLCCy8sYEK6w+c///lobW2No48+Oq6++uquU3m99NJLMXv27Dj33HNjw4YNccEFFxQ8KTvi0ksv3eKx1tbW+PjHPx7f/OY3Nzv7wcyZM2PRokUxfvz4eOCBB+Kwww4rYGLerB//+McxZ86czR7b9Fu3448/fovtS6XSNi+qVCu8UazKuZxmbbO/a8d1110XEydO3Op+7NWrV1x77bVbParHzu+Nfh3/LV/XOe3I/ravX+FIbZX74x//GBGvXE7z1fepDd4oUjs++clPxtixY+M73/lOPPTQQ7Fq1apobGyM0aNHx6c+9SnntkzM13Ft8X16xzlSCwBAei6+AABAeqIWAID0RC0AAOmJ2hrX0dERF198cXR0dBQ9Cj3Mvq4d9nVtsb9rh329bd4oVuPa29ujqakp2tratnr+UqqHfV077OvaYn/XDvt62xypBQAgPVELAEB6Lr7wVxs3boxnn302+vXrF6VSqehxKqa9vX2zP6le9nXtsK9ri/1dO2pxX5fL5Xj++efjbW972+tebc2a2r965plnorm5uegxAAD4G0uWLImhQ4ducxtHav+qX79+ERHxP7OHRJ++VmVUu76l9UWPQIU8tLal6BGooDMblxc9AhVy6vB3Fj0CFbAh1seDcU9Xp22LqP2rTUsO+vStiz79RG2161uyj2tF73r/zdWSRv9/14z60i5Fj0Al/HU9wRtZGuqrHwCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANKr2qh96qmnolQqxUc+8pGiRwEAoIdVbdQCAFA76oseoKcMGTIkHnvssWhqaip6FAAAeljVRu0uu+wSBxxwQNFjAABQAVW7/MCaWgCA2lG1UQsAQO0QtQAApFe1a2pfT0dHR3R0dHTdb29vL3AaAADejJo9Ujt58uRoamrqujU3Nxc9EgAAO6hmo3bSpEnR1tbWdVuyZEnRIwEAsINqdvlBQ0NDNDQ0FD0GAADdoGaP1AIAUD1ELQAA6YlaAADSE7UAAKRXtW8Ua2lpiXK5XPQYAABUgCO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB69UUPAEXYs1fRE1Apqzr7FD0CFbS+3Fn0CEBBHKkFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIr6qi9qmnnopSqRQf+chHih4FAIAKqqqoBQCgNolaAADS266oXb16dfTq1StOPPHEzR6fP39+lEqlKJVK8eSTT2723NixY6N3797R0dER69ati2uuuSbGjx8fzc3N0dDQEIMGDYr3ve99MW/evC3+vhtvvDFKpVLceOONcd9998Vhhx0Wb3nLW2L33XePCRMmxMqVKzfbdp999omIiJtuuqlrnlKpFNOmTdueTxMAgGTqt2fjAQMGxIgRI2LGjBnR2dkZvXr1ioiIqVOndm0zderU2G+//SIiYu3atTFnzpw47LDDoqGhIZYuXRrnn39+/OM//mMcf/zxMWDAgPjDH/4Qd955Z/z85z+P6dOnx6hRo7b4e++88864++67473vfW8cdthhMX369Lj55ptj8eLF8eCDD0ZExEEHHRQTJ06Mq666KkaMGBGnnHJK18e3tLRs778LAACJbFfURkS0trbGvHnz4te//nUceuihEfFKyA4fPjxefvnlmDp1apx99tkRETFr1qzo6OiI1tbWiHglip9++ukYMmTIZq+5cOHCGDNmTHzpS1+K+++/f4u/c8qUKTFt2rQ4/PDDIyKis7Mz3vOe98S0adNizpw5MWbMmDjooIPi/PPPj6uuuioOOuiguPjii7f5eXR0dERHR0fX/fb29u39pwAAYCex3WtqNwXq//3f/0XEK4E5ffr0aG1tjdbW1i2O2ka8sgQhIqKhoWGLoI2IOPDAA6O1tTWmT58e69ev3+L5M844oytoIyJ69eoVEyZMiIiIuXPnbu+nEBERkydPjqampq5bc3PzDr0OAADF2+6oPfLII6NXr15dwTpv3rxoa2uLcePGRWtrayxdujQee+yxiHglanv37h2jR4/u+vj58+fHGWecEXvvvXfsuuuuXetep0yZEuvWrYvnnntui7/zXe961xaPDR06NCIi1qxZs72fQkRETJo0Kdra2rpuS5Ys2aHXAQCgeNu9/KCxsTFGjhwZM2fOjPXr18fUqVOjVCpFa2trvPTSSxHxSswOGzYsHn744TjqqKNi1113jYhXliOMGzcuIiKOOeaY2H///aNv375RKpXiZz/7WSxYsGCzJQGv/ju3GLz+ldE7Ozu391OIiFeOGjc0NOzQxwIAsHPZ7qiNeGUJwty5c+Phhx+OadOmxYEHHhh77LFHRETss88+MXXq1Nh///1j/fr1XcsVIiIuv/zy6OjoiBkzZsQRRxyx2WvOmTMnFixY8CY+FQAAatUOnad2U6jed999MWPGjK6jrxER48aNi2nTpnWtud20njYiYvHixfHWt751i6B96aWX4pFHHtmRUTaz6WwMO3r0FgCAnHYoao844oior6+P6667Lp5//vnNora1tTWee+65+OEPfxh9+vTZ7BRdw4YNi9WrV8fChQu7Huvs7IzPfe5zsWLFijfxabxiwIABUSqVrI8FAKgxO7T8oG/fvjFq1KiYPXt21NXVxVFHHdX13KajuCtWrIjx48fHLrvs0vXcueeeG/fdd18cccQRcfrpp8duu+0W06ZNiz//+c8xduzYN32RhE1zTZ8+PT784Q/H/vvvH3V1dfHhD384hg0b9qZeGwCAndcOXyZ3U7wefPDB0b9//67H3/a2t8Xw4cMjYvOlBxERJ554Ytx2222x7777xo9//OO45ZZb4oADDoiHH36426LzRz/6URx33HFx1113xcUXXxwXXnhh/PGPf+yW1wYAYOdUKpfL5aKH2Bm0t7dHU1NT/O9vm6NPvx1ufZLYv37L8yFTnX7Q9s6iR6CCzh+wqOgRqJATh2x5uk+qz4by+pgW/xNtbW1bPRvWq6k3AADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQXn3RA+xs9uzVEf16af1q94cNuxY9AhXyj29ZVPQIVNDDHaWiRwAKot4AAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQXuqoXbduXVxzzTUxfvz4aG5ujoaGhhg0aFC8733vi3nz5hU9HgAAFZI6aletWhXnn39+dHR0xPHHHx+f+cxnYuzYsXHPPffEYYcdFnPnzi16RAAAKqC+6AHejAEDBsTTTz8dQ4YM2ezxhQsXxpgxY+JLX/pS3H///Vv92I6Ojujo6Oi6397e3qOzAgDQc1IfqW1oaNgiaCMiDjzwwGhtbY3p06fH+vXrt/qxkydPjqampq5bc3NzT48LAEAPSR21ERHz58+PM844I/bee+/Yddddo1QqRalUiilTpsS6deviueee2+rHTZo0Kdra2rpuS5YsqfDkAAB0l9TLD2bNmhXjxo2LiIhjjjkm9t9//+jbt2+USqX42c9+FgsWLNhsicGrNTQ0RENDQyXHBQCgh6SO2ssvvzw6OjpixowZccQRR2z23Jw5c2LBggUFTQYAQCWlXn6wePHieOtb37pF0L700kvxyCOPFDQVAACVljpqhw0bFqtXr46FCxd2PdbZ2Rmf+9znYsWKFQVOBgBAJaVefnDuuefGfffdF0cccUScfvrpsdtuu8W0adPiz3/+c4wdOzamTZtW9IgAAFRA6iO1J554Ytx2222x7777xo9//OO45ZZb4oADDoiHH344hg0bVvR4AABUSKlcLpeLHmJn0N7eHk1NTfG7RwdFv36pW583YFnnrkWPQIWsL/cqegQqqDNKRY9AhVy678iiR6ACNpTXx7T4n2hra4vGxsZtbqveAABIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAevVFD7CzWV9+5UZ126W0segRqJClnY1Fj0AFtdSvLnoEoCCO1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSe1NRO23atCiVSnHxxRd30zgAALD9HKkFACA9UQsAQHqiFgCA9Lotah988MEYO3Zs9OvXL/r37x+nnXZaPPnkk1tst3z58vjMZz4T++23XzQ0NMTAgQPjtNNOi9/97ndbfd3t2b6lpSVaWlpizZo18elPfzqam5ujvr4+brzxxu76NAEA2AnVd8eLzJkzJyZPnhzHHntsnHvuubFw4cK44447YsaMGTFnzpzYd999IyJi8eLFMXbs2HjmmWfimGOOiVNOOSWWL18et99+e9x7773xwAMPxOjRo7ted3u3j4jo6OiIcePGxQsvvBAnnXRS1NfXx+DBg7vj0wQAYCfVLVF77733xne+8534+Mc/3vXYd7/73fjEJz4REydOjClTpkRExJlnnhl/+ctf4he/+EWMHz++a9uvfOUrccghh8TZZ58dv/nNb7oe397tIyKWLl0aI0aMiJkzZ0bv3r1fc+aOjo7o6Ojout/e3r7j/wAAABSqW5YfDB8+PM4+++zNHjv77LNj//33j7vvvjtWrFgR8+bNi1mzZsWECRM2C9RXf/xvf/vbrmUF27v9q33961/fZtBGREyePDmampq6bs3NzTvyqQMAsBPoliO1hx9+eNTVbd7HdXV1cfjhh8fvf//7WLBgQfz+97+PiIhly5Zt9by2jz/+eNef73jHO2LOnDnbtf0mu+22W7zzne983ZknTZoUF1xwQdf99vZ2YQsAkFS3RO1rrVnd9HhbW1usWrUqIiLuvvvuuPvuu1/ztV588cWIiO3efpNBgwZFqVR63ZkbGhqioaHhdbcDAGDn1y3LD5YtW7bNx5uamqKxsTEiIq655pool8uveZswYUJExHZvv8kbCVoAAKpLt0TtzJkzY+PGjZs9tnHjxpg1a1aUSqUYMWJE11kKZs+e/YZec3u3BwCgdnVL1C5atCi+//3vb/bY97///Vi0aFGccMIJsccee8Shhx4ao0ePjltvvTX++7//e4vX2LhxY/zyl7/sur+92wMAULu6ZU3t+PHj47zzzot77rknDjzwwFi4cGFMmTIlBg4cGFdddVXXdrfeemu0trbGBz/4wbjyyitj5MiR0bt373j66adj9uzZsWLFili7du0Obw8AQG3qliO1Y8aMiQceeCDa2tri6quvjmnTpsUpp5wSs2fP7rrwQkTEPvvsE/PmzYuvfOUr8cILL8QNN9wQ3/3ud2P+/Plx5JFHxq233rrZ627v9gAA1KZSuVwuFz3EzqC9vT2amppi3sJB0a9ft109mJ3U8+Vu+SUFCSzZ0L/oEaiglvrVRY9AhVzQ8u6iR6ACNpTXx7T4n2hra+s6icBrUW8AAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHr1RQ+ws1m9cddYv1HrV7vBvdYVPQIVMnd9/6JHoIIOa1hV9AhAQdQbAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAIL3UUfvUU09FqVTa5q2lpaXoMQEA6GH1RQ/QHd7+9rfHv/7rv271uf79+1d2GAAAKq4qona//faLiy++uOgxAAAoSOrlBwAAECFqAQCoAlWx/ODJJ598zeUHY8aMiWOPPbayAwEAUFFVEbWLFy+OSy65ZKvPTZw4catR29HRER0dHV3329vbe2w+AAB6VlUsPxg/fnyUy+Wt3q688sqtfszkyZOjqamp69bc3FzZoQEA6DZVEbU7YtKkSdHW1tZ1W7JkSdEjAQCwg6pi+cGOaGhoiIaGhqLHAACgG9TskVoAAKqHqAUAIL2qWH6wrVN6RUR88YtfjN12261yAwEAUFFVEbXbOqVXRMT5558vagEAqljqqG1paYlyuVz0GAAAFMyaWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkF590QPsLMrlckREvPjCxoInoRLe0st+rhUvv7Sh6BGooOfLvrZrxYby+qJHoAI2xCv7eVOnbUup/Ea2qgHPPPNMNDc3Fz0GAAB/Y8mSJTF06NBtbiNq/2rjxo3x7LPPRr9+/aJUKhU9TsW0t7dHc3NzLFmyJBobG4sehx5kX9cO+7q22N+1oxb3dblcjueffz7e9ra3RV3dtlfNWn7wV3V1da/7E0A1a2xsrJkvkFpnX9cO+7q22N+1o9b2dVNT0xvazhvFAABIT9QCAJCeqK1xDQ0N8f/+3/+LhoaGokehh9nXtcO+ri32d+2wr7fNG8UAAEjPkVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOn9f9eWK/0rzK1hAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "first head of last state dec_self_attns\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAArUAAALUCAYAAAAVCMq+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAsQklEQVR4nO3de5TXdZ348dd3GBsImIFESGEESVn3WIuSCImrDLbiLcUst7VNypPbxRS7bVHpz8sq1VbrpaNdvVV6tqPHVsRS14VALkoJVKSSlIoZF7nMKMoAw/z+MOZIIArMfD+8vt/H45zv4Xy/3898eQ2fM8xzPvP+fj6l9vb29gAAgMRqih4AAAD2lKgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0aoseAOh6GzdujP/93/+Nxx9/PNavXx8XX3xxRERs2LAhWlpaol+/flFT42dcAPIqtbe3txc9BNB17r777vi3f/u3WLVqVbS3t0epVIq2traIiHjkkUfiXe96V/zoRz+Ks88+u+BJAWD3OTQDFWz27Nnxvve9L+rq6uKaa67ZLlyPOuqoOPjgg+POO+8saEIA6ByWH0AFu+KKK6JPnz7x61//Ovr16xerV6/ebpsjjzwyHn744QKmA4DO40gtVLCHH344Tj/99OjXr99rbtPY2BjLly8v41QA0PlELVSw1tbWqK+v3+k269at8yYxANLznQwq2NChQ2P+/Pk73Wbu3Llx6KGHlmkiAOgaohYq2JlnnhmzZ8+Om266aYfPf+Mb34jf/e538c///M9lngwAOpdTekEFe/HFF2P06NHx2GOPxbhx46K1tTVmz54dn/3sZ2Pu3LkxZ86cOPzww2POnDlRV1dX9LgAsNtELVS4tWvXxqc+9an46U9/2nF+2oiIUqkUZ511Vlx//fXRt2/fAicEgD0naqFKrF69OubPnx9r1qyJ+vr6GDlyZAwYMKDosQCgU4haAADS80YxqGDdunWLK664YqfbXHnllVFb6zosAOQmaqGCtbe3xxv5ZYxf2ACQnaiFKrdq1aro0aNH0WMAwB7xO0eoMLfeeus29xcuXLjdYxERbW1tsWzZsrj11lvj7W9/e7nGA4Au4Y1iUGFqamqiVCq97nZbv/R79OgRd955Z5x44oldPRoAdBlRCxXmlltuiYhXovXcc8+NCRMmxOmnn77ddt26dYu3vOUt8a53vct5agFIT9RCBfvIRz4SZ5xxRpx22mlFjwIAXUrUAgCQnjeKQRXYvHlzPPHEE7Fu3bptLpX7ascee2yZpwKAziNqoYK1t7fHJZdcEtddd1288MILO932tWIXADIQtVDBrrjiirjyyiujT58+cc4558SgQYNcPQyAimRNLVSwIUOGRKlUil/96lex7777Fj0OAHQZVxSDCrZ8+fKYMGGCoAWg4olaqGAHHXRQtLS0FD0GAHQ5UQsV7BOf+ETcc889sXLlyqJHAYAuZU0tVJBnnnlmm/vt7e3xuc99LhYsWBCXXHJJjBgxIurr63f4sQceeGA5RgSALiFqoYLU1NREqVTa7vH29vYdPr5VqVSKzZs3d+VoANClnNsHKsg555yz03gFgErlSC0AAOl5oxgAAOmJWqhgv//97+Paa6+NVatW7fD5lStXxrXXXhuPPfZYmScDgM5l+QFUsHPOOScefPDBWLZsWdTUbP8zbFtbWwwZMiTe/e53x0033VTAhADQORyphQo2a9asOP7443cYtBER3bp1i+OPPz5mzpxZ5skAoHOJWqhgy5cvj8bGxp1uM3DgwPjLX/5SpomAPTV06NA4//zzix4D9jqiFipYz549X/dqYitXrozu3buXaSJgTz3//POveREVqGaiFirYiBEj4mc/+1msW7duh8+vXbs27rrrrhgxYkR5BwN22z/8wz/EkiVLih4D9jqiFirY+eefH6tXr46mpqbt1s3+8pe/jKampli7dm186lOfKmhCYFd94QtfiKlTp8b06dOLHgX2Ks5+ABXus5/9bPzXf/1XlEqlqKuri7e+9a2xfPnyaG1tjfb29vj85z8fX/va14oeE3iDbr311vjpT38a9913X0yYMCFGjhwZAwYM2OHVBM8555wCJoRiiFqoAvfcc09cf/31MX/+/Ghubo4+ffrEUUcdFeeff36cdNJJRY8H7IKampoolUrxt9++Xx217e3tUSqVoq2trdzjQWFELQAkcsstt7zhbSdOnNiFk8DeRdQCAJBebdEDAF3vqaeeip/85CexcOHCaGlpifr6+jj88MPjgx/8YAwZMqTo8QBgjzlSCxXummuuiX//93+PzZs3b7cGb5999omvf/3rMWnSpIKmo7OtX78+1q1b95prKQ888MAyT0RXueuuu+L222+Pxx9/PF566aV48sknIyLi8ccfj7vvvjs++MEPxsCBAwueEspH1EIFu+eee+K0006Lfv36xac//eloamqK/fffP5YvXx7Tp0+Pb33rW7F69eq4++6745RTTil6XPbAD3/4w/jmN78ZTzzxxGtuUyqVYvPmzWWciq6wZcuW+Jd/+Ze44447IiKiR48e8fLLL3f8ILNixYoYNGhQXH755TF58uQiR4WyErVV6Pe//318+9vfjvnz57/mEZ1SqRRLly4tYDo607hx4+I3v/lNLFy4MAYNGrTd88uWLYsjjjgihg8fHg8++GABE9IZbrjhhjj//POjtrY2xowZE4MGDYra2h2vLrvpppvKPB2d7Zvf/GZ8/vOfj49//OPx1a9+Nb71rW/FFVdcsc3/5ePGjYuNGzfGQw89VOCkUF7W1FaZX/7yl3HiiSdGa2tr1NbWxoABA3b4zc/POpXh0UcfjQ9+8IM7DNqIiMbGxjjrrLPitttuK/NkdKarr746+vXrFw899FAMGzas6HHoYjfffHOMHDkyrr/++oiIHZ6f9uCDD45p06aVezQolKitMl/84hdj8+bN8YMf/CAmTpwY3bp1K3okutDGjRujZ8+eO92mV69esXHjxjJNRFd4+umn46Mf/aigrRJPPvlknH/++TvdZt99943Vq1eXaSLYO7hMbpVZtGhRfOADH4hzzz1X0FaBYcOGxdSpU19zHeXmzZvjnnvuEUPJ7b///k6yX0V69OgRzc3NO93m6aefjj59+pRnINhLiNoq07Nnz+jfv3/RY1Am55xzTjzxxBMxfvz4+PWvf73Nc7/61a/ipJNOiieeeMIJ2pObOHFi/PznP4/169cXPQplcMQRR8R9990XGzZs2OHza9asiV/84hcxevToMk8GxRK1Vebkk0+OWbNmFT0GZTJp0qQ47bTTYvr06XHUUUdF7969421ve1v07t07Ro0aFQ8++GCcdtppTumV3Fe+8pUYOXJk/NM//VPMnDkzXnzxxaJHogtdeOGF8eyzz8aZZ54Zzz777DbPLV26NM4444xobm6OCy+8sKAJoRjOflBlVq5cGcccc0yceOKJ8dWvfjXe/OY3Fz0SZXDrrbfGLbfcss3FF4444oiYOHFifOhDHyp6PPbQ1qVE7e3tO3zT0FZO6VU5Jk+eHF/72teiVCpFz549Y/369R3raNvb2+Piiy+Oyy67rOgxoaxEbYUbN27cdo+tW7cuFi1aFD179oxhw4ZFfX39dtuUSiWneIIkxo4du9OYfbXp06d38TSUywMPPBDf/va34+GHH441a9ZEfX19jBo1Ki688MIYP3580eNB2YnaCldTs3srTEqlkjeeVJC2trZ49tln47nnnotNmzbtcJtjjz22zFMBQOcRtVDBtmzZEldddVVcc801sWbNmp1u64cYADJznlqoYJMnT47//M//jP79+8dHPvKR2H///V/zSlMAkJkjtVWmra0t1q9fH7169drh0oStz/fs2dN5bCvAW9/61ujbt2/Mnz8/evXqVfQ4dJJzzz03SqVSXHXVVTFgwIA499xz39DHlUql+OEPf9jF09HZampqoqamJn7/+9/HsGHDoqam5g2tofbGQKqNqK0yl1xySXz961+PZcuWxX777bfd86tWrYoDDzwwJk+eHJdcckkBE9KZevXqFR//+MfjG9/4RtGj0Im2Rs1jjz3WETlvhLXyOW19I+CPfvSjGDRokDcGwmsQtVVmxIgRsf/+++/0muDvec974rnnntvuZP3kc/TRR8eQIUPitttuK3oUOtHTTz8dEREDBw6M2trajvtvxODBg7tqLIBCWVxXZf74xz9GU1PTTrf5u7/7u5g9e3aZJqIrffnLX473v//98eijj8aIESOKHodO8rdhKlSr1+rVq2PRokXR3NwcDQ0NMXz48Nh3332LHgsKIWqrzKZNm173V5WlUuk1L79ILqecckrcfPPNcdJJJ8Vpp50Ww4cP3+F5iSNeuaQueW3evDmuu+66uP322+Pxxx+Pl156qWM95cKFC+N73/teXHTRRTFs2LCCJ6UzPPXUUzFp0qSYNm1avPoXrqVSKU499dS4+uqrY8iQIcUNyB6ZO3dufPnLX4758+dHqVSKUaNGxZVXXhlHHXVU0aPt1Sw/qDLDhw+P2tranS4teOc73xkbNmyIxYsXl3EyukJra2t89KMfjdtuu63jG9/frsXbehUqay3zevnll+OEE06IOXPmRL9+/WKfffaJv/zlLx37tLm5Od761rfGZz/72fiP//iPgqdlTy1dujTGjBkTK1eujEMOOSTGjBkTAwYMiBUrVsScOXNiyZIl0b9//5gzZ04MHTq06HHZRb/97W9j1KhR2x1c6tGjRzzyyCNx2GGHFTTZ3m/3zsxPWu9973tj4cKFcckll2wXMW1tbXHxxRfHwoUL4/3vf39BE9KZPvOZz8RPfvKTeMc73hGXX355fP/7348bb7xxm9tNN90UN954Y9GjsgeuuuqqmD17dkyZMiWWL18eH/3oR7d5vqGhIY477ri47777CpqQzvSFL3whVq1aFd/5znfi8ccfjxtvvDGmTJkSN954Yzz22GNxww03xKpVq+ILX/hC0aOyG7761a/Ghg0b4stf/nIsX748li9fHhdffHG8/PLL8bWvfa3o8fZqjtRWmRdffDFGjhwZS5Ysibe97W3R1NQUAwcOjD//+c8xffr0WLp0afz93/99zJs3zymgKkD//v1j8ODBMXfuXOenrWDDhg2LxsbGjktbX3bZZXH55Zdv84PrJz/5ybjzzjtjxYoVRY1JJ+nbt2+MHTs27rrrrtfc5vTTT4+ZM2fG2rVryzgZneHAAw+MIUOGxMyZM7d5/Ljjjounnnpql94YWm18l6syvXr1ipkzZ8YnPvGJuOuuu+LJJ5/seK6mpibe9773xfXXXy9oK8SGDRuiqalJ0Fa4Z555Js4444ydbtO7d+9obm4u00R0pba2ttf9FfTb3/52p/NKasWKFfGBD3xgu8dHjRoVDz/8cAET5eE7XRXab7/94o477ogVK1bEr371q2hubo4+ffrEkUceGf379y96PDrRO9/5zm1+cKEy9e7dO1auXLnTbZYuXbrDc1OTz4gRI173PQ+LFy+OI488skwT0Zk2bdq0wwNLPXv2jE2bNhUwUR6itooNGDAgTjnllKLHoAtdddVVcfzxx8c999wTp556atHj0EVGjx4dU6dOjXXr1kWfPn22e37ZsmVx7733vu7RXHK48sor4/jjj48f/OAH262fjoj43ve+F/fdd1/HchSoFqK2wrmcZnV74IEHYuzYsXH66afHuHHjXvOUXqVSKS6++OICJqQzfP7zn4+mpqY4/vjj49prr+04lddLL70Uc+fOjQsuuCA2b94cn/nMZwqelN1x+eWXb/dYU1NTfOxjH4tvfvOb25z9YPbs2bFkyZIYP358PPjgg3H00UcXMDF76sc//nHMmzdvm8e2/tbt5JNP3m77Uqm004sqVQtvFKtwLqdZ3ezv6nHDDTfEpEmTdrgfu3XrFtdff/0Oj+qx93ujX8d/y9d1Truzv+3rVzhSW+H+9Kc/RcQrl9N89X2qgzeKVI9PfOITMXbs2PjOd74TDz/8cKxZsybq6+tj1KhR8clPftK5LRPzdVxdfJ/efY7UAgCQnosvAACQnqgFACA9UQsAQHqitsq1trbGpZdeGq2trUWPQhezr6uHfV1d7O/qYV/vnDeKVbmWlpZoaGiI5ubmHZ6/lMphX1cP+7q62N/Vw77eOUdqAQBIT9QCAJCeiy/81ZYtW+K5556L3r17R6lUKnqcsmlpadnmTyqXfV097OvqYn9Xj2rc1+3t7fHCCy/EAQcc8LpXW7Om9q+effbZaGxsLHoMAAD+xrJly2LQoEE73caR2r/q3bt3REQ8/eiQqO9lVUalO2PYO4oeAQB4HZtjUzwU93Z02s6I2r/auuSgvldN1PcWtZWutrRP0SMAAK/nr+sJ3sjSUPUGAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6VVs1D711FNRKpXiwx/+cNGjAADQxSo2agEAqB61RQ/QVQYOHBiPPfZYNDQ0FD0KAABdrGKjdp999olDDz206DEAACiDil1+YE0tAED1qNioBQCgeohaAADSq9g1ta+ntbU1WltbO+63tLQUOA0AAHuiao/UTpkyJRoaGjpujY2NRY8EAMBuqtqonTx5cjQ3N3fcli1bVvRIAADspqpdflBXVxd1dXVFjwEAQCeo2iO1AABUDlELAEB6ohYAgPRELQAA6VXsG8WGDBkS7e3tRY8BAEAZOFILAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKRXW/QAe5t3TPtw1PToXvQYdLF9pnQregTK5KDJc4seAYAycKQWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgvYqK2qeeeipKpVJ8+MMfLnoUAADKqKKiFgCA6iRqAQBIb5eidu3atdGtW7c49dRTt3l84cKFUSqVolQqxZNPPrnNc2PHjo0ePXpEa2trbNy4Ma677roYP358NDY2Rl1dXfTv3z/e+973xoIFC7b7+26++eYolUpx8803x/333x9HH310vPnNb4599903Jk6cGKtXr95m24MOOigiIm655ZaOeUqlUsyYMWNXPk0AAJKp3ZWN+/btG8OHD49Zs2ZFW1tbdOvWLSIipk+f3rHN9OnT4+CDD46IiA0bNsS8efPi6KOPjrq6uli+fHlcdNFF8Y//+I9x8sknR9++feOPf/xj3H333fHzn/88Zs6cGSNHjtzu77377rtj2rRp8Z73vCeOPvromDlzZtx6662xdOnSeOihhyIi4vDDD49JkybFNddcE8OHD48JEyZ0fPyQIUN29d8FAIBEdilqIyKamppiwYIF8etf/zqOOuqoiHglZIcNGxYvv/xyTJ8+Pc4777yIiJgzZ060trZGU1NTRLwSxc8880wMHDhwm9dcvHhxjB49Or70pS/FAw88sN3fOXXq1JgxY0aMGTMmIiLa2tri3e9+d8yYMSPmzZsXo0ePjsMPPzwuuuiiuOaaa+Lwww+PSy+9dKefR2tra7S2tnbcb2lp2dV/CgAA9hK7vKZ2a6D+3//9X0S8EpgzZ86MpqamaGpq2u6obcQrSxAiIurq6rYL2oiIww47LJqammLmzJmxadOm7Z4/++yzO4I2IqJbt24xceLEiIiYP3/+rn4KERExZcqUaGho6Lg1Njbu1usAAFC8XY7aY489Nrp169YRrAsWLIjm5uYYN25cNDU1xfLly+Oxxx6LiFeitkePHjFq1KiOj1+4cGGcffbZceCBB8ab3vSmjnWvU6dOjY0bN8bzzz+/3d/5zne+c7vHBg0aFBER69at29VPISIiJk+eHM3NzR23ZcuW7dbrAABQvF1eflBfXx8jRoyI2bNnx6ZNm2L69OlRKpWiqakpXnrppYh4JWYHDx4cjzzySBx33HHxpje9KSJeWY4wbty4iIg44YQT4pBDDolevXpFqVSKn/3sZ7Fo0aJtlgS8+u/cbvDaV0Zva2vb1U8hIl45alxXV7dbHwsAwN5ll6M24pUlCPPnz49HHnkkZsyYEYcddljst99+ERFx0EEHxfTp0+OQQw6JTZs2dSxXiIi48soro7W1NWbNmhXHHHPMNq85b968WLRo0R58KgAAVKvdOk/t1lC9//77Y9asWR1HXyMixo0bFzNmzOhYc7t1PW1ExNKlS+Mtb3nLdkH70ksvxaOPPro7o2xj69kYdvfoLQAAOe1W1B5zzDFRW1sbN9xwQ7zwwgvbRG1TU1M8//zz8cMf/jB69uy5zSm6Bg8eHGvXro3Fixd3PNbW1haf+9znYtWqVXvwabyib9++USqVrI8FAKgyu7X8oFevXjFy5MiYO3du1NTUxHHHHdfx3NajuKtWrYrx48fHPvvs0/HcBRdcEPfff38cc8wxcdZZZ0X37t1jxowZ8ec//znGjh27xxdJ2DrXzJkz40Mf+lAccsghUVNTEx/60Idi8ODBe/TaAADsvXb7Mrlb4/WII46IPn36dDx+wAEHxLBhwyJi26UHERGnnnpq3HHHHTF06ND48Y9/HLfddlsceuih8cgjj3RadP7oRz+Kk046Ke6555649NJL4+KLL44//elPnfLaAADsnUrt7e3tRQ+xN2hpaYmGhoYY9F+XR02P7kWPQxfbZ123okegTA6aPLfoEQDYTZvbN8WM+J9obm7e4dmwXm23j9QCAMDeQtQCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAerVFD7C3Ofi2DVHrX6XitQx9c9EjUCZLvjuy6BEoo2Efm1/0CEBBHKkFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqmjduPGjXHdddfF+PHjo7GxMerq6qJ///7x3ve+NxYsWFD0eAAAlEnqqF2zZk1cdNFF0draGieffHJ8+tOfjrFjx8a9994bRx99dMyfP7/oEQEAKIPaogfYE3379o1nnnkmBg4cuM3jixcvjtGjR8eXvvSleOCBB3b4sa2trdHa2tpxv6WlpUtnBQCg66Q+UltXV7dd0EZEHHbYYdHU1BQzZ86MTZs27fBjp0yZEg0NDR23xsbGrh4XAIAukjpqIyIWLlwYZ599dhx44IHxpje9KUqlUpRKpZg6dWps3Lgxnn/++R1+3OTJk6O5ubnjtmzZsjJPDgBAZ0m9/GDOnDkxbty4iIg44YQT4pBDDolevXpFqVSKn/3sZ7Fo0aJtlhi8Wl1dXdTV1ZVzXAAAukjqqL3yyiujtbU1Zs2aFcccc8w2z82bNy8WLVpU0GQAAJRT6uUHS5cujbe85S3bBe1LL70Ujz76aEFTAQBQbqmjdvDgwbF27dpYvHhxx2NtbW3xuc99LlatWlXgZAAAlFPq5QcXXHBB3H///XHMMcfEWWedFd27d48ZM2bEn//85xg7dmzMmDGj6BEBACiD1EdqTz311Ljjjjti6NCh8eMf/zhuu+22OPTQQ+ORRx6JwYMHFz0eAABlUmpvb28veoi9QUtLSzQ0NMTYkV+K2truRY9DF2sZ+uaiR6BMlh/XVvQIlNGwj7mSJFSSze2bYkb8TzQ3N0d9ff1Ot019pBYAACJELQAAFUDUAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACC92qIH2NusH9gjavfpXvQYdLEXB/p5rloMu+mlokegjNrHHF70CJRJafbCokdgL+M7OwAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACC9PYraGTNmRKlUiksvvbSTxgEAgF3nSC0AAOmJWgAA0hO1AACk12lR+9BDD8XYsWOjd+/e0adPnzjzzDPjySef3G67lStXxqc//ek4+OCDo66uLvr16xdnnnlm/O53v9vh6+7K9kOGDIkhQ4bEunXr4lOf+lQ0NjZGbW1t3HzzzZ31aQIAsBeq7YwXmTdvXkyZMiVOPPHEuOCCC2Lx4sVx1113xaxZs2LevHkxdOjQiIhYunRpjB07Np599tk44YQTYsKECbFy5cq4884747777osHH3wwRo0a1fG6u7p9RERra2uMGzcuXnzxxTjttNOitrY2BgwY0BmfJgAAe6lOidr77rsvvvOd78THPvaxjse++93vxsc//vGYNGlSTJ06NSIizjnnnPjLX/4Sv/jFL2L8+PEd237lK1+JI488Ms4777z4zW9+0/H4rm4fEbF8+fIYPnx4zJ49O3r06PGaM7e2tkZra2vH/ZaWlt3/BwAAoFCdsvxg2LBhcd55523z2HnnnReHHHJITJs2LVatWhULFiyIOXPmxMSJE7cJ1Fd//G9/+9uOZQW7uv2rff3rX99p0EZETJkyJRoaGjpujY2Nu/OpAwCwF+iUI7VjxoyJmppt+7impibGjBkTf/jDH2LRokXxhz/8ISIiVqxYscPz2j7++OMdf7797W+PefPm7dL2W3Xv3j3e8Y53vO7MkydPjs985jMd91taWoQtAEBSnRK1r7Vmdevjzc3NsWbNmoiImDZtWkybNu01X2v9+vUREbu8/Vb9+/ePUqn0ujPX1dVFXV3d624HAMDer1OWH6xYsWKnjzc0NER9fX1ERFx33XXR3t7+mreJEydGROzy9lu9kaAFAKCydErUzp49O7Zs2bLNY1u2bIk5c+ZEqVSK4cOHd5ylYO7cuW/oNXd1ewAAqlenRO2SJUvi+9///jaPff/7348lS5bEKaecEvvtt18cddRRMWrUqLj99tvjv//7v7d7jS1btsQvf/nLjvu7uj0AANWrU9bUjh8/Pi688MK4995747DDDovFixfH1KlTo1+/fnHNNdd0bHf77bdHU1NTfOADH4irr746RowYET169Ihnnnkm5s6dG6tWrYoNGzbs9vYAAFSnTjlSO3r06HjwwQejubk5rr322pgxY0ZMmDAh5s6d23HhhYiIgw46KBYsWBBf+cpX4sUXX4ybbropvvvd78bChQvj2GOPjdtvv32b193V7QEAqE6l9vb29qKH2Bu0tLREQ0NDjJxwRdTu073ocehizQd1K3oEyuSAWS8WPQJl1N6t067+zl6uNHth0SNQBpvbN8WM+J9obm7uOInAa/HVDwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkV1v0AHub3ktfiNpuG4segy7WWt+n6BEok7bu/purJrXrNxU9AmWyeeyIokegDNo2b4iY9T9vaFtHagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADppY7ap556Kkql0k5vQ4YMKXpMAAC6WG3RA3SGt73tbfGv//qvO3yuT58+5R0GAICyq4ioPfjgg+PSSy8tegwAAAqSevkBAABEiFoAACpARSw/ePLJJ19z+cHo0aPjxBNPLO9AAACUVUVE7dKlS+Oyyy7b4XOTJk3aYdS2trZGa2trx/2WlpYumw8AgK5VEcsPxo8fH+3t7Tu8XX311Tv8mClTpkRDQ0PHrbGxsbxDAwDQaSoianfH5MmTo7m5ueO2bNmyokcCAGA3VcTyg91RV1cXdXV1RY8BAEAnqNojtQAAVA5RCwBAehWx/GBnp/SKiPjiF78Y3bt3L99AAACUVUVE7c5O6RURcdFFF4laAIAKljpqhwwZEu3t7UWPAQBAwaypBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6dUWPcDeor29PSIiNre1FjwJ5dC2cUPRI1Ammzf7mq4qbZuKnoAy2by5regRKIOt/4dv7bSdKbW/ka2qwLPPPhuNjY1FjwEAwN9YtmxZDBo0aKfbiNq/2rJlSzz33HPRu3fvKJVKRY9TNi0tLdHY2BjLli2L+vr6osehC9nX1cO+ri72d/Woxn3d3t4eL7zwQhxwwAFRU7PzVbOWH/xVTU3N6/4EUMnq6+ur5guk2tnX1cO+ri72d/Wotn3d0NDwhrbzRjEAANITtQAApCdqq1xdXV38v//3/6Kurq7oUehi9nX1sK+ri/1dPezrnfNGMQAA0nOkFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAev8fenglrn6YlPsAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "first head of last state dec_enc_attns\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAArUAAALUCAYAAAAVCMq+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAshElEQVR4nO3de5zVdZ348fcZRgfiMpAIGYzgjXV/1qIkQuoqgyXeUstyW9ukfOSWmWK3LSpdL6tkW62XHtrNa6WP7aEPW1FLXRcEuSglUJGXpFRMuchlRkEGGM7vD2MeEoiCM+fL+5zn8/E4Dx/nnO+ceQ/fxziv+c7nfL+lcrlcDgAASKyu6AEAAOCtErUAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJBefdEDAF1v3bp18b//+7/x+OOPx+rVq+P888+PiIi1a9dGa2tr9O/fP+rq/I4LQF6lcrlcLnoIoOvceeed8a//+q+xbNmyKJfLUSqVor29PSIiHnnkkXjve98bP/nJT+K0004reFIA2HEOzUAVmzFjRnz4wx+OhoaGuPLKK7cI10MOOST23XffuP322wuaEAA6h+UHUMUuueSS6Nu3b/zmN7+J/v37x/Lly7fY5uCDD46HH364gOkAoPM4UgtV7OGHH46TTjop+vfv/7rbNDU1xeLFiys4FQB0PlELVaytrS369OmzzW1WrVrlTWIApOcnGVSxvffeO+bMmbPNbWbNmhX7779/hSYCgK4haqGKnXLKKTFjxoy44YYbtvr8t7/97fj9738f//RP/1ThyQCgczmlF1Sxl19+OUaPHh2PPfZYjB07Ntra2mLGjBnxxS9+MWbNmhUzZ86MAw88MGbOnBkNDQ1FjwsAO0zUQpVbuXJlfO5zn4uf//znHeenjYgolUpx6qmnxjXXXBP9+vUrcEIAeOtELdSI5cuXx5w5c2LFihXRp0+fGDlyZAwcOLDosQCgU4haAADS80YxqGLdunWLSy65ZJvbXHrppVFf7zosAOQmaqGKlcvleDN/jPEHGwCyE7VQ45YtWxY9evQoegwAeEv8zRGqzM0337zZ/Xnz5m3xWEREe3t7LFq0KG6++eZ417veVanxAKBLeKMYVJm6uroolUpvuN2mb/0ePXrE7bffHsccc0xXjwYAXUbUQpW56aabIuLVaD3jjDPi5JNPjpNOOmmL7bp16xZvf/vb473vfa/z1AKQnqiFKvbJT34yPvjBD8aJJ55Y9CgA0KVELQAA6XmjGNSADRs2xBNPPBGrVq3a7FK5r3XEEUdUeCoA6DyiFqpYuVyOCy64IK6++up46aWXtrnt68UuAGQgaqGKXXLJJXHppZdG37594/TTT4/Bgwe7ehgAVcmaWqhiQ4cOjVKpFL/+9a9jt912K3ocAOgyrigGVWzx4sVx8sknC1oAqp6ohSq21157RWtra9FjAECXE7VQxc4666y46667YunSpUWPAgBdyppaqCLPPvvsZvfL5XJ86Utfirlz58YFF1wQI0aMiD59+mz1Y/fcc89KjAgAXULUQhWpq6uLUqm0xePlcnmrj29SKpViw4YNXTkaAHQp5/aBKnL66advM14BoFo5UgsAQHreKAYAQHqiFqrYH/7wh7jqqqti2bJlW31+6dKlcdVVV8Vjjz1W4ckAoHNZfgBV7PTTT48HHnggFi1aFHV1W/4O297eHkOHDo33ve99ccMNNxQwIQB0DkdqoYpNnz49jjrqqK0GbUREt27d4qijjopp06ZVeDIA6FyiFqrY4sWLo6mpaZvbDBo0KF544YUKTQS8VXvvvXecffbZRY8BOx1RC1WsZ8+eb3g1saVLl0b37t0rNBHwVr344ouvexEVqGWiFqrYiBEj4he/+EWsWrVqq8+vXLky7rjjjhgxYkRlBwN22D/8wz/Ek08+WfQYsNMRtVDFzj777Fi+fHk0NzdvsW72wQcfjObm5li5cmV87nOfK2hCYHt95StficmTJ8eUKVOKHgV2Ks5+AFXui1/8YvzXf/1XlEqlaGhoiHe84x2xePHiaGtri3K5HF/+8pfj8ssvL3pM4E26+eab4+c//3nce++9cfLJJ8fIkSNj4MCBW72a4Omnn17AhFAMUQs14K677oprrrkm5syZEy0tLdG3b9845JBD4uyzz45jjz226PGA7VBXVxelUin+9sf3a6O2XC5HqVSK9vb2So8HhRG1AJDITTfd9Ka3HT9+fBdOAjsXUQsAQHr1RQ8AdL2nn346fvazn8W8efOitbU1+vTpEwceeGB87GMfi6FDhxY9HgC8ZY7UQpW78sor49/+7d9iw4YNW6zB22WXXeJb3/pWTJgwoaDp6GyrV6+OVatWve5ayj333LPCE9FV7rjjjrj11lvj8ccfjzVr1sRTTz0VERGPP/543HnnnfGxj30sBg0aVPCUUDmiFqrYXXfdFSeeeGL0798/Pv/5z0dzc3PssccesXjx4pgyZUp897vfjeXLl8edd94Zxx9/fNHj8hZcd9118Z3vfCeeeOKJ192mVCrFhg0bKjgVXWHjxo3xz//8z3HbbbdFRESPHj3ilVde6fhFZsmSJTF48OC4+OKLY+LEiUWOChUlamvQH/7wh/je974Xc+bMed0jOqVSKRYuXFjAdHSmsWPHxm9/+9uYN29eDB48eIvnFy1aFAcddFAMHz48HnjggQImpDNce+21cfbZZ0d9fX0cdthhMXjw4Kiv3/rqshtuuKHC09HZvvOd78SXv/zl+MxnPhPf/OY347vf/W5ccsklm/2/fOzYsbFu3bp46KGHCpwUKsua2hrz4IMPxjHHHBNtbW1RX18fAwcO3OoPP7/rVIdHH300Pvaxj201aCMimpqa4tRTT41bbrmlwpPRma644oro379/PPTQQzFs2LCix6GL3XjjjTFy5Mi45pprIiK2en7afffdN+6+++5KjwaFErU15qtf/Wps2LAhfvzjH8f48eOjW7duRY9EF1q3bl307Nlzm9v06tUr1q1bV6GJ6ArPPPNMfOpTnxK0NeKpp56Ks88+e5vb7LbbbrF8+fIKTQQ7B5fJrTHz58+Pj370o3HGGWcI2howbNiwmDx58uuuo9ywYUPcddddYii5PfbYw0n2a0iPHj2ipaVlm9s888wz0bdv38oMBDsJUVtjevbsGQMGDCh6DCrk9NNPjyeeeCLGjRsXv/nNbzZ77te//nUce+yx8cQTTzhBe3Ljx4+PX/7yl7F69eqiR6ECDjrooLj33ntj7dq1W31+xYoV8atf/SpGjx5d4cmgWKK2xhx33HExffr0osegQiZMmBAnnnhiTJkyJQ455JDo3bt37LPPPtG7d+8YNWpUPPDAA3HiiSc6pVdy3/jGN2LkyJHx/ve/P6ZNmxYvv/xy0SPRhc4999x47rnn4pRTTonnnntus+cWLlwYH/zgB6OlpSXOPffcgiaEYjj7QY1ZunRpHH744XHMMcfEN7/5zXjb295W9EhUwM033xw33XTTZhdfOOigg2L8+PHx8Y9/vOjxeIs2LSUql8tbfdPQJk7pVT0mTpwYl19+eZRKpejZs2esXr26Yx1tuVyO888/Py666KKix4SKErVVbuzYsVs8tmrVqpg/f3707Nkzhg0bFn369Nlim1Kp5BRPkMSYMWO2GbOvNWXKlC6ehkq5//7743vf+148/PDDsWLFiujTp0+MGjUqzj333Bg3blzR40HFidoqV1e3YytMSqWSN55Ukfb29njuuefi+eefj/Xr1291myOOOKLCUwFA5xG1UMU2btwYl112WVx55ZWxYsWKbW7rlxgAMnOeWqhiEydOjP/8z/+MAQMGxCc/+cnYY489XvdKUwCQmSO1Naa9vT1Wr14dvXr12urShE3P9+zZ03lsq8A73vGO6NevX8yZMyd69epV9Dh0kjPOOCNKpVJcdtllMXDgwDjjjDPe1MeVSqW47rrrung6OltdXV3U1dXFH/7whxg2bFjU1dW9qTXU3hhIrRG1NeaCCy6Ib33rW7Fo0aLYfffdt3h+2bJlseeee8bEiRPjggsuKGBCOlOvXr3iM5/5THz7298uehQ60aaoeeyxxzoi582wVj6nTW8E/MlPfhKDBw/2xkB4HaK2xowYMSL22GOPbV4T/AMf+EA8//zzW5ysn3wOPfTQGDp0aNxyyy1Fj0IneuaZZyIiYtCgQVFfX99x/80YMmRIV40FUCiL62rMn/70p2hubt7mNn/3d38XM2bMqNBEdKWvf/3r8ZGPfCQeffTRGDFiRNHj0En+NkyFau1avnx5zJ8/P1paWqKxsTGGDx8eu+22W9FjQSFEbY1Zv379G/6pslQqve7lF8nl+OOPjxtvvDGOPfbYOPHEE2P48OFbPS9xxKuX1CWvDRs2xNVXXx233nprPP7447FmzZqO9ZTz5s2LH/7wh3HeeefFsGHDCp6UzvD000/HhAkT4u67747X/sG1VCrFCSecEFdccUUMHTq0uAF5S2bNmhVf//rXY86cOVEqlWLUqFFx6aWXxiGHHFL0aDs1yw9qzPDhw6O+vn6bSwve8573xNq1a2PBggUVnIyu0NbWFp/61Kfilltu6fjB97dr8TZdhcpay7xeeeWVOProo2PmzJnRv3//2GWXXeKFF17o2KctLS3xjne8I774xS/Gf/zHfxQ8LW/VwoUL47DDDoulS5fGfvvtF4cddlgMHDgwlixZEjNnzownn3wyBgwYEDNnzoy999676HHZTr/73e9i1KhRWxxc6tGjRzzyyCNxwAEHFDTZzm/HzsxPWh/60Idi3rx5ccEFF2wRMe3t7XH++efHvHnz4iMf+UhBE9KZvvCFL8TPfvazePe73x0XX3xx/OhHP4rrr79+s9sNN9wQ119/fdGj8hZcdtllMWPGjJg0aVIsXrw4PvWpT232fGNjYxx55JFx7733FjQhnekrX/lKLFu2LL7//e/H448/Htdff31MmjQprr/++njsscfi2muvjWXLlsVXvvKVokdlB3zzm9+MtWvXxte//vVYvHhxLF68OM4///x45ZVX4vLLLy96vJ2aI7U15uWXX46RI0fGk08+Gfvss080NzfHoEGD4i9/+UtMmTIlFi5cGH//938fs2fPdgqoKjBgwIAYMmRIzJo1y/lpq9iwYcOiqamp49LWF110UVx88cWb/eL62c9+Nm6//fZYsmRJUWPSSfr16xdjxoyJO+6443W3Oemkk2LatGmxcuXKCk5GZ9hzzz1j6NChMW3atM0eP/LII+Ppp5/erjeG1ho/5WpMr169Ytq0aXHWWWfFHXfcEU899VTHc3V1dfHhD384rrnmGkFbJdauXRvNzc2Ctso9++yz8cEPfnCb2/Tu3TtaWloqNBFdqb29/Q3/BP2ud73L6bySWrJkSXz0ox/d4vFRo0bFww8/XMBEefhJV4N23333uO2222LJkiXx61//OlpaWqJv375x8MEHx4ABA4oej070nve8Z7NfXKhOvXv3jqVLl25zm4ULF2713NTkM2LEiDd8z8OCBQvi4IMPrtBEdKb169dv9cBSz549Y/369QVMlIeorWEDBw6M448/vugx6EKXXXZZHHXUUXHXXXfFCSecUPQ4dJHRo0fH5MmTY9WqVdG3b98tnl+0aFHcc889b3g0lxwuvfTSOOqoo+LHP/7xFuunIyJ++MMfxr333tuxHAVqhaitci6nWdvuv//+GDNmTJx00kkxduzY1z2lV6lUivPPP7+ACekMX/7yl6O5uTmOOuqouOqqqzpO5bVmzZqYNWtWnHPOObFhw4b4whe+UPCk7IiLL754i8eam5vj05/+dHznO9/Z7OwHM2bMiCeffDLGjRsXDzzwQBx66KEFTMxb9dOf/jRmz5692WOb/up23HHHbbF9qVTa5kWVaoU3ilU5l9OsbfZ37bj22mtjwoQJW92P3bp1i2uuuWarR/XY+b3Z7+O/5fs6px3Z3/b1qxyprXJ//vOfI+LVy2m+9j61wRtFasdZZ50VY8aMie9///vx8MMPx4oVK6JPnz4xatSo+OxnP+vclon5Pq4tfk7vOEdqAQBIz8UXAABIT9QCAJCeqAUAID1RW+Pa2triwgsvjLa2tqJHoYvZ17XDvq4t9nftsK+3zRvFalxra2s0NjZGS0vLVs9fSvWwr2uHfV1b7O/aYV9vmyO1AACkJ2oBAEjPxRf+auPGjfH8889H7969o1QqFT1OxbS2tm72X6qXfV077OvaYn/Xjlrc1+VyOV566aV45zvf+YZXW7Om9q+ee+65aGpqKnoMAAD+xqJFi2Lw4MHb3MaR2r/q3bt3RERcPvXg6N7LP0u1W1vepegRqJBZq/YpegQq6B96P1f0CFTItNE9ix6BCtgQ6+OhuKej07ZFvf3VpiUH3XvVRw9RW/VKZfu4VuyyfteiR6CCuvfyC2utqC/Z1zXhr+sJ3szSUG8UAwAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPSqNmqffvrpKJVK8YlPfKLoUQAA6GJVG7UAANSO+qIH6CqDBg2Kxx57LBobG4seBQCALla1UbvLLrvE/vvvX/QYAABUQNUuP7CmFgCgdlRt1AIAUDtELQAA6VXtmto30tbWFm1tbR33W1tbC5wGAIC3omaP1E6aNCkaGxs7bk1NTUWPBADADqrZqJ04cWK0tLR03BYtWlT0SAAA7KCaXX7Q0NAQDQ0NRY8BAEAnqNkjtQAAVA9RCwBAeqIWAID0RC0AAOlV7RvFhg4dGuVyuegxAACoAEdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID06oseYGdzYMOi6NVd61e76WuGFT0CFdKj2/qiR6CC5qwaUvQIVMyLRQ/ATka9AQCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApFdVUfv0009HqVSKT3ziE0WPAgBABVVV1AIAUJtELQAA6W1X1K5cuTK6desWJ5xwwmaPz5s3L0qlUpRKpXjqqac2e27MmDHRo0ePaGtri3Xr1sXVV18d48aNi6ampmhoaIgBAwbEhz70oZg7d+4Wn+/GG2+MUqkUN954Y9x3331x6KGHxtve9rbYbbfdYvz48bF8+fLNtt1rr70iIuKmm27qmKdUKsXUqVO358sEACCZ+u3ZuF+/fjF8+PCYPn16tLe3R7du3SIiYsqUKR3bTJkyJfbdd9+IiFi7dm3Mnj07Dj300GhoaIjFixfHeeedF//4j/8Yxx13XPTr1y/+9Kc/xZ133hm//OUvY9q0aTFy5MgtPu+dd94Zd999d3zgAx+IQw89NKZNmxY333xzLFy4MB566KGIiDjwwANjwoQJceWVV8bw4cPj5JNP7vj4oUOHbu+/CwAAiWxX1EZENDc3x9y5c+M3v/lNHHLIIRHxasgOGzYsXnnllZgyZUqceeaZERExc+bMaGtri+bm5oh4NYqfffbZGDRo0GavuWDBghg9enR87Wtfi/vvv3+Lzzl58uSYOnVqHHbYYRER0d7eHu973/ti6tSpMXv27Bg9enQceOCBcd5558WVV14ZBx54YFx44YXb/Dra2tqira2t435ra+v2/lMAALCT2O41tZsC9f/+7/8i4tXAnDZtWjQ3N0dzc/MWR20jXl2CEBHR0NCwRdBGRBxwwAHR3Nwc06ZNi/Xr12/x/GmnndYRtBER3bp1i/Hjx0dExJw5c7b3S4iIiEmTJkVjY2PHrampaYdeBwCA4m131B5xxBHRrVu3jmCdO3dutLS0xNixY6O5uTkWL14cjz32WES8GrU9evSIUaNGdXz8vHnz4rTTTos999wzdt111451r5MnT45169bFiy++uMXnfM973rPFY4MHD46IiFWrVm3vlxARERMnToyWlpaO26JFi3bodQAAKN52Lz/o06dPjBgxImbMmBHr16+PKVOmRKlUiubm5lizZk1EvBqzQ4YMiUceeSSOPPLI2HXXXSPi1eUIY8eOjYiIo48+Ovbbb7/o1atXlEql+MUvfhHz58/fbEnAaz/nFoPXvzp6e3v79n4JEfHqUeOGhoYd+lgAAHYu2x21Ea8uQZgzZ0488sgjMXXq1DjggANi9913j4iIvfbaK6ZMmRL77bdfrF+/vmO5QkTEpZdeGm1tbTF9+vQ4/PDDN3vN2bNnx/z589/ClwIAQK3aofPUbgrV++67L6ZPn95x9DUiYuzYsTF16tSONbeb1tNGRCxcuDDe/va3bxG0a9asiUcffXRHRtnMprMx7OjRWwAActqhqD388MOjvr4+rr322njppZc2i9rm5uZ48cUX47rrrouePXtudoquIUOGxMqVK2PBggUdj7W3t8eXvvSlWLZs2Vv4Ml7Vr1+/KJVK1scCANSYHVp+0KtXrxg5cmTMmjUr6urq4sgjj+x4btNR3GXLlsW4ceNil1126XjunHPOifvuuy8OP/zwOPXUU6N79+4xderU+Mtf/hJjxox5yxdJ2DTXtGnT4uMf/3jst99+UVdXFx//+MdjyJAhb+m1AQDYee3wZXI3xetBBx0Uffv27Xj8ne98ZwwbNiwiNl96EBFxwgknxG233RZ77713/PSnP41bbrkl9t9//3jkkUc6LTp/8pOfxLHHHht33XVXXHjhhXH++efHn//85055bQAAdk6lcrlcLnqInUFra2s0NjbGg78bFL1673Drk8T0NcOKHoEKefSlPYsegQp6ab2z2tSKl/5xy1OAUn02lNfH1PifaGlp2erZsF5LvQEAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPTqix5gZ/Pn9f3jbeu7FT0GXayx2+qiR6BC9u+5uOgRqKBfLf5/RY9AhewaLxY9AjsZR2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQXuqoXbduXVx99dUxbty4aGpqioaGhhgwYEB86EMfirlz5xY9HgAAFZI6alesWBHnnXdetLW1xXHHHRef//znY8yYMXHPPffEoYceGnPmzCl6RAAAKqC+6AHein79+sWzzz4bgwYN2uzxBQsWxOjRo+NrX/ta3H///Vv92La2tmhra+u439ra2qWzAgDQdVIfqW1oaNgiaCMiDjjggGhubo5p06bF+vXrt/qxkyZNisbGxo5bU1NTV48LAEAXSR21ERHz5s2L0047Lfbcc8/Yddddo1QqRalUismTJ8e6devixRdf3OrHTZw4MVpaWjpuixYtqvDkAAB0ltTLD2bOnBljx46NiIijjz469ttvv+jVq1eUSqX4xS9+EfPnz99sicFrNTQ0RENDQyXHBQCgi6SO2ksvvTTa2tpi+vTpcfjhh2/23OzZs2P+/PkFTQYAQCWlXn6wcOHCePvb375F0K5ZsyYeffTRgqYCAKDSUkftkCFDYuXKlbFgwYKOx9rb2+NLX/pSLFu2rMDJAACopNTLD84555y477774vDDD49TTz01unfvHlOnTo2//OUvMWbMmJg6dWrRIwIAUAGpj9SecMIJcdttt8Xee+8dP/3pT+OWW26J/fffPx555JEYMmRI0eMBAFAhpXK5XC56iJ1Ba2trNDY2xo2PDo+39e5W9Dh0sdUbdy16BCrkhfX9ih6BCvrV4v9X9AhUyK7vf6boEaiADeX1MTX+J1paWqJPnz7b3Db1kVoAAIgQtQAAVAFRCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID06oseYGfTv/6l6Fmv9avdHYvHFD0CFfLy+oaiR6CCzhoytegRqJDrYq+iR2Ano94AAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6b2lqJ06dWqUSqW48MILO2kcAADYfo7UAgCQnqgFACA9UQsAQHqdFrUPPfRQjBkzJnr37h19+/aNU045JZ566qkttlu6dGl8/vOfj3333TcaGhqif//+ccopp8Tvf//7rb7u9mw/dOjQGDp0aKxatSo+97nPRVNTU9TX18eNN97YWV8mAAA7ofrOeJHZs2fHpEmT4phjjolzzjknFixYEHfccUdMnz49Zs+eHXvvvXdERCxcuDDGjBkTzz33XBx99NFx8sknx9KlS+P222+Pe++9Nx544IEYNWpUx+tu7/YREW1tbTF27Nh4+eWX48QTT4z6+voYOHBgZ3yZAADspDolau+99974/ve/H5/+9Kc7HvvBD34Qn/nMZ2LChAkxefLkiIg4/fTT44UXXohf/epXMW7cuI5tv/GNb8TBBx8cZ555Zvz2t7/teHx7t4+IWLx4cQwfPjxmzJgRPXr0eN2Z29raoq2treN+a2vrjv8DAABQqE5ZfjBs2LA488wzN3vszDPPjP322y/uvvvuWLZsWcydOzdmzpwZ48eP3yxQX/vxv/vd7zqWFWzv9q/1rW99a5tBGxExadKkaGxs7Lg1NTXtyJcOAMBOoFOO1B522GFRV7d5H9fV1cVhhx0Wf/zjH2P+/Pnxxz/+MSIilixZstXz2j7++OMd/33Xu94Vs2fP3q7tN+nevXu8+93vfsOZJ06cGF/4whc67re2tgpbAICkOiVqX2/N6qbHW1paYsWKFRERcffdd8fdd9/9uq+1evXqiIjt3n6TAQMGRKlUesOZGxoaoqGh4Q23AwBg59cpyw+WLFmyzccbGxujT58+ERFx9dVXR7lcft3b+PHjIyK2e/tN3kzQAgBQXTolamfMmBEbN27c7LGNGzfGzJkzo1QqxfDhwzvOUjBr1qw39Zrbuz0AALWrU6L2ySefjB/96EebPfajH/0onnzyyTj++ONj9913j0MOOSRGjRoVt956a/z3f//3Fq+xcePGePDBBzvub+/2AADUrk5ZUztu3Lg499xz45577okDDjggFixYEJMnT47+/fvHlVde2bHdrbfeGs3NzfHRj340rrjiihgxYkT06NEjnn322Zg1a1YsW7Ys1q5du8PbAwBQmzrlSO3o0aPjgQceiJaWlrjqqqti6tSpcfLJJ8esWbM6LrwQEbHXXnvF3Llz4xvf+Ea8/PLLccMNN8QPfvCDmDdvXhxxxBFx6623bva627s9AAC1qVQul8tFD7EzaG1tjcbGxrjrt3tHz96ddvVgdlI/XDym6BGokJfXO8tJLTltj9lFj0CFXDdsr6JHoAI2lNfH1PifaGlp6TiJwOtRbwAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAevVFD7CzOXfuP0W3t3Uvegy6WLduG4segQpZ8N6fFT0CFdRe9r1dK66LvYoegZ2MI7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9FJH7dNPPx2lUmmbt6FDhxY9JgAAXay+6AE6wz777BP/8i//stXn+vbtW9lhAACouKqI2n333TcuvPDCoscAAKAgqZcfAABAhKgFAKAKVMXyg6eeeup1lx+MHj06jjnmmMoOBABARVVF1C5cuDAuuuiirT43YcKErUZtW1tbtLW1ddxvbW3tsvkAAOhaVbH8YNy4cVEul7d6u+KKK7b6MZMmTYrGxsaOW1NTU2WHBgCg01RF1O6IiRMnRktLS8dt0aJFRY8EAMAOqorlBzuioaEhGhoaih4DAIBOULNHagEAqB6iFgCA9Kpi+cG2TukVEfHVr341unfvXrmBAACoqKqI2m2d0isi4rzzzhO1AABVLHXUDh06NMrlctFjAABQMGtqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAevVFD7CzKJfLERGx8ZW2giehIrqVi56ACml9aWPRI1BB7WX7u1ZsKK8vegQqYEO8up83ddq2lMpvZqsa8Nxzz0VTU1PRYwAA8DcWLVoUgwcP3uY2ovavNm7cGM8//3z07t07SqVS0eNUTGtrazQ1NcWiRYuiT58+RY9DF7Kva4d9XVvs79pRi/u6XC7HSy+9FO985zujrm7bq2YtP/irurq6N/wNoJr16dOnZr5Bap19XTvs69pif9eOWtvXjY2Nb2o7bxQDACA9UQsAQHqitsY1NDTEv//7v0dDQ0PRo9DF7OvaYV/XFvu7dtjX2+aNYgAApOdILQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9P4/EPU1otf6aPEAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "print('first head of last state enc_self_attns')\n", "showgraph(enc_self_attns)\n", @@ -546,7 +487,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.7.13 ('ms1.9')", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -560,7 +501,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.13" + "version": "3.9.18" }, "vscode": { "interpreter": { diff --git a/5-1.Transformer/Transformer_pytorch.ipynb b/5-1.Transformer/Transformer_pytorch.ipynb index f0077c8..b9378a3 100644 --- a/5-1.Transformer/Transformer_pytorch.ipynb +++ b/5-1.Transformer/Transformer_pytorch.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "muslim-material", "metadata": {}, "outputs": [], @@ -19,14 +19,30 @@ "\n", "# S: Symbol that shows starting of decoding input\n", "# E: Symbol that shows starting of decoding output\n", - "# P: Symbol that will fill in blank sequence if current batch data size is short than time steps\n", - "\n", + "# P: Symbol that will fill in blank sequence if current batch data size is short than time steps" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "15708426-6f32-4baf-9646-2b41bf94f84a", + "metadata": {}, + "outputs": [], + "source": [ "def make_batch(sentences):\n", " input_batch = [[src_vocab[n] for n in sentences[0].split()]]\n", " output_batch = [[tgt_vocab[n] for n in sentences[1].split()]]\n", " target_batch = [[tgt_vocab[n] for n in sentences[2].split()]]\n", - " return torch.LongTensor(input_batch), torch.LongTensor(output_batch), torch.LongTensor(target_batch)\n", - "\n", + " return torch.LongTensor(input_batch), torch.LongTensor(output_batch), torch.LongTensor(target_batch)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "853b2237-f05f-4f5a-b2b6-10377cd17244", + "metadata": {}, + "outputs": [], + "source": [ "def get_sinusoid_encoding_table(n_position, d_model):\n", " def cal_angle(position, hid_idx):\n", " return position / np.power(10000, 2 * (hid_idx // 2) / d_model)\n", @@ -36,100 +52,171 @@ " sinusoid_table = np.array([get_posi_angle_vec(pos_i) for pos_i in range(n_position)])\n", " sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2]) # dim 2i\n", " sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2]) # dim 2i+1\n", - " return torch.FloatTensor(sinusoid_table)\n", - "\n", + " return torch.FloatTensor(sinusoid_table)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "392a3a9d-f02c-450c-bfe8-0432e2ef4ee7", + "metadata": {}, + "outputs": [], + "source": [ "def get_attn_pad_mask(seq_q, seq_k):\n", " batch_size, len_q = seq_q.size()\n", " batch_size, len_k = seq_k.size()\n", " # eq(zero) is PAD token\n", " pad_attn_mask = seq_k.data.eq(0).unsqueeze(1) # batch_size x 1 x len_k(=len_q), one is masking\n", - " return pad_attn_mask.expand(batch_size, len_q, len_k) # batch_size x len_q x len_k\n", - "\n", + " return pad_attn_mask.expand(batch_size, len_q, len_k) # batch_size x len_q x len_k" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "ef758198-6584-46f8-8884-63a63d7bd1d7", + "metadata": {}, + "outputs": [], + "source": [ "def get_attn_subsequent_mask(seq):\n", " attn_shape = [seq.size(0), seq.size(1), seq.size(1)]\n", " subsequent_mask = np.triu(np.ones(attn_shape), k=1)\n", " subsequent_mask = torch.from_numpy(subsequent_mask).byte()\n", - " return subsequent_mask\n", - "\n", + " return subsequent_mask" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "711b197f-55b0-4453-ad53-9dc371272784", + "metadata": {}, + "outputs": [], + "source": [ "class ScaledDotProductAttention(nn.Module):\n", - " def __init__(self):\n", + " def __init__(self, d_k):\n", " super(ScaledDotProductAttention, self).__init__()\n", + " self.softmax = nn.Softmax(dim=-1)\n", + " self.d_k = d_k\n", "\n", " def forward(self, Q, K, V, attn_mask):\n", - " scores = torch.matmul(Q, K.transpose(-1, -2)) / np.sqrt(d_k)# scores : [batch_size x n_heads x len_q(=len_k) x len_k(=len_q)]\n", + " scores = torch.matmul(Q, K.transpose(-1, -2)) / np.sqrt(self.d_k)# scores : [batch_size x n_heads x len_q(=len_k) x len_k(=len_q)]\n", " scores.masked_fill_(attn_mask, -1e9) # Fills elements of self tensor with value where mask is one.\n", - " attn = nn.Softmax(dim=-1)(scores)\n", + " attn = self.softmax(scores)\n", " context = torch.matmul(attn, V)\n", - " return context, attn\n", - "\n", + " return context, attn" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "56ec0a47-4018-48a1-8826-f58b31438502", + "metadata": {}, + "outputs": [], + "source": [ "class MultiHeadAttention(nn.Module):\n", - " def __init__(self):\n", + " def __init__(self, d_model, d_k, d_v, n_heads):\n", " super(MultiHeadAttention, self).__init__()\n", + " self.d_k = d_k\n", + " self.d_v = d_v\n", + " self.n_heads = n_heads\n", " self.W_Q = nn.Linear(d_model, d_k * n_heads)\n", " self.W_K = nn.Linear(d_model, d_k * n_heads)\n", " self.W_V = nn.Linear(d_model, d_v * n_heads)\n", " self.linear = nn.Linear(n_heads * d_v, d_model)\n", " self.layer_norm = nn.LayerNorm(d_model)\n", + " self.attention = ScaledDotProductAttention(d_k)\n", "\n", " def forward(self, Q, K, V, attn_mask):\n", " # q: [batch_size x len_q x d_model], k: [batch_size x len_k x d_model], v: [batch_size x len_k x d_model]\n", " residual, batch_size = Q, Q.size(0)\n", " # (B, S, D) -proj-> (B, S, D) -split-> (B, S, H, W) -trans-> (B, H, S, W)\n", - " q_s = self.W_Q(Q).view(batch_size, -1, n_heads, d_k).transpose(1,2) # q_s: [batch_size x n_heads x len_q x d_k]\n", - " k_s = self.W_K(K).view(batch_size, -1, n_heads, d_k).transpose(1,2) # k_s: [batch_size x n_heads x len_k x d_k]\n", - " v_s = self.W_V(V).view(batch_size, -1, n_heads, d_v).transpose(1,2) # v_s: [batch_size x n_heads x len_k x d_v]\n", + " q_s = self.W_Q(Q).view(batch_size, -1, self.n_heads, self.d_k).transpose(1,2) # q_s: [batch_size x n_heads x len_q x d_k]\n", + " k_s = self.W_K(K).view(batch_size, -1, self.n_heads, self.d_k).transpose(1,2) # k_s: [batch_size x n_heads x len_k x d_k]\n", + " v_s = self.W_V(V).view(batch_size, -1, self.n_heads, self.d_v).transpose(1,2) # v_s: [batch_size x n_heads x len_k x d_v]\n", "\n", " attn_mask = attn_mask.unsqueeze(1).repeat(1, n_heads, 1, 1) # attn_mask : [batch_size x n_heads x len_q x len_k]\n", "\n", " # context: [batch_size x n_heads x len_q x d_v], attn: [batch_size x n_heads x len_q(=len_k) x len_k(=len_q)]\n", - " context, attn = ScaledDotProductAttention()(q_s, k_s, v_s, attn_mask)\n", + " context, attn = self.attention(q_s, k_s, v_s, attn_mask)\n", " context = context.transpose(1, 2).contiguous().view(batch_size, -1, n_heads * d_v) # context: [batch_size x len_q x n_heads * d_v]\n", " output = self.linear(context)\n", - " return self.layer_norm(output + residual), attn # output: [batch_size x len_q x d_model]\n", - "\n", - "class PoswiseFeedForwardNet(nn.Module):\n", - " def __init__(self):\n", - " super(PoswiseFeedForwardNet, self).__init__()\n", + " return self.layer_norm(output + residual), attn # output: [batch_size x len_q x d_model]" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "ff8f89d4-809f-4e45-8f37-e02784fb96ab", + "metadata": {}, + "outputs": [], + "source": [ + "class PoswiseFeedForward(nn.Module):\n", + " def __init__(self, d_ff, d_model):\n", + " super(PoswiseFeedForward, self).__init__()\n", " self.conv1 = nn.Conv1d(in_channels=d_model, out_channels=d_ff, kernel_size=1)\n", " self.conv2 = nn.Conv1d(in_channels=d_ff, out_channels=d_model, kernel_size=1)\n", " self.layer_norm = nn.LayerNorm(d_model)\n", + " self.relu = nn.ReLU()\n", "\n", " def forward(self, inputs):\n", " residual = inputs # inputs : [batch_size, len_q, d_model]\n", - " output = nn.ReLU()(self.conv1(inputs.transpose(1, 2)))\n", + " output = self.relu(self.conv1(inputs.transpose(1, 2)))\n", " output = self.conv2(output).transpose(1, 2)\n", - " return self.layer_norm(output + residual)\n", - "\n", + " return self.layer_norm(output + residual)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "855a975d-37f5-4c9a-a352-debc00eb0916", + "metadata": {}, + "outputs": [], + "source": [ "class EncoderLayer(nn.Module):\n", - " def __init__(self):\n", - " super(EncoderLayer, self).__init__()\n", - " self.enc_self_attn = MultiHeadAttention()\n", - " self.pos_ffn = PoswiseFeedForwardNet()\n", + " def __init__(self, d_model, d_k, d_v, n_heads, d_ff):\n", + " super().__init__()\n", + " self.enc_self_attn = MultiHeadAttention(d_model, d_k, d_v, n_heads)\n", + " self.pos_ffn = PoswiseFeedForward(d_ff, d_model)\n", "\n", " def forward(self, enc_inputs, enc_self_attn_mask):\n", " enc_outputs, attn = self.enc_self_attn(enc_inputs, enc_inputs, enc_inputs, enc_self_attn_mask) # enc_inputs to same Q,K,V\n", " enc_outputs = self.pos_ffn(enc_outputs) # enc_outputs: [batch_size x len_q x d_model]\n", - " return enc_outputs, attn\n", - "\n", + " return enc_outputs, attn" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "c90050db-a25c-4516-b653-29ef9e82b968", + "metadata": {}, + "outputs": [], + "source": [ "class DecoderLayer(nn.Module):\n", - " def __init__(self):\n", - " super(DecoderLayer, self).__init__()\n", - " self.dec_self_attn = MultiHeadAttention()\n", - " self.dec_enc_attn = MultiHeadAttention()\n", - " self.pos_ffn = PoswiseFeedForwardNet()\n", + " def __init__(self, d_model, d_k, d_v, n_heads, d_ff):\n", + " super().__init__()\n", + " self.dec_self_attn = MultiHeadAttention(d_model, d_k, d_v, n_heads)\n", + " self.dec_enc_attn = MultiHeadAttention(d_model, d_k, d_v, n_heads)\n", + " self.pos_ffn = PoswiseFeedForward(d_ff, d_model)\n", "\n", " def forward(self, dec_inputs, enc_outputs, dec_self_attn_mask, dec_enc_attn_mask):\n", " dec_outputs, dec_self_attn = self.dec_self_attn(dec_inputs, dec_inputs, dec_inputs, dec_self_attn_mask)\n", " dec_outputs, dec_enc_attn = self.dec_enc_attn(dec_outputs, enc_outputs, enc_outputs, dec_enc_attn_mask)\n", " dec_outputs = self.pos_ffn(dec_outputs)\n", - " return dec_outputs, dec_self_attn, dec_enc_attn\n", - "\n", + " return dec_outputs, dec_self_attn, dec_enc_attn" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "497cafe9-e842-48fc-9081-52c8ec010569", + "metadata": {}, + "outputs": [], + "source": [ "class Encoder(nn.Module):\n", - " def __init__(self):\n", + " def __init__(self, src_vocab_size, d_model, d_k, d_v, n_heads, d_ff, n_layers, src_len):\n", " super(Encoder, self).__init__()\n", " self.src_emb = nn.Embedding(src_vocab_size, d_model)\n", " self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(src_len+1, d_model),freeze=True)\n", - " self.layers = nn.ModuleList([EncoderLayer() for _ in range(n_layers)])\n", + " self.layers = nn.ModuleList([EncoderLayer(d_model, d_k, d_v, n_heads, d_ff) for _ in range(n_layers)])\n", "\n", " def forward(self, enc_inputs): # enc_inputs : [batch_size x source_len]\n", " enc_outputs = self.src_emb(enc_inputs) + self.pos_emb(torch.LongTensor([[1,2,3,4,0]]))\n", @@ -138,14 +225,22 @@ " for layer in self.layers:\n", " enc_outputs, enc_self_attn = layer(enc_outputs, enc_self_attn_mask)\n", " enc_self_attns.append(enc_self_attn)\n", - " return enc_outputs, enc_self_attns\n", - "\n", + " return enc_outputs, enc_self_attns" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "7b18e014-4e63-472a-8fb0-8635aa1541b7", + "metadata": {}, + "outputs": [], + "source": [ "class Decoder(nn.Module):\n", - " def __init__(self):\n", + " def __init__(self, tgt_vocab_size, d_model, d_k, d_v, n_heads, d_ff, n_layers, tgt_len):\n", " super(Decoder, self).__init__()\n", " self.tgt_emb = nn.Embedding(tgt_vocab_size, d_model)\n", " self.pos_emb = nn.Embedding.from_pretrained(get_sinusoid_encoding_table(tgt_len+1, d_model),freeze=True)\n", - " self.layers = nn.ModuleList([DecoderLayer() for _ in range(n_layers)])\n", + " self.layers = nn.ModuleList([DecoderLayer(d_model, d_k, d_v, n_heads, d_ff) for _ in range(n_layers)])\n", "\n", " def forward(self, dec_inputs, enc_inputs, enc_outputs): # dec_inputs : [batch_size x target_len]\n", " dec_outputs = self.tgt_emb(dec_inputs) + self.pos_emb(torch.LongTensor([[5,1,2,3,4]]))\n", @@ -160,20 +255,152 @@ " dec_outputs, dec_self_attn, dec_enc_attn = layer(dec_outputs, enc_outputs, dec_self_attn_mask, dec_enc_attn_mask)\n", " dec_self_attns.append(dec_self_attn)\n", " dec_enc_attns.append(dec_enc_attn)\n", - " return dec_outputs, dec_self_attns, dec_enc_attns\n", - "\n", + " return dec_outputs, dec_self_attns, dec_enc_attns" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "73941748-7a78-4d14-90a8-0663ec9deadf", + "metadata": {}, + "outputs": [], + "source": [ "class Transformer(nn.Module):\n", - " def __init__(self):\n", + " def __init__(self, d_model, d_k, d_v, n_heads, d_ff, n_layers, src_vocab_size, tgt_vocab_size, src_len, tgt_len):\n", " super(Transformer, self).__init__()\n", - " self.encoder = Encoder()\n", - " self.decoder = Decoder()\n", + " self.encoder = Encoder(src_vocab_size, d_model, d_k, d_v, n_heads, d_ff, n_layers, src_len)\n", + " self.decoder = Decoder(tgt_vocab_size, d_model, d_k, d_v, n_heads, d_ff, n_layers, tgt_len)\n", " self.projection = nn.Linear(d_model, tgt_vocab_size, bias=False)\n", + "\n", " def forward(self, enc_inputs, dec_inputs):\n", " enc_outputs, enc_self_attns = self.encoder(enc_inputs)\n", " dec_outputs, dec_self_attns, dec_enc_attns = self.decoder(dec_inputs, enc_inputs, enc_outputs)\n", " dec_logits = self.projection(dec_outputs) # dec_logits : [batch_size x src_vocab_size x tgt_vocab_size]\n", - " return dec_logits.view(-1, dec_logits.size(-1)), enc_self_attns, dec_self_attns, dec_enc_attns\n", + " return dec_logits.view(-1, dec_logits.size(-1)), enc_self_attns, dec_self_attns, dec_enc_attns" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "be4ea15c-5147-4917-ad73-bf42f253f5a9", + "metadata": {}, + "outputs": [], + "source": [ + "sentences = ['ich mochte ein bier P', 'S i want a beer', 'i want a beer E']\n", "\n", + "# Transformer Parameters\n", + "# Padding Should be Zero\n", + "src_vocab = {'P': 0, 'ich': 1, 'mochte': 2, 'ein': 3, 'bier': 4}\n", + "src_vocab_size = len(src_vocab)\n", + "\n", + "tgt_vocab = {'P': 0, 'i': 1, 'want': 2, 'a': 3, 'beer': 4, 'S': 5, 'E': 6}\n", + "number_dict = {i: w for i, w in enumerate(tgt_vocab)}\n", + "tgt_vocab_size = len(tgt_vocab)\n", + "\n", + "src_len = 6 # length of source\n", + "tgt_len = 5 # length of target\n", + "\n", + "d_model = 512 # Embedding Size\n", + "d_ff = 2048 # FeedForward dimension\n", + "d_k = d_v = 64 # dimension of K(=Q), V\n", + "n_layers = 6 # number of Encoder of Decoder Layer\n", + "n_heads = 8 # number of heads in Multi-Head Attention" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "83f611f7-126c-4dbe-ab90-a9d203dbadee", + "metadata": {}, + "outputs": [], + "source": [ + "model = Transformer(d_model, d_k, d_v, n_heads, d_ff, n_layers, src_vocab_size, tgt_vocab_size, src_len, tgt_len)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "b818b9a6-9df9-459c-ac47-1d864e7b141b", + "metadata": {}, + "outputs": [], + "source": [ + "criterion = nn.CrossEntropyLoss()\n", + "optimizer = optim.Adam(model.parameters(), lr=0.0001)\n", + "\n", + "enc_inputs, dec_inputs, target_batch = make_batch(sentences)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "39e82729-1316-4377-bb53-58859cdca6bc", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch: 0001 cost = 2.031222\n", + "Epoch: 0002 cost = 1.260098\n", + "Epoch: 0003 cost = 0.785420\n", + "Epoch: 0004 cost = 0.526297\n", + "Epoch: 0005 cost = 0.073096\n", + "Epoch: 0006 cost = 0.020095\n", + "Epoch: 0007 cost = 0.012649\n", + "Epoch: 0008 cost = 0.010072\n", + "Epoch: 0009 cost = 0.007779\n", + "Epoch: 0010 cost = 0.005323\n", + "Epoch: 0011 cost = 0.003494\n", + "Epoch: 0012 cost = 0.002412\n", + "Epoch: 0013 cost = 0.001798\n", + "Epoch: 0014 cost = 0.001434\n", + "Epoch: 0015 cost = 0.001191\n", + "Epoch: 0016 cost = 0.001008\n", + "Epoch: 0017 cost = 0.000860\n", + "Epoch: 0018 cost = 0.000732\n", + "Epoch: 0019 cost = 0.000623\n", + "Epoch: 0020 cost = 0.000529\n" + ] + } + ], + "source": [ + "for epoch in range(20):\n", + " optimizer.zero_grad()\n", + " outputs, _, _, _ = model(enc_inputs, dec_inputs)\n", + " loss = criterion(outputs, target_batch.contiguous().view(-1))\n", + " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", + " loss.backward()\n", + " optimizer.step()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "a2685684-a92a-4019-aa95-041d97bf7096", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ich mochte ein bier P -> ['i', 'want', 'a', 'beer', 'E']\n" + ] + } + ], + "source": [ + "# Test\n", + "predict, enc_self_attns, dec_self_attns, dec_enc_attns = model(enc_inputs, dec_inputs)\n", + "predict = predict.data.max(1, keepdim=True)[1]\n", + "print(sentences[0], '->', [number_dict[n.item()] for n in predict.squeeze()])" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "8f8d43a7-f750-40d3-8bc7-f27d7336cecb", + "metadata": {}, + "outputs": [], + "source": [ "def showgraph(attn):\n", " attn = attn[-1].squeeze(0)[0]\n", " attn = attn.squeeze(0).data.numpy()\n", @@ -183,63 +410,112 @@ " ax.matshow(attn, cmap='viridis')\n", " ax.set_xticklabels(['']+sentences[0].split(), fontdict={'fontsize': 14}, rotation=90)\n", " ax.set_yticklabels(['']+sentences[2].split(), fontdict={'fontsize': 14})\n", - " plt.show()\n", - "\n", - "if __name__ == '__main__':\n", - " sentences = ['ich mochte ein bier P', 'S i want a beer', 'i want a beer E']\n", - "\n", - " # Transformer Parameters\n", - " # Padding Should be Zero\n", - " src_vocab = {'P': 0, 'ich': 1, 'mochte': 2, 'ein': 3, 'bier': 4}\n", - " src_vocab_size = len(src_vocab)\n", - "\n", - " tgt_vocab = {'P': 0, 'i': 1, 'want': 2, 'a': 3, 'beer': 4, 'S': 5, 'E': 6}\n", - " number_dict = {i: w for i, w in enumerate(tgt_vocab)}\n", - " tgt_vocab_size = len(tgt_vocab)\n", - "\n", - " src_len = 5 # length of source\n", - " tgt_len = 5 # length of target\n", - "\n", - " d_model = 512 # Embedding Size\n", - " d_ff = 2048 # FeedForward dimension\n", - " d_k = d_v = 64 # dimension of K(=Q), V\n", - " n_layers = 6 # number of Encoder of Decoder Layer\n", - " n_heads = 8 # number of heads in Multi-Head Attention\n", - "\n", - " model = Transformer()\n", - "\n", - " criterion = nn.CrossEntropyLoss()\n", - " optimizer = optim.Adam(model.parameters(), lr=0.0001)\n", - "\n", - " enc_inputs, dec_inputs, target_batch = make_batch(sentences)\n", - "\n", - " for epoch in range(20):\n", - " optimizer.zero_grad()\n", - " outputs, _, _, _ = model(enc_inputs, dec_inputs)\n", - " loss = criterion(outputs, target_batch.contiguous().view(-1))\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # Test\n", - " predict, enc_self_attns, dec_self_attns, dec_enc_attns = model(enc_inputs, dec_inputs)\n", - " predict = predict.data.max(1, keepdim=True)[1]\n", - " print(sentences[0], '->', [number_dict[n.item()] for n in predict.squeeze()])\n", - "\n", - " print('first head of last state enc_self_attns')\n", - " showgraph(enc_self_attns)\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "090b2177-65bf-4169-a2f4-afe67ab1a602", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "first head of last state enc_self_attns\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_1262415/2392112591.py:8: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n", + " ax.set_xticklabels(['']+sentences[0].split(), fontdict={'fontsize': 14}, rotation=90)\n", + "/tmp/ipykernel_1262415/2392112591.py:9: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n", + " ax.set_yticklabels(['']+sentences[2].split(), fontdict={'fontsize': 14})\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAArUAAALUCAYAAAAVCMq+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAsTElEQVR4nO3de5TXdZ348dd3GB2Iy4AiqDCCpqwda1EUIXWVwVa8pZblurZJeXK7mKJdtih1TVextlovHe2mopWe7ejPVtBC14VALkoKVOQlKRMzLnKZUZABhvn9YcyRQBSc+X54fb+Pxznf4/l+v5/5zmv4nHGe85n39/MptbW1tQUAACRWU/QAAADwdolaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIr7boAYDOt379+vjf//3feOqpp2LNmjVx2WWXRUTEunXrorm5Ofr27Rs1NX7HBSCvUltbW1vRQwCd57777ot//dd/jeXLl0dbW1uUSqVobW2NiIjHHnss3vve98aPfvSjOOeccwqeFAB2nkMzUMFmzpwZH/rQh6Kuri6uv/76rcL1yCOPjAMPPDDuueeegiYEgI5h+QFUsKuuuip69+4djz/+ePTt2zdWrFix1TZHHHFEPProowVMBwAdx5FaqGCPPvponH766dG3b9833KahoSGWLFlSxqkAoOOJWqhgLS0t0atXr+1us3r1am8SAyA9P8mggh1wwAExd+7c7W4ze/bsOPjgg8s0EQB0DlELFezMM8+MmTNnxm233bbN57/5zW/Gb3/72/inf/qnMk8GAB3LKb2ggr3yyisxcuTIePLJJ2P06NHR0tISM2fOjM9//vMxe/bsmDVrVhx66KExa9asqKurK3pcANhpohYq3KpVq+Kzn/1s/PSnP20/P21ERKlUirPOOituuumm6NOnT4ETAsDbJ2qhSqxYsSLmzp0bK1eujF69esXw4cOjf//+RY8FAB1C1AIAkJ43ikEF69KlS1x11VXb3ebqq6+O2lrXYQEgN1ELFaytrS3eyh9j/MEGgOxELVS55cuXR7du3YoeAwDeFn9zhApzxx13bHF//vz5Wz0WEdHa2hqLFy+OO+64I9797neXazwA6BTeKAYVpqamJkql0ptut/lbv1u3bnHPPffEiSee2NmjAUCnEbVQYW6//faIeC1azzvvvDjjjDPi9NNP32q7Ll26xB577BHvfe97nacWgPRELVSwj3/84/GBD3wgTjvttKJHAYBOJWoBAEjPG8WgCmzcuDGefvrpWL169RaXyn29Y489tsxTAUDHEbVQwdra2uLyyy+PG2+8MV5++eXtbvtGsQsAGYhaqGBXXXVVXH311dG7d+8499xzY+DAga4eBkBFsqYWKtjgwYOjVCrFr371q9hzzz2LHgcAOo0rikEFW7JkSZxxxhmCFoCKJ2qhgu2///7R3Nxc9BgA0OlELVSwT3/60zF58uRYtmxZ0aMAQKeyphYqyPPPP7/F/ba2tvjCF74Q8+bNi8svvzyGDRsWvXr12ubH7rfffuUYEQA6haiFClJTUxOlUmmrx9va2rb5+GalUik2btzYmaMBQKdybh+oIOeee+524xUAKpUjtQAApOeNYgAApCdqoYL97ne/ixtuuCGWL1++zeeXLVsWN9xwQzz55JNlngwAOpblB1DBzj333Hj44Ydj8eLFUVOz9e+wra2tMXjw4Hjf+94Xt912WwETAkDHcKQWKtiMGTPi+OOP32bQRkR06dIljj/++Jg+fXqZJwOAjiVqoYItWbIkGhoatrvNgAED4i9/+UuZJgLergMOOCAuuOCCoseAXY6ohQrWvXv3N72a2LJly6Jr165lmgh4u1566aU3vIgKVDNRCxVs2LBh8bOf/SxWr169zedXrVoV9957bwwbNqy8gwE77e///u/jmWeeKXoM2OWIWqhgF1xwQaxYsSIaGxu3Wjf7y1/+MhobG2PVqlXx2c9+tqAJgR31pS99KSZNmhRTp04tehTYpTj7AVS4z3/+8/Ff//VfUSqVoq6uLvbee+9YsmRJtLS0RFtbW3zxi1+Mr3/960WPCbxFd9xxR/z0pz+NKVOmxBlnnBHDhw+P/v37b/Nqgueee24BE0IxRC1UgcmTJ8dNN90Uc+fOjaampujdu3cceeSRccEFF8RJJ51U9HjADqipqYlSqRR/++P79VHb1tYWpVIpWltbyz0eFEbUAkAit99++1veduzYsZ04CexaRC0AAOnVFj0A0Pmee+65+MlPfhLz58+P5ubm6NWrVxx66KHxkY98JAYPHlz0eADwtjlSCxXu+uuvj3/7t3+LjRs3brUGb7fddotvfOMbMW7cuIKmo6OtWbMmVq9e/YZrKffbb78yT0Rnuffee+Ouu+6Kp556KtauXRvPPvtsREQ89dRTcd9998VHPvKRGDBgQMFTQvmIWqhgkydPjtNOOy369u0bl1xySTQ2NsY+++wTS5YsialTp8a3v/3tWLFiRdx3331xyimnFD0ub8Mtt9wS3/rWt+Lpp59+w21KpVJs3LixjFPRGTZt2hT//M//HHfffXdERHTr1i1effXV9l9kli5dGgMHDowrr7wyxo8fX+SoUFaitgr97ne/i+985zsxd+7cNzyiUyqVYtGiRQVMR0caPXp0/PrXv4758+fHwIEDt3p+8eLFcdhhh8XQoUPj4YcfLmBCOsLNN98cF1xwQdTW1sbRRx8dAwcOjNraba8uu+2228o8HR3tW9/6Vnzxi1+MT33qU3HttdfGt7/97bjqqqu2+H/56NGjY/369fHII48UOCmUlzW1VeaXv/xlnHjiidHS0hK1tbXRv3//bf7w87tOZXjiiSfiIx/5yDaDNiKioaEhzjrrrLjzzjvLPBkd6brrrou+ffvGI488EkOGDCl6HDrZxIkTY/jw4XHTTTdFRGzz/LQHHnhg3H///eUeDQolaqvMl7/85di4cWP88Ic/jLFjx0aXLl2KHolOtH79+ujevft2t+nRo0esX7++TBPRGf70pz/FJz7xCUFbJZ599tm44IILtrvNnnvuGStWrCjTRLBrcJncKrNgwYI4++yz47zzzhO0VWDIkCExadKkN1xHuXHjxpg8ebIYSm6fffZxkv0q0q1bt2hqatruNn/605+id+/e5RkIdhGitsp07949+vXrV/QYlMm5554bTz/9dIwZMyYef/zxLZ771a9+FSeddFI8/fTTTtCe3NixY+PnP/95rFmzpuhRKIPDDjsspkyZEuvWrdvm8ytXroxf/OIXMXLkyDJPBsUStVXm5JNPjhkzZhQ9BmUybty4OO2002Lq1Klx5JFHRs+ePeOd73xn9OzZM0aMGBEPP/xwnHbaaU7pldyll14aw4cPj3/8x3+M6dOnxyuvvFL0SHSiiy66KF544YU488wz44UXXtjiuUWLFsUHPvCBaGpqiosuuqigCaEYzn5QZZYtWxbHHHNMnHjiiXHttdfGO97xjqJHogzuuOOOuP3227e4+MJhhx0WY8eOjY9+9KNFj8fbtHkpUVtb2zbfNLSZU3pVjvHjx8fXv/71KJVK0b1791izZk37Otq2tra47LLL4mtf+1rRY0JZidoKN3r06K0eW716dSxYsCC6d+8eQ4YMiV69em21TalUcoonSGLUqFHbjdnXmzp1aidPQ7k89NBD8Z3vfCceffTRWLlyZfTq1StGjBgRF110UYwZM6bo8aDsRG2Fq6nZuRUmpVLJG08qSGtra7zwwgvx4osvxoYNG7a5zbHHHlvmqQCg44haqGCbNm2Ka665Jq6//vpYuXLldrf1SwwAmTlPLVSw8ePHx3/+539Gv3794uMf/3jss88+b3ilKQDIzJHaKtPa2hpr1qyJHj16bHNpwubnu3fv7jy2FWDvvfeOPn36xNy5c6NHjx5Fj0MHOe+886JUKsU111wT/fv3j/POO+8tfVypVIpbbrmlk6ejo9XU1ERNTU387ne/iyFDhkRNTc1bWkPtjYFUG1FbZS6//PL4xje+EYsXL4699tprq+eXL18e++23X4wfPz4uv/zyAiakI/Xo0SM+9alPxTe/+c2iR6EDbY6aJ598sj1y3gpr5XPa/EbAH/3oRzFw4EBvDIQ3IGqrzLBhw2KfffbZ7jXB3//+98eLL7641cn6yeeoo46KwYMHx5133ln0KHSgP/3pTxERMWDAgKitrW2//1YMGjSos8YCKJTFdVXmD3/4QzQ2Nm53m7/7u7+LmTNnlmkiOtNXv/rV+PCHPxxPPPFEDBs2rOhx6CB/G6ZCtXqtWLEiFixYEE1NTVFfXx9Dhw6NPffcs+ixoBCitsps2LDhTf9UWSqV3vDyi+RyyimnxMSJE+Okk06K0047LYYOHbrN8xJHvHZJXfLauHFj3HjjjXHXXXfFU089FWvXrm1fTzl//vz4/ve/HxdffHEMGTKk4EnpCM8991yMGzcu7r///nj9H1xLpVKceuqpcd1118XgwYOLG5C3Zfbs2fHVr3415s6dG6VSKUaMGBFXX311HHnkkUWPtkuz/KDKDB06NGpra7e7tODwww+PdevWxcKFC8s4GZ2hpaUlPvGJT8Sdd97Z/oPvb9fibb4KlbWWeb366qtxwgknxKxZs6Jv376x2267xV/+8pf2fdrU1BR77713fP7zn4//+I//KHha3q5FixbF0UcfHcuWLYuDDjoojj766Ojfv38sXbo0Zs2aFc8880z069cvZs2aFQcccEDR47KDfvOb38SIESO2OrjUrVu3eOyxx+KQQw4paLJd386dmZ+0PvjBD8b8+fPj8ssv3ypiWltb47LLLov58+fHhz/84YImpCN97nOfi5/85Cfxnve8J6688sr4wQ9+ELfeeusWt9tuuy1uvfXWokflbbjmmmti5syZMWHChFiyZEl84hOf2OL5+vr6OO6442LKlCkFTUhH+tKXvhTLly+P7373u/HUU0/FrbfeGhMmTIhbb701nnzyybj55ptj+fLl8aUvfanoUdkJ1157baxbty6++tWvxpIlS2LJkiVx2WWXxauvvhpf//rXix5vl+ZIbZV55ZVXYvjw4fHMM8/EO9/5zmhsbIwBAwbEn//855g6dWosWrQo3vWud8WcOXOcAqoC9OvXLwYNGhSzZ892ftoKNmTIkGhoaGi/tPXXvva1uPLKK7f4xfUzn/lM3HPPPbF06dKixqSD9OnTJ0aNGhX33nvvG25z+umnx/Tp02PVqlVlnIyOsN9++8XgwYNj+vTpWzx+3HHHxXPPPbdDbwytNn7KVZkePXrE9OnT49Of/nTce++98eyzz7Y/V1NTEx/60IfipptuErQVYt26ddHY2ChoK9zzzz8fH/jAB7a7Tc+ePaOpqalME9GZWltb3/RP0O9+97udziuppUuXxtlnn73V4yNGjIhHH320gIny8JOuCu21115x9913x9KlS+NXv/pVNDU1Re/eveOII46Ifv36FT0eHejwww/f4hcXKlPPnj1j2bJl291m0aJF2zw3NfkMGzbsTd/zsHDhwjjiiCPKNBEdacOGDds8sNS9e/fYsGFDARPlIWqrWP/+/eOUU04pegw60TXXXBPHH398TJ48OU499dSix6GTjBw5MiZNmhSrV6+O3r17b/X84sWL44EHHnjTo7nkcPXVV8fxxx8fP/zhD7daPx0R8f3vfz+mTJnSvhwFqoWorXAup1ndHnrooRg1alScfvrpMXr06Dc8pVepVIrLLrusgAnpCF/84hejsbExjj/++LjhhhvaT+W1du3amD17dlx44YWxcePG+NznPlfwpOyMK6+8cqvHGhsb45Of/GR861vf2uLsBzNnzoxnnnkmxowZEw8//HAcddRRBUzM2/XjH/845syZs8Vjm//qdvLJJ2+1falU2u5FlaqFN4pVOJfTrG72d/W4+eabY9y4cdvcj126dImbbrppm0f12PW91e/jv+X7Oqed2d/29Wscqa1wf/zjHyPitctpvv4+1cEbRarHpz/96Rg1alR897vfjUcffTRWrlwZvXr1ihEjRsRnPvMZ57ZMzPdxdfFzeuc5UgsAQHouvgAAQHqiFgCA9EQtAADpidoq19LSEldccUW0tLQUPQqdzL6uHvZ1dbG/q4d9vX3eKFblmpubo76+PpqamrZ5/lIqh31dPezr6mJ/Vw/7evscqQUAID1RCwBAei6+8FebNm2KF198MXr27BmlUqnoccqmubl5i/9Suezr6mFfVxf7u3pU475ua2uLl19+Ofbdd983vdqaNbV/9cILL0RDQ0PRYwAA8DcWL14cAwcO3O42jtT+Vc+ePSMi4vT/OTt26757wdPQ2RYu37voESiTfzrg8aJHADrBtBE9ih6BMtgYG+KReKC907ZH1P7V5iUHu3XfXdRWgS5r6ooegTLp2sP/5qAS1ZZ2K3oEyuGv6wneytJQbxQDACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9Co2ap977rkolUrxsY99rOhRAADoZBUbtQAAVI/aogfoLAMGDIgnn3wy6uvrix4FAIBOVrFRu9tuu8XBBx9c9BgAAJRBxS4/sKYWAKB6VGzUAgBQPUQtAADpVeya2jfT0tISLS0t7febm5sLnAYAgLejao/UTpgwIerr69tvDQ0NRY8EAMBOqtqoHT9+fDQ1NbXfFi9eXPRIAADspKpdflBXVxd1dXVFjwEAQAeo2iO1AABUDlELAEB6ohYAgPRELQAA6VXsG8UGDx4cbW1tRY8BAEAZOFILAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKRXW/QAu5o/rN4zumyoK3oMOlndbhuLHoEy+d+l7yp6BMpoyrsmFz0CZfK/cWjRI7CLcaQWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgvYqK2ueeey5KpVJ87GMfK3oUAADKqKKiFgCA6iRqAQBIb4eidtWqVdGlS5c49dRTt3h8/vz5USqVolQqxbPPPrvFc6NGjYpu3bpFS0tLrF+/Pm688cYYM2ZMNDQ0RF1dXfTr1y8++MEPxrx587b6fBMnToxSqRQTJ06MBx98MI466qh4xzveEXvuuWeMHTs2VqxYscW2+++/f0RE3H777e3zlEqlmDZt2o58mQAAJFO7Ixv36dMnhg4dGjNmzIjW1tbo0qVLRERMnTq1fZupU6fGgQceGBER69atizlz5sRRRx0VdXV1sWTJkrj44ovjH/7hH+Lkk0+OPn36xB/+8Ie477774uc//3lMnz49hg8fvtXnve++++L++++P97///XHUUUfF9OnT44477ohFixbFI488EhERhx56aIwbNy6uv/76GDp0aJxxxhntHz948OAd/XcBACCRHYraiIjGxsaYN29ePP7443HkkUdGxGshO2TIkHj11Vdj6tSpcf7550dExKxZs6KlpSUaGxsj4rUofv7552PAgAFbvObChQtj5MiR8ZWvfCUeeuihrT7npEmTYtq0aXH00UdHRERra2u8733vi2nTpsWcOXNi5MiRceihh8bFF18c119/fRx66KFxxRVXbPfraGlpiZaWlvb7zc3NO/pPAQDALmKH19RuDtT/+7//i4jXAnP69OnR2NgYjY2NWx21jXhtCUJERF1d3VZBGxFxyCGHRGNjY0yfPj02bNiw1fPnnHNOe9BGRHTp0iXGjh0bERFz587d0S8hIiImTJgQ9fX17beGhoadeh0AAIq3w1F77LHHRpcuXdqDdd68edHU1BSjR4+OxsbGWLJkSTz55JMR8VrUduvWLUaMGNH+8fPnz49zzjkn9ttvv9h9993b171OmjQp1q9fHy+99NJWn/Pwww/f6rGBAwdGRMTq1at39EuIiIjx48dHU1NT+23x4sU79ToAABRvh5cf9OrVK4YNGxYzZ86MDRs2xNSpU6NUKkVjY2OsXbs2Il6L2UGDBsVjjz0Wxx13XOy+++4R8dpyhNGjR0dExAknnBAHHXRQ9OjRI0qlUvzsZz+LBQsWbLEk4PWfc6vBa18bvbW1dUe/hIh47ahxXV3dTn0sAAC7lh2O2ojXliDMnTs3HnvssZg2bVoccsghsddee0VExP777x9Tp06Ngw46KDZs2NC+XCEi4uqrr46WlpaYMWNGHHPMMVu85pw5c2LBggVv40sBAKBa7dR5ajeH6oMPPhgzZsxoP/oaETF69OiYNm1a+5rbzetpIyIWLVoUe+yxx1ZBu3bt2njiiSd2ZpQtbD4bw84evQUAIKeditpjjjkmamtr4+abb46XX355i6htbGyMl156KW655Zbo3r37FqfoGjRoUKxatSoWLlzY/lhra2t84QtfiOXLl7+NL+M1ffr0iVKpZH0sAECV2anlBz169Ijhw4fH7Nmzo6amJo477rj25zYfxV2+fHmMGTMmdtttt/bnLrzwwnjwwQfjmGOOibPOOiu6du0a06ZNiz//+c8xatSot32RhM1zTZ8+PT760Y/GQQcdFDU1NfHRj340Bg0a9LZeGwCAXddOXyZ3c7wedthh0bt37/bH99133xgyZEhEbLn0ICLi1FNPjbvvvjsOOOCA+PGPfxx33nlnHHzwwfHYY491WHT+6Ec/ipNOOikmT54cV1xxRVx22WXxxz/+sUNeGwCAXVOpra2treghdgXNzc1RX18fw+6+JLp0d1aESreprVT0CJRJn66vFj0CZTTlXZOLHoEyGbPvoUWPQBlsbNsQ0+J/oqmpaZtnw3q9nT5SCwAAuwpRCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOnVFj3ArmbZ8l5R80rXosegk+3R9+WiR6BM3tf/yaJHoIxa2jYUPQJQEEdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkF7qqF2/fn3ceOONMWbMmGhoaIi6urro169ffPCDH4x58+YVPR4AAGWSOmpXrlwZF198cbS0tMTJJ58cl1xySYwaNSoeeOCBOOqoo2Lu3LlFjwgAQBnUFj3A29GnT594/vnnY8CAAVs8vnDhwhg5cmR85StfiYceemibH9vS0hItLS3t95ubmzt1VgAAOk/qI7V1dXVbBW1ExCGHHBKNjY0xffr02LBhwzY/dsKECVFfX99+a2ho6OxxAQDoJKmjNiJi/vz5cc4558R+++0Xu+++e5RKpSiVSjFp0qRYv359vPTSS9v8uPHjx0dTU1P7bfHixWWeHACAjpJ6+cGsWbNi9OjRERFxwgknxEEHHRQ9evSIUqkUP/vZz2LBggVbLDF4vbq6uqirqyvnuAAAdJLUUXv11VdHS0tLzJgxI4455pgtnpszZ04sWLCgoMkAACin1MsPFi1aFHvsscdWQbt27dp44oknCpoKAIBySx21gwYNilWrVsXChQvbH2ttbY0vfOELsXz58gInAwCgnFIvP7jwwgvjwQcfjGOOOSbOOuus6Nq1a0ybNi3+/Oc/x6hRo2LatGlFjwgAQBmkPlJ76qmnxt133x0HHHBA/PjHP44777wzDj744Hjsscdi0KBBRY8HAECZpD5SGxFx5plnxplnnrnV4xMnToyJEyeWfyAAAMou9ZFaAACIELUAAFQAUQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9GqLHmBX02X31qipay16DDrZbl02FT0CZTK82x+LHoEy+p81fYseASiII7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9N5W1E6bNi1KpVJcccUVHTQOAADsOEdqAQBIT9QCAJCeqAUAIL0Oi9pHHnkkRo0aFT179ozevXvHmWeeGc8+++xW2y1btiwuueSSOPDAA6Ouri769u0bZ555Zvz2t7/d5uvuyPaDBw+OwYMHx+rVq+Ozn/1sNDQ0RG1tbUycOLGjvkwAAHZBtR3xInPmzIkJEybEiSeeGBdeeGEsXLgw7r333pgxY0bMmTMnDjjggIiIWLRoUYwaNSpeeOGFOOGEE+KMM86IZcuWxT333BNTpkyJhx9+OEaMGNH+uju6fURES0tLjB49Ol555ZU47bTTora2Nvr3798RXyYAALuoDonaKVOmxHe/+9345Cc/2f7Y9773vfjUpz4V48aNi0mTJkVExLnnnht/+ctf4he/+EWMGTOmfdtLL700jjjiiDj//PPj17/+dfvjO7p9RMSSJUti6NChMXPmzOjWrdsbztzS0hItLS3t95ubm3f+HwAAgEJ1yPKDIUOGxPnnn7/FY+eff34cdNBBcf/998fy5ctj3rx5MWvWrBg7duwWgfr6j//Nb37TvqxgR7d/vW984xvbDdqIiAkTJkR9fX37raGhYWe+dAAAdgEdcqT26KOPjpqaLfu4pqYmjj766Pj9738fCxYsiN///vcREbF06dJtntf2qaeeav/vu9/97pgzZ84Obb9Z165d4z3vec+bzjx+/Pj43Oc+136/ublZ2AIAJNUhUftGa1Y3P97U1BQrV66MiIj7778/7r///jd8rTVr1kRE7PD2m/Xr1y9KpdKbzlxXVxd1dXVvuh0AALu+Dll+sHTp0u0+Xl9fH7169YqIiBtvvDHa2tre8DZ27NiIiB3efrO3ErQAAFSWDonamTNnxqZNm7Z4bNOmTTFr1qwolUoxdOjQ9rMUzJ49+y295o5uDwBA9eqQqH3mmWfiBz/4wRaP/eAHP4hnnnkmTjnllNhrr73iyCOPjBEjRsRdd90V//3f/73Va2zatCl++ctftt/f0e0BAKheHbKmdsyYMXHRRRfFAw88EIccckgsXLgwJk2aFH379o3rr7++fbu77rorGhsb4+yzz47rrrsuhg0bFt26dYvnn38+Zs+eHcuXL49169bt9PYAAFSnDjlSO3LkyHj44Yejqakpbrjhhpg2bVqcccYZMXv27PYLL0RE7L///jFv3ry49NJL45VXXonbbrstvve978X8+fPj2GOPjbvuumuL193R7QEAqE6ltra2tqKH2BU0NzdHfX19DLrl0qh5R9eix6GT9e39StEjUCbX/t3/K3oEymhZa8+iR6BMbhmyf9EjUAYb2zbEtPifaGpqaj+JwBvpkCO1AABQJFELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSqy16gF3NXn1ejtru64seg07Wusnvc9Xi/606vOgRKKN/2XNW0SNQNvsXPQC7GD/ZAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmljtrnnnsuSqXSdm+DBw8uekwAADpZbdEDdIR3vvOd8S//8i/bfK53797lHQYAgLKriKg98MAD44orrih6DAAACpJ6+QEAAESIWgAAKkBFLD949tln33D5wciRI+PEE08s70AAAJRVRUTtokWL4mtf+9o2nxs3btw2o7alpSVaWlra7zc3N3fafAAAdK6KWH4wZsyYaGtr2+btuuuu2+bHTJgwIerr69tvDQ0N5R0aAIAOUxFRuzPGjx8fTU1N7bfFixcXPRIAADupIpYf7Iy6urqoq6sregwAADpA1R6pBQCgcohaAADSq4jlB9s7pVdExJe//OXo2rVr+QYCAKCsKiJqt3dKr4iIiy++WNQCAFSw1FE7ePDgaGtrK3oMAAAKZk0tAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIr7boAXYVbW1tERHRural4Ekoh02b/D5XLda/sqHoESijNbtvKnoEymRjm+/tarAxXtvPmztte0ptb2WrKvDCCy9EQ0ND0WMAAPA3Fi9eHAMHDtzuNqL2rzZt2hQvvvhi9OzZM0qlUtHjlE1zc3M0NDTE4sWLo1evXkWPQyeyr6uHfV1d7O/qUY37uq2tLV5++eXYd999o6Zm+39ltfzgr2pqat70N4BK1qtXr6r5Bql29nX1sK+ri/1dPaptX9fX17+l7SwsBAAgPVELAEB6orbK1dXVxb//+79HXV1d0aPQyezr6mFfVxf7u3rY19vnjWIAAKTnSC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPT+P7NQHR0Ms2ptAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "first head of last state dec_self_attns\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_1262415/2392112591.py:8: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n", + " ax.set_xticklabels(['']+sentences[0].split(), fontdict={'fontsize': 14}, rotation=90)\n", + "/tmp/ipykernel_1262415/2392112591.py:9: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n", + " ax.set_yticklabels(['']+sentences[2].split(), fontdict={'fontsize': 14})\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAArUAAALUCAYAAAAVCMq+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAsRElEQVR4nO3de5TWdZ3A8c8zjA4EzEAiZDCABqx7rEVJhNSUwVa8pajltrZJeXK7mGK3LSpc00WqrdZLR7t6q/RsR4+tqKWuC4FcFBWoyEtQKmpc5DKjKAMMs38YcyQQBWbmx+d5Xq9znuOZ5/nOw2f4HZz3/Ob7/J5Sa2trawAAQGJVRQ8AAAB7StQCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB61UUPAHS8jRs3xv/+7//G448/HuvXr4/JkydHRMSGDRuiqakp+vTpE1VVfsYFIK9Sa2tra9FDAB3njjvuiH/913+NVatWRWtra5RKpWhpaYmIiIceeije8573xE9/+tM4++yzC54UAHafUzNQxmbPnh0f+MAHoqamJq688srtwvWII46IIUOGxG233VbQhADQPmw/gDJ22WWXRa9eveKRRx6JPn36xOrVq7dbc/jhh8eDDz5YwHQA0H6cqYUy9uCDD8Zpp50Wffr0ed019fX1sXz58k6cCgDan6iFMtbc3By1tbU7XbNu3TovEgMgPd/JoIwddNBBMX/+/J2umTt3bhx88MGdNBEAdAxRC2XszDPPjNmzZ8f111+/w8e//e1vx+9///v4p3/6p06eDADal0t6QRl76aWXYvTo0fHYY4/F2LFjo7m5OWbPnh2f//znY+7cuTFnzpw49NBDY86cOVFTU1P0uACw20QtlLm1a9fGZz7zmfjFL37Rdn3aiIhSqRRnnXVWXHPNNdG7d+8CJwSAPSdqoUKsXr065s+fH2vWrIna2toYOXJk9OvXr+ixAKBdiFoAANLzQjEoY126dInLLrtsp2umTJkS1dXehwWA3EQtlLHW1tZ4M7+M8QsbALITtVDhVq1aFd26dSt6DADYI37nCGXmpptu2ubjhQsXbndfRERLS0ssW7YsbrrppnjnO9/ZWeMBQIfwQjEoM1VVVVEqld5w3dZ/+t26dYvbbrstTjjhhI4eDQA6jKiFMnPjjTdGxKvReu6558b48ePjtNNO225dly5d4q1vfWu85z3vcZ1aANITtVDGPvaxj8Xpp58ep556atGjAECHErUAAKTnhWJQATZv3hxPPPFErFu3bpu3yn2tY445ppOnAoD2I2qhjLW2tsbFF18cV199dbz44os7Xft6sQsAGYhaKGOXXXZZTJkyJXr16hXnnHNODBgwwLuHAVCW7KmFMjZ48OAolUrx8MMPx3777Vf0OADQYbyjGJSx5cuXx/jx4wUtAGVP1EIZO/DAA6OpqanoMQCgw4laKGOf+tSn4s4774yVK1cWPQoAdCh7aqGMPPPMM9t83NraGl/4whdiwYIFcfHFF8eIESOitrZ2h587cODAzhgRADqEqIUyUlVVFaVSabv7W1tbd3j/VqVSKTZv3tyRowFAh3JtHygj55xzzk7jFQDKlTO1AACk54ViAACkJ2qhjP3hD3+Iq666KlatWrXDx1euXBlXXXVVPPbYY508GQC0L9sPoIydc845cf/998eyZcuiqmr7n2FbWlpi8ODB8b73vS+uv/76AiYEgPbhTC2UsVmzZsVxxx23w6CNiOjSpUscd9xxMXPmzE6eDADal6iFMrZ8+fKor6/f6Zr+/fvHX/7yl06aCNhTBx10UJx//vlFjwF7HVELZax79+5v+G5iK1eujK5du3bSRMCeeuGFF173TVSgkolaKGMjRoyIX/7yl7Fu3bodPr527dq4/fbbY8SIEZ07GLDb/uEf/iGefPLJoseAvY6ohTJ2/vnnx+rVq6OhoWG7fbO/+c1voqGhIdauXRuf+cxnCpoQ2FVf+tKXYtq0aTF9+vSiR4G9iqsfQJn7/Oc/H//1X/8VpVIpampq4m1ve1ssX748mpubo7W1Nb74xS/GN7/5zaLHBN6km266KX7xi1/EPffcE+PHj4+RI0dGv379dvhuguecc04BE0IxRC1UgDvvvDOuueaamD9/fjQ2NkavXr3iiCOOiPPPPz9OPPHEoscDdkFVVVWUSqX422/fr43a1tbWKJVK0dLS0tnjQWFELQAkcuONN77ptRMmTOjASWDvImoBAEivuugBgI731FNPxc9//vNYuHBhNDU1RW1tbRx66KHx4Q9/OAYPHlz0eACwx5yphTJ35ZVXxr/927/F5s2bt9uDt88++8S3vvWtmDhxYkHT0d7Wr18f69ate929lAMHDuzkiegot99+e9xyyy3x+OOPx8svvxxLliyJiIjHH3887rjjjvjwhz8c/fv3L3hK6DyiFsrYnXfeGaeeemr06dMnPvvZz0ZDQ0MccMABsXz58pg+fXp897vfjdWrV8cdd9wRJ598ctHjsgd+8pOfxHe+85144oknXndNqVSKzZs3d+JUdIQtW7bEP//zP8ett94aERHdunWLV155pe0HmRUrVsSAAQPi0ksvjUmTJhU5KnQqUVuB/vCHP8T3vve9mD9//uue0SmVSrF06dICpqM9jR07Nn7729/GwoULY8CAAds9vmzZsjjssMNi+PDhcf/99xcwIe3h2muvjfPPPz+qq6vjqKOOigEDBkR19Y53l11//fWdPB3t7Tvf+U588YtfjE9+8pPxjW98I7773e/GZZddts3/y8eOHRsbN26MBx54oMBJoXPZU1thfvOb38QJJ5wQzc3NUV1dHf369dvhNz8/65SHRx99ND784Q/vMGgjIurr6+Oss86Km2++uZMnoz1dccUV0adPn3jggQdi2LBhRY9DB7vhhhti5MiRcc0110RE7PD6tEOGDIm77rqrs0eDQonaCvPlL385Nm/eHD/+8Y9jwoQJ0aVLl6JHogNt3LgxunfvvtM1PXr0iI0bN3bSRHSEp59+Oj7+8Y8L2gqxZMmSOP/883e6Zr/99ovVq1d30kSwd/A2uRVm0aJF8aEPfSjOPfdcQVsBhg0bFtOmTXvdfZSbN2+OO++8Uwwld8ABB7jIfgXp1q1bNDY27nTN008/Hb169eqcgWAvIWorTPfu3aNv375Fj0EnOeecc+KJJ56IcePGxSOPPLLNYw8//HCceOKJ8cQTT7hAe3ITJkyIX/3qV7F+/fqiR6ETHHbYYXHPPffEhg0bdvj4mjVr4te//nWMHj26kyeDYonaCnPSSSfFrFmzih6DTjJx4sQ49dRTY/r06XHEEUdEz5494x3veEf07NkzRo0aFffff3+ceuqpLumV3Ne+9rUYOXJk/OM//mPMnDkzXnrppaJHogNdeOGF8eyzz8aZZ54Zzz777DaPLV26NE4//fRobGyMCy+8sKAJoRiuflBhVq5cGUcffXSccMIJ8Y1vfCPe8pa3FD0SneCmm26KG2+8cZs3XzjssMNiwoQJ8ZGPfKTo8dhDW7cStba27vBFQ1u5pFf5mDRpUnzzm9+MUqkU3bt3j/Xr17fto21tbY3JkyfH17/+9aLHhE4lasvc2LFjt7tv3bp1sWjRoujevXsMGzYsamtrt1tTKpVc4gmSGDNmzE5j9rWmT5/ewdPQWe6777743ve+Fw8++GCsWbMmamtrY9SoUXHhhRfGuHHjih4POp2oLXNVVbu3w6RUKnnhSRlpaWmJZ599Np5//vnYtGnTDtccc8wxnTwVALQfUQtlbMuWLXH55ZfHlVdeGWvWrNnpWj/EAJCZ69RCGZs0aVL853/+Z/Tt2zc+9rGPxQEHHPC67zQFAJk5U1thWlpaYv369dGjR48dbk3Y+nj37t1dx7YMvO1tb4vevXvH/Pnzo0ePHkWPQzs599xzo1QqxeWXXx79+vWLc8899019XqlUip/85CcdPB3traqqKqqqquIPf/hDDBs2LKqqqt7UHmovDKTSiNoKc/HFF8e3vvWtWLZsWey///7bPb5q1aoYOHBgTJo0KS6++OICJqQ99ejRIz75yU/Gt7/97aJHoR1tjZrHHnusLXLeDHvlc9r6QsCf/vSnMWDAAC8MhNchaivMiBEj4oADDtjpe4K///3vj+eff367i/WTz5FHHhmDBw+Om2++uehRaEdPP/10RET0798/qqur2z5+MwYNGtRRYwEUyua6CvOnP/0pGhoadrrm7/7u72L27NmdNBEd6atf/Wp88IMfjEcffTRGjBhR9Di0k78NU6FauVavXh2LFi2KxsbGqKuri+HDh8d+++1X9FhQCFFbYTZt2vSGv6oslUqv+/aL5HLyySfHDTfcECeeeGKceuqpMXz48B1elzji1bfUJa/NmzfH1VdfHbfccks8/vjj8fLLL7ftp1y4cGH88Ic/jIsuuiiGDRtW8KS0h6eeeiomTpwYd911V7z2F66lUilOOeWUuOKKK2Lw4MHFDcgemTt3bnz1q1+N+fPnR6lUilGjRsWUKVPiiCOOKHq0vZrtBxVm+PDhUV1dvdOtBe9+97tjw4YNsXjx4k6cjI7Q3NwcH//4x+Pmm29u+8b3t3vxtr4Llb2Web3yyitx/PHHx5w5c6JPnz6xzz77xF/+8pe2Y9rY2Bhve9vb4vOf/3z8x3/8R8HTsqeWLl0aRx11VKxcuTKGDh0aRx11VPTr1y9WrFgRc+bMiSeffDL69u0bc+bMiYMOOqjocdlFv/vd72LUqFHbnVzq1q1bPPTQQ3HIIYcUNNneb/euzE9aZ5xxRixcuDAuvvji7SKmpaUlJk+eHAsXLowPfvCDBU1Ie/rc5z4XP//5z+Nd73pXXHrppfGjH/0orrvuum1u119/fVx33XVFj8oeuPzyy2P27NkxderUWL58eXz84x/f5vG6uro49thj45577iloQtrTl770pVi1alV8//vfj8cffzyuu+66mDp1alx33XXx2GOPxbXXXhurVq2KL33pS0WPym74xje+ERs2bIivfvWrsXz58li+fHlMnjw5XnnllfjmN79Z9Hh7NWdqK8xLL70UI0eOjCeffDLe8Y53RENDQ/Tv3z+ee+65mD59eixdujT+/u//PubNm+cSUGWgb9++MWjQoJg7d67r05axYcOGRX19fdtbW3/961+PSy+9dJsfXD/96U/HbbfdFitWrChqTNpJ7969Y8yYMXH77be/7prTTjstZs6cGWvXru3EyWgPAwcOjMGDB8fMmTO3uf/YY4+Np556apdeGFppfJerMD169IiZM2fGpz71qbj99ttjyZIlbY9VVVXFBz7wgbjmmmsEbZnYsGFDNDQ0CNoy98wzz8Tpp5++0zU9e/aMxsbGTpqIjtTS0vKGv4J+5zvf6XJeSa1YsSI+9KEPbXf/qFGj4sEHHyxgojx8p6tA+++/f9x6662xYsWKePjhh6OxsTF69eoVhx9+ePTt27fo8WhH7373u7f5wYXy1LNnz1i5cuVO1yxdunSH16YmnxEjRrzhax4WL14chx9+eCdNRHvatGnTDk8sde/ePTZt2lTARHmI2grWr1+/OPnkk4segw50+eWXx3HHHRd33nlnnHLKKUWPQwcZPXp0TJs2LdatWxe9evXa7vFly5bF3Xff/YZnc8lhypQpcdxxx8WPf/zj7fZPR0T88Ic/jHvuuadtOwpUClFb5rydZmW77777YsyYMXHaaafF2LFjX/eSXqVSKSZPnlzAhLSHL37xi9HQ0BDHHXdcXHXVVW2X8nr55Zdj7ty5ccEFF8TmzZvjc5/7XMGTsjsuvfTS7e5raGiIT3ziE/Gd73xnm6sfzJ49O5588skYN25c3H///XHkkUcWMDF76mc/+1nMmzdvm/u2/tbtpJNO2m59qVTa6ZsqVQovFCtz3k6zsjnelePaa6+NiRMn7vA4dunSJa655podntVj7/dm/x3/Lf+uc9qd4+1Yv8qZ2jL35z//OSJefTvN135MZfBCkcrxqU99KsaMGRPf//7348EHH4w1a9ZEbW1tjBo1Kj796U+7tmVi/h1XFt+nd58ztQAApOfNFwAASE/UAgCQnqgFACA9UVvhmpub45JLLonm5uaiR6GDOdaVw7GuLI535XCsd84LxSpcU1NT1NXVRWNj4w6vX0r5cKwrh2NdWRzvyuFY75wztQAApCdqAQBIz5sv/NWWLVvi+eefj549e0apVCp6nE7T1NS0zX8pX4515XCsK4vjXTkq8Vi3trbGiy++GG9/+9vf8N3W7Kn9q2effTbq6+uLHgMAgL+xbNmyGDBgwE7XOFP7Vz179oyIiKcfHRy1PezKKHenD3tX0SMAAG9gc2yKB+Lutk7bGVH7V1u3HNT2qIranqK23FWX9il6BADgjfx1P8Gb2Rqq3gAASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACC9so3ap556KkqlUnz0ox8tehQAADpY2UYtAACVo7roATpK//7947HHHou6urqiRwEAoIOVbdTus88+cfDBBxc9BgAAnaBstx/YUwsAUDnKNmoBAKgcohYAgPTKdk/tG2lubo7m5ua2j5uamgqcBgCAPVGxZ2qnTp0adXV1bbf6+vqiRwIAYDdVbNROmjQpGhsb227Lli0reiQAAHZTxW4/qKmpiZqamqLHAACgHVTsmVoAAMqHqAUAID1RCwBAeqIWAID0yvaFYoMHD47W1taixwAAoBM4UwsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApFdd9AB7m3/88rlRvU/Xoseggx0wa0nRI9BJXnzvC0WPAEAncKYWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgvbKK2qeeeipKpVJ89KMfLXoUAAA6UVlFLQAAlUnUAgCQ3i5F7dq1a6NLly5xyimnbHP/woULo1QqRalUiiVLlmzz2JgxY6Jbt27R3NwcGzdujKuvvjrGjRsX9fX1UVNTE3379o0zzjgjFixYsN2fd8MNN0SpVIobbrgh7r333jjyyCPjLW95S+y3334xYcKEWL169TZrDzzwwIiIuPHGG9vmKZVKMWPGjF35MgEASKZ6Vxb37t07hg8fHrNmzYqWlpbo0qVLRERMnz69bc306dNjyJAhERGxYcOGmDdvXhx55JFRU1MTy5cvj4suuije+973xkknnRS9e/eOP/3pT3HHHXfEr371q5g5c2aMHDlyuz/3jjvuiLvuuive//73x5FHHhkzZ86Mm266KZYuXRoPPPBAREQceuihMXHixLjyyitj+PDhMX78+LbPHzx48K7+vQAAkMguRW1ERENDQyxYsCAeeeSROOKIIyLi1ZAdNmxYvPLKKzF9+vQ477zzIiJizpw50dzcHA0NDRHxahQ/88wz0b9//22ec/HixTF69Oj4yle+Evfdd992f+a0adNixowZcdRRR0VEREtLS7zvfe+LGTNmxLx582L06NFx6KGHxkUXXRRXXnllHHrooXHJJZfs9Otobm6O5ubmto+bmpp29a8CAIC9xC7vqd0aqP/3f/8XEa8G5syZM6OhoSEaGhq2O2sb8eoWhIiImpqa7YI2IuKQQw6JhoaGmDlzZmzatGm7x88+++y2oI2I6NKlS0yYMCEiIubPn7+rX0JEREydOjXq6urabvX19bv1PAAAFG+Xo/aYY46JLl26tAXrggULorGxMcaOHRsNDQ2xfPnyeOyxxyLi1ajt1q1bjBo1qu3zFy5cGGeffXYMHDgw9t1337Z9r9OmTYuNGzfGCy+8sN2f+e53v3u7+wYMGBAREevWrdvVLyEiIiZNmhSNjY1tt2XLlu3W8wAAULxd3n5QW1sbI0aMiNmzZ8emTZti+vTpUSqVoqGhIV5++eWIeDVmBw0aFA899FAce+yxse+++0bEq9sRxo4dGxERxx9/fAwdOjR69OgRpVIpfvnLX8aiRYu22RLw2j9zu8GrXx29paVlV7+EiHj1rHFNTc1ufS4AAHuXXY7aiFe3IMyfPz8eeuihmDFjRhxyyCGx//77R0TEgQceGNOnT4+hQ4fGpk2b2rYrRERMmTIlmpubY9asWXH00Udv85zz5s2LRYsW7cGXAgBApdqt69RuDdV77703Zs2a1Xb2NSJi7NixMWPGjLY9t1v300ZELF26NN761rduF7Qvv/xyPProo7szyja2Xo1hd8/eAgCQ025F7dFHHx3V1dVx7bXXxosvvrhN1DY0NMQLL7wQP/nJT6J79+7bXKJr0KBBsXbt2li8eHHbfS0tLfGFL3whVq1atQdfxqt69+4dpVLJ/lgAgAqzW9sPevToESNHjoy5c+dGVVVVHHvssW2PbT2Lu2rVqhg3blzss88+bY9dcMEFce+998bRRx8dZ511VnTt2jVmzJgRzz33XIwZM2aP3yRh61wzZ86Mj3zkIzF06NCoqqqKj3zkIzFo0KA9em4AAPZeu/02uVvj9bDDDotevXq13f/2t789hg0bFhHbbj2IiDjllFPi1ltvjYMOOih+9rOfxc033xwHH3xwPPTQQ+0WnT/96U/jxBNPjDvvvDMuueSSmDx5cvz5z39ul+cGAGDvVGptbW0teoi9QVNTU9TV1cXhZ/xHVO/Ttehx6GAHXLDkjRdRFl587/aXCQQgh82tm2JG/E80Njbu8GpYr7XbZ2oBAGBvIWoBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgveqiB9jbtNSUIvYtFT0GHWzd5IFFj0An+fMVQ4oegU405KJ5RY8AFMSZWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKSXOmo3btwYV199dYwbNy7q6+ujpqYm+vbtG2eccUYsWLCg6PEAAOgkqaN2zZo1cdFFF0Vzc3OcdNJJ8dnPfjbGjBkTd999dxx55JExf/78okcEAKATVBc9wJ7o3bt3PPPMM9G/f/9t7l+8eHGMHj06vvKVr8R99923w89tbm6O5ubmto+bmpo6dFYAADpO6jO1NTU12wVtRMQhhxwSDQ0NMXPmzNi0adMOP3fq1KlRV1fXdquvr+/ocQEA6CCpozYiYuHChXH22WfHwIEDY999941SqRSlUimmTZsWGzdujBdeeGGHnzdp0qRobGxsuy1btqyTJwcAoL2k3n4wZ86cGDt2bEREHH/88TF06NDo0aNHlEql+OUvfxmLFi3aZovBa9XU1ERNTU1njgsAQAdJHbVTpkyJ5ubmmDVrVhx99NHbPDZv3rxYtGhRQZMBANCZUm8/WLp0abz1rW/dLmhffvnlePTRRwuaCgCAzpY6agcNGhRr166NxYsXt93X0tISX/jCF2LVqlUFTgYAQGdKvf3gggsuiHvvvTeOPvroOOuss6Jr164xY8aMeO6552LMmDExY8aMokcEAKATpD5Te8opp8Stt94aBx10UPzsZz+Lm2++OQ4++OB46KGHYtCgQUWPBwBAJym1tra2Fj3E3qCpqSnq6urisH+eEl327Vr0OHSwns/s+KoYlJ8/j9+36BHoREMumlf0CEA72ty6KWbE/0RjY2PU1tbudG3qM7UAABAhagEAKAOiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpVRc9wN6mZm1LVO/TUvQYdLB1Q2uKHoFOcvAVzxU9Ap1o+SfeU/QIdJI+P5hb9AjsZZypBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKS3R1E7Y8aMKJVKcckll7TTOAAAsOucqQUAID1RCwBAeqIWAID02i1qH3jggRgzZkz07NkzevXqFWeeeWYsWbJku3UrV66Mz372szFkyJCoqamJPn36xJlnnhm///3vd/i8u7J+8ODBMXjw4Fi3bl185jOfifr6+qiuro4bbrihvb5MAAD2QtXt8STz5s2LqVOnxgknnBAXXHBBLF68OG6//faYNWtWzJs3Lw466KCIiFi6dGmMGTMmnn322Tj++ONj/PjxsXLlyrjtttvinnvuifvvvz9GjRrV9ry7uj4iorm5OcaOHRsvvfRSnHrqqVFdXR39+vVrjy8TAIC9VLtE7T333BPf//734xOf+ETbfT/4wQ/ik5/8ZEycODGmTZsWERHnnHNO/OUvf4lf//rXMW7cuLa1X/va1+Lwww+P8847L37729+23b+r6yMili9fHsOHD4/Zs2dHt27dXnfm5ubmaG5ubvu4qalp9/8CAAAoVLtsPxg2bFicd95529x33nnnxdChQ+Ouu+6KVatWxYIFC2LOnDkxYcKEbQL1tZ//u9/9rm1bwa6uf61vfetbOw3aiIipU6dGXV1d262+vn53vnQAAPYC7XKm9qijjoqqqm37uKqqKo466qj44x//GIsWLYo//vGPERGxYsWKHV7X9vHHH2/77zvf+c6YN2/eLq3fqmvXrvGud73rDWeeNGlSfO5zn2v7uKmpSdgCACTVLlH7entWt97f2NgYa9asiYiIu+66K+66667Xfa7169dHROzy+q369u0bpVLpDWeuqamJmpqaN1wHAMDer122H6xYsWKn99fV1UVtbW1ERFx99dXR2tr6urcJEyZEROzy+q3eTNACAFBe2iVqZ8+eHVu2bNnmvi1btsScOXOiVCrF8OHD265SMHfu3Df1nLu6HgCAytUuUfvkk0/Gj370o23u+9GPfhRPPvlknHzyybH//vvHEUccEaNGjYpbbrkl/vu//3u759iyZUv85je/aft4V9cDAFC52mVP7bhx4+LCCy+Mu+++Ow455JBYvHhxTJs2Lfr06RNXXnll27pbbrklGhoa4kMf+lBcccUVMWLEiOjWrVs888wzMXfu3Fi1alVs2LBht9cDAFCZ2uVM7ejRo+P++++PxsbGuOqqq2LGjBkxfvz4mDt3btsbL0REHHjggbFgwYL42te+Fi+99FJcf/318YMf/CAWLlwYxxxzTNxyyy3bPO+urgcAoDKVWltbW4seYm/Q1NQUdXV1MfqES6N6n65Fj0MHW39Al6JHoJP0u++5okegEy0f17/oEegkfX7gNTeVYHPrppgR/xONjY1tFxF4Pe1yphYAAIokagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHrVRQ+wt1k3pDq61PhrKXf9Hnml6BGADrD/w01Fj0AnaTn2sKJHoBNs2bwh4oH/eVNrnakFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApJc6ap966qkolUo7vQ0ePLjoMQEA6GDVRQ/QHt7xjnfEv/zLv+zwsV69enXuMAAAdLqyiNohQ4bEJZdcUvQYAAAUJPX2AwAAiBC1AACUgbLYfrBkyZLX3X4wevToOOGEEzp3IAAAOlVZRO3SpUvj61//+g4fmzhx4g6jtrm5OZqbm9s+bmpq6rD5AADoWGWx/WDcuHHR2tq6w9sVV1yxw8+ZOnVq1NXVtd3q6+s7d2gAANpNWUTt7pg0aVI0Nja23ZYtW1b0SAAA7Kay2H6wO2pqaqKmpqboMQAAaAcVe6YWAIDyIWoBAEivLLYf7OySXhERX/7yl6Nr166dNxAAAJ2qLKJ2Z5f0ioi46KKLRC0AQBlLHbWDBw+O1tbWoscAAKBg9tQCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0qoseYG/R2toaEREtGzcUPAmdYfNmx7lSVG1pLnoEOlFLy75Fj0Anadm8pegR6ASbN7/6//CtnbYzpdY3s6oCPPvss1FfX1/0GAAA/I1ly5bFgAEDdrpG1P7Vli1b4vnnn4+ePXtGqVQqepxO09TUFPX19bFs2bKora0tehw6kGNdORzryuJ4V45KPNatra3x4osvxtvf/vaoqtr5rlnbD/6qqqrqDX8CKGe1tbUV8w+k0jnWlcOxriyOd+WotGNdV1f3ptZ5oRgAAOmJWgAA0hO1Fa6mpib+/d//PWpqaooehQ7mWFcOx7qyON6Vw7HeOS8UAwAgPWdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACk9/8uhya2VRje1gAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "first head of last state dec_enc_attns\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_1262415/2392112591.py:8: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n", + " ax.set_xticklabels(['']+sentences[0].split(), fontdict={'fontsize': 14}, rotation=90)\n", + "/tmp/ipykernel_1262415/2392112591.py:9: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n", + " ax.set_yticklabels(['']+sentences[2].split(), fontdict={'fontsize': 14})\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAArUAAALUCAYAAAAVCMq+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAsV0lEQVR4nO3de5yWdZ34//c9jI7IYUARMhhBU9b9aaEoQuoqg614Si3LdXWT8qvbwRQ7uEWl62GVaqv10EM7Kmrptx76sxW10HUhkINiAhV5SMqEjIMcZlRkgGG+fxjzkEAUmLkv3vf9fD4e98PHXPc1N+/hiuY113zu6yq1tbW1BQAAJFZT9AAAALCjRC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKRXW/QAQOdbu3Zt/M///E88/fTT8eqrr8Zll10WERFr1qyJ5ubm6NOnT9TU+BkXgLxKbW1tbUUPAXSe++67L/71X/81li1bFm1tbVEqlaK1tTUiIh5//PF473vfG3fccUecffbZBU8KANvPqRmoYNOnT48PfehDUVdXF9dff/1m4XrEEUfE/vvvH/fcc09BEwJAx7D8ACrY1VdfHb169Ypf/epX0adPn1i+fPlm+xx++OHx2GOPFTAdAHQcZ2qhgj322GNx2mmnRZ8+fd50n4aGhli8eHEZpwKAjidqoYK1tLREz549t7rPqlWrvEkMgPR8J4MKtt9++8Xs2bO3us/MmTPjwAMPLNNEANA5RC1UsDPOOCOmT58et9566xaf/8Y3vhG//e1v45/+6Z/KPBkAdCyX9IIK9sorr8SIESPiqaeeilGjRkVLS0tMnz49Pve5z8XMmTNjxowZccghh8SMGTOirq6u6HEBYLuJWqhwK1eujE9/+tPx05/+tP36tBERpVIpzjzzzLjpppuid+/eBU4IADtO1EKVWL58ecyePTtWrFgRPXv2jGHDhkW/fv2KHgsAOoSoBQAgPW8UgwrWpUuXuPrqq7e6zzXXXBO1te7DAkBuohYqWFtbW7ydX8b4hQ0A2YlaqHLLli2Lrl27Fj0GAOwQv3OECnP77bdv8vHcuXM32xYR0draGgsXLozbb789Dj744HKNBwCdwhvFoMLU1NREqVR6y/02/tPv2rVr3HPPPXHCCSd09mgA0GlELVSY2267LSJej9bzzjsvTj/99DjttNM2269Lly6xxx57xHvf+17XqQUgPVELFexjH/tYfOADH4hTTz216FEAoFOJWgAA0vNGMagC69evj2eeeSZWrVq1ya1y3+iYY44p81QA0HFELVSwtra2uPzyy+PGG2+Ml19+eav7vlnsAkAGohYq2NVXXx3XXHNN9OrVK84999wYMGCAu4cBUJGsqYUKNmjQoCiVSvHEE0/EnnvuWfQ4ANBp3FEMKtjixYvj9NNPF7QAVDxRCxVs3333jebm5qLHAIBOJ2qhgn3yk5+M+++/P5YuXVr0KADQqayphQrywgsvbPJxW1tbfP7zn485c+bE5ZdfHkOHDo2ePXtu8XP32WefcowIAJ1C1EIFqampiVKptNn2tra2LW7fqFQqxfr16ztzNADoVK7tAxXk3HPP3Wq8AkClcqYWAID0vFEMAID0RC1UsN/97ndxww03xLJly7b4/NKlS+OGG26Ip556qsyTAUDHsvwAKti5554bjzzySCxcuDBqajb/Gba1tTUGDRoU73vf++LWW28tYEIA6BjO1EIFmzZtWhx33HFbDNqIiC5dusRxxx0XU6dOLfNkANCxRC1UsMWLF0dDQ8NW9+nfv3/85S9/KdNEwI7ab7/94sILLyx6DNjpiFqoYN26dXvLu4ktXbo0dttttzJNBOyol1566U1vogLVTNRCBRs6dGj87Gc/i1WrVm3x+ZUrV8a9994bQ4cOLe9gwHZ7z3veE88++2zRY8BOR9RCBbvwwgtj+fLl0djYuNm62V/+8pfR2NgYK1eujE9/+tMFTQhsqy984QsxceLEmDx5ctGjwE7F1Q+gwn3uc5+L//qv/4pSqRR1dXXxjne8IxYvXhwtLS3R1tYWl156aXzta18rekzgbbr99tvjpz/9aUyaNClOP/30GDZsWPTr12+LdxM899xzC5gQiiFqoQrcf//9cdNNN8Xs2bOjqakpevXqFUcccURceOGFceKJJxY9HrANampqolQqxd9++35j1La1tUWpVIrW1tZyjweFEbUAkMhtt932tvcdM2ZMJ04COxdRCwBAerVFDwB0vueffz5+/OMfx9y5c6O5uTl69uwZhxxySJxzzjkxaNCgoscDgB3mTC1UuOuvvz7+7d/+LdavX7/ZGrxddtklvv71r8fYsWMLmo6O9uqrr8aqVavedC3lPvvsU+aJ6Cz33ntv3HXXXfH000/H6tWr47nnnouIiKeffjruu+++OOecc6J///4FTwnlI2qhgt1///1x6qmnRp8+feIzn/lMNDY2xt577x2LFy+OyZMnx7e+9a1Yvnx53HfffXHyyScXPS474Ic//GF885vfjGeeeeZN9ymVSrF+/foyTkVn2LBhQ/zzP/9z3H333RER0bVr13jttdfaf5BZsmRJDBgwIK666qoYN25ckaNCWYnaKvS73/0uvv3tb8fs2bPf9IxOqVSKBQsWFDAdHWnUqFHx61//OubOnRsDBgzY7PmFCxfGoYceGkOGDIlHHnmkgAnpCDfffHNceOGFUVtbG0cddVQMGDAgamu3vLrs1ltvLfN0dLRvfvObcemll8YnPvGJ+OpXvxrf+ta34uqrr97k/8tHjRoVa9eujUcffbTASaG8rKmtMr/85S/jhBNOiJaWlqitrY1+/fpt8Zufn3Uqw5NPPhnnnHPOFoM2IqKhoSHOPPPMuPPOO8s8GR3puuuuiz59+sSjjz4agwcPLnocOtmECRNi2LBhcdNNN0VEbPH6tPvvv3888MAD5R4NCiVqq8wXv/jFWL9+ffzgBz+IMWPGRJcuXYoeiU60du3a6Nat21b36d69e6xdu7ZME9EZ/vSnP8X5558vaKvEc889FxdeeOFW99lzzz1j+fLlZZoIdg5uk1tl5s2bF2eddVacd955grYKDB48OCZOnPim6yjXr18f999/vxhKbu+993aR/SrStWvXaGpq2uo+f/rTn6JXr17lGQh2EqK2ynTr1i369u1b9BiUybnnnhvPPPNMjB49On71q19t8twTTzwRJ554YjzzzDMu0J7cmDFj4uc//3m8+uqrRY9CGRx66KExadKkWLNmzRafX7FiRfziF7+IESNGlHkyKJaorTInnXRSTJs2regxKJOxY8fGqaeeGpMnT44jjjgievToEe9617uiR48eMXz48HjkkUfi1FNPdUmv5L7yla/EsGHD4h//8R9j6tSp8corrxQ9Ep3o4osvjkWLFsUZZ5wRixYt2uS5BQsWxAc+8IFoamqKiy++uKAJoRiuflBlli5dGkcffXSccMIJ8dWvfjV23333okeiDG6//fa47bbbNrn5wqGHHhpjxoyJj3zkI0WPxw7auJSora1ti28a2sglvSrHuHHj4mtf+1qUSqXo1q1bvPrqq+3raNva2uKyyy6LK6+8sugxoaxEbYUbNWrUZttWrVoV8+bNi27dusXgwYOjZ8+em+1TKpVc4gmSGDly5FZj9o0mT57cydNQLg8//HB8+9vfjsceeyxWrFgRPXv2jOHDh8fFF18co0ePLno8KDtRW+FqarZvhUmpVPLGkwrS2toaixYtihdffDHWrVu3xX2OOeaYMk8FAB1H1EIF27BhQ1x77bVx/fXXx4oVK7a6rx9iAMjMdWqhgo0bNy7+8z//M/r27Rsf+9jHYu+9937TO00BQGbO1FaZ1tbWePXVV6N79+5bXJqw8flu3bq5jm0FeMc73hG9e/eO2bNnR/fu3Ysehw5y3nnnRalUimuvvTb69esX55133tv6vFKpFD/84Q87eTo6Wk1NTdTU1MTvfve7GDx4cNTU1LytNdTeGEi1EbVV5vLLL4+vf/3rsXDhwthrr702e37ZsmWxzz77xLhx4+Lyyy8vYEI6Uvfu3eMTn/hEfOMb3yh6FDrQxqh56qmn2iPn7bBWPqeNbwS84447YsCAAd4YCG9C1FaZoUOHxt57773Ve4K///3vjxdffHGzi/WTz5FHHhmDBg2KO++8s+hR6EB/+tOfIiKif//+UVtb2/7x2zFw4MDOGgugUBbXVZk//OEP0djYuNV9/u7v/i6mT59eponoTF/+8pfjwx/+cDz55JMxdOjQosehg/xtmArV6rV8+fKYN29eNDU1RX19fQwZMiT23HPPoseCQojaKrNu3bq3/FVlqVR609svksvJJ58cEyZMiBNPPDFOPfXUGDJkyBavSxzx+i11yWv9+vVx4403xl133RVPP/10rF69un095dy5c+N73/teXHLJJTF48OCCJ6UjPP/88zF27Nh44IEH4o2/cC2VSnHKKafEddddF4MGDSpuQHbIzJkz48tf/nLMnj07SqVSDB8+PK655po44ogjih5tp2b5QZUZMmRI1NbWbnVpwWGHHRZr1qyJ+fPnl3EyOkNLS0ucf/75ceedd7Z/4/vbtXgb70JlrWVer732Whx//PExY8aM6NOnT+yyyy7xl7/8pf2YNjU1xTve8Y743Oc+F//xH/9R8LTsqAULFsRRRx0VS5cujQMOOCCOOuqo6NevXyxZsiRmzJgRzz77bPTt2zdmzJgR++23X9Hjso1+85vfxPDhwzc7udS1a9d4/PHH46CDDiposp3f9l2Zn7Q++MEPxty5c+Pyyy/fLGJaW1vjsssui7lz58aHP/zhgiakI332s5+NH//4x/Hud787rrrqqvj+978ft9xyyyaPW2+9NW655ZaiR2UHXHvttTF9+vQYP358LF68OM4///xNnq+vr49jjz02Jk2aVNCEdKQvfOELsWzZsvjOd74TTz/9dNxyyy0xfvz4uOWWW+Kpp56Km2++OZYtWxZf+MIXih6V7fDVr3411qxZE1/+8pdj8eLFsXjx4rjsssvitddei6997WtFj7dTc6a2yrzyyisxbNiwePbZZ+Nd73pXNDY2Rv/+/ePPf/5zTJ48ORYsWBB///d/H7NmzXIJqArQt2/fGDhwYMycOdP1aSvY4MGDo6Ghof3W1ldeeWVcddVVm/zg+qlPfSruueeeWLJkSVFj0kF69+4dI0eOjHvvvfdN9znttNNi6tSpsXLlyjJORkfYZ599YtCgQTF16tRNth977LHx/PPPb9MbQ6uN73JVpnv37jF16tT45Cc/Gffee28899xz7c/V1NTEhz70objpppsEbYVYs2ZNNDY2CtoK98ILL8QHPvCBre7To0ePaGpqKtNEdKbW1ta3/BX0wQcf7HJeSS1ZsiTOOuuszbYPHz48HnvssQImysN3uiq01157xd133x1LliyJJ554IpqamqJXr15x+OGHR9++fYsejw502GGHbfKDC5WpR48esXTp0q3us2DBgi1em5p8hg4d+pbveZg/f34cfvjhZZqIjrRu3botnljq1q1brFu3roCJ8hC1Vaxfv35x8sknFz0Gnejaa6+N4447Lu6///445ZRTih6HTjJixIiYOHFirFq1Knr16rXZ8wsXLowHH3zwLc/mksM111wTxx13XPzgBz/YbP10RMT3vve9mDRpUvtyFKgWorbCuZ1mdXv44Ydj5MiRcdppp8WoUaPe9JJepVIpLrvssgImpCNceuml0djYGMcdd1zccMMN7ZfyWr16dcycOTMuuuiiWL9+fXz2s58teFK2x1VXXbXZtsbGxvj4xz8e3/zmNze5+sH06dPj2WefjdGjR8cjjzwSRx55ZAETs6N+9KMfxaxZszbZtvG3bieddNJm+5dKpa3eVKlaeKNYhXM7zermeFePm2++OcaOHbvF49ilS5e46aabtnhWj53f2/13/Lf8u85pe463Y/06Z2or3B//+MeIeP12mm/8mOrgjSLV45Of/GSMHDkyvvOd78Rjjz0WK1asiJ49e8bw4cPjU5/6lGtbJubfcXXxfXr7OVMLAEB6br4AAEB6ohYAgPRELQAA6YnaKtfS0hJXXHFFtLS0FD0Kncyxrh6OdXVxvKuHY7113ihW5Zqbm6O+vj6ampq2eP1SKodjXT0c6+rieFcPx3rrnKkFACA9UQsAQHpuvvBXGzZsiBdffDF69OgRpVKp6HHKprm5eZP/Urkc6+rhWFcXx7t6VOOxbmtri5dffjne+c53vuXd1qyp/atFixZFQ0ND0WMAAPA3Fi5cGAMGDNjqPs7U/lWPHj0iIuK0/z4rdum2a8HT0NnWtPqffrUYucczRY9AGd3468aiR6BM9v0/vyl6BMpgfayLR+PB9k7bGt/Z/2rjkoNduu0qaqtAq6itGl27O9bVpGb33YoegTKpLe1S9AiUw1/XE7ydpaHeKAYAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpVWzUPv/881EqleKjH/1o0aMAANDJKjZqAQCoHrVFD9BZ+vfvH0899VTU19cXPQoAAJ2sYqN2l112iQMPPLDoMQAAKIOKXX5gTS0AQPWo2KgFAKB6iFoAANKr2DW1b6WlpSVaWlraP25ubi5wGgAAdkTVnqkdP3581NfXtz8aGhqKHgkAgO1UtVE7bty4aGpqan8sXLiw6JEAANhOVbv8oK6uLurq6ooeAwCADlC1Z2oBAKgcohYAgPRELQAA6YlaAADSq9g3ig0aNCja2tqKHgMAgDJwphYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASK+26AF2Nk+91C+6rK4regw62Xffc0fRI1AmP1kxvOgRKKPWNb6tQbVyphYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACC9iora559/PkqlUnz0ox8tehQAAMqooqIWAIDqJGoBAEhvm6J25cqV0aVLlzjllFM22T537twolUpRKpXiueee2+S5kSNHRteuXaOlpSXWrl0bN954Y4wePToaGhqirq4u+vbtGx/84Adjzpw5m/15EyZMiFKpFBMmTIiHHnoojjzyyNh9991jzz33jDFjxsTy5cs32XffffeNiIjbbrutfZ5SqRRTpkzZli8TAIBkardl5969e8eQIUNi2rRp0draGl26dImIiMmTJ7fvM3ny5Nh///0jImLNmjUxa9asOPLII6Ouri4WL14cl1xySfzDP/xDnHTSSdG7d+/4wx/+EPfdd1/8/Oc/j6lTp8awYcM2+3Pvu+++eOCBB+L9739/HHnkkTF16tS4/fbbY8GCBfHoo49GRMQhhxwSY8eOjeuvvz6GDBkSp59+evvnDxo0aFv/XgAASGSbojYiorGxMebMmRO/+tWv4ogjjoiI10N28ODB8dprr8XkyZPjggsuiIiIGTNmREtLSzQ2NkbE61H8wgsvRP/+/Td5zfnz58eIESPiS1/6Ujz88MOb/ZkTJ06MKVOmxFFHHRUREa2trfG+970vpkyZErNmzYoRI0bEIYccEpdccklcf/31ccghh8QVV1yx1a+jpaUlWlpa2j9ubm7e1r8KAAB2Etu8pnZjoP7v//5vRLwemFOnTo3GxsZobGzc7KxtxOtLECIi6urqNgvaiIiDDjooGhsbY+rUqbFu3brNnj/77LPbgzYiokuXLjFmzJiIiJg9e/a2fgkRETF+/Pior69vfzQ0NGzX6wAAULxtjtpjjjkmunTp0h6sc+bMiaamphg1alQ0NjbG4sWL46mnnoqI16O2a9euMXz48PbPnzt3bpx99tmxzz77xK677tq+7nXixImxdu3aeOmllzb7Mw877LDNtg0YMCAiIlatWrWtX0JERIwbNy6ampraHwsXLtyu1wEAoHjbvPygZ8+eMXTo0Jg+fXqsW7cuJk+eHKVSKRobG2P16tUR8XrMDhw4MB5//PE49thjY9ddd42I15cjjBo1KiIijj/++DjggAOie/fuUSqV4mc/+1nMmzdvkyUBb/wzNxu89vXRW1tbt/VLiIjXzxrX1dVt1+cCALBz2eaojXh9CcLs2bPj8ccfjylTpsRBBx0Ue+21V0RE7LvvvjF58uQ44IADYt26de3LFSIirrnmmmhpaYlp06bF0Ucfvclrzpo1K+bNm7cDXwoAANVqu65TuzFUH3rooZg2bVr72deIiFGjRsWUKVPa19xuXE8bEbFgwYLYY489Ngva1atXx5NPPrk9o2xi49UYtvfsLQAAOW1X1B599NFRW1sbN998c7z88subRG1jY2O89NJL8cMf/jC6deu2ySW6Bg4cGCtXroz58+e3b2ttbY3Pf/7zsWzZsh34Ml7Xu3fvKJVK1scCAFSZ7Vp+0L179xg2bFjMnDkzampq4thjj21/buNZ3GXLlsXo0aNjl112aX/uoosuioceeiiOPvroOPPMM2O33XaLKVOmxJ///OcYOXLkDt8kYeNcU6dOjY985CNxwAEHRE1NTXzkIx+JgQMH7tBrAwCw89ru2+RujNdDDz00evXq1b79ne98ZwwePDgiNl16EBFxyimnxN133x377bdf/OhHP4o777wzDjzwwHj88cc7LDrvuOOOOPHEE+P++++PK664Ii677LL44x//2CGvDQDAzqnU1tbWVvQQO4Pm5uaor6+Pg/7vpdFld1dFqHTffc8dRY9AmfxkxfC33omK8f/PG1r0CJTJ4POeKHoEymB927qYEv8dTU1NW7wa1htt95laAADYWYhaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASK+26AF2Nof1eyF27b5r0WPQyY6o26XoESiTc/5neNEjUEb93r2s6BGAgjhTCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRSR+3atWvjxhtvjNGjR0dDQ0PU1dVF375944Mf/GDMmTOn6PEAACiT1FG7YsWKuOSSS6KlpSVOOumk+MxnPhMjR46MBx98MI488siYPXt20SMCAFAGtUUPsCN69+4dL7zwQvTv33+T7fPnz48RI0bEl770pXj44Ye3+LktLS3R0tLS/nFzc3OnzgoAQOdJfaa2rq5us6CNiDjooIOisbExpk6dGuvWrdvi544fPz7q6+vbHw0NDZ09LgAAnSR11EZEzJ07N84+++zYZ599Ytddd41SqRSlUikmTpwYa9eujZdeemmLnzdu3LhoampqfyxcuLDMkwMA0FFSLz+YMWNGjBo1KiIijj/++DjggAOie/fuUSqV4mc/+1nMmzdvkyUGb1RXVxd1dXXlHBcAgE6SOmqvueaaaGlpiWnTpsXRRx+9yXOzZs2KefPmFTQZAADllHr5wYIFC2KPPfbYLGhXr14dTz75ZEFTAQBQbqmjduDAgbFy5cqYP39++7bW1tb4/Oc/H8uWLStwMgAAyin18oOLLrooHnrooTj66KPjzDPPjN122y2mTJkSf/7zn2PkyJExZcqUokcEAKAMUp+pPeWUU+Luu++O/fbbL370ox/FnXfeGQceeGA8/vjjMXDgwKLHAwCgTFKfqY2IOOOMM+KMM87YbPuECRNiwoQJ5R8IAICyS32mFgAAIkQtAAAVQNQCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAIL3aogfY2ey3+0ux2+7+Wirdvj8/v+gRKJP9D19U9AiU0Yn95hc9AmUyKXoWPQI7GWdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOntUNROmTIlSqVSXHHFFR00DgAAbDtnagEASE/UAgCQnqgFACC9DovaRx99NEaOHBk9evSIXr16xRlnnBHPPffcZvstXbo0PvOZz8T+++8fdXV10adPnzjjjDPit7/97RZfd1v2HzRoUAwaNChWrVoVn/70p6OhoSFqa2tjwoQJHfVlAgCwE6rtiBeZNWtWjB8/Pk444YS46KKLYv78+XHvvffGtGnTYtasWbHffvtFRMSCBQti5MiRsWjRojj++OPj9NNPj6VLl8Y999wTkyZNikceeSSGDx/e/rrbun9EREtLS4waNSpeeeWVOPXUU6O2tjb69evXEV8mAAA7qQ6J2kmTJsV3vvOd+PjHP96+7bvf/W584hOfiLFjx8bEiRMjIuLcc8+Nv/zlL/GLX/wiRo8e3b7vV77ylTj88MPjggsuiF//+tft27d1/4iIxYsXx5AhQ2L69OnRtWvXN525paUlWlpa2j9ubm7e/r8AAAAK1SHLDwYPHhwXXHDBJtsuuOCCOOCAA+KBBx6IZcuWxZw5c2LGjBkxZsyYTQL1jZ//m9/8pn1Zwbbu/0Zf//rXtxq0ERHjx4+P+vr69kdDQ8P2fOkAAOwEOuRM7VFHHRU1NZv2cU1NTRx11FHx+9//PubNmxe///3vIyJiyZIlW7yu7dNPP93+34MPPjhmzZq1TftvtNtuu8W73/3ut5x53Lhx8dnPfrb94+bmZmELAJBUh0Ttm61Z3bi9qakpVqxYERERDzzwQDzwwANv+lqvvvpqRMQ2779R3759o1QqveXMdXV1UVdX95b7AQCw8+uQ5QdLlizZ6vb6+vro2bNnRETceOON0dbW9qaPMWPGRERs8/4bvZ2gBQCgsnRI1E6fPj02bNiwybYNGzbEjBkzolQqxZAhQ9qvUjBz5sy39Zrbuj8AANWrQ6L22Wefje9///ubbPv+978fzz77bJx88smx1157xRFHHBHDhw+Pu+66K37yk59s9hobNmyIX/7yl+0fb+v+AABUrw5ZUzt69Oi4+OKL48EHH4yDDjoo5s+fHxMnTow+ffrE9ddf377fXXfdFY2NjXHWWWfFddddF0OHDo2uXbvGCy+8EDNnzoxly5bFmjVrtnt/AACqU4ecqR0xYkQ88sgj0dTUFDfccENMmTIlTj/99Jg5c2b7jRciIvbdd9+YM2dOfOUrX4lXXnklbr311vjud78bc+fOjWOOOSbuuuuuTV53W/cHAKA6ldra2tqKHmJn0NzcHPX19XHFY8fFbt075AQ2O7GbHmssegTKZP9BW34jK5XpxH7zix6BMpl0cM+iR6AM1retiynx39HU1NR+EYE30yFnagEAoEiiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApFdb9AA7my7RFl2iregx6GS993q56BEok/pdXyt6BMpo0pL/r+gRKJtFRQ/ATsaZWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6qaP2+eefj1KptNXHoEGDih4TAIBOVlv0AB3hXe96V/zLv/zLFp/r1atXeYcBAKDsKiJq999//7jiiiuKHgMAgIKkXn4AAAARohYAgApQEcsPnnvuuTddfjBixIg44YQTyjsQAABlVRFRu2DBgrjyyiu3+NzYsWO3GLUtLS3R0tLS/nFzc3OnzQcAQOeqiOUHo0ePjra2ti0+rrvuui1+zvjx46O+vr790dDQUN6hAQDoMBURtdtj3Lhx0dTU1P5YuHBh0SMBALCdKmL5wfaoq6uLurq6oscAAKADVO2ZWgAAKoeoBQAgvYpYfrC1S3pFRHzxi1+M3XbbrXwDAQBQVhURtVu7pFdExCWXXCJqAQAqWOqoHTRoULS1tRU9BgAABbOmFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBIT9QCAJCeqAUAID1RCwBAeqIWAID0RC0AAOmJWgAA0hO1AACkJ2oBAEhP1AIAkJ6oBQAgPVELAEB6ohYAgPRELQAA6YlaAADSE7UAAKQnagEASE/UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApFdb9AA7i7a2toiIWPPK+oInoRxaV7cUPQJlsm7XtUWPQBmtX1sqegTKpW1d0RNQBuvj9eO8sdO2ptT2dvaqAosWLYqGhoaixwAA4G8sXLgwBgwYsNV9RO1fbdiwIV588cXo0aNHlErV85N+c3NzNDQ0xMKFC6Nnz55Fj0Mncqyrh2NdXRzv6lGNx7qtrS1efvnleOc73xk1NVtfNWv5wV/V1NS85U8Alaxnz55V8w+k2jnW1cOxri6Od/WotmNdX1//tvbzRjEAANITtQAApCdqq1xdXV38+7//e9TV1RU9Cp3Msa4ejnV1cbyrh2O9dd4oBgBAes7UAgCQnqgFACA9UQsAQHqiFgCA9EQtAADpiVoAANITtQAApCdqAQBI7/8BBrobLRsnoiwAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "print('first head of last state enc_self_attns')\n", + "showgraph(enc_self_attns)\n", "\n", - " print('first head of last state dec_self_attns')\n", - " showgraph(dec_self_attns)\n", + "print('first head of last state dec_self_attns')\n", + "showgraph(dec_self_attns)\n", "\n", - " print('first head of last state dec_enc_attns')\n", - " showgraph(dec_enc_attns)" + "print('first head of last state dec_enc_attns')\n", + "showgraph(dec_enc_attns)" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -253,7 +529,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.5" + "version": "3.9.18" } }, "nbformat": 4, diff --git a/5-2.BERT/BERT.ipynb b/5-2.BERT/BERT.ipynb index 6265d39..5735667 100644 --- a/5-2.BERT/BERT.ipynb +++ b/5-2.BERT/BERT.ipynb @@ -13,8 +13,7 @@ "import mindspore.nn as nn\n", "import mindspore.ops as ops\n", "import mindspore.numpy as mnp\n", - "from layers import Dense, Embedding\n", - "from mindspore import ms_function" + "from layers import Dense, Embedding" ] }, { @@ -104,7 +103,7 @@ "\n", " def construct(self, x, seg):\n", " seq_len = x.shape[1]\n", - " pos = mnp.arange(seq_len, dtype=mindspore.int64)\n", + " pos = ops.arange(seq_len, dtype=mindspore.int64)\n", " pos = pos.expand_dims(0).expand_as(x) # (seq_len,) -> (batch_size, seq_len)\n", " embedding = self.tok_embed(x) + self.pos_embed(pos) + self.seg_embed(seg)\n", " return self.norm(embedding)" @@ -226,7 +225,7 @@ " n_vocab, n_dim = embed_weight.shape\n", " self.decoder = Dense(n_dim, n_vocab, has_bias=False)\n", " self.decoder.weight = embed_weight\n", - " self.decoder_bias = mindspore.Parameter(mnp.zeros(n_vocab), 'decoder_bias')\n", + " self.decoder_bias = mindspore.Parameter(ops.zeros(n_vocab), 'decoder_bias')\n", "\n", " def construct(self, input_ids, segment_ids, masked_pos):\n", " output = self.embedding(input_ids, segment_ids)\n", @@ -340,7 +339,7 @@ "metadata": {}, "outputs": [], "source": [ - "@ms_function\n", + "@mindspore.jit\n", "def train_step(input_ids, segment_ids, masked_pos, masked_tokens, isNext):\n", " loss, grads = grad_fn(input_ids, segment_ids, masked_pos, masked_tokens, isNext)\n", " optimizer(grads)\n", @@ -353,20 +352,28 @@ "id": "81bf550b-8239-440d-9fda-c556dee4552c", "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[ERROR] CORE(1267049,7f74549fd4c0,python):2024-04-16-15:56:16.580.126 [mindspore/core/utils/file_utils.cc:253] GetRealPath] Get realpath failed, path[/tmp/ipykernel_1267049/3083615623.py]\n", + "[ERROR] CORE(1267049,7f74549fd4c0,python):2024-04-16-15:56:16.580.172 [mindspore/core/utils/file_utils.cc:253] GetRealPath] Get realpath failed, path[/tmp/ipykernel_1267049/3083615623.py]\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "Epoch: 0010 cost = 54.808880\n", - "Epoch: 0020 cost = 43.177929\n", - "Epoch: 0030 cost = 24.757572\n", - "Epoch: 0040 cost = 17.764444\n", - "Epoch: 0050 cost = 10.951277\n", - "Epoch: 0060 cost = 7.573495\n", - "Epoch: 0070 cost = 5.663257\n", - "Epoch: 0080 cost = 4.734429\n", - "Epoch: 0090 cost = 3.930883\n", - "Epoch: 0100 cost = 3.159842\n" + "Epoch: 0010 cost = 46.552399\n", + "Epoch: 0020 cost = 19.055964\n", + "Epoch: 0030 cost = 15.114850\n", + "Epoch: 0040 cost = 9.543916\n", + "Epoch: 0050 cost = 6.100155\n", + "Epoch: 0060 cost = 2.962293\n", + "Epoch: 0070 cost = 3.004694\n", + "Epoch: 0080 cost = 2.631464\n", + "Epoch: 0090 cost = 2.321460\n", + "Epoch: 0100 cost = 2.230808\n" ] } ], @@ -383,28 +390,10 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "f7da833d-9efb-475f-9aa1-93be15e3ea73", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Hello, how are you? I am Romeo.\n", - "Hello, Romeo My name is Juliet. Nice to meet you.\n", - "Nice meet you too. How are you today?\n", - "Great. My baseball team won the competition.\n", - "Oh Congratulations, Juliet\n", - "Thanks you Romeo\n", - "['[CLS]', '[MASK]', 'congratulations', '[MASK]', '[SEP]', 'nice', 'meet', 'you', 'too', 'how', 'are', 'you', 'today', '[SEP]']\n", - "masked tokens list : [Tensor(shape=[], dtype=Int64, value= 28), Tensor(shape=[], dtype=Int64, value= 12)]\n", - "predict masked tokens list : []\n", - "isNext : False\n", - "predict isNext : False\n" - ] - } - ], + "outputs": [], "source": [ "# Predict mask tokens ans isNext\n", "input_ids, segment_ids, masked_tokens, masked_pos, isNext = map(mindspore.Tensor, zip(batch[0]))\n", @@ -424,7 +413,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.7.13 ('ms1.8')", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -438,7 +427,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.13" + "version": "3.9.18" }, "vscode": { "interpreter": { diff --git a/5-2.BERT/BERT_pytorch.ipynb b/5-2.BERT/BERT_pytorch.ipynb index 215f860..e483616 100644 --- a/5-2.BERT/BERT_pytorch.ipynb +++ b/5-2.BERT/BERT_pytorch.ipynb @@ -1,271 +1,271 @@ { - "cells": [ - { - "cell_type": "code", - "metadata": {}, - "source": [ - "# code by Tae Hwan Jung(Jeff Jung) @graykode\n", - "# Reference : https://github.com/jadore801120/attention-is-all-you-need-pytorch\n", - "# https://github.com/JayParks/transformer, https://github.com/dhlee347/pytorchic-bert\n", - "import math\n", - "import re\n", - "from random import *\n", - "import numpy as np\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.optim as optim\n", - "\n", - "# sample IsNext and NotNext to be same in small batch size\n", - "def make_batch():\n", - " batch = []\n", - " positive = negative = 0\n", - " while positive != batch_size/2 or negative != batch_size/2:\n", - " tokens_a_index, tokens_b_index= randrange(len(sentences)), randrange(len(sentences)) # sample random index in sentences\n", - " tokens_a, tokens_b= token_list[tokens_a_index], token_list[tokens_b_index]\n", - " input_ids = [word_dict['[CLS]']] + tokens_a + [word_dict['[SEP]']] + tokens_b + [word_dict['[SEP]']]\n", - " segment_ids = [0] * (1 + len(tokens_a) + 1) + [1] * (len(tokens_b) + 1)\n", - "\n", - " # MASK LM\n", - " n_pred = min(max_pred, max(1, int(round(len(input_ids) * 0.15)))) # 15 % of tokens in one sentence\n", - " cand_maked_pos = [i for i, token in enumerate(input_ids)\n", - " if token != word_dict['[CLS]'] and token != word_dict['[SEP]']]\n", - " shuffle(cand_maked_pos)\n", - " masked_tokens, masked_pos = [], []\n", - " for pos in cand_maked_pos[:n_pred]:\n", - " masked_pos.append(pos)\n", - " masked_tokens.append(input_ids[pos])\n", - " if random() < 0.8: # 80%\n", - " input_ids[pos] = word_dict['[MASK]'] # make mask\n", - " elif random() < 0.5: # 10%\n", - " index = randint(0, vocab_size - 1) # random index in vocabulary\n", - " input_ids[pos] = word_dict[number_dict[index]] # replace\n", - "\n", - " # Zero Paddings\n", - " n_pad = maxlen - len(input_ids)\n", - " input_ids.extend([0] * n_pad)\n", - " segment_ids.extend([0] * n_pad)\n", - "\n", - " # Zero Padding (100% - 15%) tokens\n", - " if max_pred > n_pred:\n", - " n_pad = max_pred - n_pred\n", - " masked_tokens.extend([0] * n_pad)\n", - " masked_pos.extend([0] * n_pad)\n", - "\n", - " if tokens_a_index + 1 == tokens_b_index and positive < batch_size/2:\n", - " batch.append([input_ids, segment_ids, masked_tokens, masked_pos, True]) # IsNext\n", - " positive += 1\n", - " elif tokens_a_index + 1 != tokens_b_index and negative < batch_size/2:\n", - " batch.append([input_ids, segment_ids, masked_tokens, masked_pos, False]) # NotNext\n", - " negative += 1\n", - " return batch\n", - "# Proprecessing Finished\n", - "\n", - "def get_attn_pad_mask(seq_q, seq_k):\n", - " batch_size, len_q = seq_q.size()\n", - " batch_size, len_k = seq_k.size()\n", - " # eq(zero) is PAD token\n", - " pad_attn_mask = seq_k.data.eq(0).unsqueeze(1) # batch_size x 1 x len_k(=len_q), one is masking\n", - " return pad_attn_mask.expand(batch_size, len_q, len_k) # batch_size x len_q x len_k\n", - "\n", - "def gelu(x):\n", - " \"Implementation of the gelu activation function by Hugging Face\"\n", - " return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))\n", - "\n", - "class Embedding(nn.Module):\n", - " def __init__(self):\n", - " super(Embedding, self).__init__()\n", - " self.tok_embed = nn.Embedding(vocab_size, d_model) # token embedding\n", - " self.pos_embed = nn.Embedding(maxlen, d_model) # position embedding\n", - " self.seg_embed = nn.Embedding(n_segments, d_model) # segment(token type) embedding\n", - " self.norm = nn.LayerNorm(d_model)\n", - "\n", - " def forward(self, x, seg):\n", - " seq_len = x.size(1)\n", - " pos = torch.arange(seq_len, dtype=torch.long)\n", - " pos = pos.unsqueeze(0).expand_as(x) # (seq_len,) -> (batch_size, seq_len)\n", - " embedding = self.tok_embed(x) + self.pos_embed(pos) + self.seg_embed(seg)\n", - " return self.norm(embedding)\n", - "\n", - "class ScaledDotProductAttention(nn.Module):\n", - " def __init__(self):\n", - " super(ScaledDotProductAttention, self).__init__()\n", - "\n", - " def forward(self, Q, K, V, attn_mask):\n", - " scores = torch.matmul(Q, K.transpose(-1, -2)) / np.sqrt(d_k) # scores : [batch_size x n_heads x len_q(=len_k) x len_k(=len_q)]\n", - " scores.masked_fill_(attn_mask, -1e9) # Fills elements of self tensor with value where mask is one.\n", - " attn = nn.Softmax(dim=-1)(scores)\n", - " context = torch.matmul(attn, V)\n", - " return context, attn\n", - "\n", - "class MultiHeadAttention(nn.Module):\n", - " def __init__(self):\n", - " super(MultiHeadAttention, self).__init__()\n", - " self.W_Q = nn.Linear(d_model, d_k * n_heads)\n", - " self.W_K = nn.Linear(d_model, d_k * n_heads)\n", - " self.W_V = nn.Linear(d_model, d_v * n_heads)\n", - " def forward(self, Q, K, V, attn_mask):\n", - " # q: [batch_size x len_q x d_model], k: [batch_size x len_k x d_model], v: [batch_size x len_k x d_model]\n", - " residual, batch_size = Q, Q.size(0)\n", - " # (B, S, D) -proj-> (B, S, D) -split-> (B, S, H, W) -trans-> (B, H, S, W)\n", - " q_s = self.W_Q(Q).view(batch_size, -1, n_heads, d_k).transpose(1,2) # q_s: [batch_size x n_heads x len_q x d_k]\n", - " k_s = self.W_K(K).view(batch_size, -1, n_heads, d_k).transpose(1,2) # k_s: [batch_size x n_heads x len_k x d_k]\n", - " v_s = self.W_V(V).view(batch_size, -1, n_heads, d_v).transpose(1,2) # v_s: [batch_size x n_heads x len_k x d_v]\n", - "\n", - " attn_mask = attn_mask.unsqueeze(1).repeat(1, n_heads, 1, 1) # attn_mask : [batch_size x n_heads x len_q x len_k]\n", - "\n", - " # context: [batch_size x n_heads x len_q x d_v], attn: [batch_size x n_heads x len_q(=len_k) x len_k(=len_q)]\n", - " context, attn = ScaledDotProductAttention()(q_s, k_s, v_s, attn_mask)\n", - " context = context.transpose(1, 2).contiguous().view(batch_size, -1, n_heads * d_v) # context: [batch_size x len_q x n_heads * d_v]\n", - " output = nn.Linear(n_heads * d_v, d_model)(context)\n", - " return nn.LayerNorm(d_model)(output + residual), attn # output: [batch_size x len_q x d_model]\n", - "\n", - "class PoswiseFeedForwardNet(nn.Module):\n", - " def __init__(self):\n", - " super(PoswiseFeedForwardNet, self).__init__()\n", - " self.fc1 = nn.Linear(d_model, d_ff)\n", - " self.fc2 = nn.Linear(d_ff, d_model)\n", - "\n", - " def forward(self, x):\n", - " # (batch_size, len_seq, d_model) -> (batch_size, len_seq, d_ff) -> (batch_size, len_seq, d_model)\n", - " return self.fc2(gelu(self.fc1(x)))\n", - "\n", - "class EncoderLayer(nn.Module):\n", - " def __init__(self):\n", - " super(EncoderLayer, self).__init__()\n", - " self.enc_self_attn = MultiHeadAttention()\n", - " self.pos_ffn = PoswiseFeedForwardNet()\n", - "\n", - " def forward(self, enc_inputs, enc_self_attn_mask):\n", - " enc_outputs, attn = self.enc_self_attn(enc_inputs, enc_inputs, enc_inputs, enc_self_attn_mask) # enc_inputs to same Q,K,V\n", - " enc_outputs = self.pos_ffn(enc_outputs) # enc_outputs: [batch_size x len_q x d_model]\n", - " return enc_outputs, attn\n", - "\n", - "class BERT(nn.Module):\n", - " def __init__(self):\n", - " super(BERT, self).__init__()\n", - " self.embedding = Embedding()\n", - " self.layers = nn.ModuleList([EncoderLayer() for _ in range(n_layers)])\n", - " self.fc = nn.Linear(d_model, d_model)\n", - " self.activ1 = nn.Tanh()\n", - " self.linear = nn.Linear(d_model, d_model)\n", - " self.activ2 = gelu\n", - " self.norm = nn.LayerNorm(d_model)\n", - " self.classifier = nn.Linear(d_model, 2)\n", - " # decoder is shared with embedding layer\n", - " embed_weight = self.embedding.tok_embed.weight\n", - " n_vocab, n_dim = embed_weight.size()\n", - " self.decoder = nn.Linear(n_dim, n_vocab, bias=False)\n", - " self.decoder.weight = embed_weight\n", - " self.decoder_bias = nn.Parameter(torch.zeros(n_vocab))\n", - "\n", - " def forward(self, input_ids, segment_ids, masked_pos):\n", - " output = self.embedding(input_ids, segment_ids)\n", - " enc_self_attn_mask = get_attn_pad_mask(input_ids, input_ids)\n", - " for layer in self.layers:\n", - " output, enc_self_attn = layer(output, enc_self_attn_mask)\n", - " # output : [batch_size, len, d_model], attn : [batch_size, n_heads, d_mode, d_model]\n", - " # it will be decided by first token(CLS)\n", - " h_pooled = self.activ1(self.fc(output[:, 0])) # [batch_size, d_model]\n", - " logits_clsf = self.classifier(h_pooled) # [batch_size, 2]\n", - "\n", - " masked_pos = masked_pos[:, :, None].expand(-1, -1, output.size(-1)) # [batch_size, max_pred, d_model]\n", - " # get masked position from final output of transformer.\n", - " h_masked = torch.gather(output, 1, masked_pos) # masking position [batch_size, max_pred, d_model]\n", - " h_masked = self.norm(self.activ2(self.linear(h_masked)))\n", - " logits_lm = self.decoder(h_masked) + self.decoder_bias # [batch_size, max_pred, n_vocab]\n", - "\n", - " return logits_lm, logits_clsf\n", - "\n", - "if __name__ == '__main__':\n", - " # BERT Parameters\n", - " maxlen = 30 # maximum of length\n", - " batch_size = 6\n", - " max_pred = 5 # max tokens of prediction\n", - " n_layers = 6 # number of Encoder of Encoder Layer\n", - " n_heads = 12 # number of heads in Multi-Head Attention\n", - " d_model = 768 # Embedding Size\n", - " d_ff = 768 * 4 # 4*d_model, FeedForward dimension\n", - " d_k = d_v = 64 # dimension of K(=Q), V\n", - " n_segments = 2\n", - "\n", - " text = (\n", - " 'Hello, how are you? I am Romeo.\\n'\n", - " 'Hello, Romeo My name is Juliet. Nice to meet you.\\n'\n", - " 'Nice meet you too. How are you today?\\n'\n", - " 'Great. My baseball team won the competition.\\n'\n", - " 'Oh Congratulations, Juliet\\n'\n", - " 'Thanks you Romeo'\n", - " )\n", - " sentences = re.sub(\"[.,!?\\\\-]\", '', text.lower()).split('\\n') # filter '.', ',', '?', '!'\n", - " word_list = list(set(\" \".join(sentences).split()))\n", - " word_dict = {'[PAD]': 0, '[CLS]': 1, '[SEP]': 2, '[MASK]': 3}\n", - " for i, w in enumerate(word_list):\n", - " word_dict[w] = i + 4\n", - " number_dict = {i: w for i, w in enumerate(word_dict)}\n", - " vocab_size = len(word_dict)\n", - "\n", - " token_list = list()\n", - " for sentence in sentences:\n", - " arr = [word_dict[s] for s in sentence.split()]\n", - " token_list.append(arr)\n", - "\n", - " model = BERT()\n", - " criterion = nn.CrossEntropyLoss()\n", - " optimizer = optim.Adam(model.parameters(), lr=0.001)\n", - "\n", - " batch = make_batch()\n", - " input_ids, segment_ids, masked_tokens, masked_pos, isNext = map(torch.LongTensor, zip(*batch))\n", - "\n", - " for epoch in range(100):\n", - " optimizer.zero_grad()\n", - " logits_lm, logits_clsf = model(input_ids, segment_ids, masked_pos)\n", - " loss_lm = criterion(logits_lm.transpose(1, 2), masked_tokens) # for masked LM\n", - " loss_lm = (loss_lm.float()).mean()\n", - " loss_clsf = criterion(logits_clsf, isNext) # for sentence classification\n", - " loss = loss_lm + loss_clsf\n", - " if (epoch + 1) % 10 == 0:\n", - " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " # Predict mask tokens ans isNext\n", - " input_ids, segment_ids, masked_tokens, masked_pos, isNext = map(torch.LongTensor, zip(batch[0]))\n", - " print(text)\n", - " print([number_dict[w.item()] for w in input_ids[0] if number_dict[w.item()] != '[PAD]'])\n", - "\n", - " logits_lm, logits_clsf = model(input_ids, segment_ids, masked_pos)\n", - " logits_lm = logits_lm.data.max(2)[1][0].data.numpy()\n", - " print('masked tokens list : ',[pos.item() for pos in masked_tokens[0] if pos.item() != 0])\n", - " print('predict masked tokens list : ',[pos for pos in logits_lm if pos != 0])\n", - "\n", - " logits_clsf = logits_clsf.data.max(1)[1].data.numpy()[0]\n", - " print('isNext : ', True if isNext else False)\n", - " print('predict isNext : ',True if logits_clsf else False)\n" - ], - "outputs": [], - "execution_count": null - } - ], - "metadata": { - "anaconda-cloud": {}, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.1" - } + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# code by Tae Hwan Jung(Jeff Jung) @graykode\n", + "# Reference : https://github.com/jadore801120/attention-is-all-you-need-pytorch\n", + "# https://github.com/JayParks/transformer, https://github.com/dhlee347/pytorchic-bert\n", + "import math\n", + "import re\n", + "from random import *\n", + "import numpy as np\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.optim as optim\n", + "\n", + "# sample IsNext and NotNext to be same in small batch size\n", + "def make_batch():\n", + " batch = []\n", + " positive = negative = 0\n", + " while positive != batch_size/2 or negative != batch_size/2:\n", + " tokens_a_index, tokens_b_index= randrange(len(sentences)), randrange(len(sentences)) # sample random index in sentences\n", + " tokens_a, tokens_b= token_list[tokens_a_index], token_list[tokens_b_index]\n", + " input_ids = [word_dict['[CLS]']] + tokens_a + [word_dict['[SEP]']] + tokens_b + [word_dict['[SEP]']]\n", + " segment_ids = [0] * (1 + len(tokens_a) + 1) + [1] * (len(tokens_b) + 1)\n", + "\n", + " # MASK LM\n", + " n_pred = min(max_pred, max(1, int(round(len(input_ids) * 0.15)))) # 15 % of tokens in one sentence\n", + " cand_maked_pos = [i for i, token in enumerate(input_ids)\n", + " if token != word_dict['[CLS]'] and token != word_dict['[SEP]']]\n", + " shuffle(cand_maked_pos)\n", + " masked_tokens, masked_pos = [], []\n", + " for pos in cand_maked_pos[:n_pred]:\n", + " masked_pos.append(pos)\n", + " masked_tokens.append(input_ids[pos])\n", + " if random() < 0.8: # 80%\n", + " input_ids[pos] = word_dict['[MASK]'] # make mask\n", + " elif random() < 0.5: # 10%\n", + " index = randint(0, vocab_size - 1) # random index in vocabulary\n", + " input_ids[pos] = word_dict[number_dict[index]] # replace\n", + "\n", + " # Zero Paddings\n", + " n_pad = maxlen - len(input_ids)\n", + " input_ids.extend([0] * n_pad)\n", + " segment_ids.extend([0] * n_pad)\n", + "\n", + " # Zero Padding (100% - 15%) tokens\n", + " if max_pred > n_pred:\n", + " n_pad = max_pred - n_pred\n", + " masked_tokens.extend([0] * n_pad)\n", + " masked_pos.extend([0] * n_pad)\n", + "\n", + " if tokens_a_index + 1 == tokens_b_index and positive < batch_size/2:\n", + " batch.append([input_ids, segment_ids, masked_tokens, masked_pos, True]) # IsNext\n", + " positive += 1\n", + " elif tokens_a_index + 1 != tokens_b_index and negative < batch_size/2:\n", + " batch.append([input_ids, segment_ids, masked_tokens, masked_pos, False]) # NotNext\n", + " negative += 1\n", + " return batch\n", + "# Proprecessing Finished\n", + "\n", + "def get_attn_pad_mask(seq_q, seq_k):\n", + " batch_size, len_q = seq_q.size()\n", + " batch_size, len_k = seq_k.size()\n", + " # eq(zero) is PAD token\n", + " pad_attn_mask = seq_k.data.eq(0).unsqueeze(1) # batch_size x 1 x len_k(=len_q), one is masking\n", + " return pad_attn_mask.expand(batch_size, len_q, len_k) # batch_size x len_q x len_k\n", + "\n", + "def gelu(x):\n", + " \"Implementation of the gelu activation function by Hugging Face\"\n", + " return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))\n", + "\n", + "class Embedding(nn.Module):\n", + " def __init__(self):\n", + " super(Embedding, self).__init__()\n", + " self.tok_embed = nn.Embedding(vocab_size, d_model) # token embedding\n", + " self.pos_embed = nn.Embedding(maxlen, d_model) # position embedding\n", + " self.seg_embed = nn.Embedding(n_segments, d_model) # segment(token type) embedding\n", + " self.norm = nn.LayerNorm(d_model)\n", + "\n", + " def forward(self, x, seg):\n", + " seq_len = x.size(1)\n", + " pos = torch.arange(seq_len, dtype=torch.long)\n", + " pos = pos.unsqueeze(0).expand_as(x) # (seq_len,) -> (batch_size, seq_len)\n", + " embedding = self.tok_embed(x) + self.pos_embed(pos) + self.seg_embed(seg)\n", + " return self.norm(embedding)\n", + "\n", + "class ScaledDotProductAttention(nn.Module):\n", + " def __init__(self):\n", + " super(ScaledDotProductAttention, self).__init__()\n", + "\n", + " def forward(self, Q, K, V, attn_mask):\n", + " scores = torch.matmul(Q, K.transpose(-1, -2)) / np.sqrt(d_k) # scores : [batch_size x n_heads x len_q(=len_k) x len_k(=len_q)]\n", + " scores.masked_fill_(attn_mask, -1e9) # Fills elements of self tensor with value where mask is one.\n", + " attn = nn.Softmax(dim=-1)(scores)\n", + " context = torch.matmul(attn, V)\n", + " return context, attn\n", + "\n", + "class MultiHeadAttention(nn.Module):\n", + " def __init__(self):\n", + " super(MultiHeadAttention, self).__init__()\n", + " self.W_Q = nn.Linear(d_model, d_k * n_heads)\n", + " self.W_K = nn.Linear(d_model, d_k * n_heads)\n", + " self.W_V = nn.Linear(d_model, d_v * n_heads)\n", + " def forward(self, Q, K, V, attn_mask):\n", + " # q: [batch_size x len_q x d_model], k: [batch_size x len_k x d_model], v: [batch_size x len_k x d_model]\n", + " residual, batch_size = Q, Q.size(0)\n", + " # (B, S, D) -proj-> (B, S, D) -split-> (B, S, H, W) -trans-> (B, H, S, W)\n", + " q_s = self.W_Q(Q).view(batch_size, -1, n_heads, d_k).transpose(1,2) # q_s: [batch_size x n_heads x len_q x d_k]\n", + " k_s = self.W_K(K).view(batch_size, -1, n_heads, d_k).transpose(1,2) # k_s: [batch_size x n_heads x len_k x d_k]\n", + " v_s = self.W_V(V).view(batch_size, -1, n_heads, d_v).transpose(1,2) # v_s: [batch_size x n_heads x len_k x d_v]\n", + "\n", + " attn_mask = attn_mask.unsqueeze(1).repeat(1, n_heads, 1, 1) # attn_mask : [batch_size x n_heads x len_q x len_k]\n", + "\n", + " # context: [batch_size x n_heads x len_q x d_v], attn: [batch_size x n_heads x len_q(=len_k) x len_k(=len_q)]\n", + " context, attn = ScaledDotProductAttention()(q_s, k_s, v_s, attn_mask)\n", + " context = context.transpose(1, 2).contiguous().view(batch_size, -1, n_heads * d_v) # context: [batch_size x len_q x n_heads * d_v]\n", + " output = nn.Linear(n_heads * d_v, d_model)(context)\n", + " return nn.LayerNorm(d_model)(output + residual), attn # output: [batch_size x len_q x d_model]\n", + "\n", + "class PoswiseFeedForwardNet(nn.Module):\n", + " def __init__(self):\n", + " super(PoswiseFeedForwardNet, self).__init__()\n", + " self.fc1 = nn.Linear(d_model, d_ff)\n", + " self.fc2 = nn.Linear(d_ff, d_model)\n", + "\n", + " def forward(self, x):\n", + " # (batch_size, len_seq, d_model) -> (batch_size, len_seq, d_ff) -> (batch_size, len_seq, d_model)\n", + " return self.fc2(gelu(self.fc1(x)))\n", + "\n", + "class EncoderLayer(nn.Module):\n", + " def __init__(self):\n", + " super(EncoderLayer, self).__init__()\n", + " self.enc_self_attn = MultiHeadAttention()\n", + " self.pos_ffn = PoswiseFeedForwardNet()\n", + "\n", + " def forward(self, enc_inputs, enc_self_attn_mask):\n", + " enc_outputs, attn = self.enc_self_attn(enc_inputs, enc_inputs, enc_inputs, enc_self_attn_mask) # enc_inputs to same Q,K,V\n", + " enc_outputs = self.pos_ffn(enc_outputs) # enc_outputs: [batch_size x len_q x d_model]\n", + " return enc_outputs, attn\n", + "\n", + "class BERT(nn.Module):\n", + " def __init__(self):\n", + " super(BERT, self).__init__()\n", + " self.embedding = Embedding()\n", + " self.layers = nn.ModuleList([EncoderLayer() for _ in range(n_layers)])\n", + " self.fc = nn.Linear(d_model, d_model)\n", + " self.activ1 = nn.Tanh()\n", + " self.linear = nn.Linear(d_model, d_model)\n", + " self.activ2 = gelu\n", + " self.norm = nn.LayerNorm(d_model)\n", + " self.classifier = nn.Linear(d_model, 2)\n", + " # decoder is shared with embedding layer\n", + " embed_weight = self.embedding.tok_embed.weight\n", + " n_vocab, n_dim = embed_weight.size()\n", + " self.decoder = nn.Linear(n_dim, n_vocab, bias=False)\n", + " self.decoder.weight = embed_weight\n", + " self.decoder_bias = nn.Parameter(torch.zeros(n_vocab))\n", + "\n", + " def forward(self, input_ids, segment_ids, masked_pos):\n", + " output = self.embedding(input_ids, segment_ids)\n", + " enc_self_attn_mask = get_attn_pad_mask(input_ids, input_ids)\n", + " for layer in self.layers:\n", + " output, enc_self_attn = layer(output, enc_self_attn_mask)\n", + " # output : [batch_size, len, d_model], attn : [batch_size, n_heads, d_mode, d_model]\n", + " # it will be decided by first token(CLS)\n", + " h_pooled = self.activ1(self.fc(output[:, 0])) # [batch_size, d_model]\n", + " logits_clsf = self.classifier(h_pooled) # [batch_size, 2]\n", + "\n", + " masked_pos = masked_pos[:, :, None].expand(-1, -1, output.size(-1)) # [batch_size, max_pred, d_model]\n", + " # get masked position from final output of transformer.\n", + " h_masked = torch.gather(output, 1, masked_pos) # masking position [batch_size, max_pred, d_model]\n", + " h_masked = self.norm(self.activ2(self.linear(h_masked)))\n", + " logits_lm = self.decoder(h_masked) + self.decoder_bias # [batch_size, max_pred, n_vocab]\n", + "\n", + " return logits_lm, logits_clsf\n", + "\n", + "if __name__ == '__main__':\n", + " # BERT Parameters\n", + " maxlen = 30 # maximum of length\n", + " batch_size = 6\n", + " max_pred = 5 # max tokens of prediction\n", + " n_layers = 6 # number of Encoder of Encoder Layer\n", + " n_heads = 12 # number of heads in Multi-Head Attention\n", + " d_model = 768 # Embedding Size\n", + " d_ff = 768 * 4 # 4*d_model, FeedForward dimension\n", + " d_k = d_v = 64 # dimension of K(=Q), V\n", + " n_segments = 2\n", + "\n", + " text = (\n", + " 'Hello, how are you? I am Romeo.\\n'\n", + " 'Hello, Romeo My name is Juliet. Nice to meet you.\\n'\n", + " 'Nice meet you too. How are you today?\\n'\n", + " 'Great. My baseball team won the competition.\\n'\n", + " 'Oh Congratulations, Juliet\\n'\n", + " 'Thanks you Romeo'\n", + " )\n", + " sentences = re.sub(\"[.,!?\\\\-]\", '', text.lower()).split('\\n') # filter '.', ',', '?', '!'\n", + " word_list = list(set(\" \".join(sentences).split()))\n", + " word_dict = {'[PAD]': 0, '[CLS]': 1, '[SEP]': 2, '[MASK]': 3}\n", + " for i, w in enumerate(word_list):\n", + " word_dict[w] = i + 4\n", + " number_dict = {i: w for i, w in enumerate(word_dict)}\n", + " vocab_size = len(word_dict)\n", + "\n", + " token_list = list()\n", + " for sentence in sentences:\n", + " arr = [word_dict[s] for s in sentence.split()]\n", + " token_list.append(arr)\n", + "\n", + " model = BERT()\n", + " criterion = nn.CrossEntropyLoss()\n", + " optimizer = optim.Adam(model.parameters(), lr=0.001)\n", + "\n", + " batch = make_batch()\n", + " input_ids, segment_ids, masked_tokens, masked_pos, isNext = map(torch.LongTensor, zip(*batch))\n", + "\n", + " for epoch in range(100):\n", + " optimizer.zero_grad()\n", + " logits_lm, logits_clsf = model(input_ids, segment_ids, masked_pos)\n", + " loss_lm = criterion(logits_lm.transpose(1, 2), masked_tokens) # for masked LM\n", + " loss_lm = (loss_lm.float()).mean()\n", + " loss_clsf = criterion(logits_clsf, isNext) # for sentence classification\n", + " loss = loss_lm + loss_clsf\n", + " if (epoch + 1) % 10 == 0:\n", + " print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # Predict mask tokens ans isNext\n", + " input_ids, segment_ids, masked_tokens, masked_pos, isNext = map(torch.LongTensor, zip(batch[0]))\n", + " print(text)\n", + " print([number_dict[w.item()] for w in input_ids[0] if number_dict[w.item()] != '[PAD]'])\n", + "\n", + " logits_lm, logits_clsf = model(input_ids, segment_ids, masked_pos)\n", + " logits_lm = logits_lm.data.max(2)[1][0].data.numpy()\n", + " print('masked tokens list : ',[pos.item() for pos in masked_tokens[0] if pos.item() != 0])\n", + " print('predict masked tokens list : ',[pos for pos in logits_lm if pos != 0])\n", + "\n", + " logits_clsf = logits_clsf.data.max(1)[1].data.numpy()[0]\n", + " print('isNext : ', True if isNext else False)\n", + " print('predict isNext : ',True if logits_clsf else False)\n" + ] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" }, - "nbformat": 4, - "nbformat_minor": 4 -} \ No newline at end of file + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}