diff --git a/Workshops/Workshop_How_to_Fine_tuning_Gemma_Transformers_Edition.ipynb b/Workshops/Workshop_How_to_Fine_tuning_Gemma_Transformers_Edition.ipynb index 5a2ed1e..02caf27 100644 --- a/Workshops/Workshop_How_to_Fine_tuning_Gemma_Transformers_Edition.ipynb +++ b/Workshops/Workshop_How_to_Fine_tuning_Gemma_Transformers_Edition.ipynb @@ -4,6 +4,7 @@ "cell_type": "code", "execution_count": null, "metadata": { + "cellView": "form", "id": "i1PHqD-ZY4-c" }, "outputs": [], @@ -92,30 +93,14 @@ }, { "cell_type": "code", - "source": [ - "import os\n", - "from google.colab import userdata, drive\n", - "from huggingface_hub import login\n", - "\n", - "login(userdata.get(\"HUGGING_FACE\"))\n", - "\n", - "access_token = userdata.get(\"HUGGING_FACE\")\n", - "my_hf_username = userdata.get(\"HUGGING_FACE_UN\")\n", - "os.environ[\"HF_USER\"] = my_hf_username\n", - "os.environ[\"HF_TOKEN\"] = userdata.get(\"HUGGING_FACE\")" - ], + "execution_count": null, "metadata": { - "id": "TT7GexJnZZCj", - "outputId": "5f5cc93f-d9c8-4bb4-d828-b670ed352b35", - "colab": { - "base_uri": "https://localhost:8080/" - } + "id": "TT7GexJnZZCj" }, - "execution_count": null, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.\n", "Token is valid (permission: read).\n", @@ -123,6 +108,18 @@ "Login successful\n" ] } + ], + "source": [ + "import os\n", + "from google.colab import userdata, drive\n", + "from huggingface_hub import login\n", + "\n", + "login(userdata.get(\"HUGGING_FACE\"))\n", + "\n", + "access_token = userdata.get(\"HUGGING_FACE\")\n", + "my_hf_username = userdata.get(\"HUGGING_FACE_UN\")\n", + "os.environ[\"HF_USER\"] = my_hf_username\n", + "os.environ[\"HF_TOKEN\"] = userdata.get(\"HUGGING_FACE\")" ] }, { @@ -138,34 +135,14 @@ }, { "cell_type": "code", - "source": [ - "!pip install transformers torch\n", - "# Set the backbend before importing Keras\n", - "os.environ[\"KERAS_BACKEND\"] = \"jax\"\n", - "# Avoid memory fragmentation on JAX backend.\n", - "os.environ[\"XLA_PYTHON_CLIENT_MEM_FRACTION\"] = \"1.00\"\n", - "\n", - "# Training Configurations\n", - "token_limit = 128\n", - "num_data_limit = 100\n", - "lora_name = \"my_lora\"\n", - "lora_rank = 4\n", - "lr_value = 1e-3\n", - "train_epoch = 5\n", - "model_id = \"google/gemma-2-2b-it\"" - ], + "execution_count": null, "metadata": { - "id": "WNn86PiiXTNf", - "outputId": "5ac7a598-0773-4596-a393-ba53cbfd2470", - "colab": { - "base_uri": "https://localhost:8080/" - } + "id": "WNn86PiiXTNf" }, - "execution_count": null, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.44.2)\n", "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.4.1+cu121)\n", @@ -192,6 +169,22 @@ "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch) (1.3.0)\n" ] } + ], + "source": [ + "!pip install transformers torch\n", + "# Set the backbend before importing Keras\n", + "os.environ[\"KERAS_BACKEND\"] = \"jax\"\n", + "# Avoid memory fragmentation on JAX backend.\n", + "os.environ[\"XLA_PYTHON_CLIENT_MEM_FRACTION\"] = \"1.00\"\n", + "\n", + "# Training Configurations\n", + "token_limit = 128\n", + "num_data_limit = 100\n", + "lora_name = \"my_lora\"\n", + "lora_rank = 4\n", + "lr_value = 1e-3\n", + "train_epoch = 5\n", + "model_id = \"google/gemma-2-2b-it\"" ] }, { @@ -217,182 +210,10 @@ }, { "cell_type": "code", - "source": [ - "from transformers import AutoModelForCausalLM, AutoTokenizer\n", - "import time\n", - "\n", - "# Load a pretrained model and tokenizer from Hugging Face\n", - "gemma_lm = AutoModelForCausalLM.from_pretrained(model_id, token=access_token)\n", - "tokenizer = AutoTokenizer.from_pretrained(model_id, token=access_token)\n", - "\n", - "# Summarize the model\n", - "print(gemma_lm)\n", - "\n", - "tick_start = 0\n", - "\n", - "def tick():\n", - " global tick_start\n", - " tick_start = time.time()\n", - "\n", - "def tock():\n", - " print(f\"TOTAL TIME ELAPSED: {time.time() - tick_start:.2f}s\")\n", - "\n", - "def text_gen(prompt, token_limit=100): # You can set your token limit\n", - " tick()\n", - "\n", - " # Format input, same as your original code\n", - " input_text = f\"user\\n{prompt}\\nmodel\\n\"\n", - "\n", - " # Tokenize input\n", - " inputs = tokenizer(input_text, return_tensors=\"pt\")\n", - "\n", - " # Generate text using the model\n", - " output_tokens = gemma_lm.generate(\n", - " inputs[\"input_ids\"],\n", - " max_length=token_limit,\n", - " pad_token_id=tokenizer.eos_token_id # Prevent errors if the input length exceeds the model's limit\n", - " )\n", - "\n", - " # Decode the generated tokens back to text\n", - " output = tokenizer.decode(output_tokens[0], skip_special_tokens=True)\n", - "\n", - " print(\"\\nGemma output:\")\n", - " print(output)\n", - "\n", - " tock()" - ], + "execution_count": null, "metadata": { - "id": "ywcDWVhAXb_9", - "outputId": "d9af537b-6c7a-4018-ada6-7b5dc58328d8", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 880, - "referenced_widgets": [ - "5dcc5fb3ad9e41eeb877fdd04f09ba7d", - "eeda0cc4233f41c7a1bb077b335922af", - "7786f72663544d3b964f8706ee4c87db", - "16489924a7194b6780f51cb3fa808c5e", - "8a87e9b8ba9d41ce8b44f2fef3244a81", - "e84b2d9e1e9e46c28e788385d64bc7c6", - "60c77f65806f4ff688453439fe7c9d83", - "de9354062f994aaabb104890709fc34a", - "9ae9adefe60b4511a90cdb2b3ef6e03d", - "3a29897fd3b3454ca752f63f45007d7f", - "1ef2aa04eaba4620a49c748ca4380d99", - "f4188acdf47f4cce86c02cc0d2bf46c7", - "54c456ec68684e57aa57aa7715921bfb", - "4ad2c31035574f26bf6118ef0d1413a9", - "a12cf321b8ae4075a4c3b0677d7bdf45", - "f8234ec2555b4e5684816aed66b63651", - "d816729082e94c41988d467654ef7841", - "b2d33d479a634d5c807e2a1860f4bc8c", - "b542562fcd6c466c9aa7f12b1b881207", - "2e8af71a4d85433a8e68f5c3737d7fb9", - "cdf16eb2e8ca487b9e9c7de16381c5c8", - "15332ef69c9244ccaaf3030dc9cb113a", - "eee1dd11b43e435cb226ee6df594291d", - "cc3c5ae5ebb74b2dbc81333533cc78cf", - "2f79a5c2a23b4808a181904593072dc5", - "a1c592e1355844adaa9ea80718499e4c", - "9805d9bec3f14c41a7a87fced47bb6ca", - "aed3fca1af5648e3b681d0c44f9c4f86", - "5960f282615e4932af3d9b7f69de2efa", - "151d7e95158d4d8aa2faf2886c8e0604", - "b5e8f1d688ea4694ac83a1bf89e27ae0", - "02ddb4d4c134415fa89a98984ae45c5f", - "7ba48bd1bbb148ff9dc7a2c33b835a35", - "9dc263e2dbfa4d9d9a56d62a69a57a8f", - "6a3171fe96744527805d8dc9eb159006", - "0bd0296c84a248259b5c48ac4e6bb760", - "20bc2639f53d4701aeafdb30e60f47da", - "372294f99f90486391b77296724916cc", - "2d9816c05a3544c0afc5a41c74ad6179", - "52132d841f1f48dd8c1d87ee0d308764", - "2fd5a5bfe878469c8f6b710c672fda9c", - "3a0c665acdbb4c83918c0ac469114235", - "eebc1808719c45b3bc7ad525798ce120", - "5d5a094d9cb545788f75992cbd859257", - "ecb1c6f7effa46b296e3574cddcb88de", - "cf5ec90af2944a63a0f83d5520307b82", - "28c7d5e1e84848c29b3d533930ccc198", - "b2831f93a9d148f4871c435382c6302e", - "9a827d1a0a424d6381b3e1eaac796b1e", - "afda84f0f4a445ee9dee2ccf047ce5ce", - "fad3963f838c44d98af0fdebb787741c", - "2dbc25a62598422b9c0768e109f8fd92", - "f97ae90efc3f44498db9203ef90d232e", - "707955f6553648fbb75ea65f2df2eaf6", - "c2fd92147e044b96ace6530ae67ca181", - "64324fda5c814874aaa2e97fa78ab3f0", - "abc4f6598a1f4c57a74a55f854ddc66d", - "6bfe81a71ff54a1390ff1d90d7d78864", - "25c94e5770e84ecea0e0f68f5bd64905", - "d15f3939e8b143a5b2deeb82435f9d26", - "a006f1ee609b47dda37389b5478a803d", - "0254a7b1482545cba26b5708a8920b34", - "977e2dc71b224a05823aa8120fa472bd", - "f6bdb5a59a044f4bb7332068179adce1", - "ecb124db56ea4925b83e91206c53f89b", - "e29c06ee25f84198bbea49aa0b19bb65", - "43bccde2f2804b089e4e40081058855f", - "f96a8568f4dc409090a123ff63766e58", - "163aee402b454733b1e74d43834849fb", - "8ebf2a32eac047cf9367f615ec01f0a8", - "760de7e3cef94be8b82cf8fc99385844", - "22d61c177e4e4d09a62ac08785682151", - "7ad0cea5c87e40da9b1dd1966b222f5a", - "a1e5eef9348f4bfeb456e5f096301778", - "4d94e337748343bb8e26d7ad7d361906", - "bc1dcd6dcd16462999359b579146c856", - "7c79134c76334c8680e81b58b9f75819", - "488f3d23e6ee4370b0e3750c4610acc1", - "5b1b61f335594a4487667b9d60395218", - "350ac704ca744a859527627fa92f9aa7", - "5a25dc05639c4cb282b1a2c4bbfdb572", - "4aaae0e141b14af78d9207b5f99a4516", - "182f4c00d1e94a4081057f34462704b7", - "62bb5bafdf2e47628a3ff1b114bac45a", - "79fac9dcae554d6f885608bd29fb904e", - "3b43ab0605644de9a852218edf3516fb", - "67261f6d01b14c50a4d7413ae8a4d113", - "110e4a8116e847dc9e4cea9a9df9e2b5", - "9ac1990111d64f919644a292b4ea2d36", - "494574a90f844172b5dce66aba248564", - "6b30ac7eab3346e8978274b8bcb03371", - "bdee91edaed5469a98ed808735bb30b0", - "eb7cfbc69c4848b3b4274203a70b6f1e", - "adb76c55b90d4f5aadd92d51b526fc27", - "5dd424f1051645abb00a3a3aa7d93990", - "018cb6becc764afb8bc6d9e9f2196fb9", - "bdd09cce6f4d4cf5b555678537c029c9", - "41ffd80053b6402f8aba20e1413e30cd", - "015fabd8f4934029900eec53ef02bdb8", - "b91c257be15341179ce8a262b3962ef0", - "e207f4774be44704a804dfacd628ad5a", - "559c60d219f24907b2b3b8e7c78deed8", - "88189b7d8769412685e197c8c8b7ec5c", - "2c8f45c2c1624865a698eb8c5527b0e4", - "d55c5f99d58942c2ba0506090f9755c2", - "9b5c6869f25945fa9424593255af07fc", - "435dfab7d6c94fb7b8b1a9dbb81bdbe7", - "9c0fa484394142c3beee7e272f626b08", - "3ad6acbc511b4417b5ca4e56a8688828", - "2d693baa6ea04b58965141813bd0ceca", - "379696127cc841a08cb6ca5000180032", - "9d03f9f959254badae3c94b71e84f2cc", - "65d1d113a0a8446c83563d9aefd8b112", - "9f61fd282f104094bc821e8a4b9f27ce", - "fd0fc02ef4b545c0a459527aca8bc81e", - "01cf0405f63a4d38ad8d1fe31721806f", - "ae20add968524517a261a81be45cde48", - "dc76053f6ae847469aab55949ad182be", - "4d98195ba52f48a29a237777eeef35cf", - "ffeafe7d69484d64af7484311e32703d", - "b66e7b11b2404e3daeaed1272994adb5" - ] - } + "id": "ywcDWVhAXb_9" }, - "execution_count": null, "outputs": [ { "data": { @@ -479,78 +300,78 @@ "output_type": "display_data" }, { - "output_type": "display_data", "data": { - "text/plain": [ - "generation_config.json: 0%| | 0.00/187 [00:00user\\n{prompt}\\nmodel\\n\"\n", + "\n", + " # Tokenize input\n", + " inputs = tokenizer(input_text, return_tensors=\"pt\")\n", + "\n", + " # Generate text using the model\n", + " output_tokens = gemma_lm.generate(\n", + " inputs[\"input_ids\"],\n", + " max_length=token_limit,\n", + " pad_token_id=tokenizer.eos_token_id # Prevent errors if the input length exceeds the model's limit\n", + " )\n", + "\n", + " # Decode the generated tokens back to text\n", + " output = tokenizer.decode(output_tokens[0], skip_special_tokens=True)\n", + "\n", + " print(\"\\nGemma output:\")\n", + " print(output)\n", + "\n", + " tock()" ] }, { @@ -631,16 +496,12 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "xR1riO7NURFF", - "outputId": "f9277cf5-9ea4-4509-a50b-260941c24ae3", - "colab": { - "base_uri": "https://localhost:8080/" - } + "id": "xR1riO7NURFF" }, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "[2, 177383, 235265, 6235, 3004, 2190, 26809, 235260, 235442, 235290, 545, 235265, 687, 11030, 5365, 14693, 12318, 43897, 235265]\n", " 2 -> \n", @@ -728,16 +589,12 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "ZiS-KU9osh_N", - "outputId": "dedf9024-ff52-4a36-e586-6fea9139a53f", - "colab": { - "base_uri": "https://localhost:8080/" - } + "id": "ZiS-KU9osh_N" }, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "15\n", "user\n", @@ -807,61 +664,25 @@ }, { "cell_type": "code", - "source": [ - "!pip install datasets" - ], + "execution_count": null, "metadata": { "id": "ZS9zT92tiKHu" }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "!pip install datasets" + ] }, { "cell_type": "code", - "source": [ - "# from datasets import load_dataset\n", - "\n", - "# # Load the dataset\n", - "# ds = load_dataset(\"bebechien/korean_cake_boss\", split=\"train\")\n", - "# print(ds)\n", - "\n", - "# # Prepare the dataset for tokenization\n", - "# train = []\n", - "\n", - "# # Iterate through the dataset and format the prompts\n", - "# for x in ds:\n", - "# # Create the formatted input-output text\n", - "# item = f\"user\\n다음에 대한 이메일 답장을 작성해줘.\\n\\\"{x['input']}\\\"\\nmodel\\n{x['output']}\"\n", - "\n", - "# # Tokenize the item and get its length\n", - "# length = len(tokenizer(item)[\"input_ids\"])\n", - "# print(length)\n", - "# # Skip if the tokenized item is longer than the token limit\n", - "# if length < token_limit:\n", - "# train.append(item)\n", - "\n", - "# # Stop if we have reached the desired data limit\n", - "# if len(train) >= num_data_limit:\n", - "# break\n", - "\n", - "# # Output the results\n", - "# print(f\"Number of training examples: {len(train)}\")\n", - "# print(f\"First example: {train[0]}\")\n", - "# print(f\"Second example: {train[1]}\")\n", - "# print(f\"Third example: {train[2]}\")" - ], + "execution_count": null, "metadata": { - "id": "Vh9s8m_PgoAH", - "outputId": "ccdf0c19-c3b5-40d5-d7ae-dc7a2c59fe0b", - "colab": { - "base_uri": "https://localhost:8080/" - } + "id": "Vh9s8m_PgoAH" }, - "execution_count": null, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "Dataset({\n", " features: ['input', 'output'],\n", @@ -923,6 +744,38 @@ "[가게 이름] 드림\n" ] } + ], + "source": [ + "# from datasets import load_dataset\n", + "\n", + "# # Load the dataset\n", + "# ds = load_dataset(\"bebechien/korean_cake_boss\", split=\"train\")\n", + "# print(ds)\n", + "\n", + "# # Prepare the dataset for tokenization\n", + "# train = []\n", + "\n", + "# # Iterate through the dataset and format the prompts\n", + "# for x in ds:\n", + "# # Create the formatted input-output text\n", + "# item = f\"user\\n다음에 대한 이메일 답장을 작성해줘.\\n\\\"{x['input']}\\\"\\nmodel\\n{x['output']}\"\n", + "\n", + "# # Tokenize the item and get its length\n", + "# length = len(tokenizer(item)[\"input_ids\"])\n", + "# print(length)\n", + "# # Skip if the tokenized item is longer than the token limit\n", + "# if length < token_limit:\n", + "# train.append(item)\n", + "\n", + "# # Stop if we have reached the desired data limit\n", + "# if len(train) >= num_data_limit:\n", + "# break\n", + "\n", + "# # Output the results\n", + "# print(f\"Number of training examples: {len(train)}\")\n", + "# print(f\"First example: {train[0]}\")\n", + "# print(f\"Second example: {train[1]}\")\n", + "# print(f\"Third example: {train[2]}\")" ] }, { @@ -973,53 +826,14 @@ }, { "cell_type": "code", - "source": [ - "!pip install peft\n", - "from peft import get_peft_model, LoraConfig, TaskType\n", - "import torch.nn as nn\n", - "\n", - "lora_config = LoraConfig(\n", - " task_type=TaskType.CAUSAL_LM,\n", - " r=lora_rank, # Using your predefined lora_rank\n", - " lora_alpha=32,\n", - " lora_dropout=0.1\n", - ")\n", - "gemma_lm = get_peft_model(gemma_lm, lora_config) # Enable LoRA for the model\n", - "\n", - "print(gemma_lm) # Hugging Face models don't have a summary method; use print() instead\n", - "\n", - "tokenizer.model_max_length = token_limit # Set token limit in the tokenizer\n", - "\n", - "from transformers import AdamW\n", - "\n", - "optimizer_grouped_parameters = [\n", - " {'params': [p for n, p in gemma_lm.named_parameters() if not any(nd in n for nd in [\"bias\", \"LayerNorm.weight\"])], 'weight_decay': 0.01},\n", - " {'params': [p for n, p in gemma_lm.named_parameters() if any(nd in n for nd in [\"bias\", \"LayerNorm.weight\"])], 'weight_decay': 0.0}\n", - "]\n", - "optimizer = AdamW(optimizer_grouped_parameters, lr=lr_value) # Use AdamW optimizer\n", - "\n", - "\n", - "loss_fn = nn.CrossEntropyLoss() # Define the loss function\n", - "\n", - "def forward_pass(input_text):\n", - " inputs = tokenizer(input_text, return_tensors=\"pt\", max_length=token_limit, truncation=True)\n", - " outputs = gemma_lm(**inputs, labels=inputs[\"input_ids\"])\n", - " loss = outputs.loss\n", - " return loss\n", - "\n" - ], + "execution_count": null, "metadata": { - "id": "YQiQxLFKfyzx", - "outputId": "5a5b1398-61cd-48ba-d354-71e4eee37213", - "colab": { - "base_uri": "https://localhost:8080/" - } + "id": "YQiQxLFKfyzx" }, - "execution_count": null, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "Collecting peft\n", " Downloading peft-0.13.2-py3-none-any.whl.metadata (13 kB)\n", @@ -1115,13 +929,47 @@ ] }, { - "output_type": "stream", "name": "stderr", + "output_type": "stream", "text": [ "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:591: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", " warnings.warn(\n" ] } + ], + "source": [ + "!pip install peft\n", + "from peft import get_peft_model, LoraConfig, TaskType\n", + "import torch.nn as nn\n", + "\n", + "lora_config = LoraConfig(\n", + " task_type=TaskType.CAUSAL_LM,\n", + " r=lora_rank, # Using your predefined lora_rank\n", + " lora_alpha=32,\n", + " lora_dropout=0.1\n", + ")\n", + "gemma_lm = get_peft_model(gemma_lm, lora_config) # Enable LoRA for the model\n", + "\n", + "print(gemma_lm) # Hugging Face models don't have a summary method; use print() instead\n", + "\n", + "tokenizer.model_max_length = token_limit # Set token limit in the tokenizer\n", + "\n", + "from transformers import AdamW\n", + "\n", + "optimizer_grouped_parameters = [\n", + " {'params': [p for n, p in gemma_lm.named_parameters() if not any(nd in n for nd in [\"bias\", \"LayerNorm.weight\"])], 'weight_decay': 0.01},\n", + " {'params': [p for n, p in gemma_lm.named_parameters() if any(nd in n for nd in [\"bias\", \"LayerNorm.weight\"])], 'weight_decay': 0.0}\n", + "]\n", + "optimizer = AdamW(optimizer_grouped_parameters, lr=lr_value) # Use AdamW optimizer\n", + "\n", + "\n", + "loss_fn = nn.CrossEntropyLoss() # Define the loss function\n", + "\n", + "def forward_pass(input_text):\n", + " inputs = tokenizer(input_text, return_tensors=\"pt\", max_length=token_limit, truncation=True)\n", + " outputs = gemma_lm(**inputs, labels=inputs[\"input_ids\"])\n", + " loss = outputs.loss\n", + " return loss\n" ] }, { @@ -1139,81 +987,21 @@ }, { "cell_type": "code", - "source": [ - "import torch\n", - "import os\n", - "import matplotlib.pyplot as plt\n", - "\n", - "# Define a custom callback-like function to handle actions at the end of each epoch\n", - "class CustomCallback:\n", - " def __init__(self, model, lora_name, lora_rank, text_gen):\n", - " self.model = model\n", - " self.lora_name = lora_name\n", - " self.lora_rank = lora_rank\n", - " self.text_gen = text_gen # text_gen function for evaluation\n", - "\n", - " def on_epoch_end(self, epoch):\n", - " # Save LoRA weights at the end of each epoch\n", - " model_name = f\"./{self.lora_name}_{self.lora_rank}_epoch{epoch+1}.lora.pt\"\n", - " self.model.save_pretrained(model_name, token=access_token) # Save model with LoRA weights locally\n", - "\n", - " # Evaluate the model using text generation\n", - " print(f\"Epoch {epoch + 1} finished. Running evaluation:\")\n", - " self.text_gen(\"Write a title\")\n", - " self.text_gen(\"Write a poem\")\n", - "\n", - "# Assuming train is your DataLoader and gemma_lm is your model\n", - "callback = CustomCallback(gemma_lm, lora_name, lora_rank, text_gen)\n", - "\n", - "# Training loop with callback-like behavior\n", - "losses = []\n", - "for epoch in range(train_epoch):\n", - " epoch_loss = 0\n", - " for batch in train: # Assuming `train` is a DataLoader or similar iterable\n", - " optimizer.zero_grad()\n", - "\n", - " inputs = tokenizer(batch, return_tensors=\"pt\", max_length=token_limit, truncation=True, padding=True)\n", - " labels = inputs[\"input_ids\"]\n", - " outputs = gemma_lm(**inputs, labels=labels)\n", - " loss = outputs.loss\n", - "\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " epoch_loss += loss.item()\n", - "\n", - " losses.append(epoch_loss / len(train)) # Store average loss per epoch\n", - "\n", - " # Run custom callback at the end of each epoch\n", - " callback.on_epoch_end(epoch)\n", - "\n", - "# Plot training loss over epochs\n", - "plt.plot(losses)\n", - "plt.xlabel(\"Epoch\")\n", - "plt.ylabel(\"Loss\")\n", - "plt.title(\"Training Loss Over Epochs\")\n", - "plt.show()" - ], + "execution_count": null, "metadata": { - "id": "YKpmDIfXh1Kx", - "outputId": "2422fc27-260d-4e7d-ce4b-bfa4234bec7d", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 1000 - } + "id": "YKpmDIfXh1Kx" }, - "execution_count": null, "outputs": [ { - "output_type": "stream", "name": "stderr", + "output_type": "stream", "text": [ "The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n" ] }, { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "Epoch 1 finished. Running evaluation:\n", "\n", @@ -1293,15 +1081,70 @@ ] }, { - "output_type": "display_data", "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABTlUlEQVR4nO3deVwTZ/4H8E/CEQ5JAJVLIqhYUFFUPADPVq1V60pr16MqaD22W+zqtt2ubH+97Hbpsd3eq1arVK31rLr1xgNPqIpi0aotlVMIqBzhPpL5/YGkTYFwCEwSPu/Xa15tJs8k34ch5sPMM89IBEEQQERERGQmpGIXQERERNSaGG6IiIjIrDDcEBERkVlhuCEiIiKzwnBDREREZoXhhoiIiMwKww0RERGZFYYbIiIiMisMN0RERGRWGG6IjMT8+fPh7e3dom3feOMNSCSS1i2IqBG1v3d3794VuxQiPQw3RI2QSCRNWmJjY8UuVRTz589Hp06dxC6jSQRBwKZNmzB69Gg4OjrCzs4O/fv3x8qVK1FSUiJ2eXXUhoeGFpVKJXaJREbJUuwCiIzdpk2b9B5v3LgRMTExddb36dPngd5n7dq10Gq1Ldr2//7v/7BixYoHen9zp9Fo8PTTT2P79u0YNWoU3njjDdjZ2eH06dN48803sWPHDhw9ehSurq5il1rHqlWr6g2Qjo6O7V8MkQlguCFqxNy5c/Uex8fHIyYmps763ystLYWdnV2T38fKyqpF9QGApaUlLC35cTbkvffew/bt2/HSSy/h/fff161fsmQJZsyYgdDQUMyfPx8HDx5s17qa8nvy1FNPoUuXLu1UEZHp42kpolYwduxY+Pv7IyEhAaNHj4adnR3+8Y9/AAD27t2LKVOmwMPDAzKZDL169cJbb70FjUaj9xq/H3OTmpoKiUSCf//73/jiiy/Qq1cvyGQyDB06FBcuXNDbtr4xNxKJBEuXLsWePXvg7+8PmUyGfv364dChQ3Xqj42NxZAhQ2BjY4NevXphzZo1rT6OZ8eOHQgMDIStrS26dOmCuXPn4vbt23ptVCoVFixYAE9PT8hkMri7u2PatGlITU3Vtbl48SImTpyILl26wNbWFj169MAzzzxj8L3Lysrw/vvv46GHHkJUVFSd56dOnYrw8HAcOnQI8fHxAIDHH38cPXv2rPf1goODMWTIEL11mzdv1vXP2dkZs2bNQkZGhl4bQ78nDyI2NhYSiQTbtm3DP/7xD7i5ucHe3h5/+MMf6tQANG1fAMCNGzcwY8YMdO3aFba2tvD19cUrr7xSp11BQQHmz58PR0dHKBQKLFiwAKWlpXptYmJiMHLkSDg6OqJTp07w9fVtlb4T1Yd/6hG1knv37mHSpEmYNWsW5s6dqzu9ER0djU6dOuGFF15Ap06dcPz4cbz22mtQq9V6RxAasmXLFhQVFeFPf/oTJBIJ3nvvPTz55JO4detWo0d7zpw5g2+//RbPPfccHBwc8Mknn2D69OlIT09H586dAQCXL1/GY489Bnd3d7z55pvQaDRYuXIlunbt+uA/lPuio6OxYMECDB06FFFRUcjJycHHH3+Ms2fP4vLly7rTK9OnT8e1a9fw/PPPw9vbG7m5uYiJiUF6erru8aOPPoquXbtixYoVcHR0RGpqKr799ttGfw75+flYtmxZg0e4wsLCsGHDBuzbtw9BQUGYOXMmwsLCcOHCBQwdOlTXLi0tDfHx8Xr77u2338arr76KGTNmYNGiRbhz5w4+/fRTjB49Wq9/QMO/J4bk5eXVWWdpaVnntNTbb78NiUSCv//978jNzcVHH32E8ePHIzExEba2tgCavi9++OEHjBo1ClZWVliyZAm8vb3xyy+/4LvvvsPbb7+t974zZsxAjx49EBUVhUuXLmHdunVwcXHBu+++CwC4du0aHn/8cQwYMAArV66ETCZDcnIyzp4922jfiVpEIKJmiYiIEH7/0RkzZowAQFi9enWd9qWlpXXW/elPfxLs7OyE8vJy3brw8HDBy8tL9zglJUUAIHTu3FnIy8vTrd+7d68AQPjuu+90615//fU6NQEQrK2theTkZN26K1euCACETz/9VLdu6tSpgp2dnXD79m3dup9//lmwtLSs85r1CQ8PF+zt7Rt8vrKyUnBxcRH8/f2FsrIy3fp9+/YJAITXXntNEARByM/PFwAI77//foOvtXv3bgGAcOHChUbr+q2PPvpIACDs3r27wTZ5eXkCAOHJJ58UBEEQCgsLBZlMJrz44ot67d577z1BIpEIaWlpgiAIQmpqqmBhYSG8/fbbeu2SkpIES0tLvfWGfk/qU7tf61t8fX117U6cOCEAELp16yao1Wrd+u3btwsAhI8//lgQhKbvC0EQhNGjRwsODg66ftbSarV16nvmmWf02jzxxBNC586ddY8//PBDAYBw586dJvWb6EHxtBRRK5HJZFiwYEGd9bV/MQNAUVER7t69i1GjRqG0tBQ3btxo9HVnzpwJJycn3eNRo0YBAG7dutXotuPHj0evXr10jwcMGAC5XK7bVqPR4OjRowgNDYWHh4eunY+PDyZNmtTo6zfFxYsXkZubi+eeew42Nja69VOmTIGfnx/2798PoObnZG1tjdjYWOTn59f7WrVHFfbt24eqqqom11BUVAQAcHBwaLBN7XNqtRoAIJfLMWnSJGzfvh2CIOjabdu2DUFBQejevTsA4Ntvv4VWq8WMGTNw9+5d3eLm5obevXvjxIkTeu/T0O+JIbt27UJMTIzesmHDhjrtwsLC9Pr41FNPwd3dHQcOHADQ9H1x584dnDp1Cs8884yun7XqO1X57LPP6j0eNWoU7t27p/tZ1u63vXv3tnjQPFFzMNwQtZJu3brB2tq6zvpr167hiSeegEKhgFwuR9euXXWDkQsLCxt93d9/udQGnYYCgKFta7ev3TY3NxdlZWXw8fGp066+dS2RlpYGAPD19a3znJ+fn+55mUyGd999FwcPHoSrqytGjx6N9957T+9y5zFjxmD69Ol488030aVLF0ybNg0bNmxARUWFwRpqv/BrQ0596gtAM2fOREZGBuLi4gAAv/zyCxISEjBz5kxdm59//hmCIKB3797o2rWr3nL9+nXk5ubqvU9DvyeGjB49GuPHj9dbgoOD67Tr3bu33mOJRAIfHx/dmKWm7ova8Ovv79+k+hr7HZ05cyZGjBiBRYsWwdXVFbNmzcL27dsZdKjNMNwQtZLfHqGpVVBQgDFjxuDKlStYuXIlvvvuO8TExOjGIjTlH3cLC4t61//2aEJbbCuG5cuX46effkJUVBRsbGzw6quvok+fPrh8+TKAmi/rnTt3Ii4uDkuXLsXt27fxzDPPIDAwEMXFxQ2+bu1l+j/88EODbWqf69u3r27d1KlTYWdnh+3btwMAtm/fDqlUij/+8Y+6NlqtFhKJBIcOHapzdCUmJgZr1qzRe5/6fk9MXWO/Z7a2tjh16hSOHj2KefPm4YcffsDMmTMxYcKEOgPriVoDww1RG4qNjcW9e/cQHR2NZcuW4fHHH8f48eP1TjOJycXFBTY2NkhOTq7zXH3rWsLLywsAcPPmzTrP3bx5U/d8rV69euHFF1/EkSNHcPXqVVRWVuKDDz7QaxMUFIS3334bFy9exNdff41r165h69atDdZQe5XOli1bGvwy3bhxI4Caq6Rq2dvb4/HHH8eOHTug1Wqxbds2jBo1Su8UXq9evSAIAnr06FHn6Mr48eMRFBTUyE+o9fz88896jwVBQHJysu4qvKbui9qrxK5evdpqtUmlUowbNw7/+c9/8OOPP+Ltt9/G8ePH65y2I2oNDDdEbaj2L9rfHimprKzEf//7X7FK0mNhYYHx48djz549yMrK0q1PTk5utflehgwZAhcXF6xevVrv9NHBgwdx/fp1TJkyBUDNfC/l5eV62/bq1QsODg667fLz8+scdRo4cCAAGDw1ZWdnh5deegk3b96s91Lm/fv3Izo6GhMnTqwTRmbOnImsrCysW7cOV65c0TslBQBPPvkkLCws8Oabb9apTRAE3Lt3r8G6WtvGjRv1Tr3t3LkT2dnZuvFTTd0XXbt2xejRo7F+/Xqkp6frvUdLjvrVd7VXU/YbUUvxUnCiNhQSEgInJyeEh4fjL3/5CyQSCTZt2mRUp4XeeOMNHDlyBCNGjMCf//xnaDQafPbZZ/D390diYmKTXqOqqgr//Oc/66x3dnbGc889h3fffRcLFizAmDFjMHv2bN3lx97e3vjrX/8KAPjpp58wbtw4zJgxA3379oWlpSV2796NnJwczJo1CwDw1Vdf4b///S+eeOIJ9OrVC0VFRVi7di3kcjkmT55ssMYVK1bg8uXLePfddxEXF4fp06fD1tYWZ86cwebNm9GnTx989dVXdbabPHkyHBwc8NJLL8HCwgLTp0/Xe75Xr1745z//icjISKSmpiI0NBQODg5ISUnB7t27sWTJErz00ktN+jk2ZOfOnfXOUDxhwgS9S8mdnZ0xcuRILFiwADk5Ofjoo4/g4+ODxYsXA6iZKLIp+wIAPvnkE4wcORKDBw/GkiVL0KNHD6SmpmL//v1N/r2otXLlSpw6dQpTpkyBl5cXcnNz8d///heenp4YOXJky34oRIaIco0WkQlr6FLwfv361dv+7NmzQlBQkGBrayt4eHgIL7/8snD48GEBgHDixAldu4YuBa/v0mgAwuuvv6573NCl4BEREXW29fLyEsLDw/XWHTt2TBg0aJBgbW0t9OrVS1i3bp3w4osvCjY2Ng38FH4VHh7e4OXKvXr10rXbtm2bMGjQIEEmkwnOzs7CnDlzhMzMTN3zd+/eFSIiIgQ/Pz/B3t5eUCgUwvDhw4Xt27fr2ly6dEmYPXu20L17d0EmkwkuLi7C448/Lly8eLHROgVBEDQajbBhwwZhxIgRglwuF2xsbIR+/foJb775plBcXNzgdnPmzBEACOPHj2+wza5du4SRI0cK9vb2gr29veDn5ydEREQIN2/e1LUx9HtSH0OXgv/296f2UvBvvvlGiIyMFFxcXARbW1thypQpdS7lFoTG90Wtq1evCk888YTg6Ogo2NjYCL6+vsKrr75ap77fX+K9YcMGAYCQkpIiCELN79e0adMEDw8PwdraWvDw8BBmz54t/PTTT03+WRA1h0QQjOhPSCIyGqGhobh27VqdcRxkfGJjY/Hwww9jx44deOqpp8Quh0h0HHNDRCgrK9N7/PPPP+PAgQMYO3asOAURET0AjrkhIvTs2RPz589Hz549kZaWhlWrVsHa2hovv/yy2KURETUbww0R4bHHHsM333wDlUoFmUyG4OBg/Otf/6ozKRwRkSngmBsiIiIyKxxzQ0RERGaF4YaIiIjMSocbc6PVapGVlQUHB4d6725LRERExkcQBBQVFcHDwwNSqeFjMx0u3GRlZUGpVIpdBhEREbVARkYGPD09DbYxmnDzzjvvIDIyEsuWLcNHH31Ub5vo6GgsWLBAb51MJqtzPxpDHBwcANT8cORyeYvrJSIiovajVquhVCp13+OGGEW4uXDhAtasWYMBAwY02lYul+vd0ba5p5Zq28vlcoYbIiIiE9OU733RBxQXFxdjzpw5WLt2LZycnBptL5FI4Obmplt+e9M4IiIiItHDTUREBKZMmYLx48c3qX1xcTG8vLygVCoxbdo0XLt2zWD7iooKqNVqvYWIiIjMl6jhZuvWrbh06RKioqKa1N7X1xfr16/H3r17sXnzZmi1WoSEhCAzM7PBbaKioqBQKHQLBxMTERGZN9FmKM7IyMCQIUMQExOjG2szduxYDBw4sMEBxb9XVVWFPn36YPbs2XjrrbfqbVNRUYGKigrd49oBSYWFhRxzQ0REZCLUajUUCkWTvr9FG1CckJCA3NxcDB48WLdOo9Hg1KlT+Oyzz1BRUQELCwuDr2FlZYVBgwYhOTm5wTYymQwymazV6iYiIiLjJlq4GTduHJKSkvTWLViwAH5+fvj73//eaLABasJQUlISJk+e3FZlEhERkYkRLdw4ODjA399fb529vT06d+6sWx8WFoZu3brpxuSsXLkSQUFB8PHxQUFBAd5//32kpaVh0aJF7V4/ERERGSejmOemIenp6XpTLOfn52Px4sVQqVRwcnJCYGAgzp07h759+4pYJRERERkT0QYUi6U5A5KIiIjIODTn+1v0eW6IiIiIWhPDDREREZkVhhsiIiIyKww3rUhVWI7r2by9AxERkZgYblrJwaRsjHrvOF7ZndR4YyIiImozDDetJNDbCRJIcCm9AAlpeWKXQ0RE1GEx3LQSFwcbhA7yAACsPZUicjVEREQdF8NNK1o0qicA4PCPKqTdKxG5GiIioo6J4aYVPeTqgLG+XSEIwPozPHpDREQkBoabVrb4/tGb7RczUVBaKXI1REREHQ/DTSsL6dUZfd3lKKvS4Ovv08Uuh4iIqMNhuGllEokEi0f3AABEn0tFRbVG5IqIiIg6FoabNvD4AA+4yW1wp6gCexOzxC6HiIioQ2G4aQNWFlIsGOENAPjydAo62I3XiYiIRMVw00ZmDesOe2sL3Mwpwqmf74pdDhERUYfBcNNGFLZWmDm0OwBg7albIldDRETUcTDctKEFI7xhIZXgTPJd/JjFG2oSERG1B4abNqR0tsMkfzcAwLozPHpDRETUHhhu2ljtpH7/S8yCqrBc5GqIiIjMH8NNGwtQOmJYD2dUawVEn0sVuxwiIiKzx3DTDmqP3mz5Pg3FFdUiV0NERGTeGG7awTg/F/TsYg91eTW2X8gQuxwiIiKzxnDTDqRSCRaOqrklw/qzKajWaEWuiIiIyHwx3LST6YM94Wxvjcz8Mhy+liN2OURERGaL4aad2FhZYG6QFwDgi9O3eEsGIiKiNsJw047Cgr1gbSnFlYwCXEzLF7scIiIis8Rw0466dJJh+uBuAHhLBiIiorbCcNPOFo6suSw85noOUu6WiFwNERGR+WG4aWc+Lp3wiJ8LBAH4krdkICIianUMNyKondRvZ0Im8ksqRa6GiIjIvDDciCCopzP8u8lRXqXF5vg0scshIiIyK0YTbt555x1IJBIsX77cYLsdO3bAz88PNjY26N+/Pw4cONA+BbYiiUSiO3rzVVwqyqs0IldERERkPowi3Fy4cAFr1qzBgAEDDLY7d+4cZs+ejYULF+Ly5csIDQ1FaGgorl692k6Vtp7J/d3hobDB3eJK7E28LXY5REREZkP0cFNcXIw5c+Zg7dq1cHJyMtj2448/xmOPPYa//e1v6NOnD9566y0MHjwYn332WTtV23qsLKRYMKLmlgxrT6dAq+WkfkRERK1B9HATERGBKVOmYPz48Y22jYuLq9Nu4sSJiIuLa3CbiooKqNVqvcVYzBymRCeZJZJzi3Hypztil0NERGQWRA03W7duxaVLlxAVFdWk9iqVCq6urnrrXF1doVKpGtwmKioKCoVCtyiVygequTXJbawwe1hNPWtP87JwIiKi1iBauMnIyMCyZcvw9ddfw8bGps3eJzIyEoWFhbolIyOjzd6rJeaP6AELqQTnfrmHq7cLxS6HiIjI5IkWbhISEpCbm4vBgwfD0tISlpaWOHnyJD755BNYWlpCo6l7BZGbmxtycvTvqJ2TkwM3N7cG30cmk0Eul+stxqSboy2m9HcHAKzj0RsiIqIHJlq4GTduHJKSkpCYmKhbhgwZgjlz5iAxMREWFhZ1tgkODsaxY8f01sXExCA4OLi9ym4TtZeF7/shG9mFZSJXQ0REZNosxXpjBwcH+Pv7662zt7dH586ddevDwsLQrVs33ZicZcuWYcyYMfjggw8wZcoUbN26FRcvXsQXX3zR7vW3pv6eCgT1dEb8rTxEn01F5OQ+YpdERERkskS/WsqQ9PR0ZGdn6x6HhIRgy5Yt+OKLLxAQEICdO3diz549dUKSKao9erPl+3QUlVeJXA0REZHpkgiC0KEmWFGr1VAoFCgsLDSq8TdarYAJH57EL3dK8H9T+mDR/bBDREREzfv+NuojNx2JVCrRBZoNZ1NRrdGKXBEREZFpYrgxIk8M6obO9ta4XVCGg1cbnruHiIiIGsZwY0RsrCwwL9gLQM2kfh3sjCEREVGrYLgxMvOCvCCzlOKHzEKcT8kTuxwiIiKTw3BjZDp3kmF6oCeAmhtqEhERUfMw3BihhSNr7hZ+9HoOfrlTLHI1REREpoXhxgj16toJ4/vU3CD0yzM8ekNERNQcDDdGavGomqM3uxIyca+4QuRqiIiITAfDjZEa1sMZAzwVqKjWYlN8mtjlEBERmQyGGyMlkUh0t2TYFJeG8qq6d0knIiKiuhhujNgkfzd0c7TFvZJK7L58W+xyiIiITALDjRGztJBiwQhvADWT+mm1nNSPiIioMQw3Rm7WsO5wsLHErTslOHEzV+xyiIiIjB7DjZHrJLPE08O6A6g5ekNERESGMdyYgPkjvGEplSD+Vh6SMgvFLoeIiMioMdyYAHeFLaYGeADg0RsiIqLGMNyYiEX3J/Xbn5SN2wVlIldDRERkvBhuTEQ/DwVCenWGRisg+ixvyUBERNQQhhsTsnh0zaR+35zPgLq8SuRqiIiIjBPDjQkZ+1BX9HbphOKKamw7nyF2OUREREaJ4caESCQS3dibDWdTUKXRilwRERGR8WG4MTHTBnZDl07WyCosx4GkbLHLISIiMjoMNybGxsoC4cHeAGouCxcE3pKBiIjotxhuTNDcIC/YWElx9bYa8bfyxC6HiIjIqDDcmCAne2s8FegJgJP6ERER/R7DjYlaOLInJBLg+I1cJOcWiV0OERGR0WC4MVE9uthjQh9XAMCXZzipHxERUS2GGxNWO6nfrku3caeoQuRqiIiIjAPDjQkb4uWEgUpHVFZrsSk+TexyiIiIjALDjQmTSCRYPKrm6M3m+DSUVWpEroiIiEh8DDcmbmI/V3g62SKvpBK7LmWKXQ4REZHoGG5MnKWFFAtH1tySYf2ZFGi1nNSPiIg6NlHDzapVqzBgwADI5XLI5XIEBwfj4MGDDbaPjo6GRCLRW2xsbNqxYuM0Y4gSchtL3LpbgmM3csUuh4iISFSihhtPT0+88847SEhIwMWLF/HII49g2rRpuHbtWoPbyOVyZGdn65a0NA6ktZdZ4unhXgCAtac4qR8REXVsooabqVOnYvLkyejduzceeughvP322+jUqRPi4+Mb3EYikcDNzU23uLq6tmPFxmt+iDesLCQ4n5qHKxkFYpdDREQkGqMZc6PRaLB161aUlJQgODi4wXbFxcXw8vKCUqls9CgPAFRUVECtVust5shNYYOpAR4AeEsGIiLq2EQPN0lJSejUqRNkMhmeffZZ7N69G3379q23ra+vL9avX4+9e/di8+bN0Gq1CAkJQWZmw1cJRUVFQaFQ6BalUtlWXRHdopE1l4UfvKpCRl6pyNUQERGJQyIIgqiX11RWViI9PR2FhYXYuXMn1q1bh5MnTzYYcH6rqqoKffr0wezZs/HWW2/V26aiogIVFb/O3qtWq6FUKlFYWAi5XN5q/TAW8778Hqd/votnRvTAa1Mb/xkSERGZArVaDYVC0aTvb9GP3FhbW8PHxweBgYGIiopCQEAAPv744yZta2VlhUGDBiE5ObnBNjKZTHc1Vu1izhbdn9Rv24V0FJZViVwNERFR+xM93PyeVqvVO9JiiEajQVJSEtzd3du4KtMxuncX+Lo6oKRSg63n08Uuh4iIqN2JGm4iIyNx6tQppKamIikpCZGRkYiNjcWcOXMAAGFhYYiMjNS1X7lyJY4cOYJbt27h0qVLmDt3LtLS0rBo0SKxumB0JBIJFo6qmdRvw9lUVFZrRa6IiIiofVmK+ea5ubkICwtDdnY2FAoFBgwYgMOHD2PChAkAgPT0dEilv+av/Px8LF68GCqVCk5OTggMDMS5c+eaND6nI5k20APvH74Jlboc+5Oy8MQgT7FLIiIiajeiDyhub80ZkGTKPj+RjPcP30Rfdzn2/2UkJBKJ2CURERG1mEkNKKa2MWd4d9haWeDHbDXO/XJP7HKIiIjaDcONmXK0s8aMITWnozipHxERdSQMN2bsmZE9IJEAsTfv4KecIrHLISIiahcMN2bMq7M9JvZ1AwCs49EbIiLqIBhuzNzi0TWT+u25nIXconKRqyEiImp7DDdmLtDLCYO7O6JSo8WmuDSxyyEiImpzDDcdwOL7t2TYFJ+G0spqkashIiJqWww3HcCj/dzQ3dkOBaVV2JXQ8B3UiYiIzAHDTQdgIZVg4ciaWzJ8eSYFGm2HmreRiIg6GIabDuKPQzyhsLVC6r1SHL2eI3Y5REREbYbhpoOws7bE3KDuAIC1p3hZOBERmS+Gmw4kPNgb1hZSXEzLx+X0fLHLISIiahMMNx2Ii9wGfxjoAQBYdzpF5GqIiIjaBsNNB1N7WfjBq9nIyCsVuRoiIqLWx3DTwfi6OWD0Q12hFWqunCIiIjI3DDcd0OJRNZeFb7+YgcLSKpGrISIial0MNx3QSJ8u8HNzQGmlBl+f5y0ZiIjIvDDcdEASiUQ39uarc6morNaKXBEREVHrYbjpoKYGeMBVLkOOugLfXckSuxwiIqJWw3DTQVlbShEe4g0AWHv6FgSBt2QgIiLzwHDTgc0Z5gU7awvcUBXhTPJdscshIiJqFQw3HZjCzgozhigBAGs5qR8REZkJhpsObuHIHpBKgFM/3cENlVrscoiIiB4Yw00Hp3S2wyR/dwC8JQMREZkHhhvCovuT+u1NvI1cdbnI1RARET0YhhvCoO5OGOLlhCqNgOhzqWKXQ0RE9EAYbggAsHh0zaR+X3+fjtLKapGrISIiajmGGwIAjO/jCu/Odigsq8KOi5lil0NERNRiDDcEALCQSrBwZM3Ymy/PpECj5aR+RERkmhhuSOepQCWc7KyQnleKI9dUYpdDRETUIgw3pGNrbYG5QV4Aam7JQEREZIpEDTerVq3CgAEDIJfLIZfLERwcjIMHDxrcZseOHfDz84ONjQ369++PAwcOtFO1HcO8YC9YW0hxKb0ACWl5YpdDRETUbKKGG09PT7zzzjtISEjAxYsX8cgjj2DatGm4du1ave3PnTuH2bNnY+HChbh8+TJCQ0MRGhqKq1evtnPl5svFwQahgzwAAGtPcVI/IiIyPRLByG4H7ezsjPfffx8LFy6s89zMmTNRUlKCffv26dYFBQVh4MCBWL16dZNeX61WQ6FQoLCwEHK5vNXqNic/5RTh0Q9PQSIBYl8aC6/O9mKXREREHVxzvr+NZsyNRqPB1q1bUVJSguDg4HrbxMXFYfz48XrrJk6ciLi4uAZft6KiAmq1Wm8hwx5ydcBY364QBGD9GR69ISIi0yJ6uElKSkKnTp0gk8nw7LPPYvfu3ejbt2+9bVUqFVxdXfXWubq6QqVq+MqeqKgoKBQK3aJUKlu1fnO1eFTNpH7bL2aioLRS5GqIiIiaTvRw4+vri8TERHz//ff485//jPDwcPz444+t9vqRkZEoLCzULRkZGa322uYspFdn9HWXo6xKg6+/Txe7HCIioiYTPdxYW1vDx8cHgYGBiIqKQkBAAD7++ON627q5uSEnJ0dvXU5ODtzc3Bp8fZlMprsaq3ahxkkkEiweXTOpX/S5VFRUa0SuiIiIqGlEDze/p9VqUVFRUe9zwcHBOHbsmN66mJiYBsfo0IN5fIAH3OQ2uFNUgb2JWWKXQ0RE1CSihpvIyEicOnUKqampSEpKQmRkJGJjYzFnzhwAQFhYGCIjI3Xtly1bhkOHDuGDDz7AjRs38MYbb+DixYtYunSpWF0wa1YWUiwY4Q0A+PJ0CozswjoiIqJ6iRpucnNzERYWBl9fX4wbNw4XLlzA4cOHMWHCBABAeno6srOzde1DQkKwZcsWfPHFFwgICMDOnTuxZ88e+Pv7i9UFszdrWHfYW1vgZk4RTv18V+xyiIiIGmV089y0Nc5z03wrv/sR68+mYKRPF2xeNFzscoiIqAMyyXluyHgtGOENC6kEZ5Lv4scszhNERETGjeGGGqV0tsMk/5or0tad4Q01iYjIuDHcUJPUTur3v8QsqArLRa6GiIioYQw31CQBSkcM6+GMaq2A6HOpYpdDRETUIIYbarLaozdbvk9DSUW1yNUQERHVj+GGmmycnwt6drGHurwa2y/yNhZERGScGG6oyaRSCRaOqrklw5dnUlCt0YpcERERUV0MN9Qs0wd7wtneGpn5ZTh8LafxDYiIiNoZww01i42VBeYGeQEAvjh9i7dkICIio8NwQ80WFuwFa0sprmQU4GJavtjlEBER6WG4oWbr0kmG6YO7AQDWnuKkfkREZFwYbqhFFo6suSw85noOUu6WiFwNERHRrxhuqEV8XDrhET8XCALwJW/JQERERoThhlqsdlK/nQmZyC+pFLkaIiKiGgw31GJBPZ3h302O8iotNseniV0OERERAIYbegASiUR39OaruFSUV2lEroiIiIjhhh7Q5P7u8FDY4G5xJfYm3ha7HCIiIoYbejBWFlIsGFFzS4a1p1Og1XJSPyIiEhfDDT2wmcOU6CSzRHJuMU7+dEfscoiIqINjuKEHJrexwuxhSgDA2tO8LJyIiMTFcEOtYv6IHrCQSnDul3u4llUodjlERNSBMdxQq+jmaIsp/d0BAOtOp4hcDRERdWQMN9Rqai8L/+5KFrILy0SuhoiIOiqGG2o1/T0VCOrpjGqtgOizqWKXQ0REHRTDDbWq2qM3W75PR1F5lcjVEBFRR8RwQ63qYV8X9Opqj6KKamy7kCF2OURE1AEx3FCrkkolWHT/6M2Gs6mo1mhFroiIiDoahhtqdU8M6obO9ta4XVCGg1dVYpdDREQdDMMNtTobKwvMC/YCUDOpnyDwlgxERNR+GG6oTcwL8oLMUoofMgtxPiVP7HKIiKgDYbihNtG5kwzTAz0B1NxQk4iIqL2IGm6ioqIwdOhQODg4wMXFBaGhobh586bBbaKjoyGRSPQWGxubdqqYmmPhyJq7hR+9noNf7hSLXA0REXUUooabkydPIiIiAvHx8YiJiUFVVRUeffRRlJSUGNxOLpcjOztbt6SlpbVTxdQcvbp2wvg+rgCAL8/w6A0REbUPSzHf/NChQ3qPo6Oj4eLigoSEBIwePbrB7SQSCdzc3Nq6PGoFi0f1wNHrOdiVkIkXJzyEzp1kYpdERERmrkVHbjIyMpCZmal7fP78eSxfvhxffPHFAxVTWFhzN2lnZ2eD7YqLi+Hl5QWlUolp06bh2rVrD/S+1HaG9XDGAE8FKqq12BTPI2xERNT2WhRunn76aZw4cQIAoFKpMGHCBJw/fx6vvPIKVq5c2aJCtFotli9fjhEjRsDf37/Bdr6+vli/fj327t2LzZs3Q6vVIiQkRC9s/VZFRQXUarXeQu1HIpHobsmwKS4N5VUakSsiIiJz16Jwc/XqVQwbNgwAsH37dvj7++PcuXP4+uuvER0d3aJCIiIicPXqVWzdutVgu+DgYISFhWHgwIEYM2YMvv32W3Tt2hVr1qypt31UVBQUCoVuUSqVLaqPWm6Svxu6OdriXkkldl++LXY5RERk5loUbqqqqiCT1YydOHr0KP7whz8AAPz8/JCdnd3s11u6dCn27duHEydOwNPTs1nbWllZYdCgQUhOTq73+cjISBQWFuqWjAze76i9WVpIsWCEN4CaSf20Wk7qR0REbadF4aZfv35YvXo1Tp8+jZiYGDz22GMAgKysLHTu3LnJryMIApYuXYrdu3fj+PHj6NGjR7Nr0Wg0SEpKgru7e73Py2QyyOVyvYXa36xh3eFgY4lbd0pw4mau2OUQEZEZa1G4effdd7FmzRqMHTsWs2fPRkBAAADgf//7n+50VVNERERg8+bN2LJlCxwcHKBSqaBSqVBWVqZrExYWhsjISN3jlStX4siRI7h16xYuXbqEuXPnIi0tDYsWLWpJV6iddJJZ4ulh3QHUHL0hIiJqKy26FHzs2LG4e/cu1Go1nJycdOuXLFkCOzu7Jr/OqlWrdK/3Wxs2bMD8+fMBAOnp6ZBKf81g+fn5WLx4MVQqFZycnBAYGIhz586hb9++LekKtaP5I7zx5ZkUxN/KQ1JmIfp7KsQuiYiIzJBEaMFdDcvKyiAIgi7IpKWlYffu3ejTpw8mTpzY6kW2JrVaDYVCgcLCQp6iEsFftyVi9+Xb+EOABz6ZPUjscoiIyEQ05/u7Raelpk2bho0bNwIACgoKMHz4cHzwwQcIDQ3VHY0hqs+iUTXjqvYnZeN2QVkjrYmIiJqvReHm0qVLGDVqFABg586dcHV1RVpaGjZu3IhPPvmkVQsk89LPQ4GQXp2h0QqIPstbMhARUetrUbgpLS2Fg4MDAODIkSN48sknIZVKERQUxPs8UaMWj66Z1O+b8xlQl1eJXA0REZmbFoUbHx8f7NmzBxkZGTh8+DAeffRRAEBubi7HsVCjxj7UFb1dOqG4ohrbznPeISIial0tCjevvfYaXnrpJXh7e2PYsGEIDg4GUHMUZ9AgDhIlwyQSiW7szYazKajSaEWuiIiIzEmLws1TTz2F9PR0XLx4EYcPH9atHzduHD788MNWK47M17SB3dClkzWyCstxIKn5s1oTERE1pEXhBgDc3NwwaNAgZGVl6W5aOWzYMPj5+bVacWS+bKwsEB7sDaBmUr8WzEhARERUrxaFG61Wi5UrV0KhUMDLywteXl5wdHTEW2+9Ba2WpxioaeYGecHGSoqrt9WIv5UndjlERGQmWhRuXnnlFXz22Wd45513cPnyZVy+fBn/+te/8Omnn+LVV19t7RrJTDnZW+OpwJobpfKWDERE1FpaNEOxh4cHVq9erbsbeK29e/fiueeew+3bt1utwNbGGYqNS8rdEjzyQSwEATj6wmj4uDiIXRIRERmhNp+hOC8vr96xNX5+fsjL4+kFaroeXewxoY8rAODLM5zUj4iIHlyLwk1AQAA+++yzOus/++wzDBgw4IGLoo6ldlK/XZdu405RhcjVEBGRqWvRXcHfe+89TJkyBUePHtXNcRMXF4eMjAwcOHCgVQsk8zfEywkDlY5IzCjApvg0vDDhIbFLIiIiE9aiIzdjxozBTz/9hCeeeAIFBQUoKCjAk08+iWvXrmHTpk2tXSOZOYlEgsWjao7ebI5PQ3mVRuSKiIjIlLVoQHFDrly5gsGDB0OjMd4vJw4oNk7VGi3G/jsWmfllePsJf8wZ7iV2SUREZETafEAxUWuztJBi4ciaWzJ8eToFWi0n9SMiopZhuCGjMWOIEnIbS9y6W4JjN3LFLoeIiEwUww0ZDXuZJZ6+fzqKk/oREVFLNetqqSeffNLg8wUFBQ9SCxHmh3jjyzO3cD4lD1cyChCgdBS7JCIiMjHNOnKjUCgMLl5eXggLC2urWqkDcFPYYGqABwAevSEiopZp1pGbDRs2tFUdRDqLRvbEt5du4+BVFTLySqF0thO7JCIiMiEcc0NGp6+HHKN6d4FGK2DD2VSxyyEiIhPDcENGadH9Sf22XUhHYVmVyNUQEZEpYbghozS6dxf4ujqgpFKDrefTxS6HiIhMCMMNGSWJRIKFo2om9dtwNhWV1VqRKyIiIlPBcENGa9pAD3R1kEGlLsf+pCyxyyEiIhPBcENGS2Zpgfkh3gCAtadS0Iq3QSMiIjPGcENGbc7w7rC1ssCP2Wqc++We2OUQEZEJYLgho+ZoZ40ZQzwBcFI/IiJqGoYbMnrPjOwBiQSIvXkHP+UUiV0OEREZOYYbMnpene0xsa8bAGAdj94QEVEjGG7IJCweXTOp357LWcgtKhe5GiIiMmaihpuoqCgMHToUDg4OcHFxQWhoKG7evNnodjt27ICfnx9sbGzQv39/HDhwoB2qJTEFejlhcHdHVGq02BSXJnY5RERkxEQNNydPnkRERATi4+MRExODqqoqPProoygpKWlwm3PnzmH27NlYuHAhLl++jNDQUISGhuLq1avtWDmJYfH9WzJsik9DWaVG5GqIiMhYSQQjmjzkzp07cHFxwcmTJzF69Oh628ycORMlJSXYt2+fbl1QUBAGDhyI1atXN/oearUaCoUChYWFkMvlrVY7tT2NVsDD/45Fel4p3prWD/OCvcUuiYiI2klzvr+NasxNYWEhAMDZ2bnBNnFxcRg/frzeuokTJyIuLq7e9hUVFVCr1XoLmSYLqQQLR9bckuHLMynQaI0mlxMRkRExmnCj1WqxfPlyjBgxAv7+/g22U6lUcHV11Vvn6uoKlUpVb/uoqCgoFArdolQqW7Vual9/HOIJha0VUu+V4uj1HLHLISIiI2Q04SYiIgJXr17F1q1bW/V1IyMjUVhYqFsyMjJa9fWpfdlZW2JuUHcAwNpTvCyciIjqMopws3TpUuzbtw8nTpyAp6enwbZubm7IydH/iz0nJwdubm71tpfJZJDL5XoLmbbwYG9YW0hxMS0fl9PzxS6HiIiMjKjhRhAELF26FLt378bx48fRo0ePRrcJDg7GsWPH9NbFxMQgODi4rcokI+Mit8EfBnoAANadThG5GiIiMjaihpuIiAhs3rwZW7ZsgYODA1QqFVQqFcrKynRtwsLCEBkZqXu8bNkyHDp0CB988AFu3LiBN954AxcvXsTSpUvF6AKJZNGomiB88Go2MvJKRa6GiIiMiajhZtWqVSgsLMTYsWPh7u6uW7Zt26Zrk56ejuzsbN3jkJAQbNmyBV988QUCAgKwc+dO7Nmzx+AgZDI/fm5yjH6oK7RCzZVTREREtYxqnpv2wHluzMfpn+9g3pfnYWdtgbgV46CwsxK7JCIiaiMmO88NUXOM9OkCPzcHlFZq8PV53pKBiIhqMNyQyZJIJLpbMnx1LhWV1VqRKyIiImPAcEMmbWqAB1zlMuSoK/DdlSyxyyEiIiPAcEMmzdpSivAQbwDA2tO30MGGkBERUT0YbsjkzRnmBTtrC9xQFeFM8l2xyyEiIpEx3JDJU9hZYcaQmnuGreWkfkREHR7DDZmFhSN7QCoBTv10BzdVRWKXQ0REImK4IbOgdLbDJH93ADVjb4iIqONiuCGzUXtLhr2Jt5GrLhe5GiIiEgvDDZmNQd2dMMTLCVUaAV/FpYpdDhERiYThhszK4tE1k/ptjk9HaWW1yNUQEZEYGG7IrIzv4wrvznYoLKvCjouZYpdDREQiYLghs2IhlWDhyJqxN1+eSYFGy0n9iIg6GoYbMjtPBSrhZGeF9LxSHLmmErscIiJqZww3ZHZsrS0wN8gLAC8LJyLqiBhuyCzNC/aCtYUUl9ILkJCWJ3Y5RETUjhhuyCy5ONggdJAHAGDtKd6SgYioI2G4IbO1aFTNZeGHf1Qh7V6JyNUQEVF7Ybghs/WQqwPG+naFIADrz/DoDRFRR8FwQ2Zt8f2jN9svZqKgtFLkaoiIqD0w3JBZC+nVGX3d5Sir0uDr79PFLoeIiNoBww2ZNYlEgsWjayb1iz6XiopqjcgVERFRW2O4IbP3+AAPuMltcKeoAnsTs8Quh4iI2hjDDZk9KwspFozwBgB8eToFgsBbMhARmTOGG+oQZg3rDntrC9zMKcKpn++KXQ4REbUhhhvqEBS2Vpg5tDsAYB1vyUBEZNYYbqjDWDDCGxZSCU7/fBc/ZqnFLoeIiNoIww11GEpnO0zydwMArDvDozdEROaK4YY6lNpJ/b67kgVVYbnI1RARUVtguKEOJUDpiGE9nFGlERB9LlXscoiIqA0w3FCHU3v0Zsv3aSipqBa5GiIiam2ihptTp05h6tSp8PDwgEQiwZ49ewy2j42NhUQiqbOoVKr2KZjMwjg/F/TsYg91eTW2X8wQuxwiImplooabkpISBAQE4PPPP2/Wdjdv3kR2drZucXFxaaMKyRxJpRIsHFVzS4Yvz6SgWqMVuSIiImpNlmK++aRJkzBp0qRmb+fi4gJHR8fWL4g6jOmDPfHBkZ+QmV+Gw9dyMGWAu9glERFRKzHJMTcDBw6Eu7s7JkyYgLNnz4pdDpkgGysLzA3yAgB8cfoWb8lARGRGTCrcuLu7Y/Xq1di1axd27doFpVKJsWPH4tKlSw1uU1FRAbVarbcQAUBYsBesLaW4klGAi2n5YpdDREStxKTCja+vL/70pz8hMDAQISEhWL9+PUJCQvDhhx82uE1UVBQUCoVuUSqV7VgxGbMunWSYPrgbAGDtKU7qR0RkLkwq3NRn2LBhSE5ObvD5yMhIFBYW6paMDF4dQ79aOLLmsvCY6zlIuVsicjVERNQaTD7cJCYmwt294cGgMpkMcrlcbyGq5ePSCY/4uUAQgC95SwYiIrMg6tVSxcXFekddUlJSkJiYCGdnZ3Tv3h2RkZG4ffs2Nm7cCAD46KOP0KNHD/Tr1w/l5eVYt24djh8/jiNHjojVBTIDi0f1xPEbudiZkIkXJ/jCyd5a7JKIiOgBiHrk5uLFixg0aBAGDRoEAHjhhRcwaNAgvPbaawCA7OxspKen69pXVlbixRdfRP/+/TFmzBhcuXIFR48exbhx40Spn8xDUE9n+HeTo7xKi83xaWKXQ0RED0gidLBrYNVqNRQKBQoLC3mKinT2Jt7Gsq2J6NJJhjN/fxg2VhZil0RERL/RnO9vkx9zQ9QaJvd3h4fCBneLK/DK7qtI5eBiIiKTxXBDBMDKQoqIR3wAALsuZWLsv2Mxf8N5HL+RA622Qx3cJCIyeaIOKCYyJnOGe8HD0RYbz6Ui9qc7iL1Zs3R3tsO8IC/8cYgnHO042JiIyNhxzA1RPVLvlmBzfBq2X8yAurwaACCzlCJ0YDfMC/aCfzeFyBUSEXUszfn+ZrghMqCsUoO9ibexMS4NP2b/euuOQC8nhAV7YZK/O6wteXaXiKitMdwYwHBDLSEIAhLS8rExLg0HkrJRfX8cTpdOMswepsTTw7vDXWErcpVEROaL4cYAhht6ULlF5dh6PgNff5+GHHUFAMBCKsHEfq6YF+SNoJ7OkEgkIldJRGReGG4MYLih1lKl0eLItRxsjEvF9yl5uvUPuXbCvGBvPDmoG+xlHLNPRNQaGG4MYLihtnBDpcamuDTsvnwbpZUaAICDzBLTAz0xL9gLvbp2ErlCIiLTxnBjAMMNtSV1eRV2JWRiU1wabv1mIsBRvbtgXpAXxvVxhYWUp6yIiJqL4cYAhhtqD1qtgLO/3MVX59Jw7EYOaj9l3RxtMSeoO2YN7Q5n3qCTiKjJGG4MYLih9paRV4qvv0/HtgvpyC+tAgBYW0oxdYAHwoK9EKB0FLdAIiITwHBjAMMNiaW8SoPvrmRhY1wakm4X6tYHKB0RFuSFKQPcecNOIqIGMNwYwHBDYhMEAYkZBdgUl4Z9P2SjUqMFADjbW2PmUCXmDO8OTyc7kaskIjIuDDcGMNyQMblbXIFtFzLwdXwasgrLAQBSCTCujyvCg70xwqcz58whIgLDjUEMN2SMqjVaHLuRi41xqTibfE+3vldXe8wL8sL0QE842FiJWCERkbgYbgxguCFjl5xbhE1xadiZkImS+3Pm2Ftb4InB3RAW7I2HXB1ErpCIqP0x3BjAcEOmoriiGrsvZeKruDQk5xbr1gf37IywYC9M6OsKSwvetJOIOgaGGwMYbsjUCIKAuFv3sPFcGmKu50Bz/6ad7gobPD2sO2YN646uDjKRqyQialsMNwYw3JApyyoow5bv0/HN+XTcK6kEAFhZSDC5vzvCgr0xuLsjByATkVliuDGA4YbMQUW1BgeTVPgqLhWX0wt06/27yREW5I0/DPTgnDlEZFYYbgxguCFzk5RZiI1xqfjflSxUVNfMmeNoZ4UZQ5SYO9wL3TtzzhwiMn0MNwYw3JC5yi+pxPaLGdgUn4bM/DIAgEQCPOzrgrBgL4zu3RVS3rSTiEwUw40BDDdk7jRaAbE3c/FVXBpO/XRHt967sx3mBnnhj4FKKOw4Zw4RmRaGGwMYbqgjuXWnGJvj07EjIQNF5dUAAFsrC4QO8sC8IG/09eBngIhMA8ONAQw31BGVVlZjz+UsbIxLxQ1VkW79MG9nzAv2wmP+brDinDlEZMQYbgxguKGOTBAEXEjNx1dxqTh8VYXq+3PmuDjIMHtYdzw9vDtc5TYiV0lEVBfDjQEMN0Q1ctTl2PJ9OracT8edogoAgKVUgon+bggP9sZQbyfOmUNERoPhxgCGGyJ9ldVaHL6mwsa4VFxIzdet93NzQFiwN0IHecDO2lLEComIGG4MYrghatiPWWpsik/FnstZKKuquWmng40l/hioxLxgL/ToYi9yhUTUUTHcGMBwQ9S4wtIq7EjIwOb4NKTeK9WtH/1QV4QHe2GsrwssOGcOEbUjhhsDGG6Imk6rFXDq5zvYGJeGEzdzUfuvhdLZFnOHe2HGECWc7K3FLZKIOoTmfH+Leu3nqVOnMHXqVHh4eEAikWDPnj2NbhMbG4vBgwdDJpPBx8cH0dHRbV4nUUcllUow1tcF6+cPxcmXHsaS0T2hsLVCRl4Zog7eQFDUMfxtxxVcvV0odqlERDqihpuSkhIEBATg888/b1L7lJQUTJkyBQ8//DASExOxfPlyLFq0CIcPH27jSomoe2c7/GNyH8RHjsN70wegn4ccFdVa7EjIxOOfnsET/z2LPZdvo6JaI3apRNTBGc1pKYlEgt27dyM0NLTBNn//+9+xf/9+XL16Vbdu1qxZKCgowKFDh5r0PjwtRdQ6BEHApfQCbIpLxf6kbFRpav4p6dLJGrOG1syZ4+FoK3KVRGQuTOa0VHPFxcVh/PjxeusmTpyIuLi4BrepqKiAWq3WW4jowUkkEgR6OeGjWYNwbsU4vDjhIbjJbXC3uBKfnUjGqPdO4NlNCTj3y10Yyd9QRNRBmFS4UalUcHV11Vvn6uoKtVqNsrKyereJioqCQqHQLUqlsj1KJepQujrI8Py43jjz94exas5gBPV0hkYr4NA1FZ5e+z0e/fAUNsWloriiWuxSiagDMKlw0xKRkZEoLCzULRkZGWKXRGS2LC2kmNTfHVuXBOPIX0djblB32Flb4OfcYry69xqC/nUMr++9iuTcYrFLJSIzZlLTjrq5uSEnJ0dvXU5ODuRyOWxt6z+3L5PJIJPJ2qM8IvqNh1wd8M/Q/nj5MT98m5CJjfFpuHWnBF/FpeGruDSM8OmMsGBvjPNzgSVv2klErcikwk1wcDAOHDigty4mJgbBwcEiVUREjZHbWGH+iB4ID/HG2eR72BiXiqPXc3A2+R7OJt+Dh8IGc4K8MGuoEp078Q8RInpwol4tVVxcjOTkZADAoEGD8J///AcPP/wwnJ2d0b17d0RGRuL27dvYuHEjgJpLwf39/REREYFnnnkGx48fx1/+8hfs378fEydObNJ78mopIvFl5pfi6+/Tse1CBvJKKgEA1hZSPD7AHWEh3hiodBS3QCIyOiYzQ3FsbCwefvjhOuvDw8MRHR2N+fPnIzU1FbGxsXrb/PWvf8WPP/4IT09PvPrqq5g/f36T35Phhsh4lFdpsP+HbGyMT8OVjALd+gGeCoQFe+PxAe6wsbIQr0AiMhomE27EwHBDZJwSMwqwMS4V+37IRmW1FgDgZGeFmUO7Y87w7lA624lcIRGJieHGAIYbIuOWV1KJbRdqbtp5u6BmigepBHjEzxXhIV4Y0asLpLxpJ1GHw3BjAMMNkWnQaAUcu56DTfFpOP3zXd36nl3sMS/YC9MDPSG3sRKxQiJqTww3BjDcEJme5NxibI5Pw86ETN1EgHbWFnhiUDeEBXvD181B5AqJqK0x3BjAcENkuoorqrH78m1sikvFTzm/TgQ4vIczwkO8MaGvK6w4Zw6RWWK4MYDhhsj0CYKA+Ft52BSfisPXcqDR1vwz5iqXYeYQJYJ6doa/p4KnrYjMCMONAQw3ROYlu7AM33yfji3n03G3uFLvuV5d7RGgdMRApSMCPB3h5+4AmSUvLScyRQw3BjDcEJmnimoNDl1V4ciPOfghswAZeXVvpmttIUUfDzkGeioQoHREgNIRPTrb8+orIhPAcGMAww1Rx3CvuAI/ZBYiMaMAVzILcCWjAPmlVXXaOcgsMUCpQICno+4oj6vcRoSKicgQhhsDGG6IOiZBEJCZX1YTdu4HnqTbhSiv0tZp6ya3QYCy5ujOQE9Hjt8hMgIMNwYw3BBRrWqNFj/lFOuO7CRmFOCnnCJo6/lXkeN3iMTFcGMAww0RGVJaWY1rWWpd2LnC8TtERoHhxgCGGyJqLo7fIRIfw40BDDdE9KAEQUBGXhkSMwvwA8fvELULhhsDGG6IqC1w/A5R22K4MYDhhojaS2llNa7eVuuuzuL4HaKWY7gxgOGGiMTE8TtELcNwYwDDDREZE47fIWoahhsDGG6IyNhx/A5RXQw3BjDcEJEp4vgd6ugYbgxguCEic8HxO9SRMNwYwHBDROaK43fInDHcGMBwQ0QdCcfvkLlguDGA4YaIOjqO3yFTxHBjAMMNEVFdHL9Dxo7hxgCGGyKixnH8DhkbhhsDGG6IiFqG43dITAw3BjDcEBG1nt+O30nMLMAPHL9DbYThxgCGGyKitsXxO9QWGG4MYLghImpfvx2/cyWjZrma1fD4nT7uDlA620HpZFfzX2dbKJ3tOI6ng2O4MYDhhohIfM0Zv1NLYWuF7rVhx8kOns52NY+dbNHNyZZjeswcw40BDDdERMapdvzOL3eKkZFXivS8UmTklyEzrxT3SioNbiuRAK4ONujubAfP++FHWRt+nG3h6mDD8T0mzuTCzeeff473338fKpUKAQEB+PTTTzFs2LB620ZHR2PBggV662QyGcrLy5v0Xgw3RESmp6SiGhn5pcjIK0NGXun9/7//OL8UpZUag9tbW0jRzcn2/ukuW91pr9rwo7C1gkTC8GPMmvP9bdlONTVo27ZteOGFF7B69WoMHz4cH330ESZOnIibN2/CxcWl3m3kcjlu3rype8xfSCIi82Yvs4Sfmxx+bnW/1ARBQF5Jpe5IT0ZeKTLz7x/5yStDVkEZKjVapNwtQcrdknpf30Fmef80l63eWJ/uznbwdLKDjRVPeZkS0Y/cDB8+HEOHDsVnn30GANBqtVAqlXj++eexYsWKOu2jo6OxfPlyFBQUtOj9eOSGiKhjqdZooVKXIz2vFJn3j/T89rTXnaKKRl+jq4MMSifb+0d67O6P+akJQu4KG1haSNuhJx2byRy5qaysREJCAiIjI3XrpFIpxo8fj7i4uAa3Ky4uhpeXF7RaLQYPHox//etf6NevX71tKyoqUFHx6y+uWq1uvQ4QEZHRs7SQwtOp5ggMetV9vrxKo3ekp/a0V3pezXifoopq3CmqwJ2iClxKL6j7+lIJPBxtdQOdlboAVHP6q7O9Nc8wtDNRw83du3eh0Wjg6uqqt97V1RU3btyodxtfX1+sX78eAwYMQGFhIf79738jJCQE165dg6enZ532UVFRePPNN9ukfiIiMn02VhbwcXGAj4tDnecEQUBhWRUy8sruH+m5P9bn/umv2/k1p7zS7x8JAu7VeQ07a4v7occWnrpxPr9e9WUvE32EiNkR9bRUVlYWunXrhnPnziE4OFi3/uWXX8bJkyfx/fffN/oaVVVV6NOnD2bPno233nqrzvP1HblRKpU8LUVERA9MqxWQU1T+a/i5H4Ay7z/OKSpHY9+yne2t4fmbIz3ddXP82MLD0RZWPOUFwIROS3Xp0gUWFhbIycnRW5+TkwM3N7cmvYaVlRUGDRqE5OTkep+XyWSQyWQPXCsREdHvSaUSuCts4a6wxbAeznWer6jW4HZ+GTLyy+6P+SnVXfWVnleKwrIq3CupxL2SSlzJKKj7+hLAXaF/yuu3c/10dZDxlFc9RA031tbWCAwMxLFjxxAaGgqgZkDxsWPHsHTp0ia9hkajQVJSEiZPntyGlRIRETWfzNICPbt2Qs+unep9Xl1e9esl7fWc9qqo1uJ2QRluF5QhHnn1vL60zuXtnNXZCC4Ff+GFFxAeHo4hQ4Zg2LBh+Oijj1BSUqKbyyYsLAzdunVDVFQUAGDlypUICgqCj48PCgoK8P777yMtLQ2LFi0SsxtERETNJrexQj8PBfp5KOo8p9UKuFtcoXekJ+M3R36yC8tQUa1Fcm4xknOL6339jjqrs+jhZubMmbhz5w5ee+01qFQqDBw4EIcOHdINMk5PT4dU+uv5xvz8fCxevBgqlQpOTk4IDAzEuXPn0LdvX7G6QERE1OqkUglc5DZwkdsg0Kvu81UaLbIKynQTGf4afn6d1bmwrApJtwuRdLuwzvbmPKuz6PPctDfOc0NERB3B72d1Tr8/uaGpzupsMgOKiYiIqG00NqvzvZJKvfE9vz3lZeqzOvPIDREREemp1miRXViud1n7bwc7Nzars49LJxx9YUyr1sQjN0RERNRilhZS3UzL9c3qXFZZM6tz3cHONeN9lE627V/0bzDcEBERUbPYWlugt6sDervWP6tzRbVWhKp+xWkPiYiIqNVIJBLR76LOcENERERmheGGiIiIzArDDREREZkVhhsiIiIyKww3REREZFYYboiIiMisMNwQERGRWWG4ISIiIrPCcENERERmheGGiIiIzArDDREREZkVhhsiIiIyKww3REREZFYsxS6gvQmCAABQq9UiV0JERERNVfu9Xfs9bkiHCzdFRUUAAKVSKXIlRERE1FxFRUVQKBQG20iEpkQgM6LVapGVlQUHBwdIJJJWfW21Wg2lUomMjAzI5fJWfW1jYO79A8y/j+yf6TP3PrJ/pq+t+igIAoqKiuDh4QGp1PComg535EYqlcLT07NN30Mul5vtLy1g/v0DzL+P7J/pM/c+sn+mry362NgRm1ocUExERERmheGGiIiIzArDTSuSyWR4/fXXIZPJxC6lTZh7/wDz7yP7Z/rMvY/sn+kzhj52uAHFREREZN545IaIiIjMCsMNERERmRWGGyIiIjIrDDdERERkVhhumunzzz+Ht7c3bGxsMHz4cJw/f95g+x07dsDPzw82Njbo378/Dhw40E6Vtkxz+hcdHQ2JRKK32NjYtGO1zXPq1ClMnToVHh4ekEgk2LNnT6PbxMbGYvDgwZDJZPDx8UF0dHSb19lSze1fbGxsnf0nkUigUqnap+BmioqKwtChQ+Hg4AAXFxeEhobi5s2bjW5nSp/BlvTRlD6Hq1atwoABA3STuwUHB+PgwYMGtzGl/dfc/pnSvqvPO++8A4lEguXLlxtsJ8Y+ZLhphm3btuGFF17A66+/jkuXLiEgIAATJ05Ebm5uve3PnTuH2bNnY+HChbh8+TJCQ0MRGhqKq1evtnPlTdPc/gE1M1BmZ2frlrS0tHasuHlKSkoQEBCAzz//vEntU1JSMGXKFDz88MNITEzE8uXLsWjRIhw+fLiNK22Z5vav1s2bN/X2oYuLSxtV+GBOnjyJiIgIxMfHIyYmBlVVVXj00UdRUlLS4Dam9hlsSR8B0/kcenp64p133kFCQgIuXryIRx55BNOmTcO1a9fqbW9q+6+5/QNMZ9/93oULF7BmzRoMGDDAYDvR9qFATTZs2DAhIiJC91ij0QgeHh5CVFRUve1nzJghTJkyRW/d8OHDhT/96U9tWmdLNbd/GzZsEBQKRTtV17oACLt37zbY5uWXXxb69eunt27mzJnCxIkT27Cy1tGU/p04cUIAIOTn57dLTa0tNzdXACCcPHmywTam9hn8vab00ZQ/h4IgCE5OTsK6devqfc7U958gGO6fqe67oqIioXfv3kJMTIwwZswYYdmyZQ22FWsf8shNE1VWViIhIQHjx4/XrZNKpRg/fjzi4uLq3SYuLk6vPQBMnDixwfZiakn/AKC4uBheXl5QKpWN/oViakxp/z2IgQMHwt3dHRMmTMDZs2fFLqfJCgsLAQDOzs4NtjH1fdiUPgKm+TnUaDTYunUrSkpKEBwcXG8bU95/TekfYJr7LiIiAlOmTKmzb+oj1j5kuGmiu3fvQqPRwNXVVW+9q6trg2MUVCpVs9qLqSX98/X1xfr167F3715s3rwZWq0WISEhyMzMbI+S21xD+0+tVqOsrEykqlqPu7s7Vq9ejV27dmHXrl1QKpUYO3YsLl26JHZpjdJqtVi+fDlGjBgBf3//BtuZ0mfw95raR1P7HCYlJaFTp06QyWR49tlnsXv3bvTt27fetqa4/5rTP1PbdwCwdetWXLp0CVFRUU1qL9Y+7HB3BafWExwcrPcXSUhICPr06YM1a9bgrbfeErEyagpfX1/4+vrqHoeEhOCXX37Bhx9+iE2bNolYWeMiIiJw9epVnDlzRuxS2kxT+2hqn0NfX18kJiaisLAQO3fuRHh4OE6ePNlgADA1zemfqe27jIwMLFu2DDExMUY/8Jnhpom6dOkCCwsL5OTk6K3PycmBm5tbvdu4ubk1q72YWtK/37OyssKgQYOQnJzcFiW2u4b2n1wuh62trUhVta1hw4YZfWBYunQp9u3bh1OnTsHT09NgW1P6DP5Wc/r4e8b+ObS2toaPjw8AIDAwEBcuXMDHH3+MNWvW1GlrivuvOf37PWPfdwkJCcjNzcXgwYN16zQaDU6dOoXPPvsMFRUVsLCw0NtGrH3I01JNZG1tjcDAQBw7dky3TqvV4tixYw2eTw0ODtZrDwAxMTEGz7+KpSX9+z2NRoOkpCS4u7u3VZntypT2X2tJTEw02v0nCAKWLl2K3bt34/jx4+jRo0ej25jaPmxJH3/P1D6HWq0WFRUV9T5navuvPob693vGvu/GjRuHpKQkJCYm6pYhQ4Zgzpw5SExMrBNsABH3YZsOVzYzW7duFWQymRAdHS38+OOPwpIlSwRHR0dBpVIJgiAI8+bNE1asWKFrf/bsWcHS0lL497//LVy/fl14/fXXBSsrKyEpKUmsLhjU3P69+eabwuHDh4VffvlFSEhIEGbNmiXY2NgI165dE6sLBhUVFQmXL18WLl++LAAQ/vOf/wiXL18W0tLSBEEQhBUrVgjz5s3Ttb9165ZgZ2cn/O1vfxOuX78ufP7554KFhYVw6NAhsbpgUHP79+GHHwp79uwRfv75ZyEpKUlYtmyZIJVKhaNHj4rVBYP+/Oc/CwqFQoiNjRWys7N1S2lpqa6NqX8GW9JHU/ocrlixQjh58qSQkpIi/PDDD8KKFSsEiUQiHDlyRBAE099/ze2fKe27hvz+ailj2YcMN8306aefCt27dxesra2FYcOGCfHx8brnxowZI4SHh+u13759u/DQQw8J1tbWQr9+/YT9+/e3c8XN05z+LV++XNfW1dVVmDx5snDp0iURqm6a2kuff7/U9ik8PFwYM2ZMnW0GDhwoWFtbCz179hQ2bNjQ7nU3VXP79+677wq9evUSbGxsBGdnZ2Hs2LHC8ePHxSm+CerrGwC9fWLqn8GW9NGUPofPPPOM4OXlJVhbWwtdu3YVxo0bp/viFwTT33/N7Z8p7buG/D7cGMs+lAiCILTtsSEiIiKi9sMxN0RERGRWGG6IiIjIrDDcEBERkVlhuCEiIiKzwnBDREREZoXhhoiIiMwKww0RERGZFYYbIurwJBIJ9uzZI3YZRNRKGG6ISFTz58+HRCKpszz22GNil0ZEJop3BSci0T322GPYsGGD3jqZTCZSNURk6njkhohEJ5PJ4Obmprc4OTkBqDlltGrVKkyaNAm2trbo2bMndu7cqbd9UlISHnnkEdja2qJz585YsmQJiouL9dqsX78e/fr1g0wmg7u7O5YuXar3/N27d/HEE0/Azs4OvXv3xv/+97+27TQRtRmGGyIyeq+++iqmT5+OK1euYM6cOZg1axauX78OACgpKcHEiRPh5OSECxcuYMeOHTh69KheeFm1ahUiIiKwZMkSJCUl4X//+x98fHz03uPNN9/EjBkz8MMPP2Dy5MmYM2cO8vLy2rWfRNRK2vzWnEREBoSHhwsWFhaCvb293vL2228LglBzp+xnn31Wb5vhw4cLf/7znwVBEIQvvvhCcHJyEoqLi3XP79+/X5BKpYJKpRIEQRA8PDyEV155pcEaAAj/93//p3tcXFwsABAOHjzYav0kovbDMTdEJLqHH34Yq1at0lvn7Oys+//g4GC954KDg5GYmAgAuH79OgICAmBvb697fsSIEdBqtbh58yYkEgmysrIwbtw4gzUMGDBA9//29vaQy+XIzc1taZeISEQMN0QkOnt7+zqniVqLra1tk9pZWVnpPZZIJNBqtW1REhG1MY65ISKjFx8fX+dxnz59AAB9+vTBlStXUFJSonv+7NmzkEql8PX1hYODA7y9vXHs2LF2rZmIxMMjN0QkuoqKCqhUKr11lpaW6NKlCwBgx44dGDJkCEaOHImvv/4a58+fx5dffgkAmDNnDl5//XWEh4fjjTfewJ07d/D8889j3rx5cHV1BQC88cYbePbZZ+Hi4oJJkyahqKgIZ8+exfPPP9++HSWidsFwQ0SiO3ToENzd3fXW+fr64saNGwBqrmTaunUrnnvuObi7u+Obb75B3759AQB2dnY4fPgwli1bhqFDh8LOzg7Tp0/Hf/7zH91rhYeHo7y8HB9++CFeeukldOnSBU899VT7dZCI2pVEEARB7CKIiBoikUiwe/duhIaGil0KEZkIjrkhIiIis8JwQ0RERGaFY26IyKjxzDkRNReP3BAREZFZYbghIiIis8JwQ0RERGaF4YaIiIjMCsMNERERmRWGGyIiIjIrDDdERERkVhhuiIiIyKww3BAREZFZ+X9611boj/snDQAAAABJRU5ErkJggg==\n", "text/plain": [ "
" - ], - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHHCAYAAABDUnkqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABTlUlEQVR4nO3deVwTZ/4H8E/CEQ5JAJVLIqhYUFFUPADPVq1V60pr16MqaD22W+zqtt2ubH+97Hbpsd3eq1arVK31rLr1xgNPqIpi0aotlVMIqBzhPpL5/YGkTYFwCEwSPu/Xa15tJs8k34ch5sPMM89IBEEQQERERGQmpGIXQERERNSaGG6IiIjIrDDcEBERkVlhuCEiIiKzwnBDREREZoXhhoiIiMwKww0RERGZFYYbIiIiMisMN0RERGRWGG6IjMT8+fPh7e3dom3feOMNSCSS1i2IqBG1v3d3794VuxQiPQw3RI2QSCRNWmJjY8UuVRTz589Hp06dxC6jSQRBwKZNmzB69Gg4OjrCzs4O/fv3x8qVK1FSUiJ2eXXUhoeGFpVKJXaJREbJUuwCiIzdpk2b9B5v3LgRMTExddb36dPngd5n7dq10Gq1Ldr2//7v/7BixYoHen9zp9Fo8PTTT2P79u0YNWoU3njjDdjZ2eH06dN48803sWPHDhw9ehSurq5il1rHqlWr6g2Qjo6O7V8MkQlguCFqxNy5c/Uex8fHIyYmps763ystLYWdnV2T38fKyqpF9QGApaUlLC35cTbkvffew/bt2/HSSy/h/fff161fsmQJZsyYgdDQUMyfPx8HDx5s17qa8nvy1FNPoUuXLu1UEZHp42kpolYwduxY+Pv7IyEhAaNHj4adnR3+8Y9/AAD27t2LKVOmwMPDAzKZDL169cJbb70FjUaj9xq/H3OTmpoKiUSCf//73/jiiy/Qq1cvyGQyDB06FBcuXNDbtr4xNxKJBEuXLsWePXvg7+8PmUyGfv364dChQ3Xqj42NxZAhQ2BjY4NevXphzZo1rT6OZ8eOHQgMDIStrS26dOmCuXPn4vbt23ptVCoVFixYAE9PT8hkMri7u2PatGlITU3Vtbl48SImTpyILl26wNbWFj169MAzzzxj8L3Lysrw/vvv46GHHkJUVFSd56dOnYrw8HAcOnQI8fHxAIDHH38cPXv2rPf1goODMWTIEL11mzdv1vXP2dkZs2bNQkZGhl4bQ78nDyI2NhYSiQTbtm3DP/7xD7i5ucHe3h5/+MMf6tQANG1fAMCNGzcwY8YMdO3aFba2tvD19cUrr7xSp11BQQHmz58PR0dHKBQKLFiwAKWlpXptYmJiMHLkSDg6OqJTp07w9fVtlb4T1Yd/6hG1knv37mHSpEmYNWsW5s6dqzu9ER0djU6dOuGFF15Ap06dcPz4cbz22mtQq9V6RxAasmXLFhQVFeFPf/oTJBIJ3nvvPTz55JO4detWo0d7zpw5g2+//RbPPfccHBwc8Mknn2D69OlIT09H586dAQCXL1/GY489Bnd3d7z55pvQaDRYuXIlunbt+uA/lPuio6OxYMECDB06FFFRUcjJycHHH3+Ms2fP4vLly7rTK9OnT8e1a9fw/PPPw9vbG7m5uYiJiUF6erru8aOPPoquXbtixYoVcHR0RGpqKr799ttGfw75+flYtmxZg0e4wsLCsGHDBuzbtw9BQUGYOXMmwsLCcOHCBQwdOlTXLi0tDfHx8Xr77u2338arr76KGTNmYNGiRbhz5w4+/fRTjB49Wq9/QMO/J4bk5eXVWWdpaVnntNTbb78NiUSCv//978jNzcVHH32E8ePHIzExEba2tgCavi9++OEHjBo1ClZWVliyZAm8vb3xyy+/4LvvvsPbb7+t974zZsxAjx49EBUVhUuXLmHdunVwcXHBu+++CwC4du0aHn/8cQwYMAArV66ETCZDcnIyzp4922jfiVpEIKJmiYiIEH7/0RkzZowAQFi9enWd9qWlpXXW/elPfxLs7OyE8vJy3brw8HDBy8tL9zglJUUAIHTu3FnIy8vTrd+7d68AQPjuu+90615//fU6NQEQrK2theTkZN26K1euCACETz/9VLdu6tSpgp2dnXD79m3dup9//lmwtLSs85r1CQ8PF+zt7Rt8vrKyUnBxcRH8/f2FsrIy3fp9+/YJAITXXntNEARByM/PFwAI77//foOvtXv3bgGAcOHChUbr+q2PPvpIACDs3r27wTZ5eXkCAOHJJ58UBEEQCgsLBZlMJrz44ot67d577z1BIpEIaWlpgiAIQmpqqmBhYSG8/fbbeu2SkpIES0tLvfWGfk/qU7tf61t8fX117U6cOCEAELp16yao1Wrd+u3btwsAhI8//lgQhKbvC0EQhNGjRwsODg66ftbSarV16nvmmWf02jzxxBNC586ddY8//PBDAYBw586dJvWb6EHxtBRRK5HJZFiwYEGd9bV/MQNAUVER7t69i1GjRqG0tBQ3btxo9HVnzpwJJycn3eNRo0YBAG7dutXotuPHj0evXr10jwcMGAC5XK7bVqPR4OjRowgNDYWHh4eunY+PDyZNmtTo6zfFxYsXkZubi+eeew42Nja69VOmTIGfnx/2798PoObnZG1tjdjYWOTn59f7WrVHFfbt24eqqqom11BUVAQAcHBwaLBN7XNqtRoAIJfLMWnSJGzfvh2CIOjabdu2DUFBQejevTsA4Ntvv4VWq8WMGTNw9+5d3eLm5obevXvjxIkTeu/T0O+JIbt27UJMTIzesmHDhjrtwsLC9Pr41FNPwd3dHQcOHADQ9H1x584dnDp1Cs8884yun7XqO1X57LPP6j0eNWoU7t27p/tZ1u63vXv3tnjQPFFzMNwQtZJu3brB2tq6zvpr167hiSeegEKhgFwuR9euXXWDkQsLCxt93d9/udQGnYYCgKFta7ev3TY3NxdlZWXw8fGp066+dS2RlpYGAPD19a3znJ+fn+55mUyGd999FwcPHoSrqytGjx6N9957T+9y5zFjxmD69Ol488030aVLF0ybNg0bNmxARUWFwRpqv/BrQ0596gtAM2fOREZGBuLi4gAAv/zyCxISEjBz5kxdm59//hmCIKB3797o2rWr3nL9+nXk5ubqvU9DvyeGjB49GuPHj9dbgoOD67Tr3bu33mOJRAIfHx/dmKWm7ova8Ovv79+k+hr7HZ05cyZGjBiBRYsWwdXVFbNmzcL27dsZdKjNMNwQtZLfHqGpVVBQgDFjxuDKlStYuXIlvvvuO8TExOjGIjTlH3cLC4t61//2aEJbbCuG5cuX46effkJUVBRsbGzw6quvok+fPrh8+TKAmi/rnTt3Ii4uDkuXLsXt27fxzDPPIDAwEMXFxQ2+bu1l+j/88EODbWqf69u3r27d1KlTYWdnh+3btwMAtm/fDqlUij/+8Y+6NlqtFhKJBIcOHapzdCUmJgZr1qzRe5/6fk9MXWO/Z7a2tjh16hSOHj2KefPm4YcffsDMmTMxYcKEOgPriVoDww1RG4qNjcW9e/cQHR2NZcuW4fHHH8f48eP1TjOJycXFBTY2NkhOTq7zXH3rWsLLywsAcPPmzTrP3bx5U/d8rV69euHFF1/EkSNHcPXqVVRWVuKDDz7QaxMUFIS3334bFy9exNdff41r165h69atDdZQe5XOli1bGvwy3bhxI4Caq6Rq2dvb4/HHH8eOHTug1Wqxbds2jBo1Su8UXq9evSAIAnr06FHn6Mr48eMRFBTUyE+o9fz88896jwVBQHJysu4qvKbui9qrxK5evdpqtUmlUowbNw7/+c9/8OOPP+Ltt9/G8ePH65y2I2oNDDdEbaj2L9rfHimprKzEf//7X7FK0mNhYYHx48djz549yMrK0q1PTk5utflehgwZAhcXF6xevVrv9NHBgwdx/fp1TJkyBUDNfC/l5eV62/bq1QsODg667fLz8+scdRo4cCAAGDw1ZWdnh5deegk3b96s91Lm/fv3Izo6GhMnTqwTRmbOnImsrCysW7cOV65c0TslBQBPPvkkLCws8Oabb9apTRAE3Lt3r8G6WtvGjRv1Tr3t3LkT2dnZuvFTTd0XXbt2xejRo7F+/Xqkp6frvUdLjvrVd7VXU/YbUUvxUnCiNhQSEgInJyeEh4fjL3/5CyQSCTZt2mRUp4XeeOMNHDlyBCNGjMCf//xnaDQafPbZZ/D390diYmKTXqOqqgr//Oc/66x3dnbGc889h3fffRcLFizAmDFjMHv2bN3lx97e3vjrX/8KAPjpp58wbtw4zJgxA3379oWlpSV2796NnJwczJo1CwDw1Vdf4b///S+eeOIJ9OrVC0VFRVi7di3kcjkmT55ssMYVK1bg8uXLePfddxEXF4fp06fD1tYWZ86cwebNm9GnTx989dVXdbabPHkyHBwc8NJLL8HCwgLTp0/Xe75Xr1745z//icjISKSmpiI0NBQODg5ISUnB7t27sWTJErz00ktN+jk2ZOfOnfXOUDxhwgS9S8mdnZ0xcuRILFiwADk5Ofjoo4/g4+ODxYsXA6iZKLIp+wIAPvnkE4wcORKDBw/GkiVL0KNHD6SmpmL//v1N/r2otXLlSpw6dQpTpkyBl5cXcnNz8d///heenp4YOXJky34oRIaIco0WkQlr6FLwfv361dv+7NmzQlBQkGBrayt4eHgIL7/8snD48GEBgHDixAldu4YuBa/v0mgAwuuvv6573NCl4BEREXW29fLyEsLDw/XWHTt2TBg0aJBgbW0t9OrVS1i3bp3w4osvCjY2Ng38FH4VHh7e4OXKvXr10rXbtm2bMGjQIEEmkwnOzs7CnDlzhMzMTN3zd+/eFSIiIgQ/Pz/B3t5eUCgUwvDhw4Xt27fr2ly6dEmYPXu20L17d0EmkwkuLi7C448/Lly8eLHROgVBEDQajbBhwwZhxIgRglwuF2xsbIR+/foJb775plBcXNzgdnPmzBEACOPHj2+wza5du4SRI0cK9vb2gr29veDn5ydEREQIN2/e1LUx9HtSH0OXgv/296f2UvBvvvlGiIyMFFxcXARbW1thypQpdS7lFoTG90Wtq1evCk888YTg6Ogo2NjYCL6+vsKrr75ap77fX+K9YcMGAYCQkpIiCELN79e0adMEDw8PwdraWvDw8BBmz54t/PTTT03+WRA1h0QQjOhPSCIyGqGhobh27VqdcRxkfGJjY/Hwww9jx44deOqpp8Quh0h0HHNDRCgrK9N7/PPPP+PAgQMYO3asOAURET0AjrkhIvTs2RPz589Hz549kZaWhlWrVsHa2hovv/yy2KURETUbww0R4bHHHsM333wDlUoFmUyG4OBg/Otf/6ozKRwRkSngmBsiIiIyKxxzQ0RERGaF4YaIiIjMSocbc6PVapGVlQUHB4d6725LRERExkcQBBQVFcHDwwNSqeFjMx0u3GRlZUGpVIpdBhEREbVARkYGPD09DbYxmnDzzjvvIDIyEsuWLcNHH31Ub5vo6GgsWLBAb51MJqtzPxpDHBwcANT8cORyeYvrJSIiovajVquhVCp13+OGGEW4uXDhAtasWYMBAwY02lYul+vd0ba5p5Zq28vlcoYbIiIiE9OU733RBxQXFxdjzpw5WLt2LZycnBptL5FI4Obmplt+e9M4IiIiItHDTUREBKZMmYLx48c3qX1xcTG8vLygVCoxbdo0XLt2zWD7iooKqNVqvYWIiIjMl6jhZuvWrbh06RKioqKa1N7X1xfr16/H3r17sXnzZmi1WoSEhCAzM7PBbaKioqBQKHQLBxMTERGZN9FmKM7IyMCQIUMQExOjG2szduxYDBw4sMEBxb9XVVWFPn36YPbs2XjrrbfqbVNRUYGKigrd49oBSYWFhRxzQ0REZCLUajUUCkWTvr9FG1CckJCA3NxcDB48WLdOo9Hg1KlT+Oyzz1BRUQELCwuDr2FlZYVBgwYhOTm5wTYymQwymazV6iYiIiLjJlq4GTduHJKSkvTWLViwAH5+fvj73//eaLABasJQUlISJk+e3FZlEhERkYkRLdw4ODjA399fb529vT06d+6sWx8WFoZu3brpxuSsXLkSQUFB8PHxQUFBAd5//32kpaVh0aJF7V4/ERERGSejmOemIenp6XpTLOfn52Px4sVQqVRwcnJCYGAgzp07h759+4pYJRERERkT0QYUi6U5A5KIiIjIODTn+1v0eW6IiIiIWhPDDREREZkVhhsiIiIyKww3rUhVWI7r2by9AxERkZgYblrJwaRsjHrvOF7ZndR4YyIiImozDDetJNDbCRJIcCm9AAlpeWKXQ0RE1GEx3LQSFwcbhA7yAACsPZUicjVEREQdF8NNK1o0qicA4PCPKqTdKxG5GiIioo6J4aYVPeTqgLG+XSEIwPozPHpDREQkBoabVrb4/tGb7RczUVBaKXI1REREHQ/DTSsL6dUZfd3lKKvS4Ovv08Uuh4iIqMNhuGllEokEi0f3AABEn0tFRbVG5IqIiIg6FoabNvD4AA+4yW1wp6gCexOzxC6HiIioQ2G4aQNWFlIsGOENAPjydAo62I3XiYiIRMVw00ZmDesOe2sL3Mwpwqmf74pdDhERUYfBcNNGFLZWmDm0OwBg7albIldDRETUcTDctKEFI7xhIZXgTPJd/JjFG2oSERG1B4abNqR0tsMkfzcAwLozPHpDRETUHhhu2ljtpH7/S8yCqrBc5GqIiIjMH8NNGwtQOmJYD2dUawVEn0sVuxwiIiKzx3DTDmqP3mz5Pg3FFdUiV0NERGTeGG7awTg/F/TsYg91eTW2X8gQuxwiIiKzxnDTDqRSCRaOqrklw/qzKajWaEWuiIiIyHwx3LST6YM94Wxvjcz8Mhy+liN2OURERGaL4aad2FhZYG6QFwDgi9O3eEsGIiKiNsJw047Cgr1gbSnFlYwCXEzLF7scIiIis8Rw0466dJJh+uBuAHhLBiIiorbCcNPOFo6suSw85noOUu6WiFwNERGR+WG4aWc+Lp3wiJ8LBAH4krdkICIianUMNyKondRvZ0Im8ksqRa6GiIjIvDDciCCopzP8u8lRXqXF5vg0scshIiIyK0YTbt555x1IJBIsX77cYLsdO3bAz88PNjY26N+/Pw4cONA+BbYiiUSiO3rzVVwqyqs0IldERERkPowi3Fy4cAFr1qzBgAEDDLY7d+4cZs+ejYULF+Ly5csIDQ1FaGgorl692k6Vtp7J/d3hobDB3eJK7E28LXY5REREZkP0cFNcXIw5c+Zg7dq1cHJyMtj2448/xmOPPYa//e1v6NOnD9566y0MHjwYn332WTtV23qsLKRYMKLmlgxrT6dAq+WkfkRERK1B9HATERGBKVOmYPz48Y22jYuLq9Nu4sSJiIuLa3CbiooKqNVqvcVYzBymRCeZJZJzi3Hypztil0NERGQWRA03W7duxaVLlxAVFdWk9iqVCq6urnrrXF1doVKpGtwmKioKCoVCtyiVygequTXJbawwe1hNPWtP87JwIiKi1iBauMnIyMCyZcvw9ddfw8bGps3eJzIyEoWFhbolIyOjzd6rJeaP6AELqQTnfrmHq7cLxS6HiIjI5IkWbhISEpCbm4vBgwfD0tISlpaWOHnyJD755BNYWlpCo6l7BZGbmxtycvTvqJ2TkwM3N7cG30cmk0Eul+stxqSboy2m9HcHAKzj0RsiIqIHJlq4GTduHJKSkpCYmKhbhgwZgjlz5iAxMREWFhZ1tgkODsaxY8f01sXExCA4OLi9ym4TtZeF7/shG9mFZSJXQ0REZNosxXpjBwcH+Pv7662zt7dH586ddevDwsLQrVs33ZicZcuWYcyYMfjggw8wZcoUbN26FRcvXsQXX3zR7vW3pv6eCgT1dEb8rTxEn01F5OQ+YpdERERkskS/WsqQ9PR0ZGdn6x6HhIRgy5Yt+OKLLxAQEICdO3diz549dUKSKao9erPl+3QUlVeJXA0REZHpkgiC0KEmWFGr1VAoFCgsLDSq8TdarYAJH57EL3dK8H9T+mDR/bBDREREzfv+NuojNx2JVCrRBZoNZ1NRrdGKXBEREZFpYrgxIk8M6obO9ta4XVCGg1cbnruHiIiIGsZwY0RsrCwwL9gLQM2kfh3sjCEREVGrYLgxMvOCvCCzlOKHzEKcT8kTuxwiIiKTw3BjZDp3kmF6oCeAmhtqEhERUfMw3BihhSNr7hZ+9HoOfrlTLHI1REREpoXhxgj16toJ4/vU3CD0yzM8ekNERNQcDDdGavGomqM3uxIyca+4QuRqiIiITAfDjZEa1sMZAzwVqKjWYlN8mtjlEBERmQyGGyMlkUh0t2TYFJeG8qq6d0knIiKiuhhujNgkfzd0c7TFvZJK7L58W+xyiIiITALDjRGztJBiwQhvADWT+mm1nNSPiIioMQw3Rm7WsO5wsLHErTslOHEzV+xyiIiIjB7DjZHrJLPE08O6A6g5ekNERESGMdyYgPkjvGEplSD+Vh6SMgvFLoeIiMioMdyYAHeFLaYGeADg0RsiIqLGMNyYiEX3J/Xbn5SN2wVlIldDRERkvBhuTEQ/DwVCenWGRisg+ixvyUBERNQQhhsTsnh0zaR+35zPgLq8SuRqiIiIjBPDjQkZ+1BX9HbphOKKamw7nyF2OUREREaJ4caESCQS3dibDWdTUKXRilwRERGR8WG4MTHTBnZDl07WyCosx4GkbLHLISIiMjoMNybGxsoC4cHeAGouCxcE3pKBiIjotxhuTNDcIC/YWElx9bYa8bfyxC6HiIjIqDDcmCAne2s8FegJgJP6ERER/R7DjYlaOLInJBLg+I1cJOcWiV0OERGR0WC4MVE9uthjQh9XAMCXZzipHxERUS2GGxNWO6nfrku3caeoQuRqiIiIjAPDjQkb4uWEgUpHVFZrsSk+TexyiIiIjALDjQmTSCRYPKrm6M3m+DSUVWpEroiIiEh8DDcmbmI/V3g62SKvpBK7LmWKXQ4REZHoGG5MnKWFFAtH1tySYf2ZFGi1nNSPiIg6NlHDzapVqzBgwADI5XLI5XIEBwfj4MGDDbaPjo6GRCLRW2xsbNqxYuM0Y4gSchtL3LpbgmM3csUuh4iISFSihhtPT0+88847SEhIwMWLF/HII49g2rRpuHbtWoPbyOVyZGdn65a0NA6ktZdZ4unhXgCAtac4qR8REXVsooabqVOnYvLkyejduzceeughvP322+jUqRPi4+Mb3EYikcDNzU23uLq6tmPFxmt+iDesLCQ4n5qHKxkFYpdDREQkGqMZc6PRaLB161aUlJQgODi4wXbFxcXw8vKCUqls9CgPAFRUVECtVust5shNYYOpAR4AeEsGIiLq2EQPN0lJSejUqRNkMhmeffZZ7N69G3379q23ra+vL9avX4+9e/di8+bN0Gq1CAkJQWZmw1cJRUVFQaFQ6BalUtlWXRHdopE1l4UfvKpCRl6pyNUQERGJQyIIgqiX11RWViI9PR2FhYXYuXMn1q1bh5MnTzYYcH6rqqoKffr0wezZs/HWW2/V26aiogIVFb/O3qtWq6FUKlFYWAi5XN5q/TAW8778Hqd/votnRvTAa1Mb/xkSERGZArVaDYVC0aTvb9GP3FhbW8PHxweBgYGIiopCQEAAPv744yZta2VlhUGDBiE5ObnBNjKZTHc1Vu1izhbdn9Rv24V0FJZViVwNERFR+xM93PyeVqvVO9JiiEajQVJSEtzd3du4KtMxuncX+Lo6oKRSg63n08Uuh4iIqN2JGm4iIyNx6tQppKamIikpCZGRkYiNjcWcOXMAAGFhYYiMjNS1X7lyJY4cOYJbt27h0qVLmDt3LtLS0rBo0SKxumB0JBIJFo6qmdRvw9lUVFZrRa6IiIiofVmK+ea5ubkICwtDdnY2FAoFBgwYgMOHD2PChAkAgPT0dEilv+av/Px8LF68GCqVCk5OTggMDMS5c+eaND6nI5k20APvH74Jlboc+5Oy8MQgT7FLIiIiajeiDyhub80ZkGTKPj+RjPcP30Rfdzn2/2UkJBKJ2CURERG1mEkNKKa2MWd4d9haWeDHbDXO/XJP7HKIiIjaDcONmXK0s8aMITWnozipHxERdSQMN2bsmZE9IJEAsTfv4KecIrHLISIiahcMN2bMq7M9JvZ1AwCs49EbIiLqIBhuzNzi0TWT+u25nIXconKRqyEiImp7DDdmLtDLCYO7O6JSo8WmuDSxyyEiImpzDDcdwOL7t2TYFJ+G0spqkashIiJqWww3HcCj/dzQ3dkOBaVV2JXQ8B3UiYiIzAHDTQdgIZVg4ciaWzJ8eSYFGm2HmreRiIg6GIabDuKPQzyhsLVC6r1SHL2eI3Y5REREbYbhpoOws7bE3KDuAIC1p3hZOBERmS+Gmw4kPNgb1hZSXEzLx+X0fLHLISIiahMMNx2Ii9wGfxjoAQBYdzpF5GqIiIjaBsNNB1N7WfjBq9nIyCsVuRoiIqLWx3DTwfi6OWD0Q12hFWqunCIiIjI3DDcd0OJRNZeFb7+YgcLSKpGrISIial0MNx3QSJ8u8HNzQGmlBl+f5y0ZiIjIvDDcdEASiUQ39uarc6morNaKXBEREVHrYbjpoKYGeMBVLkOOugLfXckSuxwiIqJWw3DTQVlbShEe4g0AWHv6FgSBt2QgIiLzwHDTgc0Z5gU7awvcUBXhTPJdscshIiJqFQw3HZjCzgozhigBAGs5qR8REZkJhpsObuHIHpBKgFM/3cENlVrscoiIiB4Yw00Hp3S2wyR/dwC8JQMREZkHhhvCovuT+u1NvI1cdbnI1RARET0YhhvCoO5OGOLlhCqNgOhzqWKXQ0RE9EAYbggAsHh0zaR+X3+fjtLKapGrISIiajmGGwIAjO/jCu/Odigsq8KOi5lil0NERNRiDDcEALCQSrBwZM3Ymy/PpECj5aR+RERkmhhuSOepQCWc7KyQnleKI9dUYpdDRETUIgw3pGNrbYG5QV4Aam7JQEREZIpEDTerVq3CgAEDIJfLIZfLERwcjIMHDxrcZseOHfDz84ONjQ369++PAwcOtFO1HcO8YC9YW0hxKb0ACWl5YpdDRETUbKKGG09PT7zzzjtISEjAxYsX8cgjj2DatGm4du1ave3PnTuH2bNnY+HChbh8+TJCQ0MRGhqKq1evtnPl5svFwQahgzwAAGtPcVI/IiIyPRLByG4H7ezsjPfffx8LFy6s89zMmTNRUlKCffv26dYFBQVh4MCBWL16dZNeX61WQ6FQoLCwEHK5vNXqNic/5RTh0Q9PQSIBYl8aC6/O9mKXREREHVxzvr+NZsyNRqPB1q1bUVJSguDg4HrbxMXFYfz48XrrJk6ciLi4uAZft6KiAmq1Wm8hwx5ydcBY364QBGD9GR69ISIi0yJ6uElKSkKnTp0gk8nw7LPPYvfu3ejbt2+9bVUqFVxdXfXWubq6QqVq+MqeqKgoKBQK3aJUKlu1fnO1eFTNpH7bL2aioLRS5GqIiIiaTvRw4+vri8TERHz//ff485//jPDwcPz444+t9vqRkZEoLCzULRkZGa322uYspFdn9HWXo6xKg6+/Txe7HCIioiYTPdxYW1vDx8cHgYGBiIqKQkBAAD7++ON627q5uSEnJ0dvXU5ODtzc3Bp8fZlMprsaq3ahxkkkEiweXTOpX/S5VFRUa0SuiIiIqGlEDze/p9VqUVFRUe9zwcHBOHbsmN66mJiYBsfo0IN5fIAH3OQ2uFNUgb2JWWKXQ0RE1CSihpvIyEicOnUKqampSEpKQmRkJGJjYzFnzhwAQFhYGCIjI3Xtly1bhkOHDuGDDz7AjRs38MYbb+DixYtYunSpWF0wa1YWUiwY4Q0A+PJ0CozswjoiIqJ6iRpucnNzERYWBl9fX4wbNw4XLlzA4cOHMWHCBABAeno6srOzde1DQkKwZcsWfPHFFwgICMDOnTuxZ88e+Pv7i9UFszdrWHfYW1vgZk4RTv18V+xyiIiIGmV089y0Nc5z03wrv/sR68+mYKRPF2xeNFzscoiIqAMyyXluyHgtGOENC6kEZ5Lv4scszhNERETGjeGGGqV0tsMk/5or0tad4Q01iYjIuDHcUJPUTur3v8QsqArLRa6GiIioYQw31CQBSkcM6+GMaq2A6HOpYpdDRETUIIYbarLaozdbvk9DSUW1yNUQERHVj+GGmmycnwt6drGHurwa2y/yNhZERGScGG6oyaRSCRaOqrklw5dnUlCt0YpcERERUV0MN9Qs0wd7wtneGpn5ZTh8LafxDYiIiNoZww01i42VBeYGeQEAvjh9i7dkICIio8NwQ80WFuwFa0sprmQU4GJavtjlEBER6WG4oWbr0kmG6YO7AQDWnuKkfkREZFwYbqhFFo6suSw85noOUu6WiFwNERHRrxhuqEV8XDrhET8XCALwJW/JQERERoThhlqsdlK/nQmZyC+pFLkaIiKiGgw31GJBPZ3h302O8iotNseniV0OERERAIYbegASiUR39OaruFSUV2lEroiIiIjhhh7Q5P7u8FDY4G5xJfYm3ha7HCIiIoYbejBWFlIsGFFzS4a1p1Og1XJSPyIiEhfDDT2wmcOU6CSzRHJuMU7+dEfscoiIqINjuKEHJrexwuxhSgDA2tO8LJyIiMTFcEOtYv6IHrCQSnDul3u4llUodjlERNSBMdxQq+jmaIsp/d0BAOtOp4hcDRERdWQMN9Rqai8L/+5KFrILy0SuhoiIOiqGG2o1/T0VCOrpjGqtgOizqWKXQ0REHRTDDbWq2qM3W75PR1F5lcjVEBFRR8RwQ63qYV8X9Opqj6KKamy7kCF2OURE1AEx3FCrkkolWHT/6M2Gs6mo1mhFroiIiDoahhtqdU8M6obO9ta4XVCGg1dVYpdDREQdDMMNtTobKwvMC/YCUDOpnyDwlgxERNR+GG6oTcwL8oLMUoofMgtxPiVP7HKIiKgDYbihNtG5kwzTAz0B1NxQk4iIqL2IGm6ioqIwdOhQODg4wMXFBaGhobh586bBbaKjoyGRSPQWGxubdqqYmmPhyJq7hR+9noNf7hSLXA0REXUUooabkydPIiIiAvHx8YiJiUFVVRUeffRRlJSUGNxOLpcjOztbt6SlpbVTxdQcvbp2wvg+rgCAL8/w6A0REbUPSzHf/NChQ3qPo6Oj4eLigoSEBIwePbrB7SQSCdzc3Nq6PGoFi0f1wNHrOdiVkIkXJzyEzp1kYpdERERmrkVHbjIyMpCZmal7fP78eSxfvhxffPHFAxVTWFhzN2lnZ2eD7YqLi+Hl5QWlUolp06bh2rVrD/S+1HaG9XDGAE8FKqq12BTPI2xERNT2WhRunn76aZw4cQIAoFKpMGHCBJw/fx6vvPIKVq5c2aJCtFotli9fjhEjRsDf37/Bdr6+vli/fj327t2LzZs3Q6vVIiQkRC9s/VZFRQXUarXeQu1HIpHobsmwKS4N5VUakSsiIiJz16Jwc/XqVQwbNgwAsH37dvj7++PcuXP4+uuvER0d3aJCIiIicPXqVWzdutVgu+DgYISFhWHgwIEYM2YMvv32W3Tt2hVr1qypt31UVBQUCoVuUSqVLaqPWm6Svxu6OdriXkkldl++LXY5RERk5loUbqqqqiCT1YydOHr0KP7whz8AAPz8/JCdnd3s11u6dCn27duHEydOwNPTs1nbWllZYdCgQUhOTq73+cjISBQWFuqWjAze76i9WVpIsWCEN4CaSf20Wk7qR0REbadF4aZfv35YvXo1Tp8+jZiYGDz22GMAgKysLHTu3LnJryMIApYuXYrdu3fj+PHj6NGjR7Nr0Wg0SEpKgru7e73Py2QyyOVyvYXa36xh3eFgY4lbd0pw4mau2OUQEZEZa1G4effdd7FmzRqMHTsWs2fPRkBAAADgf//7n+50VVNERERg8+bN2LJlCxwcHKBSqaBSqVBWVqZrExYWhsjISN3jlStX4siRI7h16xYuXbqEuXPnIi0tDYsWLWpJV6iddJJZ4ulh3QHUHL0hIiJqKy26FHzs2LG4e/cu1Go1nJycdOuXLFkCOzu7Jr/OqlWrdK/3Wxs2bMD8+fMBAOnp6ZBKf81g+fn5WLx4MVQqFZycnBAYGIhz586hb9++LekKtaP5I7zx5ZkUxN/KQ1JmIfp7KsQuiYiIzJBEaMFdDcvKyiAIgi7IpKWlYffu3ejTpw8mTpzY6kW2JrVaDYVCgcLCQp6iEsFftyVi9+Xb+EOABz6ZPUjscoiIyEQ05/u7Raelpk2bho0bNwIACgoKMHz4cHzwwQcIDQ3VHY0hqs+iUTXjqvYnZeN2QVkjrYmIiJqvReHm0qVLGDVqFABg586dcHV1RVpaGjZu3IhPPvmkVQsk89LPQ4GQXp2h0QqIPstbMhARUetrUbgpLS2Fg4MDAODIkSN48sknIZVKERQUxPs8UaMWj66Z1O+b8xlQl1eJXA0REZmbFoUbHx8f7NmzBxkZGTh8+DAeffRRAEBubi7HsVCjxj7UFb1dOqG4ohrbznPeISIial0tCjevvfYaXnrpJXh7e2PYsGEIDg4GUHMUZ9AgDhIlwyQSiW7szYazKajSaEWuiIiIzEmLws1TTz2F9PR0XLx4EYcPH9atHzduHD788MNWK47M17SB3dClkzWyCstxIKn5s1oTERE1pEXhBgDc3NwwaNAgZGVl6W5aOWzYMPj5+bVacWS+bKwsEB7sDaBmUr8WzEhARERUrxaFG61Wi5UrV0KhUMDLywteXl5wdHTEW2+9Ba2WpxioaeYGecHGSoqrt9WIv5UndjlERGQmWhRuXnnlFXz22Wd45513cPnyZVy+fBn/+te/8Omnn+LVV19t7RrJTDnZW+OpwJobpfKWDERE1FpaNEOxh4cHVq9erbsbeK29e/fiueeew+3bt1utwNbGGYqNS8rdEjzyQSwEATj6wmj4uDiIXRIRERmhNp+hOC8vr96xNX5+fsjL4+kFaroeXewxoY8rAODLM5zUj4iIHlyLwk1AQAA+++yzOus/++wzDBgw4IGLoo6ldlK/XZdu405RhcjVEBGRqWvRXcHfe+89TJkyBUePHtXNcRMXF4eMjAwcOHCgVQsk8zfEywkDlY5IzCjApvg0vDDhIbFLIiIiE9aiIzdjxozBTz/9hCeeeAIFBQUoKCjAk08+iWvXrmHTpk2tXSOZOYlEgsWjao7ebI5PQ3mVRuSKiIjIlLVoQHFDrly5gsGDB0OjMd4vJw4oNk7VGi3G/jsWmfllePsJf8wZ7iV2SUREZETafEAxUWuztJBi4ciaWzJ8eToFWi0n9SMiopZhuCGjMWOIEnIbS9y6W4JjN3LFLoeIiEwUww0ZDXuZJZ6+fzqKk/oREVFLNetqqSeffNLg8wUFBQ9SCxHmh3jjyzO3cD4lD1cyChCgdBS7JCIiMjHNOnKjUCgMLl5eXggLC2urWqkDcFPYYGqABwAevSEiopZp1pGbDRs2tFUdRDqLRvbEt5du4+BVFTLySqF0thO7JCIiMiEcc0NGp6+HHKN6d4FGK2DD2VSxyyEiIhPDcENGadH9Sf22XUhHYVmVyNUQEZEpYbghozS6dxf4ujqgpFKDrefTxS6HiIhMCMMNGSWJRIKFo2om9dtwNhWV1VqRKyIiIlPBcENGa9pAD3R1kEGlLsf+pCyxyyEiIhPBcENGS2Zpgfkh3gCAtadS0Iq3QSMiIjPGcENGbc7w7rC1ssCP2Wqc++We2OUQEZEJYLgho+ZoZ40ZQzwBcFI/IiJqGoYbMnrPjOwBiQSIvXkHP+UUiV0OEREZOYYbMnpene0xsa8bAGAdj94QEVEjGG7IJCweXTOp357LWcgtKhe5GiIiMmaihpuoqCgMHToUDg4OcHFxQWhoKG7evNnodjt27ICfnx9sbGzQv39/HDhwoB2qJTEFejlhcHdHVGq02BSXJnY5RERkxEQNNydPnkRERATi4+MRExODqqoqPProoygpKWlwm3PnzmH27NlYuHAhLl++jNDQUISGhuLq1avtWDmJYfH9WzJsik9DWaVG5GqIiMhYSQQjmjzkzp07cHFxwcmTJzF69Oh628ycORMlJSXYt2+fbl1QUBAGDhyI1atXN/oearUaCoUChYWFkMvlrVY7tT2NVsDD/45Fel4p3prWD/OCvcUuiYiI2klzvr+NasxNYWEhAMDZ2bnBNnFxcRg/frzeuokTJyIuLq7e9hUVFVCr1XoLmSYLqQQLR9bckuHLMynQaI0mlxMRkRExmnCj1WqxfPlyjBgxAv7+/g22U6lUcHV11Vvn6uoKlUpVb/uoqCgoFArdolQqW7Vual9/HOIJha0VUu+V4uj1HLHLISIiI2Q04SYiIgJXr17F1q1bW/V1IyMjUVhYqFsyMjJa9fWpfdlZW2JuUHcAwNpTvCyciIjqMopws3TpUuzbtw8nTpyAp6enwbZubm7IydH/iz0nJwdubm71tpfJZJDL5XoLmbbwYG9YW0hxMS0fl9PzxS6HiIiMjKjhRhAELF26FLt378bx48fRo0ePRrcJDg7GsWPH9NbFxMQgODi4rcokI+Mit8EfBnoAANadThG5GiIiMjaihpuIiAhs3rwZW7ZsgYODA1QqFVQqFcrKynRtwsLCEBkZqXu8bNkyHDp0CB988AFu3LiBN954AxcvXsTSpUvF6AKJZNGomiB88Go2MvJKRa6GiIiMiajhZtWqVSgsLMTYsWPh7u6uW7Zt26Zrk56ejuzsbN3jkJAQbNmyBV988QUCAgKwc+dO7Nmzx+AgZDI/fm5yjH6oK7RCzZVTREREtYxqnpv2wHluzMfpn+9g3pfnYWdtgbgV46CwsxK7JCIiaiMmO88NUXOM9OkCPzcHlFZq8PV53pKBiIhqMNyQyZJIJLpbMnx1LhWV1VqRKyIiImPAcEMmbWqAB1zlMuSoK/DdlSyxyyEiIiPAcEMmzdpSivAQbwDA2tO30MGGkBERUT0YbsjkzRnmBTtrC9xQFeFM8l2xyyEiIpEx3JDJU9hZYcaQmnuGreWkfkREHR7DDZmFhSN7QCoBTv10BzdVRWKXQ0REImK4IbOgdLbDJH93ADVjb4iIqONiuCGzUXtLhr2Jt5GrLhe5GiIiEgvDDZmNQd2dMMTLCVUaAV/FpYpdDhERiYThhszK4tE1k/ptjk9HaWW1yNUQEZEYGG7IrIzv4wrvznYoLKvCjouZYpdDREQiYLghs2IhlWDhyJqxN1+eSYFGy0n9iIg6GoYbMjtPBSrhZGeF9LxSHLmmErscIiJqZww3ZHZsrS0wN8gLAC8LJyLqiBhuyCzNC/aCtYUUl9ILkJCWJ3Y5RETUjhhuyCy5ONggdJAHAGDtKd6SgYioI2G4IbO1aFTNZeGHf1Qh7V6JyNUQEVF7Ybghs/WQqwPG+naFIADrz/DoDRFRR8FwQ2Zt8f2jN9svZqKgtFLkaoiIqD0w3JBZC+nVGX3d5Sir0uDr79PFLoeIiNoBww2ZNYlEgsWjayb1iz6XiopqjcgVERFRW2O4IbP3+AAPuMltcKeoAnsTs8Quh4iI2hjDDZk9KwspFozwBgB8eToFgsBbMhARmTOGG+oQZg3rDntrC9zMKcKpn++KXQ4REbUhhhvqEBS2Vpg5tDsAYB1vyUBEZNYYbqjDWDDCGxZSCU7/fBc/ZqnFLoeIiNoIww11GEpnO0zydwMArDvDozdEROaK4YY6lNpJ/b67kgVVYbnI1RARUVtguKEOJUDpiGE9nFGlERB9LlXscoiIqA0w3FCHU3v0Zsv3aSipqBa5GiIiam2ihptTp05h6tSp8PDwgEQiwZ49ewy2j42NhUQiqbOoVKr2KZjMwjg/F/TsYg91eTW2X8wQuxwiImplooabkpISBAQE4PPPP2/Wdjdv3kR2drZucXFxaaMKyRxJpRIsHFVzS4Yvz6SgWqMVuSIiImpNlmK++aRJkzBp0qRmb+fi4gJHR8fWL4g6jOmDPfHBkZ+QmV+Gw9dyMGWAu9glERFRKzHJMTcDBw6Eu7s7JkyYgLNnz4pdDpkgGysLzA3yAgB8cfoWb8lARGRGTCrcuLu7Y/Xq1di1axd27doFpVKJsWPH4tKlSw1uU1FRAbVarbcQAUBYsBesLaW4klGAi2n5YpdDREStxKTCja+vL/70pz8hMDAQISEhWL9+PUJCQvDhhx82uE1UVBQUCoVuUSqV7VgxGbMunWSYPrgbAGDtKU7qR0RkLkwq3NRn2LBhSE5ObvD5yMhIFBYW6paMDF4dQ79aOLLmsvCY6zlIuVsicjVERNQaTD7cJCYmwt294cGgMpkMcrlcbyGq5ePSCY/4uUAQgC95SwYiIrMg6tVSxcXFekddUlJSkJiYCGdnZ3Tv3h2RkZG4ffs2Nm7cCAD46KOP0KNHD/Tr1w/l5eVYt24djh8/jiNHjojVBTIDi0f1xPEbudiZkIkXJ/jCyd5a7JKIiOgBiHrk5uLFixg0aBAGDRoEAHjhhRcwaNAgvPbaawCA7OxspKen69pXVlbixRdfRP/+/TFmzBhcuXIFR48exbhx40Spn8xDUE9n+HeTo7xKi83xaWKXQ0RED0gidLBrYNVqNRQKBQoLC3mKinT2Jt7Gsq2J6NJJhjN/fxg2VhZil0RERL/RnO9vkx9zQ9QaJvd3h4fCBneLK/DK7qtI5eBiIiKTxXBDBMDKQoqIR3wAALsuZWLsv2Mxf8N5HL+RA622Qx3cJCIyeaIOKCYyJnOGe8HD0RYbz6Ui9qc7iL1Zs3R3tsO8IC/8cYgnHO042JiIyNhxzA1RPVLvlmBzfBq2X8yAurwaACCzlCJ0YDfMC/aCfzeFyBUSEXUszfn+ZrghMqCsUoO9ibexMS4NP2b/euuOQC8nhAV7YZK/O6wteXaXiKitMdwYwHBDLSEIAhLS8rExLg0HkrJRfX8cTpdOMswepsTTw7vDXWErcpVEROaL4cYAhht6ULlF5dh6PgNff5+GHHUFAMBCKsHEfq6YF+SNoJ7OkEgkIldJRGReGG4MYLih1lKl0eLItRxsjEvF9yl5uvUPuXbCvGBvPDmoG+xlHLNPRNQaGG4MYLihtnBDpcamuDTsvnwbpZUaAICDzBLTAz0xL9gLvbp2ErlCIiLTxnBjAMMNtSV1eRV2JWRiU1wabv1mIsBRvbtgXpAXxvVxhYWUp6yIiJqL4cYAhhtqD1qtgLO/3MVX59Jw7EYOaj9l3RxtMSeoO2YN7Q5n3qCTiKjJGG4MYLih9paRV4qvv0/HtgvpyC+tAgBYW0oxdYAHwoK9EKB0FLdAIiITwHBjAMMNiaW8SoPvrmRhY1wakm4X6tYHKB0RFuSFKQPcecNOIqIGMNwYwHBDYhMEAYkZBdgUl4Z9P2SjUqMFADjbW2PmUCXmDO8OTyc7kaskIjIuDDcGMNyQMblbXIFtFzLwdXwasgrLAQBSCTCujyvCg70xwqcz58whIgLDjUEMN2SMqjVaHLuRi41xqTibfE+3vldXe8wL8sL0QE842FiJWCERkbgYbgxguCFjl5xbhE1xadiZkImS+3Pm2Ftb4InB3RAW7I2HXB1ErpCIqP0x3BjAcEOmoriiGrsvZeKruDQk5xbr1gf37IywYC9M6OsKSwvetJOIOgaGGwMYbsjUCIKAuFv3sPFcGmKu50Bz/6ad7gobPD2sO2YN646uDjKRqyQialsMNwYw3JApyyoow5bv0/HN+XTcK6kEAFhZSDC5vzvCgr0xuLsjByATkVliuDGA4YbMQUW1BgeTVPgqLhWX0wt06/27yREW5I0/DPTgnDlEZFYYbgxguCFzk5RZiI1xqfjflSxUVNfMmeNoZ4UZQ5SYO9wL3TtzzhwiMn0MNwYw3JC5yi+pxPaLGdgUn4bM/DIAgEQCPOzrgrBgL4zu3RVS3rSTiEwUw40BDDdk7jRaAbE3c/FVXBpO/XRHt967sx3mBnnhj4FKKOw4Zw4RmRaGGwMYbqgjuXWnGJvj07EjIQNF5dUAAFsrC4QO8sC8IG/09eBngIhMA8ONAQw31BGVVlZjz+UsbIxLxQ1VkW79MG9nzAv2wmP+brDinDlEZMQYbgxguKGOTBAEXEjNx1dxqTh8VYXq+3PmuDjIMHtYdzw9vDtc5TYiV0lEVBfDjQEMN0Q1ctTl2PJ9OracT8edogoAgKVUgon+bggP9sZQbyfOmUNERoPhxgCGGyJ9ldVaHL6mwsa4VFxIzdet93NzQFiwN0IHecDO2lLEComIGG4MYrghatiPWWpsik/FnstZKKuquWmng40l/hioxLxgL/ToYi9yhUTUUTHcGMBwQ9S4wtIq7EjIwOb4NKTeK9WtH/1QV4QHe2GsrwssOGcOEbUjhhsDGG6Imk6rFXDq5zvYGJeGEzdzUfuvhdLZFnOHe2HGECWc7K3FLZKIOoTmfH+Leu3nqVOnMHXqVHh4eEAikWDPnj2NbhMbG4vBgwdDJpPBx8cH0dHRbV4nUUcllUow1tcF6+cPxcmXHsaS0T2hsLVCRl4Zog7eQFDUMfxtxxVcvV0odqlERDqihpuSkhIEBATg888/b1L7lJQUTJkyBQ8//DASExOxfPlyLFq0CIcPH27jSomoe2c7/GNyH8RHjsN70wegn4ccFdVa7EjIxOOfnsET/z2LPZdvo6JaI3apRNTBGc1pKYlEgt27dyM0NLTBNn//+9+xf/9+XL16Vbdu1qxZKCgowKFDh5r0PjwtRdQ6BEHApfQCbIpLxf6kbFRpav4p6dLJGrOG1syZ4+FoK3KVRGQuTOa0VHPFxcVh/PjxeusmTpyIuLi4BrepqKiAWq3WW4jowUkkEgR6OeGjWYNwbsU4vDjhIbjJbXC3uBKfnUjGqPdO4NlNCTj3y10Yyd9QRNRBmFS4UalUcHV11Vvn6uoKtVqNsrKyereJioqCQqHQLUqlsj1KJepQujrI8Py43jjz94exas5gBPV0hkYr4NA1FZ5e+z0e/fAUNsWloriiWuxSiagDMKlw0xKRkZEoLCzULRkZGWKXRGS2LC2kmNTfHVuXBOPIX0djblB32Flb4OfcYry69xqC/nUMr++9iuTcYrFLJSIzZlLTjrq5uSEnJ0dvXU5ODuRyOWxt6z+3L5PJIJPJ2qM8IvqNh1wd8M/Q/nj5MT98m5CJjfFpuHWnBF/FpeGruDSM8OmMsGBvjPNzgSVv2klErcikwk1wcDAOHDigty4mJgbBwcEiVUREjZHbWGH+iB4ID/HG2eR72BiXiqPXc3A2+R7OJt+Dh8IGc4K8MGuoEp078Q8RInpwol4tVVxcjOTkZADAoEGD8J///AcPP/wwnJ2d0b17d0RGRuL27dvYuHEjgJpLwf39/REREYFnnnkGx48fx1/+8hfs378fEydObNJ78mopIvFl5pfi6+/Tse1CBvJKKgEA1hZSPD7AHWEh3hiodBS3QCIyOiYzQ3FsbCwefvjhOuvDw8MRHR2N+fPnIzU1FbGxsXrb/PWvf8WPP/4IT09PvPrqq5g/f36T35Phhsh4lFdpsP+HbGyMT8OVjALd+gGeCoQFe+PxAe6wsbIQr0AiMhomE27EwHBDZJwSMwqwMS4V+37IRmW1FgDgZGeFmUO7Y87w7lA624lcIRGJieHGAIYbIuOWV1KJbRdqbtp5u6BmigepBHjEzxXhIV4Y0asLpLxpJ1GHw3BjAMMNkWnQaAUcu56DTfFpOP3zXd36nl3sMS/YC9MDPSG3sRKxQiJqTww3BjDcEJme5NxibI5Pw86ETN1EgHbWFnhiUDeEBXvD181B5AqJqK0x3BjAcENkuoorqrH78m1sikvFTzm/TgQ4vIczwkO8MaGvK6w4Zw6RWWK4MYDhhsj0CYKA+Ft52BSfisPXcqDR1vwz5iqXYeYQJYJ6doa/p4KnrYjMCMONAQw3ROYlu7AM33yfji3n03G3uFLvuV5d7RGgdMRApSMCPB3h5+4AmSUvLScyRQw3BjDcEJmnimoNDl1V4ciPOfghswAZeXVvpmttIUUfDzkGeioQoHREgNIRPTrb8+orIhPAcGMAww1Rx3CvuAI/ZBYiMaMAVzILcCWjAPmlVXXaOcgsMUCpQICno+4oj6vcRoSKicgQhhsDGG6IOiZBEJCZX1YTdu4HnqTbhSiv0tZp6ya3QYCy5ujOQE9Hjt8hMgIMNwYw3BBRrWqNFj/lFOuO7CRmFOCnnCJo6/lXkeN3iMTFcGMAww0RGVJaWY1rWWpd2LnC8TtERoHhxgCGGyJqLo7fIRIfw40BDDdE9KAEQUBGXhkSMwvwA8fvELULhhsDGG6IqC1w/A5R22K4MYDhhojaS2llNa7eVuuuzuL4HaKWY7gxgOGGiMTE8TtELcNwYwDDDREZE47fIWoahhsDGG6IyNhx/A5RXQw3BjDcEJEp4vgd6ugYbgxguCEic8HxO9SRMNwYwHBDROaK43fInDHcGMBwQ0QdCcfvkLlguDGA4YaIOjqO3yFTxHBjAMMNEVFdHL9Dxo7hxgCGGyKixnH8DhkbhhsDGG6IiFqG43dITAw3BjDcEBG1nt+O30nMLMAPHL9DbYThxgCGGyKitsXxO9QWGG4MYLghImpfvx2/cyWjZrma1fD4nT7uDlA620HpZFfzX2dbKJ3tOI6ng2O4MYDhhohIfM0Zv1NLYWuF7rVhx8kOns52NY+dbNHNyZZjeswcw40BDDdERMapdvzOL3eKkZFXivS8UmTklyEzrxT3SioNbiuRAK4ONujubAfP++FHWRt+nG3h6mDD8T0mzuTCzeeff473338fKpUKAQEB+PTTTzFs2LB620ZHR2PBggV662QyGcrLy5v0Xgw3RESmp6SiGhn5pcjIK0NGXun9/7//OL8UpZUag9tbW0jRzcn2/ukuW91pr9rwo7C1gkTC8GPMmvP9bdlONTVo27ZteOGFF7B69WoMHz4cH330ESZOnIibN2/CxcWl3m3kcjlu3rype8xfSCIi82Yvs4Sfmxx+bnW/1ARBQF5Jpe5IT0ZeKTLz7x/5yStDVkEZKjVapNwtQcrdknpf30Fmef80l63eWJ/uznbwdLKDjRVPeZkS0Y/cDB8+HEOHDsVnn30GANBqtVAqlXj++eexYsWKOu2jo6OxfPlyFBQUtOj9eOSGiKhjqdZooVKXIz2vFJn3j/T89rTXnaKKRl+jq4MMSifb+0d67O6P+akJQu4KG1haSNuhJx2byRy5qaysREJCAiIjI3XrpFIpxo8fj7i4uAa3Ky4uhpeXF7RaLQYPHox//etf6NevX71tKyoqUFHx6y+uWq1uvQ4QEZHRs7SQwtOp5ggMetV9vrxKo3ekp/a0V3pezXifoopq3CmqwJ2iClxKL6j7+lIJPBxtdQOdlboAVHP6q7O9Nc8wtDNRw83du3eh0Wjg6uqqt97V1RU3btyodxtfX1+sX78eAwYMQGFhIf79738jJCQE165dg6enZ532UVFRePPNN9ukfiIiMn02VhbwcXGAj4tDnecEQUBhWRUy8sruH+m5P9bn/umv2/k1p7zS7x8JAu7VeQ07a4v7occWnrpxPr9e9WUvE32EiNkR9bRUVlYWunXrhnPnziE4OFi3/uWXX8bJkyfx/fffN/oaVVVV6NOnD2bPno233nqrzvP1HblRKpU8LUVERA9MqxWQU1T+a/i5H4Ay7z/OKSpHY9+yne2t4fmbIz3ddXP82MLD0RZWPOUFwIROS3Xp0gUWFhbIycnRW5+TkwM3N7cmvYaVlRUGDRqE5OTkep+XyWSQyWQPXCsREdHvSaUSuCts4a6wxbAeznWer6jW4HZ+GTLyy+6P+SnVXfWVnleKwrIq3CupxL2SSlzJKKj7+hLAXaF/yuu3c/10dZDxlFc9RA031tbWCAwMxLFjxxAaGgqgZkDxsWPHsHTp0ia9hkajQVJSEiZPntyGlRIRETWfzNICPbt2Qs+unep9Xl1e9esl7fWc9qqo1uJ2QRluF5QhHnn1vL60zuXtnNXZCC4Ff+GFFxAeHo4hQ4Zg2LBh+Oijj1BSUqKbyyYsLAzdunVDVFQUAGDlypUICgqCj48PCgoK8P777yMtLQ2LFi0SsxtERETNJrexQj8PBfp5KOo8p9UKuFtcoXekJ+M3R36yC8tQUa1Fcm4xknOL6339jjqrs+jhZubMmbhz5w5ee+01qFQqDBw4EIcOHdINMk5PT4dU+uv5xvz8fCxevBgqlQpOTk4IDAzEuXPn0LdvX7G6QERE1OqkUglc5DZwkdsg0Kvu81UaLbIKynQTGf4afn6d1bmwrApJtwuRdLuwzvbmPKuz6PPctDfOc0NERB3B72d1Tr8/uaGpzupsMgOKiYiIqG00NqvzvZJKvfE9vz3lZeqzOvPIDREREemp1miRXViud1n7bwc7Nzars49LJxx9YUyr1sQjN0RERNRilhZS3UzL9c3qXFZZM6tz3cHONeN9lE627V/0bzDcEBERUbPYWlugt6sDervWP6tzRbVWhKp+xWkPiYiIqNVIJBLR76LOcENERERmheGGiIiIzArDDREREZkVhhsiIiIyKww3REREZFYYboiIiMisMNwQERGRWWG4ISIiIrPCcENERERmheGGiIiIzArDDREREZkVhhsiIiIyKww3REREZFYsxS6gvQmCAABQq9UiV0JERERNVfu9Xfs9bkiHCzdFRUUAAKVSKXIlRERE1FxFRUVQKBQG20iEpkQgM6LVapGVlQUHBwdIJJJWfW21Wg2lUomMjAzI5fJWfW1jYO79A8y/j+yf6TP3PrJ/pq+t+igIAoqKiuDh4QGp1PComg535EYqlcLT07NN30Mul5vtLy1g/v0DzL+P7J/pM/c+sn+mry362NgRm1ocUExERERmheGGiIiIzArDTSuSyWR4/fXXIZPJxC6lTZh7/wDz7yP7Z/rMvY/sn+kzhj52uAHFREREZN545IaIiIjMCsMNERERmRWGGyIiIjIrDDdERERkVhhumunzzz+Ht7c3bGxsMHz4cJw/f95g+x07dsDPzw82Njbo378/Dhw40E6Vtkxz+hcdHQ2JRKK32NjYtGO1zXPq1ClMnToVHh4ekEgk2LNnT6PbxMbGYvDgwZDJZPDx8UF0dHSb19lSze1fbGxsnf0nkUigUqnap+BmioqKwtChQ+Hg4AAXFxeEhobi5s2bjW5nSp/BlvTRlD6Hq1atwoABA3STuwUHB+PgwYMGtzGl/dfc/pnSvqvPO++8A4lEguXLlxtsJ8Y+ZLhphm3btuGFF17A66+/jkuXLiEgIAATJ05Ebm5uve3PnTuH2bNnY+HChbh8+TJCQ0MRGhqKq1evtnPlTdPc/gE1M1BmZ2frlrS0tHasuHlKSkoQEBCAzz//vEntU1JSMGXKFDz88MNITEzE8uXLsWjRIhw+fLiNK22Z5vav1s2bN/X2oYuLSxtV+GBOnjyJiIgIxMfHIyYmBlVVVXj00UdRUlLS4Dam9hlsSR8B0/kcenp64p133kFCQgIuXryIRx55BNOmTcO1a9fqbW9q+6+5/QNMZ9/93oULF7BmzRoMGDDAYDvR9qFATTZs2DAhIiJC91ij0QgeHh5CVFRUve1nzJghTJkyRW/d8OHDhT/96U9tWmdLNbd/GzZsEBQKRTtV17oACLt37zbY5uWXXxb69eunt27mzJnCxIkT27Cy1tGU/p04cUIAIOTn57dLTa0tNzdXACCcPHmywTam9hn8vab00ZQ/h4IgCE5OTsK6devqfc7U958gGO6fqe67oqIioXfv3kJMTIwwZswYYdmyZQ22FWsf8shNE1VWViIhIQHjx4/XrZNKpRg/fjzi4uLq3SYuLk6vPQBMnDixwfZiakn/AKC4uBheXl5QKpWN/oViakxp/z2IgQMHwt3dHRMmTMDZs2fFLqfJCgsLAQDOzs4NtjH1fdiUPgKm+TnUaDTYunUrSkpKEBwcXG8bU95/TekfYJr7LiIiAlOmTKmzb+oj1j5kuGmiu3fvQqPRwNXVVW+9q6trg2MUVCpVs9qLqSX98/X1xfr167F3715s3rwZWq0WISEhyMzMbI+S21xD+0+tVqOsrEykqlqPu7s7Vq9ejV27dmHXrl1QKpUYO3YsLl26JHZpjdJqtVi+fDlGjBgBf3//BtuZ0mfw95raR1P7HCYlJaFTp06QyWR49tlnsXv3bvTt27fetqa4/5rTP1PbdwCwdetWXLp0CVFRUU1qL9Y+7HB3BafWExwcrPcXSUhICPr06YM1a9bgrbfeErEyagpfX1/4+vrqHoeEhOCXX37Bhx9+iE2bNolYWeMiIiJw9epVnDlzRuxS2kxT+2hqn0NfX18kJiaisLAQO3fuRHh4OE6ePNlgADA1zemfqe27jIwMLFu2DDExMUY/8Jnhpom6dOkCCwsL5OTk6K3PycmBm5tbvdu4ubk1q72YWtK/37OyssKgQYOQnJzcFiW2u4b2n1wuh62trUhVta1hw4YZfWBYunQp9u3bh1OnTsHT09NgW1P6DP5Wc/r4e8b+ObS2toaPjw8AIDAwEBcuXMDHH3+MNWvW1GlrivuvOf37PWPfdwkJCcjNzcXgwYN16zQaDU6dOoXPPvsMFRUVsLCw0NtGrH3I01JNZG1tjcDAQBw7dky3TqvV4tixYw2eTw0ODtZrDwAxMTEGz7+KpSX9+z2NRoOkpCS4u7u3VZntypT2X2tJTEw02v0nCAKWLl2K3bt34/jx4+jRo0ej25jaPmxJH3/P1D6HWq0WFRUV9T5navuvPob693vGvu/GjRuHpKQkJCYm6pYhQ4Zgzpw5SExMrBNsABH3YZsOVzYzW7duFWQymRAdHS38+OOPwpIlSwRHR0dBpVIJgiAI8+bNE1asWKFrf/bsWcHS0lL497//LVy/fl14/fXXBSsrKyEpKUmsLhjU3P69+eabwuHDh4VffvlFSEhIEGbNmiXY2NgI165dE6sLBhUVFQmXL18WLl++LAAQ/vOf/wiXL18W0tLSBEEQhBUrVgjz5s3Ttb9165ZgZ2cn/O1vfxOuX78ufP7554KFhYVw6NAhsbpgUHP79+GHHwp79uwRfv75ZyEpKUlYtmyZIJVKhaNHj4rVBYP+/Oc/CwqFQoiNjRWys7N1S2lpqa6NqX8GW9JHU/ocrlixQjh58qSQkpIi/PDDD8KKFSsEiUQiHDlyRBAE099/ze2fKe27hvz+ailj2YcMN8306aefCt27dxesra2FYcOGCfHx8brnxowZI4SHh+u13759u/DQQw8J1tbWQr9+/YT9+/e3c8XN05z+LV++XNfW1dVVmDx5snDp0iURqm6a2kuff7/U9ik8PFwYM2ZMnW0GDhwoWFtbCz179hQ2bNjQ7nU3VXP79+677wq9evUSbGxsBGdnZ2Hs2LHC8ePHxSm+CerrGwC9fWLqn8GW9NGUPofPPPOM4OXlJVhbWwtdu3YVxo0bp/viFwTT33/N7Z8p7buG/D7cGMs+lAiCILTtsSEiIiKi9sMxN0RERGRWGG6IiIjIrDDcEBERkVlhuCEiIiKzwnBDREREZoXhhoiIiMwKww0RERGZFYYbIurwJBIJ9uzZI3YZRNRKGG6ISFTz58+HRCKpszz22GNil0ZEJop3BSci0T322GPYsGGD3jqZTCZSNURk6njkhohEJ5PJ4Obmprc4OTkBqDlltGrVKkyaNAm2trbo2bMndu7cqbd9UlISHnnkEdja2qJz585YsmQJiouL9dqsX78e/fr1g0wmg7u7O5YuXar3/N27d/HEE0/Azs4OvXv3xv/+97+27TQRtRmGGyIyeq+++iqmT5+OK1euYM6cOZg1axauX78OACgpKcHEiRPh5OSECxcuYMeOHTh69KheeFm1ahUiIiKwZMkSJCUl4X//+x98fHz03uPNN9/EjBkz8MMPP2Dy5MmYM2cO8vLy2rWfRNRK2vzWnEREBoSHhwsWFhaCvb293vL2228LglBzp+xnn31Wb5vhw4cLf/7znwVBEIQvvvhCcHJyEoqLi3XP79+/X5BKpYJKpRIEQRA8PDyEV155pcEaAAj/93//p3tcXFwsABAOHjzYav0kovbDMTdEJLqHH34Yq1at0lvn7Oys+//g4GC954KDg5GYmAgAuH79OgICAmBvb697fsSIEdBqtbh58yYkEgmysrIwbtw4gzUMGDBA9//29vaQy+XIzc1taZeISEQMN0QkOnt7+zqniVqLra1tk9pZWVnpPZZIJNBqtW1REhG1MY65ISKjFx8fX+dxnz59AAB9+vTBlStXUFJSonv+7NmzkEql8PX1hYODA7y9vXHs2LF2rZmIxMMjN0QkuoqKCqhUKr11lpaW6NKlCwBgx44dGDJkCEaOHImvv/4a58+fx5dffgkAmDNnDl5//XWEh4fjjTfewJ07d/D8889j3rx5cHV1BQC88cYbePbZZ+Hi4oJJkyahqKgIZ8+exfPPP9++HSWidsFwQ0SiO3ToENzd3fXW+fr64saNGwBqrmTaunUrnnvuObi7u+Obb75B3759AQB2dnY4fPgwli1bhqFDh8LOzg7Tp0/Hf/7zH91rhYeHo7y8HB9++CFeeukldOnSBU899VT7dZCI2pVEEARB7CKIiBoikUiwe/duhIaGil0KEZkIjrkhIiIis8JwQ0RERGaFY26IyKjxzDkRNReP3BAREZFZYbghIiIis8JwQ0RERGaF4YaIiIjMCsMNERERmRWGGyIiIjIrDDdERERkVhhuiIiIyKww3BAREZFZ+X9611boj/snDQAAAABJRU5ErkJggg==\n" + ] }, - "metadata": {} + "metadata": {}, + "output_type": "display_data" } + ], + "source": [ + "import torch\n", + "import os\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# Define a custom callback-like function to handle actions at the end of each epoch\n", + "class CustomCallback:\n", + " def __init__(self, model, lora_name, lora_rank, text_gen):\n", + " self.model = model\n", + " self.lora_name = lora_name\n", + " self.lora_rank = lora_rank\n", + " self.text_gen = text_gen # text_gen function for evaluation\n", + "\n", + " def on_epoch_end(self, epoch):\n", + " # Save LoRA weights at the end of each epoch\n", + " model_name = f\"./{self.lora_name}_{self.lora_rank}_epoch{epoch+1}.lora.pt\"\n", + " self.model.save_pretrained(model_name, token=access_token) # Save model with LoRA weights locally\n", + "\n", + " # Evaluate the model using text generation\n", + " print(f\"Epoch {epoch + 1} finished. Running evaluation:\")\n", + " self.text_gen(\"Write a title\")\n", + " self.text_gen(\"Write a poem\")\n", + "\n", + "# Assuming train is your DataLoader and gemma_lm is your model\n", + "callback = CustomCallback(gemma_lm, lora_name, lora_rank, text_gen)\n", + "\n", + "# Training loop with callback-like behavior\n", + "losses = []\n", + "for epoch in range(train_epoch):\n", + " epoch_loss = 0\n", + " for batch in train: # Assuming `train` is a DataLoader or similar iterable\n", + " optimizer.zero_grad()\n", + "\n", + " inputs = tokenizer(batch, return_tensors=\"pt\", max_length=token_limit, truncation=True, padding=True)\n", + " labels = inputs[\"input_ids\"]\n", + " outputs = gemma_lm(**inputs, labels=labels)\n", + " loss = outputs.loss\n", + "\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " epoch_loss += loss.item()\n", + "\n", + " losses.append(epoch_loss / len(train)) # Store average loss per epoch\n", + "\n", + " # Run custom callback at the end of each epoch\n", + " callback.on_epoch_end(epoch)\n", + "\n", + "# Plot training loss over epochs\n", + "plt.plot(losses)\n", + "plt.xlabel(\"Epoch\")\n", + "plt.ylabel(\"Loss\")\n", + "plt.title(\"Training Loss Over Epochs\")\n", + "plt.show()" ] }, { @@ -1328,6 +1171,11 @@ }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kVe4vjgCngsd" + }, + "outputs": [], "source": [ "# Example Code for Load LoRA\n", "\n", @@ -1339,12 +1187,7 @@ "\n", "# # Load the LoRA adapter into the model using PeftModel\n", "# gemma_lm = PeftModel.from_pretrained(gemma_lm, lora_weights_path)" - ], - "metadata": { - "id": "kVe4vjgCngsd" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", @@ -1359,6 +1202,24 @@ }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "K2JUE2IilwNi" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Gemma output:\n", + "user\n", + "Write a title\n", + "model\n", + "Capitães da Areia\n" + ] + } + ], "source": [ "import torch\n", "\n", @@ -1389,28 +1250,6 @@ "\n", "# Generate text 5 times using the top_k sampling strategy\n", "text_gen_with_top_k(\"Write a title\", token_limit=100, top_k=50)\n" - ], - "metadata": { - "id": "K2JUE2IilwNi", - "outputId": "1ffdd099-1e0f-4a66-a7c6-81b43cc643e3", - "colab": { - "base_uri": "https://localhost:8080/" - } - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "\n", - "Gemma output:\n", - "user\n", - "Write a title\n", - "model\n", - "Capitães da Areia\n" - ] - } ] }, { @@ -1426,16 +1265,12 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "qC-MLxYWM1HU", - "outputId": "f26faf29-ce26-4a5c-e6ff-f2125148249b", - "colab": { - "base_uri": "https://localhost:8080/" - } + "id": "qC-MLxYWM1HU" }, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "\n", "Gemma output:\n", @@ -1500,15 +1335,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "4TcvzBH995FE", - "outputId": "26a4be25-9a51-41f9-ad28-ef1708bf440f", - "colab": { - "base_uri": "https://localhost:8080/" - } + "id": "4TcvzBH995FE" }, "outputs": [ { - "output_type": "execute_result", "data": { "text/plain": [ "('./my_gemma2_lt_pt/tokenizer_config.json',\n", @@ -1518,8 +1348,9 @@ " './my_gemma2_lt_pt/tokenizer.json')" ] }, + "execution_count": 18, "metadata": {}, - "execution_count": 18 + "output_type": "execute_result" } ], "source": [ @@ -1546,4551 +1377,73 @@ }, { "cell_type": "code", - "source": [ - "# Upload the model to Hugging Face Hub\n", - "my_model_name = \"my_gemma2_pt\"\n", - "writeToken = userdata.get(\"HF_WRITE_TOKEN\")\n", - "hf_repo_id = f\"{my_hf_username}/{my_model_name}\" # Correct format\n", - "gemma_lm.push_to_hub(hf_repo_id, token=writeToken)" - ], + "execution_count": null, "metadata": { - "id": "AK31-LuXpwen", - "outputId": "e96865e3-4e8c-4acf-e256-c5ec31b6f931", - "colab": { - "base_uri": "https://localhost:8080/", - "height": 136, - "referenced_widgets": [ - "84cd1a70535f4272a0206041373ff281", - "325cf823a7244a4b8e25462ae4912ea3", - "325f38ff3fef43cbbd248ff3ff714c43", - "de4fae4a6e3e420d86414a572846a9e0", - "4d729ba11c9442528af79d6e377a292c", - "49a42d4bcb8b4684a69656ee8b0449df", - "50b06db300ce432c9235071f48a18432", - "a31850a948e344848d43caae277f3200", - "fe5291b55e7147b682b671dad99d16e6", - "b485e94c1509478da6dd7a04c41e055d", - "98e4874bfa904312a4a9b00ca2ea8577", - "a163fe0a8b594568bf102caf54856484", - "dd53c0e07382441fbad818b0c3852006", - "3c316ae120e445c8b4bbc8260cf14ab4", - "9387a006080845ce9ef9aeb41fbd6c6e", - "c49cd450c3d24511a1efe272820c5f0a", - "99a5ad5c04b4456ba65118367cd2ea5a", - "d74745e493704920b56855e0cb73a39e", - "bd53b02c9808436ead8b213886e76bed", - "3f1bd9e04dc64bca88b99998339df788", - "d082ae773106445aa6ec56167b2a5cb9", - "030123f151674251ac0f0be0f8f713dc" - ] - } + "id": "AK31-LuXpwen" }, - "execution_count": null, "outputs": [ { - "output_type": "display_data", "data": { - "text/plain": [ - "README.md: 0%| | 0.00/5.17k [00:00