From 331c09f824082c78be72b52d6f7ae45d9012d1a8 Mon Sep 17 00:00:00 2001 From: "Md. Khairul Islam" Date: Tue, 19 Sep 2023 11:48:45 -0400 Subject: [PATCH] basic interpretation codes --- interpret.ipynb | 390 ++++++++ interpret.py | 101 +++ models/DLinear.py | 5 +- models/Transformer.py | 5 +- requirements.txt | 2 +- run.py | 169 ++-- scripts/Covid/Transformer.sh | 2 +- scripts/Exchange_script/Transformer.sh | 16 +- scripts/ILI_script/Transformer.sh | 24 +- scripts/ILI_script/Transformer_windows.sh | 2 +- scripts/Traffic_script/Transformer.sh | 16 +- tsai.ipynb | 1003 --------------------- 12 files changed, 614 insertions(+), 1121 deletions(-) create mode 100644 interpret.ipynb create mode 100644 interpret.py delete mode 100644 tsai.ipynb diff --git a/interpret.ipynb b/interpret.ipynb new file mode 100644 index 0000000..b5f3a5a --- /dev/null +++ b/interpret.ipynb @@ -0,0 +1,390 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "from run import *\n", + "from tint.attr import FeatureAblation, Occlusion, Fit\n", + "from tint.metrics import mse, mae" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "parser = get_parser()\n", + "argv = \"\"\"\n", + " --root_path ./dataset/illness/ \\\n", + " --data_path national_illness.csv \\\n", + " --model_id ili_36_24 \\\n", + " --model Transformer \\\n", + " --data custom \\\n", + " --use_gpu\n", + " --features MS \\\n", + " --seq_len 36 \\\n", + " --label_len 18 \\\n", + " --pred_len 24 \\\n", + " --e_layers 2 \\\n", + " --d_layers 1 \\\n", + " --factor 3 \\\n", + " --enc_in 7 \\\n", + " --dec_in 7 \\\n", + " --c_out 7 \\\n", + " --des Exp \\\n", + " --itr 1\n", + "\"\"\".split()\n", + "args = parser.parse_args(argv)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "set_random_seed(args.seed)\n", + "args.use_gpu = True if torch.cuda.is_available() and args.use_gpu else False\n", + "\n", + "if args.use_gpu and args.use_multi_gpu:\n", + " args.devices = args.devices.replace(' ', '')\n", + " device_ids = args.devices.split(',')\n", + " args.device_ids = [int(id_) for id_ in device_ids]\n", + " args.gpu = args.device_ids[0]\n", + " \n", + "if args.task_name == 'classification':\n", + " Exp = Exp_Classification\n", + "else:\n", + " Exp = Exp_Long_Term_Forecast" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "setting = stringify_setting(args, 0)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Use GPU: cuda:0\n", + "test 170\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "exp = Exp(args) # set experiments\n", + "_, dataloader = exp._get_data('test')\n", + "exp.model.load_state_dict(\n", + " torch.load(os.path.join('checkpoints/' + setting, 'checkpoint.pth'))\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "model = exp.model\n", + "model.eval()\n", + "model.zero_grad()\n", + "\n", + "# only need to output targets, sinec interpretation is based on outputs\n", + "assert not exp.args.output_attention\n", + "\n", + "explainer = FeatureAblation(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 25%|██▌ | 6/24 [00:55<02:46, 9.22s/it]\n", + "0it [00:55, ?it/s]\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[1;32mc:\\Softwares\\SA-Timeseries\\interpret.ipynb Cell 7\u001b[0m line \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[0;32m 18\u001b[0m mean_score \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m 19\u001b[0m \u001b[39mfor\u001b[39;00m target \u001b[39min\u001b[39;00m tqdm(\u001b[39mrange\u001b[39m(args\u001b[39m.\u001b[39mpred_len)):\n\u001b[1;32m---> 20\u001b[0m score \u001b[39m=\u001b[39m explainer\u001b[39m.\u001b[39;49mattribute(\n\u001b[0;32m 21\u001b[0m inputs\u001b[39m=\u001b[39;49m(batch_x),\n\u001b[0;32m 22\u001b[0m baselines\u001b[39m=\u001b[39;49m\u001b[39m0\u001b[39;49m,\n\u001b[0;32m 23\u001b[0m target\u001b[39m=\u001b[39;49mtarget,\n\u001b[0;32m 24\u001b[0m additional_forward_args\u001b[39m=\u001b[39;49m(batch_x_mark, dec_inp, batch_y_mark)\n\u001b[0;32m 25\u001b[0m )\n\u001b[0;32m 26\u001b[0m \u001b[39mif\u001b[39;00m target\u001b[39m==\u001b[39m\u001b[39m0\u001b[39m: mean_score \u001b[39m=\u001b[39m score\n\u001b[0;32m 27\u001b[0m \u001b[39melse\u001b[39;00m: mean_score \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m score\n", + "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\captum\\log\\__init__.py:42\u001b[0m, in \u001b[0;36mlog_usage.._log_usage..wrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m 40\u001b[0m \u001b[39m@wraps\u001b[39m(func)\n\u001b[0;32m 41\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mwrapper\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m---> 42\u001b[0m \u001b[39mreturn\u001b[39;00m func(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n", + "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\tint\\attr\\feature_ablation.py:368\u001b[0m, in \u001b[0;36mFeatureAblation.attribute\u001b[1;34m(self, inputs, baselines, target, additional_forward_args, feature_mask, perturbations_per_eval, attributions_fn, show_progress, **kwargs)\u001b[0m\n\u001b[0;32m 349\u001b[0m \u001b[39mcontinue\u001b[39;00m\n\u001b[0;32m 351\u001b[0m \u001b[39mfor\u001b[39;00m (\n\u001b[0;32m 352\u001b[0m current_inputs,\n\u001b[0;32m 353\u001b[0m current_add_args,\n\u001b[1;32m (...)\u001b[0m\n\u001b[0;32m 366\u001b[0m \u001b[39m# modified_eval dimensions: 1D tensor with length\u001b[39;00m\n\u001b[0;32m 367\u001b[0m \u001b[39m# equal to #num_examples * #features in batch\u001b[39;00m\n\u001b[1;32m--> 368\u001b[0m modified_eval \u001b[39m=\u001b[39m _run_forward(\n\u001b[0;32m 369\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mforward_func,\n\u001b[0;32m 370\u001b[0m current_inputs,\n\u001b[0;32m 371\u001b[0m current_target,\n\u001b[0;32m 372\u001b[0m current_add_args,\n\u001b[0;32m 373\u001b[0m )\n\u001b[0;32m 375\u001b[0m \u001b[39mif\u001b[39;00m show_progress:\n\u001b[0;32m 376\u001b[0m attr_progress\u001b[39m.\u001b[39mupdate()\n", + "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\captum\\_utils\\common.py:482\u001b[0m, in \u001b[0;36m_run_forward\u001b[1;34m(forward_func, inputs, target, additional_forward_args)\u001b[0m\n\u001b[0;32m 479\u001b[0m inputs \u001b[39m=\u001b[39m _format_inputs(inputs)\n\u001b[0;32m 480\u001b[0m additional_forward_args \u001b[39m=\u001b[39m _format_additional_forward_args(additional_forward_args)\n\u001b[1;32m--> 482\u001b[0m output \u001b[39m=\u001b[39m forward_func(\n\u001b[0;32m 483\u001b[0m \u001b[39m*\u001b[39;49m(\u001b[39m*\u001b[39;49minputs, \u001b[39m*\u001b[39;49madditional_forward_args)\n\u001b[0;32m 484\u001b[0m \u001b[39mif\u001b[39;49;00m additional_forward_args \u001b[39mis\u001b[39;49;00m \u001b[39mnot\u001b[39;49;00m \u001b[39mNone\u001b[39;49;00m\n\u001b[0;32m 485\u001b[0m \u001b[39melse\u001b[39;49;00m inputs\n\u001b[0;32m 486\u001b[0m )\n\u001b[0;32m 487\u001b[0m \u001b[39mreturn\u001b[39;00m _select_targets(output, target)\n", + "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\torch\\nn\\modules\\module.py:1194\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1190\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m 1191\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m 1192\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m 1193\u001b[0m \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1194\u001b[0m \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39m\u001b[39minput\u001b[39m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m 1195\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[0;32m 1196\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n", + "File \u001b[1;32mc:\\Softwares\\SA-Timeseries\\models\\Transformer.py:96\u001b[0m, in \u001b[0;36mModel.forward\u001b[1;34m(self, x_enc, x_mark_enc, x_dec, x_mark_dec, mask)\u001b[0m\n\u001b[0;32m 94\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, x_enc, x_mark_enc, x_dec, x_mark_dec, mask\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m):\n\u001b[0;32m 95\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtask_name \u001b[39m==\u001b[39m \u001b[39m'\u001b[39m\u001b[39mlong_term_forecast\u001b[39m\u001b[39m'\u001b[39m:\n\u001b[1;32m---> 96\u001b[0m dec_out \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mforecast(x_enc, x_mark_enc, x_dec, x_mark_dec)\n\u001b[0;32m 98\u001b[0m f_dim \u001b[39m=\u001b[39m \u001b[39m-\u001b[39m\u001b[39m1\u001b[39m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mconfigs\u001b[39m.\u001b[39mfeatures \u001b[39m==\u001b[39m \u001b[39m'\u001b[39m\u001b[39mMS\u001b[39m\u001b[39m'\u001b[39m \u001b[39melse\u001b[39;00m \u001b[39m0\u001b[39m\n\u001b[0;32m 99\u001b[0m \u001b[39mreturn\u001b[39;00m dec_out[:, \u001b[39m-\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpred_len:, f_dim:] \u001b[39m# [B, L, D]\u001b[39;00m\n", + "File \u001b[1;32mc:\\Softwares\\SA-Timeseries\\models\\Transformer.py:78\u001b[0m, in \u001b[0;36mModel.forecast\u001b[1;34m(self, x_enc, x_mark_enc, x_dec, x_mark_dec)\u001b[0m\n\u001b[0;32m 75\u001b[0m enc_out, attns \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mencoder(enc_out, attn_mask\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m)\n\u001b[0;32m 77\u001b[0m dec_out \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdec_embedding(x_dec, x_mark_dec)\n\u001b[1;32m---> 78\u001b[0m dec_out \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdecoder(dec_out, enc_out, x_mask\u001b[39m=\u001b[39;49m\u001b[39mNone\u001b[39;49;00m, cross_mask\u001b[39m=\u001b[39;49m\u001b[39mNone\u001b[39;49;00m)\n\u001b[0;32m 79\u001b[0m \u001b[39mreturn\u001b[39;00m dec_out\n", + "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\torch\\nn\\modules\\module.py:1194\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1190\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m 1191\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m 1192\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m 1193\u001b[0m \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1194\u001b[0m \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39m\u001b[39minput\u001b[39m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m 1195\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[0;32m 1196\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n", + "File \u001b[1;32mc:\\Softwares\\SA-Timeseries\\layers\\Transformer_EncDec.py:128\u001b[0m, in \u001b[0;36mDecoder.forward\u001b[1;34m(self, x, cross, x_mask, cross_mask, tau, delta)\u001b[0m\n\u001b[0;32m 126\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, x, cross, x_mask\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, cross_mask\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, tau\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, delta\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m):\n\u001b[0;32m 127\u001b[0m \u001b[39mfor\u001b[39;00m layer \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mlayers:\n\u001b[1;32m--> 128\u001b[0m x \u001b[39m=\u001b[39m layer(x, cross, x_mask\u001b[39m=\u001b[39;49mx_mask, cross_mask\u001b[39m=\u001b[39;49mcross_mask, tau\u001b[39m=\u001b[39;49mtau, delta\u001b[39m=\u001b[39;49mdelta)\n\u001b[0;32m 130\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mnorm \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m 131\u001b[0m x \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mnorm(x)\n", + "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\torch\\nn\\modules\\module.py:1194\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1190\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m 1191\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m 1192\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m 1193\u001b[0m \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1194\u001b[0m \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39m\u001b[39minput\u001b[39m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m 1195\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[0;32m 1196\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n", + "File \u001b[1;32mc:\\Softwares\\SA-Timeseries\\layers\\Transformer_EncDec.py:99\u001b[0m, in \u001b[0;36mDecoderLayer.forward\u001b[1;34m(self, x, cross, x_mask, cross_mask, tau, delta)\u001b[0m\n\u001b[0;32m 98\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, x, cross, x_mask\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, cross_mask\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, tau\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, delta\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m):\n\u001b[1;32m---> 99\u001b[0m x \u001b[39m=\u001b[39m x \u001b[39m+\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdropout(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mself_attention(\n\u001b[0;32m 100\u001b[0m x, x, x,\n\u001b[0;32m 101\u001b[0m attn_mask\u001b[39m=\u001b[39;49mx_mask,\n\u001b[0;32m 102\u001b[0m tau\u001b[39m=\u001b[39;49mtau, delta\u001b[39m=\u001b[39;49m\u001b[39mNone\u001b[39;49;00m\n\u001b[0;32m 103\u001b[0m )[\u001b[39m0\u001b[39m])\n\u001b[0;32m 104\u001b[0m x \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mnorm1(x)\n\u001b[0;32m 106\u001b[0m x \u001b[39m=\u001b[39m x \u001b[39m+\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdropout(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcross_attention(\n\u001b[0;32m 107\u001b[0m x, cross, cross,\n\u001b[0;32m 108\u001b[0m attn_mask\u001b[39m=\u001b[39mcross_mask,\n\u001b[0;32m 109\u001b[0m tau\u001b[39m=\u001b[39mtau, delta\u001b[39m=\u001b[39mdelta\n\u001b[0;32m 110\u001b[0m )[\u001b[39m0\u001b[39m])\n", + "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\torch\\nn\\modules\\module.py:1194\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1190\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m 1191\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m 1192\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m 1193\u001b[0m \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1194\u001b[0m \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39m\u001b[39minput\u001b[39m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m 1195\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[0;32m 1196\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n", + "File \u001b[1;32mc:\\Softwares\\SA-Timeseries\\layers\\SelfAttention_Family.py:201\u001b[0m, in \u001b[0;36mAttentionLayer.forward\u001b[1;34m(self, queries, keys, values, attn_mask, tau, delta)\u001b[0m\n\u001b[0;32m 198\u001b[0m keys \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mkey_projection(keys)\u001b[39m.\u001b[39mview(B, S, H, \u001b[39m-\u001b[39m\u001b[39m1\u001b[39m)\n\u001b[0;32m 199\u001b[0m values \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mvalue_projection(values)\u001b[39m.\u001b[39mview(B, S, H, \u001b[39m-\u001b[39m\u001b[39m1\u001b[39m)\n\u001b[1;32m--> 201\u001b[0m out, attn \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49minner_attention(\n\u001b[0;32m 202\u001b[0m queries,\n\u001b[0;32m 203\u001b[0m keys,\n\u001b[0;32m 204\u001b[0m values,\n\u001b[0;32m 205\u001b[0m attn_mask,\n\u001b[0;32m 206\u001b[0m tau\u001b[39m=\u001b[39;49mtau,\n\u001b[0;32m 207\u001b[0m delta\u001b[39m=\u001b[39;49mdelta\n\u001b[0;32m 208\u001b[0m )\n\u001b[0;32m 209\u001b[0m out \u001b[39m=\u001b[39m out\u001b[39m.\u001b[39mview(B, L, \u001b[39m-\u001b[39m\u001b[39m1\u001b[39m)\n\u001b[0;32m 211\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mout_projection(out), attn\n", + "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python310\\site-packages\\torch\\nn\\modules\\module.py:1194\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[1;34m(self, *input, **kwargs)\u001b[0m\n\u001b[0;32m 1190\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[0;32m 1191\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[0;32m 1192\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[0;32m 1193\u001b[0m \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[1;32m-> 1194\u001b[0m \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39m\u001b[39minput\u001b[39m, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m 1195\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[0;32m 1196\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n", + "File \u001b[1;32mc:\\Softwares\\SA-Timeseries\\layers\\SelfAttention_Family.py:63\u001b[0m, in \u001b[0;36mFullAttention.forward\u001b[1;34m(self, queries, keys, values, attn_mask, tau, delta)\u001b[0m\n\u001b[0;32m 61\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmask_flag:\n\u001b[0;32m 62\u001b[0m \u001b[39mif\u001b[39;00m attn_mask \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m---> 63\u001b[0m attn_mask \u001b[39m=\u001b[39m TriangularCausalMask(B, L, device\u001b[39m=\u001b[39;49mqueries\u001b[39m.\u001b[39;49mdevice)\n\u001b[0;32m 65\u001b[0m scores\u001b[39m.\u001b[39mmasked_fill_(attn_mask\u001b[39m.\u001b[39mmask, \u001b[39m-\u001b[39mnp\u001b[39m.\u001b[39minf)\n\u001b[0;32m 67\u001b[0m A \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdropout(torch\u001b[39m.\u001b[39msoftmax(scale \u001b[39m*\u001b[39m scores, dim\u001b[39m=\u001b[39m\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m))\n", + "File \u001b[1;32mc:\\Softwares\\SA-Timeseries\\utils\\masking.py:8\u001b[0m, in \u001b[0;36mTriangularCausalMask.__init__\u001b[1;34m(self, B, L, device)\u001b[0m\n\u001b[0;32m 6\u001b[0m mask_shape \u001b[39m=\u001b[39m [B, \u001b[39m1\u001b[39m, L, L]\n\u001b[0;32m 7\u001b[0m \u001b[39mwith\u001b[39;00m torch\u001b[39m.\u001b[39mno_grad():\n\u001b[1;32m----> 8\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_mask \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39;49mtriu(torch\u001b[39m.\u001b[39;49mones(mask_shape, dtype\u001b[39m=\u001b[39;49mtorch\u001b[39m.\u001b[39;49mbool), diagonal\u001b[39m=\u001b[39;49m\u001b[39m1\u001b[39;49m)\u001b[39m.\u001b[39;49mto(device)\n", + "\u001b[1;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "from tqdm import tqdm\n", + "\n", + "results = {\n", + " 'mae':[], 'mse':[]\n", + "}\n", + "\n", + "for i, (batch_x, batch_y, batch_x_mark, batch_y_mark) in tqdm(enumerate(dataloader)):\n", + " batch_x = batch_x.float().to(exp.device)\n", + " batch_y = batch_y.float().to(exp.device)\n", + "\n", + " batch_x_mark = batch_x_mark.float().to(exp.device)\n", + " batch_y_mark = batch_y_mark.float().to(exp.device)\n", + "\n", + " # decoder input\n", + " dec_inp = torch.zeros_like(batch_y[:, -exp.args.pred_len:, :]).float()\n", + " dec_inp = torch.cat([batch_y[:, :exp.args.label_len, :], dec_inp], dim=1).float().to(exp.device)\n", + " \n", + " mean_score = None\n", + " for target in tqdm(range(args.pred_len)):\n", + " score = explainer.attribute(\n", + " inputs=(batch_x),\n", + " baselines=0,\n", + " target=target,\n", + " additional_forward_args=(batch_x_mark, dec_inp, batch_y_mark)\n", + " )\n", + " if target==0: mean_score = score\n", + " else: mean_score += score\n", + " mean_score /= args.pred_len\n", + " \n", + " # temp = score.reshape(\n", + " # (batch_x.shape[0], args.pred_len, args.seq_len, -1)\n", + " # ).mean(axis=1).float().to(exp.device)\n", + " \n", + " mae_error = mae(\n", + " model, inputs=batch_x, \n", + " attributions=mean_score, baselines=0, \n", + " additional_forward_args=(batch_x_mark, dec_inp, batch_y_mark),\n", + " topk=0.2\n", + " )\n", + " mse_error = mse(\n", + " model, inputs=batch_x, \n", + " attributions=mean_score, baselines=0, \n", + " additional_forward_args=(batch_x_mark, dec_inp, batch_y_mark),\n", + " topk=0.2\n", + " )\n", + " results['mae'].append(mae_error)\n", + " results['mse'].append(mse_error)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(torch.Size([10, 36, 7]), torch.Size([10, 36, 7]))" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "batch_x.shape, score.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'mae': 10.65552536646525, 'mse': 4.753488858540853}\n" + ] + } + ], + "source": [ + "for key in results.keys():\n", + " results[key] = np.mean(results[key])\n", + "print(results)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "# outputs = model(batch_x, batch_x_mark, dec_inp, batch_y_mark)\n", + "# outputs.shape, outputs.numel(), outputs[0].numel()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "# score = explainer.attribute(\n", + "# inputs=(batch_x),\n", + "# baselines=0,\n", + "# additional_forward_args=(batch_x_mark, dec_inp, batch_y_mark)\n", + "# )\n", + "# print(batch_x.shape, score.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "# score_targeted = explainer.attribute(\n", + "# inputs=(batch_x),\n", + "# baselines=0,\n", + "# additional_forward_args=(batch_x_mark, dec_inp, batch_y_mark),\n", + "# target=0\n", + "# )\n", + "# print(batch_x.shape, score_targeted.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [], + "source": [ + "# temp = score.reshape((batch_x.shape[0], args.pred_len, args.seq_len, -1)).mean(axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "# mse_error = mse(\n", + "# model, inputs=batch_x, \n", + "# attributions=temp, baselines=0, \n", + "# additional_forward_args=(batch_x_mark, dec_inp, batch_y_mark),\n", + "# topk=0.2\n", + "# )\n", + "# print(mse_error)\n", + "\n", + "# mae_error = mae(\n", + "# model, inputs=batch_x, \n", + "# attributions=temp, baselines=0, \n", + "# additional_forward_args=(batch_x_mark, dec_inp, batch_y_mark),\n", + "# # target=0,\n", + "# topk=0.2\n", + "# )\n", + "# print(mae_error)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "# temporal_mask = torch.zeros_like(batch_x, dtype=int)\n", + "# for t in range(batch_x.shape[1]):\n", + "# temporal_mask[:, t] = t\n", + "\n", + "# explainer = FeatureAblation(model)\n", + "# time_score = explainer.attribute(\n", + "# inputs=(batch_x),\n", + "# baselines=(batch_x*0),\n", + "# additional_forward_args=(batch_x_mark, dec_inp, batch_y_mark),\n", + "# target=0,\n", + "# feature_mask=temporal_mask\n", + "# )\n", + "# print(score.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "# from tint.attr import Occlusion\n", + "\n", + "# temporal_mask = torch.zeros(size=(1, *batch_x.shape[1:]), dtype=int)\n", + "# for t in range(batch_x.shape[1]):\n", + "# temporal_mask[:, t, :] = t\n", + "\n", + "# explainer = Occlusion(model)\n", + "# time_score = explainer.attribute(\n", + "# inputs=(batch_x),\n", + "# baselines=(batch_x*0),\n", + "# additional_forward_args=(batch_x_mark, dec_inp, batch_y_mark),\n", + "# sliding_window_shapes=(1, 1)\n", + "# # feature_mask=temporal_mask.to(exp.device)\n", + "# )\n", + "# print(time_score.shape)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/interpret.py b/interpret.py new file mode 100644 index 0000000..f96be0f --- /dev/null +++ b/interpret.py @@ -0,0 +1,101 @@ +from run import * +from tint.attr import FeatureAblation +from tint.metrics import mse, mae +from tqdm import tqdm + +parser = get_parser() +argv = """ + --root_path ./dataset/illness/ \ + --data_path national_illness.csv \ + --model_id ili_36_24 \ + --model Transformer \ + --data custom \ + --features MS \ + --use_gpu \ + --seq_len 36 \ + --label_len 18 \ + --pred_len 24 \ + --e_layers 2 \ + --d_layers 1 \ + --factor 3 \ + --enc_in 7 \ + --dec_in 7 \ + --c_out 7 \ + --des Exp \ + --itr 1 +""".split() +args = parser.parse_args(argv) + +set_random_seed(args.seed) +# Disable cudnn if using cuda accelerator. + # Please see https://captum.ai/docs/faq#how-can-i-resolve-cudnn-rnn-backward-error-for-rnn-or-lstm-network +args.use_gpu = False + +assert args.task_name == 'long_term_forecast', "Only long_term_forecast is supported for now" + +Exp = Exp_Long_Term_Forecast + +setting = stringify_setting(args, 0) +exp = Exp(args) # set experiments +_, dataloader = exp._get_data('test') + +exp.model.load_state_dict( + torch.load(os.path.join('checkpoints/' + setting, 'checkpoint.pth')) +) + +model = exp.model +model.eval() +model.zero_grad() +explainer = FeatureAblation(model) +assert not exp.args.output_attention + +if args.use_gpu: + torch.backends.cudnn.enabled = False + +topk = 0.2 +error_results = { + 'mae':[], 'mse':[] +} + +for i, (batch_x, batch_y, batch_x_mark, batch_y_mark) in tqdm(enumerate(dataloader)): + batch_x = batch_x.float().to(exp.device) + batch_y = batch_y.float().to(exp.device) + + batch_x_mark = batch_x_mark.float().to(exp.device) + batch_y_mark = batch_y_mark.float().to(exp.device) + + # decoder input + dec_inp = torch.zeros_like(batch_y[:, -exp.args.pred_len:, :]).float() + dec_inp = torch.cat([batch_y[:, :exp.args.label_len, :], dec_inp], dim=1).float().to(exp.device) + + # batch size x pred_len x seq_len x n_features if target = None + # batch size x seq_len x n_features if target specified + score = explainer.attribute( + inputs=(batch_x), baselines=0, # target=0, + additional_forward_args=(batch_x_mark, dec_inp, batch_y_mark) + ) + + # batch size x seq_len x n_features + # take mean score across all output horizon + mean_score = score.reshape( + (batch_x.shape[0], args.pred_len, args.seq_len, -1) + ).mean(axis=1) + + mae_error = mae( + model, inputs=batch_x, topk=topk, mask_largest=True, + attributions=mean_score, baselines=0, + additional_forward_args=(batch_x_mark, dec_inp, batch_y_mark) + ) + + mse_error = mse( + model, inputs=batch_x, topk=topk, mask_largest=True, + attributions=mean_score, baselines=0, + additional_forward_args=(batch_x_mark, dec_inp, batch_y_mark) + ) + error_results['mae'].append(mae_error) + error_results['mse'].append(mse_error) + +for key in error_results.keys(): + error_results[key] = np.mean(error_results[key]) + +print(error_results) \ No newline at end of file diff --git a/models/DLinear.py b/models/DLinear.py index c826a43..ce3b9b2 100644 --- a/models/DLinear.py +++ b/models/DLinear.py @@ -14,6 +14,7 @@ def __init__(self, configs, individual=False): individual: Bool, whether shared model among different variates. """ super(Model, self).__init__() + self.configs = configs self.task_name = configs.task_name self.seq_len = configs.seq_len if self.task_name == 'classification': @@ -91,7 +92,9 @@ def classification(self, x_enc): def forward(self, x_enc, x_mark_enc, x_dec, x_mark_dec, mask=None): if self.task_name == 'long_term_forecast': dec_out = self.forecast(x_enc) - return dec_out[:, -self.pred_len:, :] # [B, L, D] + + f_dim = -1 if self.configs.features == 'MS' else 0 + return dec_out[:, -self.pred_len:, f_dim:] # [B, L, D] if self.task_name == 'classification': dec_out = self.classification(x_enc) return dec_out # [B, N] diff --git a/models/Transformer.py b/models/Transformer.py index 512bcc3..a86d9df 100644 --- a/models/Transformer.py +++ b/models/Transformer.py @@ -16,6 +16,7 @@ class Model(nn.Module): def __init__(self, configs): super(Model, self).__init__() + self.configs = configs self.task_name = configs.task_name self.pred_len = configs.pred_len self.output_attention = configs.output_attention @@ -93,7 +94,9 @@ def classification(self, x_enc, x_mark_enc): def forward(self, x_enc, x_mark_enc, x_dec, x_mark_dec, mask=None): if self.task_name == 'long_term_forecast': dec_out = self.forecast(x_enc, x_mark_enc, x_dec, x_mark_dec) - return dec_out[:, -self.pred_len:, :] # [B, L, D] + + f_dim = -1 if self.configs.features == 'MS' else 0 + return dec_out[:, -self.pred_len:, f_dim:] # [B, L, D] if self.task_name == 'classification': dec_out = self.classification(x_enc, x_mark_enc) return dec_out # [B, N] diff --git a/requirements.txt b/requirements.txt index dd97dc1..b2729ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ pyunpack SALib shutil scikit-learn -tensorflow-gpu==2.9.1 +time-interpret==0.3.0 torch==1.13.1+cu116 tqdm wget \ No newline at end of file diff --git a/run.py b/run.py index 2ce4784..1338b2d 100644 --- a/run.py +++ b/run.py @@ -6,21 +6,81 @@ import random import numpy as np -if __name__ == '__main__': - fix_seed = 2021 - random.seed(fix_seed) - torch.manual_seed(fix_seed) - np.random.seed(fix_seed) +def main(args): + set_random_seed(args.seed) + args.use_gpu = True if torch.cuda.is_available() and args.use_gpu else False + + if args.use_gpu and args.use_multi_gpu: + args.devices = args.devices.replace(' ', '') + device_ids = args.devices.split(',') + args.device_ids = [int(id_) for id_ in device_ids] + args.gpu = args.device_ids[0] + + print('Args in experiment:') + print(args) + + if args.task_name == 'classification': + Exp = Exp_Classification + else: + Exp = Exp_Long_Term_Forecast - parser = argparse.ArgumentParser(description='Run Timeseries') + if args.is_training: + for ii in range(args.itr): + # setting record of experiments + setting = stringify_setting(args, ii) + + exp = Exp(args) # set experiments + print('>>>>>>>start training : {}>>>>>>>>>>>>>>>>>>>>>>>>>>'.format(setting)) + exp.train(setting) + + print('>>>>>>>testing : {}<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'.format(setting)) + exp.test(setting) + else: + setting = stringify_setting(args, 0) + + exp = Exp(args) # set experiments + print('>>>>>>>testing : {}<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'.format(setting)) + exp.test(setting, test=1) + + torch.cuda.empty_cache() + + +def stringify_setting(args, iteration): + setting = '{}_{}_{}_{}_ft{}_sl{}_ll{}_pl{}_dm{}_nh{}_el{}_dl{}_df{}_fc{}_eb{}_dt{}_{}_{}'.format( + args.task_name, + args.model_id, + args.model, + args.data, + args.features, + args.seq_len, + args.label_len, + args.pred_len, + args.d_model, + args.n_heads, + args.e_layers, + args.d_layers, + args.d_ff, + args.factor, + args.embed, + args.distil, + args.des, iteration + ) + return setting + +def get_parser(): + parser = argparse.ArgumentParser( + description='Run Timeseries', + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) # basic config - parser.add_argument('--task_name', type=str, required=True, default='long_term_forecast', - help='task name, options:[long_term_forecast, short_term_forecast, imputation, classification, anomaly_detection]') - parser.add_argument('--is_training', type=int, required=True, default=1, help='status') + parser.add_argument('--task_name', type=str, default='long_term_forecast', + choices=['long_term_forecast', 'classification'], help='task name') + parser.add_argument('--is_training', action='store_true', help='status') parser.add_argument('--model_id', type=str, required=True, default='test', help='model id') parser.add_argument('--model', type=str, required=True, default='Transformer', choices=['Transformer', 'DLinear'], help='model name') + parser.add_argument('--seed', default=7, help='random seed') # data loader parser.add_argument('--data', type=str, required=True, default='ETTm1', help='dataset type') @@ -33,7 +93,7 @@ help='freq for time features encoding, options:[s:secondly, t:minutely, h:hourly, d:daily, b:business days, w:weekly, m:monthly], you can also use more detailed freq like 15min or 3h') parser.add_argument('--checkpoints', type=str, default='./checkpoints/', help='location of model checkpoints') parser.add_argument('--no-scale', action='store_true', help='do not scale the dataset') - parser.add_argument('--group-id', type=str, default=None, help='Group identifier id for multiple timeseries') + parser.add_argument('--group-id', type=str, default=None, help='group identifier id for multiple timeseries') # forecasting task parser.add_argument('--seq_len', type=int, default=96, help='input sequence length') @@ -44,9 +104,9 @@ # model define parser.add_argument('--top_k', type=int, default=5, help='for TimesBlock') parser.add_argument('--num_kernels', type=int, default=6, help='for Inception') - parser.add_argument('--enc_in', type=int, default=7, help='encoder input size') - parser.add_argument('--dec_in', type=int, default=7, help='decoder input size') - parser.add_argument('--c_out', type=int, default=7, help='output size') + parser.add_argument('--enc_in', type=int, default=7, help='encoder input size, equal to number of input fetures.') + parser.add_argument('--dec_in', type=int, default=7, help='decoder input size, same as enc_in') + parser.add_argument('--c_out', type=int, default=7, help='output size, same as enc_in') parser.add_argument('--d_model', type=int, default=512, help='dimension of model') parser.add_argument('--n_heads', type=int, default=8, help='num of heads') parser.add_argument('--e_layers', type=int, default=2, help='num of encoder layers') @@ -64,7 +124,7 @@ parser.add_argument('--output_attention', action='store_true', help='whether to output attention in ecoder') # optimization - parser.add_argument('--num_workers', type=int, default=10, help='data loader num workers') + parser.add_argument('--num_workers', type=int, default=0, help='data loader num workers') parser.add_argument('--itr', type=int, default=1, help='experiments times') parser.add_argument('--train_epochs', type=int, default=10, help='train epochs') parser.add_argument('--batch_size', type=int, default=32, help='batch size of train input data') @@ -76,7 +136,7 @@ parser.add_argument('--use_amp', action='store_true', help='use automatic mixed precision training', default=False) # GPU - parser.add_argument('--use_gpu', type=bool, default=True, help='use gpu') + parser.add_argument('--use_gpu', action='store_true', help='use gpu') parser.add_argument('--gpu', type=int, default=0, help='gpu') parser.add_argument('--use_multi_gpu', action='store_true', help='use multiple gpus', default=False) parser.add_argument('--devices', type=str, default='0,1,2,3', help='device ids of multile gpus') @@ -85,76 +145,15 @@ parser.add_argument('--p_hidden_dims', type=int, nargs='+', default=[128, 128], help='hidden layer dimensions of projector (List)') parser.add_argument('--p_hidden_layers', type=int, default=2, help='number of hidden layers in projector') + + return parser +def set_random_seed(seed): + random.seed(seed) + torch.manual_seed(seed) + np.random.seed(seed) +if __name__ == '__main__': + parser = get_parser() args = parser.parse_args() - args.use_gpu = True if torch.cuda.is_available() and args.use_gpu else False - - if args.use_gpu and args.use_multi_gpu: - args.devices = args.devices.replace(' ', '') - device_ids = args.devices.split(',') - args.device_ids = [int(id_) for id_ in device_ids] - args.gpu = args.device_ids[0] - - print('Args in experiment:') - print(args) - - if args.task_name == 'classification': - Exp = Exp_Classification - else: - Exp = Exp_Long_Term_Forecast - - if args.is_training: - for ii in range(args.itr): - # setting record of experiments - setting = '{}_{}_{}_{}_ft{}_sl{}_ll{}_pl{}_dm{}_nh{}_el{}_dl{}_df{}_fc{}_eb{}_dt{}_{}_{}'.format( - args.task_name, - args.model_id, - args.model, - args.data, - args.features, - args.seq_len, - args.label_len, - args.pred_len, - args.d_model, - args.n_heads, - args.e_layers, - args.d_layers, - args.d_ff, - args.factor, - args.embed, - args.distil, - args.des, ii) - - exp = Exp(args) # set experiments - print('>>>>>>>start training : {}>>>>>>>>>>>>>>>>>>>>>>>>>>'.format(setting)) - exp.train(setting) - - print('>>>>>>>testing : {}<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'.format(setting)) - exp.test(setting) - torch.cuda.empty_cache() - else: - ii = 0 - setting = '{}_{}_{}_{}_ft{}_sl{}_ll{}_pl{}_dm{}_nh{}_el{}_dl{}_df{}_fc{}_eb{}_dt{}_{}_{}'.format( - args.task_name, - args.model_id, - args.model, - args.data, - args.features, - args.seq_len, - args.label_len, - args.pred_len, - args.d_model, - args.n_heads, - args.e_layers, - args.d_layers, - args.d_ff, - args.factor, - args.embed, - args.distil, - args.des, ii) - - exp = Exp(args) # set experiments - print('>>>>>>>testing : {}<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'.format(setting)) - exp.test(setting, test=1) - torch.cuda.empty_cache() + main(args) diff --git a/scripts/Covid/Transformer.sh b/scripts/Covid/Transformer.sh index 75b5ebe..473317f 100644 --- a/scripts/Covid/Transformer.sh +++ b/scripts/Covid/Transformer.sh @@ -1 +1 @@ -python run.py --is_training 1 --root_path ./dataset/covid/ --data_path Top_20.csv --target Cases --model_id covid_14_14 --model Transformer --data covid --features MS --seq_len 14 --label_len 7 --pred_len 14 --e_layers 2 --d_layers 1 --factor 3 --enc_in 10 --dec_in 10 --c_out 10 --des 'Exp' --freq d --group-id 'FIPS' --train_epochs 2 --itr 1 --task_name long_term_forecast \ No newline at end of file +python run.py --is_training --root_path ./dataset/covid/ --data_path Top_20.csv --target Cases --model_id covid_14_14 --model Transformer --data covid --features MS --seq_len 14 --label_len 7 --pred_len 14 --e_layers 2 --d_layers 1 --factor 3 --enc_in 10 --dec_in 10 --c_out 10 --des Exp --freq d --group-id 'FIPS' --train_epochs 2 --itr 1 --task_name long_term_forecast \ No newline at end of file diff --git a/scripts/Exchange_script/Transformer.sh b/scripts/Exchange_script/Transformer.sh index 8488afc..4316b92 100644 --- a/scripts/Exchange_script/Transformer.sh +++ b/scripts/Exchange_script/Transformer.sh @@ -2,7 +2,7 @@ export CUDA_VISIBLE_DEVICES=4 python -u run.py \ - --is_training 1 \ + --is_training \ --root_path ./dataset/exchange_rate/ \ --data_path exchange_rate.csv \ --model_id Exchange_96_96 \ @@ -18,11 +18,11 @@ python -u run.py \ --enc_in 8 \ --dec_in 8 \ --c_out 8 \ - --des 'Exp' \ + --des Exp \ --itr 1 python -u run.py \ - --is_training 1 \ + --is_training \ --root_path ./dataset/exchange_rate/ \ --data_path exchange_rate.csv \ --model_id Exchange_96_192 \ @@ -38,11 +38,11 @@ python -u run.py \ --enc_in 8 \ --dec_in 8 \ --c_out 8 \ - --des 'Exp' \ + --des Exp \ --itr 1 python -u run.py \ - --is_training 1 \ + --is_training \ --root_path ./dataset/exchange_rate/ \ --data_path exchange_rate.csv \ --model_id Exchange_96_336 \ @@ -58,12 +58,12 @@ python -u run.py \ --enc_in 8 \ --dec_in 8 \ --c_out 8 \ - --des 'Exp' \ + --des Exp \ --itr 1 \ --train_epochs 1 python -u run.py \ - --is_training 1 \ + --is_training \ --root_path ./dataset/exchange_rate/ \ --data_path exchange_rate.csv \ --model_id Exchange_96_720 \ @@ -79,5 +79,5 @@ python -u run.py \ --enc_in 8 \ --dec_in 8 \ --c_out 8 \ - --des 'Exp' \ + --des Exp \ --itr 1 \ No newline at end of file diff --git a/scripts/ILI_script/Transformer.sh b/scripts/ILI_script/Transformer.sh index a7d21c2..d8c4e06 100644 --- a/scripts/ILI_script/Transformer.sh +++ b/scripts/ILI_script/Transformer.sh @@ -1,13 +1,13 @@ export CUDA_VISIBLE_DEVICES=0 python -u run.py \ - --is_training 1 \ + --is_training \ --root_path ./dataset/illness/ \ --data_path national_illness.csv \ --model_id ili_36_24 \ --model Transformer \ --data custom \ - --features M \ + --features MS \ --seq_len 36 \ --label_len 18 \ --pred_len 24 \ @@ -17,17 +17,17 @@ python -u run.py \ --enc_in 7 \ --dec_in 7 \ --c_out 7 \ - --des 'Exp' \ + --des Exp \ --itr 1 python -u run.py \ - --is_training 1 \ + --is_training \ --root_path ./dataset/illness/ \ --data_path national_illness.csv \ --model_id ili_36_36 \ --model Transformer \ --data custom \ - --features M \ + --features MS \ --seq_len 36 \ --label_len 18 \ --pred_len 36 \ @@ -37,17 +37,17 @@ python -u run.py \ --enc_in 7 \ --dec_in 7 \ --c_out 7 \ - --des 'Exp' \ + --des Exp \ --itr 1 python -u run.py \ - --is_training 1 \ + --is_training \ --root_path ./dataset/illness/ \ --data_path national_illness.csv \ --model_id ili_36_48 \ --model Transformer \ --data custom \ - --features M \ + --features MS \ --seq_len 36 \ --label_len 18 \ --pred_len 48 \ @@ -57,17 +57,17 @@ python -u run.py \ --enc_in 7 \ --dec_in 7 \ --c_out 7 \ - --des 'Exp' \ + --des Exp \ --itr 1 python -u run.py \ - --is_training 1 \ + --is_training \ --root_path ./dataset/illness/ \ --data_path national_illness.csv \ --model_id ili_36_60 \ --model Transformer \ --data custom \ - --features M \ + --features MS \ --seq_len 36 \ --label_len 18 \ --pred_len 60 \ @@ -77,5 +77,5 @@ python -u run.py \ --enc_in 7 \ --dec_in 7 \ --c_out 7 \ - --des 'Exp' \ + --des Exp \ --itr 1 \ No newline at end of file diff --git a/scripts/ILI_script/Transformer_windows.sh b/scripts/ILI_script/Transformer_windows.sh index bc373d4..3fda134 100644 --- a/scripts/ILI_script/Transformer_windows.sh +++ b/scripts/ILI_script/Transformer_windows.sh @@ -1 +1 @@ -python run.py --is_training 1 --root_path ./dataset/illness/ --data_path national_illness.csv --model_id ili_36_24 --model Transformer --data custom --features M --seq_len 36 --label_len 18 --pred_len 24 --e_layers 2 --d_layers 1 --factor 3 --enc_in 7 --dec_in 7 --c_out 7 --des 'Exp' --itr 1 --task_name long_term_forecast \ No newline at end of file +python run.py --is_training --root_path ./dataset/illness/ --data_path national_illness.csv --model_id ili_36_24 --model Transformer --data custom --features M --seq_len 36 --label_len 18 --pred_len 24 --e_layers 2 --d_layers 1 --factor 3 --enc_in 7 --dec_in 7 --c_out 7 --des Exp --itr 1 --task_name long_term_forecast \ No newline at end of file diff --git a/scripts/Traffic_script/Transformer.sh b/scripts/Traffic_script/Transformer.sh index 916d021..e86f810 100644 --- a/scripts/Traffic_script/Transformer.sh +++ b/scripts/Traffic_script/Transformer.sh @@ -1,7 +1,7 @@ export CUDA_VISIBLE_DEVICES=5 python -u run.py \ - --is_training 1 \ + --is_training \ --root_path ./dataset/traffic/ \ --data_path traffic.csv \ --model_id traffic_96_96 \ @@ -17,12 +17,12 @@ python -u run.py \ --enc_in 862 \ --dec_in 862 \ --c_out 862 \ - --des 'Exp' \ + --des Exp \ --itr 1 \ --train_epochs 3 python -u run.py \ - --is_training 1 \ + --is_training \ --root_path ./dataset/traffic/ \ --data_path traffic.csv \ --model_id traffic_96_192 \ @@ -38,12 +38,12 @@ python -u run.py \ --enc_in 862 \ --dec_in 862 \ --c_out 862 \ - --des 'Exp' \ + --des Exp \ --itr 1 \ --train_epochs 3 python -u run.py \ - --is_training 1 \ + --is_training \ --root_path ./dataset/traffic/ \ --data_path traffic.csv \ --model_id traffic_96_336 \ @@ -59,12 +59,12 @@ python -u run.py \ --enc_in 862 \ --dec_in 862 \ --c_out 862 \ - --des 'Exp' \ + --des Exp \ --itr 1 \ --train_epochs 3 python -u run.py \ - --is_training 1 \ + --is_training \ --root_path ./dataset/traffic/ \ --data_path traffic.csv \ --model_id traffic_96_720 \ @@ -80,6 +80,6 @@ python -u run.py \ --enc_in 862 \ --dec_in 862 \ --c_out 862 \ - --des 'Exp' \ + --des Exp \ --itr 1 \ --train_epochs 3 diff --git a/tsai.ipynb b/tsai.ipynb deleted file mode 100644 index 890fabe..0000000 --- a/tsai.ipynb +++ /dev/null @@ -1,1003 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Imports" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "os : Windows-10-10.0.19045-SP0\n", - "python : 3.10.11\n", - "tsai : 0.3.7\n", - "fastai : 2.7.12\n", - "fastcore : 1.5.29\n", - "torch : 1.13.1+cu117\n", - "device : 1 gpu (['NVIDIA GeForce RTX 3060 Laptop GPU'])\n", - "cpu cores : 14\n", - "threads per cpu : 1\n", - "RAM : 31.69 GB\n", - "GPU memory : [6.0] GB\n" - ] - } - ], - "source": [ - "from tsai.all import *\n", - "my_setup()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## UCI Electricity " - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "electricity : (26304, 322) ██████████| 100.01% [54001664/53995526 00:05<00:00]\n" - ] - } - ], - "source": [ - "# https://forecastingdata.org/\n", - "# https://archive.ics.uci.edu/dataset/321/electricityloaddiagrams20112014\n", - "# dsid = \"electricity\"\n", - "# try:\n", - "# df = get_long_term_forecasting_data(\n", - "# dsid, target_dir='datasets/forecasting/', \n", - "# force_download=False, return_df=True\n", - "# )\n", - "# print(f\"{dsid:15}: {str(df.shape):15}\")\n", - "# remove_dir('./data/forecasting/', False)\n", - "# except Exception as e:\n", - "# print(f\"{dsid:15}: {str(e):15}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "from tsai.data.external import get_Monash_forecasting_data\n", - "import pandas as pd" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Dataset: electricity_hourly_dataset\n", - "converting data to dataframe...\n", - "...done\n", - "\n", - "freq : hourly\n", - "forecast_horizon : 168\n", - "contain_missing_values : False\n", - "contain_equal_length : True\n", - "\n", - "exploding dataframe...\n", - "...done\n", - "\n", - "\n", - "data.shape: (8443584, 3)\n", - "electricity_hourly_dataset: (8443584, 3) \n" - ] - } - ], - "source": [ - "dsid = \"electricity_hourly_dataset\"\n", - "try:\n", - " df = get_Monash_forecasting_data(\n", - " dsid, path='datasets/forecasting/'\n", - " )\n", - " print(f\"{dsid:15}: {str(df.shape):15}\")\n", - " # del df; gc.collect()\n", - " # remove_dir('datasets/forecasting/', False)\n", - "except Exception as e:\n", - " print(f\"{dsid:15}: {str(e):15}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "def add_time_encoding(data:pd.DataFrame, time_column:str=\"date\"):\n", - " df = data.copy()\n", - "\n", - " date = pd.to_datetime(df[time_column])\n", - " earliest_date = date.min()\n", - "\n", - " delta = (date - earliest_date).dt\n", - " df['hours_from_start'] = delta.seconds / 60 / 60 + delta.days * 24\n", - " # df['days_from_start'] = delta.days\n", - " df['hour'] = date.dt.hour\n", - " df['day'] = date.dt.day\n", - " df['weekday'] = date.dt.weekday\n", - " df['month'] = date.dt.month\n", - "\n", - " return df" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
series_nametimestampseries_value
0T12012-01-01 00:00:0114.0
1T12012-01-01 01:00:0118.0
2T12012-01-01 02:00:0121.0
3T12012-01-01 03:00:0120.0
4T12012-01-01 04:00:0122.0
\n", - "
" - ], - "text/plain": [ - " series_name timestamp series_value\n", - "0 T1 2012-01-01 00:00:01 14.0\n", - "1 T1 2012-01-01 01:00:01 18.0\n", - "2 T1 2012-01-01 02:00:01 21.0\n", - "3 T1 2012-01-01 03:00:01 20.0\n", - "4 T1 2012-01-01 04:00:01 22.0" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "time_column = 'timestamp'\n", - "id_column = 'series_name'\n", - "target_column = 'series_value'\n", - "\n", - "df = df[df[time_column] >= pd.to_datetime('2012-01-01')].reset_index(drop=True)\n", - "df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "df = add_time_encoding(df, time_column='timestamp')\n", - "df.to_csv('datasets/forecasting/electricity_hourly_dataset.csv', index=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 107, - "metadata": {}, - "outputs": [], - "source": [ - "def summary(df:pd.DataFrame, time_column, id_column):\n", - " T = df[time_column].nunique()\n", - " n_ids = df[id_column].nunique()\n", - " n_samples = df.shape[0]\n", - "\n", - " output = f\"\\\n", - " The dataset has {T} time steps, {n_ids} ids.\\n\\\n", - " Sample size {n_samples}, per user {n_samples/n_ids}.\\n\\\n", - " Start {df[time_column].min()}, end {df[time_column].max()}.\\n\"\n", - " \n", - " print(output)" - ] - }, - { - "cell_type": "code", - "execution_count": 106, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " The dataset has 26304 time steps, 321 ids.\n", - " Sample size 8443584, per user 26304.0.\n", - " Start 2012-01-01 00:00:01, end 2014-12-31 23:00:01.\n", - "\n" - ] - } - ], - "source": [ - "summary(df, time_column, id_column)" - ] - }, - { - "cell_type": "code", - "execution_count": 91, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
series_nametimestampseries_valuehours_from_starthourdayweekdaymonth
0T12014-01-01 00:00:0112.00.00121
1T12014-01-01 01:00:0113.01.01121
2T12014-01-01 02:00:0113.02.02121
\n", - "
" - ], - "text/plain": [ - " series_name timestamp series_value hours_from_start hour day \\\n", - "0 T1 2014-01-01 00:00:01 12.0 0.0 0 1 \n", - "1 T1 2014-01-01 01:00:01 13.0 1.0 1 1 \n", - "2 T1 2014-01-01 02:00:01 13.0 2.0 2 1 \n", - "\n", - " weekday month \n", - "0 2 1 \n", - "1 2 1 \n", - "2 2 1 " - ] - }, - "execution_count": 91, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.head(3)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "fcst_history = 168\n", - "fcst_horizon = 24\n", - "stride = 1\n", - "valid_size=0.1\n", - "test_size=0.2" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['hours_from_start', 'hour', 'day', 'weekday', 'month'] series_value\n" - ] - } - ], - "source": [ - "x_vars = [col for col in df.columns if col not in [time_column, id_column, target_column]]\n", - "y_vars = target_column\n", - "print(x_vars, y_vars)" - ] - }, - { - "cell_type": "code", - "execution_count": 118, - "metadata": {}, - "outputs": [], - "source": [ - "from tsai.data.preparation import prepare_forecasting_data\n", - "from tsai.data.validation import get_forecasting_splits" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(78912, 8)\n" - ] - } - ], - "source": [ - "temp = df[df[id_column].isin([f'T{num}' for num in range(301, 304)])]\n", - "print(temp.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "from fastai.learner import Learner\n", - "from tsai.models import FCN\n", - "from tsai.all import *\n", - "from fastai.metrics import mse, mae\n", - "\n", - "# https://docs.fast.ai/callback.tracker.html\n", - "from fastai.callback.tracker import EarlyStoppingCallback, ReduceLROnPlateau, SaveModelCallback\n", - "from tsai.utils import cat2int" - ] - }, - { - "cell_type": "code", - "execution_count": 328, - "metadata": {}, - "outputs": [], - "source": [ - "cat_names = [] # ['series_name']\n", - "cont_names = ['hours_from_start', 'hour', 'day', 'weekday', 'month', target_column]\n", - "\n", - "for feature in cat_names:\n", - " temp[feature] = cat2int(temp[feature].astype(str).values)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABAgAAABiCAYAAADdueE1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWDElEQVR4nO3de3RV5ZnH8d+TBAgaREIit2ixYICQGmIEtaWIOFK1QgcRRVCkA9rFOGMtLaK2VYuM48xyjY63esEL3hAv1AvitSJ4WbUmyPECCQkKBSWQEAIigrk888fZsWfSXAiccDic72etrJz97ne/+zl5kr1ynvPu95i7CwAAAAAAJLakWAcAAAAAAABijwIBAAAAAACgQAAAAAAAACgQAAAAAAAAUSAAAAAAAACiQAAAAAAAAESBAAAQx8zsLTObHjyebGav7cdYfc3MzSwl2H7ZzC6JUpw/NrOSiO11ZvZP0Rg7GO9TMxsZrfEAAEBiokAAAIgpMxtuZu+Z2XYzqzKzd81saFvHcffH3X10xLhuZv33NS53P8vd57fWb2/O4+5vu/uAfY2l0fkeNrO5jcYf7O5vRWN8AACQuFJiHQAAIHGZ2RGSFkuaIekpSR0l/VjSnljGFU1mluLutbGOAwAAoDXMIAAAxFK2JLn7Anevc/dv3P01d/9IksxsajCj4M5ghkGxmZ3e1EBB33eCx8uD5pCZ7TSzC5ron2xmt5hZpZl9JumnjfZH3r7Q38yWBTFUmtnC5s5jZiPNbKOZzTazckkPNbQ1CmGoma0ys21m9pCZpTZ+HhGxeBDDZZImS7oqON+Lwf7vblkws05mdpuZfRl83WZmnYJ9DbH92sy2mNkmM/t5q1kCAAAJgQIBACCW1kiqM7P5ZnaWmXVros9JktZKypB0vaRFZpbe0qDuPiJ4mOfuae6+sIlul0o6R1K+pBMlndfCkDdKek1SN0lZku5o5Tw9JaVL+p6ky5oZc7Kkn0jqp3Ch5HctPafgfPdJelzSfwfnG9NEt99KOlnSEEl5koY1GrunpK6S+kiaJumuZn7uAAAgwVAgAADEjLvvkDRckku6X1KFmb1gZj0ium2RdJu71wQvwEvU6N3+fXR+MO4Gd6+S9J8t9K1R+MV+b3ff7e7vtNBXkuolXe/ue9z9m2b63Blx7v+QdGFbn0AzJkua4+5b3L1C0h8kXRyxvybYX+PuSyTtlBSV9REAAEB8o0AAAIgpd1/t7lPdPUtSrqTekm6L6PKFu3vE9vqgz/7qLWlDo3Gbc5Ukk/TX4BMD/qWVsSvcfXcrfRqfOxrPScE4kc+l8dhbG62JsEtSWpTODQAA4hgFAgDAQcPdiyU9rHChoEEfM7OI7WMkfRmF022SdHSjcZuLq9zdL3X33pJ+IenuVj65wFvY16DxuRue09eSDmvYYWY92zj2lwrPdmhqbAAAgGZRIAAAxIyZDQwWzMsKto9WeKr9XyK6HSXpCjPrYGYTJA2StGQvht8s6fst7H8qGDcruAf/6hbinNAQo6RtCr9Ir9/L8zTn8uDc6QqvG9CwfkFI0mAzGxIsXHhDo+NaO98CSb8zs0wzy5B0naTH9iE+AACQYCgQAABi6SuFFyF838y+Vrgw8ImkX0f0eV/ScZIqFb5X/zx337oXY98gab6ZVZvZ+U3sv1/Sqwq/IF8haVELYw0NYtwp6QVJv3T3z/byPM15QuGFDz9TeBHGuZLk7mskzZH0hqRSSY3XO3hAUk5wvueaGHeupEJJH0n6OHhuc9sQFwAASFD2/2/rBADg4GFmUyVNd/fhsY4FAADgUMcMAgAAAAAAQIEAAAAAAABwiwEAAAAAABAzCAAAAAAAgCgQAAAAAAAASSntMahZhkt922NoAAAAtIOCY4tiHUJCWJ16WKxDAPbbrtW7Kt09M9ZxIPrapUAQLg4Uts/QAAAAiLrCuRbrEBJCwcCBsQ4B2G8rClasj3UMaB/cYgAAAAAAACgQAAAAAAAACgQAAAAAAEDttgYBAAAAAAAHr6KioqNSUlLmScpVYrx5Xi/pk9ra2ukFBQVbmupAgQAAAAAAkHBSUlLm9ezZc1BmZua2pKQkj3U87a2+vt4qKipyysvL50ka21SfRKiSAAAAAADQWG5mZuaORCgOSFJSUpJnZmZuV3jGRNN9DmA8AAAAAAAcLJISpTjQIHi+zdYBKBAAAAAAABADZlZw6aWXZjVsX3fddT1mzpzZO1bxtLoGgZk9KOkcSVvcvdmpCAAAAAAAxKu1a9cWRHO8fv36FbXWp2PHjr5kyZJumzZtKu/Vq1dtNM+/L/ZmBsHDks5s5zgAAAAAAEgoycnJPmXKlIqbbrqpR+N9JSUlHU8++eTs7OzsnFNOOSW7tLS0oySNHz++79SpU4/Oz88fmJWV9YOHHnqoW8Mxv//973vk5uYOys7OzvnVr37V5pkIrRYI3H25pKq2DgwAAAAAAFo2a9asLYsWLUrfunVrcmT7jBkzjpk8efLWNWvWrLrgggu2zpgx4+iGfZs3b+5QWFhY/Pzzz5def/31fSRp0aJFR5SVlaV+9NFHq1evXr1q5cqVh7388stpbYklamsQmNllZlZoZoVSRbSGBQAAAADgkJWenl4/YcKErTfffPNRke0ffvjh4ZdddlmVJM2YMaOqqKjouxf7Y8eOrU5OTlZBQcHurVu3dpCkV1555Yjly5cfkZOTkzN48OCctWvXphYXF6e2JZZW1yDYW+5+n6T7JMnsxIRaCRIAAAAAgH11zTXXbD7hhBNyJk6cWLk3/VNTU797ze3u332/8sorN82aNWuvxmgKn2IAAAAAAEAM9ejRo27MmDHbnnjiiYyGtvz8/K/nzZvXTZLuvffe9BNPPHFnS2OcddZZOx599NGM7du3J0nS559/3uGLL75o06QACgQAAAAAAMTYb3/72/Lq6urvXtDfc889f3v00UczsrOzcxYsWND97rvv3tDS8eeee+6OCRMmVA0dOnRgdnZ2zrhx4/pVV1cnt3RMY9YwHaHZDmYLJI2UlCFps6Tr3f2Blo850aXCtsQBAACAGPLHLdYhJISCgSfEOgRgv60oWFHk7ifGOo79FQqF1uXl5e3zdPx4FQqFMvLy8vo2ta/V6QbufmHUIwIAAAAAAAcVbjEAAAAAAAAUCAAAAAAAAAUCAAAAAAAgCgQAAAAAAEAUCAAAAAAAgPbiUwwAAAAAAEB0lZeXJ48cOXKAJFVWVnZISkry9PT0WklauXLl6tTUVG/u2OXLlx/24IMPdn/44Yc3RDMmCgQAAAAAgIRnpoJojueuopb29+zZs664uHiVJM2cObN3Wlpa3Zw5czY37K+pqVGHDh2aPHbEiBG7RowYsSua8UrcYgAAAAAAwEFh/PjxfSdNmnTM8ccfP3DGjBlZS5cuPWzIkCEDBw0alJOfnz8wFAp1kqTFixd3Oe200/pL4eLChAkT+g4bNmxAVlbWD+bOnXvUvp6/nWYQFO2UrKR9xsYBkiGpMtZBYL+Qw0MDeYx/5PDQcMjn0SbHOoJ2d5DkcEWsA4hnB0kOIel7sQ7gULZp06aOK1asKE5JSVFVVVXSBx98UNyhQwc999xzXa666qqsV199dW3jY8rKylLfe++9kurq6uRBgwblzpo1q6JTp07N3qLQnPa6xaDE3U9sp7FxAJhZITmMb+Tw0EAe4x85PDSQx/hHDuMfOUSiOPfcc7elpIRfqldVVSVfcMEFx65bty7VzLympsaaOmb06NHVnTt39s6dO9emp6fXbNy4MaVfv341bT03txgAAAAAAHCQSEtLq294PHv27D6nnnrqV6WlpZ+++OKLZd9++22Tr+EjZwskJyertra2yUJCaygQAAAAAABwENqxY0dyVlbWt5J07733ZrT3+dqrQHBfO42LA4ccxj9yeGggj/GPHB4ayGP8I4fxjxwi4cyePbv8hhtuyBo0aFBObW1tu5/P3Nu8bgEAAAAAAHEtFAqty8vLS7iFL0OhUEZeXl7fpvZxiwEAAAAAAIhugcDMzjSzEjMrM7Orozk22s7MHjSzLWb2SURbupm9bmalwfduQbuZ2e1B7j4ysxMijrkk6F9qZpdEtBeY2cfBMbeb2T4thIHmmdnRZrbUzFaZ2adm9sugnTzGETNLNbO/mlkoyOMfgvZjzez94Ge/0Mw6Bu2dgu2yYH/fiLGuCdpLzOwnEe1cfw8AM0s2sw/NbHGwTQ7jjJmtC655K82sMGjjmhpHzOxIM3vGzIrNbLWZnUIO44uZDQj+Bhu+dpjZleQRiL2oFQjMLFnSXZLOkpQj6UIzy4nW+NgnD0s6s1Hb1ZL+7O7HSfpzsC2F83Zc8HWZpD9K4X+aJF0v6SRJwyRd33CxDvpcGnFc43Nh/9VK+rW750g6WdLlwd8VeYwveySNcvc8SUMknWlmJ0v6L0m3unt/SdskTQv6T5O0LWi/NeinIPcTJQ1WOE93By9Yuf4eOL+UtDpimxzGp9PcfUjEx6VxTY0v/yvpFXcfKClP4b9JchhH3L0k+BscIqlA0i5JfxJ5BGIumjMIhkkqc/fP3P1bSU9K+lkUx0cbuftySVWNmn8maX7weL6kf45of8TD/iLpSDPrJeknkl539yp33ybpdYVf3PSSdIS7/8XDC1k8EjEWosTdN7n7iuDxVwr/E9RH5DGuBPnYGWx2CL5c0ihJzwTtjfPYkN9nJJ0evPPxM0lPuvsed/9cUpnC116uvweAmWVJ+qmkecG2iRweKrimxgkz6ypphKQHJMndv3X3apHDeHa6pLXuvl7kEYi5aBYI+kjaELG9MWjDwaWHu28KHpdL6hE8bi5/LbVvbKId7cTCU5TzJb0v8hh3gneJV0raovA/MGslVbt7w3K0kT/77/IV7N8uqbvanl9E122SrpLU8NnE3UUO45FLes3MiszssqCNa2r8OFZShaSHLHy7zzwzO1zkMJ5NlLQgeEwegRhjkcIEFlRU+RiLOGBmaZKelXSlu++I3Ece44O71wVTKbMUfrd4YGwjQluY2TmStrh7UaxjwX4b7u4nKDxl+XIzGxG5k2vqQS9F0gmS/uju+ZK+1t+noUsih/HEwuu2jJX0dON95BGIjWgWCL6QdHTEdlbQhoPL5mDalYLvW4L25vLXUntWE+2IMjProHBx4HF3XxQ0k8c4FUyFXSrpFIWnSKYEuyJ/9t/lK9jfVdJWtT2/iJ4fSRprZusUnv4/SuH7oMlhnHH3L4LvWxS+53mYuKbGk42SNrr7+8H2MwoXDMhhfDpL0gp33xxsk0cklJNOOin72WefPSKybc6cOUdNnjz5mKb6Dxs2bMDy5csPk6RTTz21f2VlZXLjPjNnzux93XXX9fjHo/dONAsEH0g6zsIrOndUeLrQC1EcH9HxgqSGFV4vkfR8RPuUYJXYkyVtD6Z4vSpptJl1CxZ9GS3p1WDfDjM7ObivdkrEWIiS4Gf7gKTV7v4/EbvIYxwxs0wzOzJ43FnSGQqvJ7FU0nlBt8Z5bMjveZLeDN5JeUHSRAuvkH+swosu/VVcf9udu1/j7lnu3lfhn++b7j5Z5DCumNnhZtal4bHC18JPxDU1brh7uaQNZjYgaDpd0iqRw3h1of5+e4FEHhFrT1hBVL9aMWHChKoFCxakR7Y9++yz6RdddFHjdeT+wbJly8oyMjLq9ufpNiWl9S57x91rzezfFP5DTZb0oLt/Gq3x0XZmtkDSSEkZZrZR4VVeb5b0lJlNk7Re0vlB9yWSzlZ4waxdkn4uSe5eZWY3KvzPqyTNcfeGX9h/VfiTEjpLejn4QnT9SNLFkj4O7l+XpGtFHuNNL0nzLbxSfZKkp9x9sZmtkvSkmc2V9KGCRbeC74+aWZnCC41OlCR3/9TMnlL4n+FaSZe7e50kcf2Nmdkih/Gkh6Q/hV8vKEXSE+7+ipl9IK6p8eTfJT0eFNM+UzgvSSKHcSUo0p0h6RcRzfx/g4Ry8cUXb7vpppv67N6921JTU72kpKTjli1bOjz22GPps2bNOnr37t1JY8aM2Xbrrbd+2fjYPn36/KCwsHB1r169amfPnt1z4cKFGd27d6/p3bv3t/n5+bv2NSYLv6EBAAAAAEDiCIVC6/Ly8iq/a9iLd/3bZFLraxeddtpp/adNm1Z50UUXVV977bU9KysrU2688cZNPXr0qKutrdUPf/jDAXfcccffTjrppG+GDRs24JZbbtkwYsSIXQ0FgrKyso7Tpk3rW1RUVFxTU6MhQ4bkTJ06tWLOnDmbmztnKBTKyMvL69vUPhYpBAAAAAAgBs4///yqhQsXdpOkRYsWpV988cVV8+fPT8/JyRmUk5OTU1pamhoKhVKbO37p0qVpZ599dnWXLl3q09PT60ePHl29P/FQIAAAAAAAIAYmTZpU/e677x7xzjvvHLZ79+6kzMzM2jvvvLPHsmXL1qxZs2bVqFGjtu/evfuAvW6nQAAAAAAAQAx07dq1/pRTTvlq+vTpfceNG1e1bdu25M6dO9enp6fXbdiwIeWtt97q2tLxo0aN2rlkyZIjd+7cadu2bUt6/fXXj9yfeKK2SCEAAAAAAGibiRMnVk2ZMqXfggULPsvPz9+dm5u7q1+/frm9evX6tqCgYGdLxw4fPnzXuHHjqnJzcwd379695vjjj/96f2JhkUIAAAAAQML5h0UKEwSLFAIAAAAAgBZRIAAAAAAAABQIAAAAAAAABQIAAAAAACAKBAAAAAAAQBQIAAAAAACApJRYBwAAAAAAQKIpLy9PHjly5ABJqqys7JCUlOTp6em1krRy5crVqamp3tLxixcv7tKpU6f6M8444+toxUSBAAAAAACQ8ApWFBREc7yiE4qKWtrfs2fPuuLi4lWSNHPmzN5paWl1c+bM2by347/55ptd0tLS6qJZIOAWAwAAAAAADgJvv/32YUOHDh0wePDgQcOHDz9u/fr1HSRp7ty5R/Xr129wdnZ2zjnnnPP9kpKSjo888kjmPffc02PgwIE5r7zySlo0zs8MAgAAAAAAYszddcUVVxzz0ksvlfXu3bv2/vvv7/ab3/ymz9NPP73u9ttv77l+/fqPO3fu7JWVlckZGRl1U6ZMqWjrrIPWUCAAAAAAACDG9uzZk1RaWtp51KhR2ZJUX1+vzMzMGkkaMGDAN+PGjTt27Nix1ZMnT65urxgoEAAAAAAAEGPurv79+3+zcuXK4sb7li5dWvryyy93ef7557vecsstvUpKSj5tjxhYgwAAAAAAgBjr1KlTfVVVVcobb7xxuCTt2bPHCgsLU+vq6rR27dqOY8aM+equu+76YufOncnbt29P7tKlS91XX32VHM0YKBAAAAAAABBjSUlJevLJJ9deffXVWQMGDMgZPHhwzrJly9Jqa2tt0qRJx2ZnZ+fk5ubmTJ8+fUtGRkbd+PHjq1966aUjo7lIobm3+NGKAAAAAAAcckKh0Lq8vLzKWMdxoIVCoYy8vLy+Te1jBgEAAAAAAKBAAAAAAAAAKBAAAAAAAABRIAAAAAAAJKb6+vp6i3UQB1LwfOub20+BAAAAAACQiD6pqKjomihFgvr6equoqOgq6ZPm+qQcwHgAAAAAADgo1NbWTi8vL59XXl6eq8R487xe0ie1tbXTm+vAxxwCAAAAAICEqJIAAAAAAIBWUCAAAAAAAAAUCAAAAAAAAAUCAAAAAAAgCgQAAAAAAEDS/wH1axAcdPPKjwAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "X, y = prepare_forecasting_data(\n", - " temp, fcst_history, fcst_horizon, \n", - " x_vars=x_vars, y_vars=target_column\n", - ")\n", - "splits = get_forecasting_splits(\n", - " temp, fcst_history, fcst_horizon, valid_size=valid_size, \n", - " test_size=test_size, show_plot=True\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [], - "source": [ - "# We'll use inplace=True to preprocess data at dataset initialization. \n", - "# This will significantly speed up training.\n", - "from tsai.data.core import TSDatasets, TSDataLoaders\n", - "from tsai.data.preprocessing import TSStandardize\n", - "\n", - "tfms = [None, [TSRegression()]]\n", - "batch_tfms = TSStandardize(by_sample=True, by_var=True)\n", - "batch_size = 64\n", - "\n", - "datasets = TSDatasets(X, y, splits=splits, tfms=tfms)\n", - "dataloaders = TSDataLoaders.from_dsets(\n", - " datasets.train, datasets.valid, bs=[batch_size, batch_size*2],\n", - " batch_tfms=batch_tfms, \n", - " # num_workers=0\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "arch: TSTPlus(c_in=5 c_out=1 seq_len=168 arch_config={} kwargs={'custom_head': functools.partial(, d=[1, 24])})\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "SuggestedLRs(valley=0.04786301031708717)" - ] - }, - "execution_count": 54, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEVCAYAAAD6u3K7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAA/8ElEQVR4nO3dd3zU9f3A8df7MsneCYRAmGETICCKAwUBi4p1oWIdtVq7rNaftv46tHaPX61WrbvW1oW4FSeioCgQ9l4hhAQyIIPs+fn9cZd4hEu4kPveXZL38/G4R5LPd70Twr3z2WKMQSmllGrP5usAlFJK+SdNEEoppVzSBKGUUsolTRBKKaVc0gShlFLKJU0QSimlXOp1CUJEnhGRYhHZ6ub5V4rIdhHZJiIvWB2fUkr1FNLb5kGIyNlAFfCcMWbcSc4dASwGzjPGlIlIkjGm2BtxKqWUv+t1NQhjzAqg1LlMRIaJyPsisk5EVorIKMehm4FHjDFljms1OSillEOvSxAdeAL4kTFmCvA/wKOO8pHASBH5QkS+EpF5PotQKaX8TKCvA7CaiEQAZwCviEhrcYjjYyAwApgJDARWiMh4Y0y5l8NUSim/0+sTBPZaUrkxJtPFsXxgtTGmEdgvIruxJ4y1XoxPKaX8Uq9vYjLGHMP+5n8FgNhNdBx+A3vtARFJwN7klOODMJVSyu/0ugQhIi8CXwIZIpIvIjcBi4CbRGQTsA1Y4Dj9A+CoiGwHlgN3GWOO+iJupZTyN71umKtSSinP6HU1CKWUUp6hCUIppZRLvWoUU0JCgklPT/d1GEop1WOsW7fuiDEm0dWxXpUg0tPTyc7O9nUYSinVY4jIgY6OaROTUkoplzRBKKWUckkThFJKKZd6VR+EUkp1pLGxkfz8fOrq6nwdik+EhoYycOBAgoKC3L5GE4RSqk/Iz88nMjKS9PR0nBbu7BOMMRw9epT8/HyGDBni9nXaxKSU6hPq6uqIj4/vc8kBQESIj4/vcu1JE0QvZoxhc345upyKUnZ9MTm0OpXvXRNEL7ZizxEufvgL3thY4OtQlFJdFBERAUBubi7jxnW6e7JlNEH0Yh9vLwLg8c9ytBahVFdtXgwPjIP7YuwfNy/2dURepwmilzLGsHxXMZGhgewsrOTT3SW+DkmpnmPzYnj7Nqg4CBj7x7dv61aS+NnPfsYjjzzS9vV9993Hb3/7W2bNmsXkyZMZP348b775Zqf3aG5u5q677mLq1KlMmDCBxx9/HIDrrruON954o+28RYsWnfRe7rA0QYjIHSKyTUS2isiLIhLa7vgNIlIiIhsdr+84HbteRPY4XtdbGWdvtK+kivyyWu48fyT9o0N5/LN9vg5JqZ5j2f3QWHt8WWOtvfwULVy4kMWLv04wixcv5vrrr+f1119n/fr1LF++nDvvvLPT2v7TTz9NdHQ0a9euZe3atTz55JPs37+fm266iWeffRaAiooKVq1axfz580851laWDXMVkVTgNmCMMaZWRBYDVwHPtjv1ZWPMD9tdGwfcC2QBBlgnIm8ZY8qsire3+WRnMQDnj02hqcXw23d3sPFgOZlpMb4NTKmeoCK/a+VumDRpEsXFxRw6dIiSkhJiY2NJSUnhjjvuYMWKFdhsNgoKCigqKiIlJcXlPT788EM2b97MkiVL7OFUVLBnzx7mzJnD97//fUpKSnj11Ve57LLLCAzs/tu71fMgAoF+ItIIhAGH3LxuLvCRMaYUQEQ+AuYBL1oSZS+0fGcJGcmRpMb046ppg3ho2R6eWLGPRxdN8XVoSvm/6IGO5iUX5d1wxRVXsGTJEgoLC1m4cCHPP/88JSUlrFu3jqCgINLT0zsdimqM4R//+Adz58494dh1113Hf//7X1566SX+9a9/dSvOVpY1MRljCoC/AnnAYaDCGPOhi1MvE5HNIrJERNIcZamA879OvqNMuaGyrpG1uaWcOyoJgIiQQL51+mDe21rI/iPVPo5OqR5g1q8gqN/xZUH97OXdsHDhQl566SWWLFnCFVdcQUVFBUlJSQQFBbF8+XIOHOhwYVUA5s6dyz//+U8aGxsB2L17N9XV9v/TN9xwA3//+98BGDNmTLfibGVZghCRWOx7Pw8BBgDhInJtu9PeBtKNMROAj4B/n8JzbhGRbBHJLinpekdsY3MLf/1gF8t2FHX5Wn/1+Z4jNLUYzs34eon3689IJyjAxk9f3UyuJgmlOjfhSrjoIYhOA8T+8aKH7OXdMHbsWCorK0lNTaV///4sWrSI7Oxsxo8fz3PPPceoUaM6vf473/kOY8aMYfLkyYwbN47vfve7NDU1AZCcnMzo0aO58cYbuxWjM8v2pBaRK4B5xpibHF9fB0w3xny/g/MDgFJjTLSIXA3MNMZ813HsceBTY0ynTUxZWVmmq/tBGGOY8tuPmTMmmT9eNqFL1/qru5ds4r2thaz/5fkEBXz9N8DLa/O4/+3tNDS38O0ZQ/jhecOJDHV/XRalerIdO3YwevRoX4dhmZqaGsaPH8/69euJjo52eY6rn4GIrDPGZLk638pRTHnAdBEJE/sUvlnAjnaB9Xf68mKn4x8Ac0Qk1lETmeMo8zgRYWRyBLuKKq24vdfZh7eWcPbIxOOSA8DCqYNY/j8zuSQzlSdW5jDngRUcq2v0UaRKKU/5+OOPGT16ND/60Y86TA6nwrJOamPMahFZAqwHmoANwBMicj+QbYx5C7hNRC52HC8FbnBcWyoivwHWOm53f2uHtRVGpUTxSvZBWloMNlvPnoq/7dAxSirrOTcjyeXxpKhQ/nLFROaNS+Gmf2ezfGcxCzK1e0epnmz27Nkn7b84FZaOYjLG3It9uKqzXzkdvwe4p4NrnwGesS66r41MjqS6oZmC8lrS4sK88UjLLHcMbz1npMstZtucm5FEYmQIH24r0gShlHJJZ1IDGSmRAOwqPLGZaX1eGfVNzd4O6ZTUNDTx/Oo8pqXHkRgZ0um5Nptw/phkPt1VTF1jz/j+lOquvrzkzKl875oggJHJ9kWx2vdD7C2u4tJHV/Hu5sO+CKvLnliRQ+GxOu6el+HW+XPGJFPd0MyqfUcsjkwp3wsNDeXo0aN9Mkm07gcRGhp68pOd6IZBQGRoEKkx/U6oQazefxSAwxX+vwNVYUUdj3+Ww/zx/clKj3PrmtOHxRMREsiH24o4b1SyxREq5VsDBw4kPz+fUxkO3xu07ijXFZogHEalRJ6QINbut/eLl1Y3+CKkLvnrh7tobjH8dF7n46idhQQGcO6oJD7aXsTvvmkI6OEd9Ep1JigoqEu7qSltYmozMiWSfSVVNDS1tJWtzbUv/VTm5wlia0EFr67P58YZ6QyK71on+5wxyRytbmB9ni5zpZQ6niYIh1EpkTS1mLalKA6V11JQbl/N8agfJwhjDL95ZzuxYcH84LzhXb5+ZkYiwQE2PtxWaEF0SqmeTBOEw8hk+0imnYXHAFiba29eSooM8esmppLKelbvL+U7Zw0h6hRmRUeGBnHG8Hg+2FbUJzvvlFId0wThMCwxgkCbsNsxkmltbikRIYGcPizerxNE0bF6AIYnRpzyPeaMSSGvtKbXzCZXSnmGJgiH4EAbQxLC2zqq1+4vY/LgWBIj/LwGUWUfYXWyeQ+dmT0mCRF47kvPz8RUSvVcmiCcZKREsquokoqaRnYVVTJ1cCyx4cHUNjZT2+Cfk8lKKu01iKSoro1vdpYUGcqNZwzhhdV5vL7h1DdEUUr1LpognIxKieRgaS2f7bGPk546JI748GAASmv8sxZR7GhiSogI7tZ97vnGKE4bEsc9r21ha0GFJ0JTSvVwmiCctHZUP//VAYIChMy0GOJaE0SVfyaIkqp6ovsFERIY0K37BAXYeGTRZGLDgvnuf9b5dbOaUso7NEE4GZUSBcDq/aWMT40mNCjg6wThpzWIksr6bvU/OEuICOGxa6dQUlXPj1/aoKOalOrjNEE4GRjbj7Bg+1/iU4fYl6toSxDV9T6LqzMllfUkeShBAExMi+GX80ezcs8R3ukha1AppayhCcKJzSaMcDQzTR1sTxDx4fY336N+2sRU7MEaRKtrThvM2AFR/GHpDmoamjx6b6VUz6EJop0Mx8quWemxAESGBhJgE8r8sInJGGNvYorwbIIIsAn3XTyWQxV1PPbpPo/eWynVc2iCaOfms4by58smEBNmb1qy2YTYsGC/7LStbmimtrGZpCjPJgiAqelxXDxxAI+vyOFgaY3H76+U8n+aINoZkRzJlVPTjiuLDw/2yyam4mPdnyTXmXu+MQqbCL9fuuPkJyuleh1NEG6IDQ/yyyam1klyiRGnPkmuM/2j+/H9mcN4b2sh6w5YtiW4UspPaYJwQ3x4iF+u6FpS5UgQFtUgAL595hACbMKnu/rmJitK9WWaINwQF+6ffRBty2xYmCDCQwLJSI5k48Fyy56hlPJPliYIEblDRLaJyFYReVFEQtsd/4mIbBeRzSKyTEQGOx1rFpGNjtdbVsZ5MrHhwVTUNtLU3HLyk72ouLKeoAAhul/Xl/nuisxBMWzMK6elRSfOKdWXWJYgRCQVuA3IMsaMAwKAq9qdtsFxfAKwBPiz07FaY0ym43WxVXG6Iz48GGOgvLbRl2GcoKSynoSIEGwWbxU6KS2Gyvomco5UWfocpZR/sbqJKRDoJyKBQBhwyPmgMWa5MaZ1DOVXQNd21PaSr2dT+1czkyeX2ejMpEExAGzIK7f8WUop/2FZgjDGFAB/BfKAw0CFMebDTi65CXjP6etQEckWka9E5BKr4nSHPycIK/sfWg1NiCAyNJAN2g+hVJ9iZRNTLLAAGAIMAMJF5NoOzr0WyAL+4lQ82BiTBVwD/F1EhnVw7S2ORJJdUmLNSBurEkRzi+Gj7UXUN53aXhNWLLPhis1mX9l2o9YglOpTrGximg3sN8aUGGMagdeAM9qfJCKzgZ8DFxtj2lbEc9RAMMbkAJ8Ck1w9xBjzhDEmyxiTlZiY6PnvAtr2hPD0UNc/LN3Bzc9l8+G2oi5f29xiKK32/DIbHclMi2FXUaWuzaRUH2JlgsgDpotImIgIMAs4bkquiEwCHseeHIqdymNFJMTxeQIwA9huYaydal12o8yDCeL51Qd46vP9AOw/Ut3l649W19NirJ0D4WzSoBiaWwxb8nUzIaX6Civ7IFZjH5m0HtjieNYTInK/iLSOSvoLEAG80m4462ggW0Q2AcuBPxpjfJYgggNtRIYGeqyJaeWeEn715jbOzUgkOSqE3KNdTxCtO8klRlozi7q9iQNjAHQ+hFJ9SKCVNzfG3Avc2674V07HZ3dw3SpgvIWhdVl8eLBHmpj2Flfy/efXMyIpgn9cM5nv/HsteUe7vhieN2ZRO4uPCGFQXJgmCKX6EJ1J7abY8GCPNDE9tXI/xsDTN0wlIiSQwXHh5J5KgvDCLOr2Jg2K0aGuSvUhmiDc5KkaREF5LcMSw0mN6QfA4IQwjlTVU1Xftc7ftoX6vJggMtNiKDxWR2FFndeeqZTyHU0QbrLvCdH9bUcLK+pIjvq63yA9PhyAA13shyiprCcyNJDQoIBux+SuSYPsmyhtPFjmtWeqjr25sYB739yqS6Aoy2iCcFNcRDBl1Y0Y073/jIXH6kiJ/jpBDI4PA+BAF5uZvDWL2tno/pEEB9i0mckPvLQmjx+/tJF/f3mAl7MP+joc1UtpgnBTfHgwDc0tXW4KclbT0ERlXVO7BGGvQXR1JJMVW42eTEhgAGNTo1ifpzUIb/n3qlwm/vpDfvvOdoocG0S9sDqPn722hXNGJjItPY4/vb/T72b5q95BE4SbYsO6P5u6te0+xamJKSIkkISI4C6PZCqpqicpyjtDXJ1NS49j08EK6hpPbfa3ct+7mw9z39vbiA8P5l+rcjnrT8u5+bls/vf1LZybkcjj35rCb785jqq6Jv703k5fh6t6IU0QboqP8ECCOHZiggB7LaKrNYjiY3Ver0EATB8WT0NzC+sOaC3CSqtzjnLHyxuZPCiWpT8+i+V3zuSyKQP5bFcJs0Yl8di3phAaFMDI5EhuOmsIL2cf1F3/lMdpgnBTXLj9zbg7CaK1iSA5un2CCOtSH0R1fRPVDc1e74MAmJoeR4BN+HLfUa8/u6/YXVTJzc9lkxbXj6euyyI0KIBB8WH84dLxrPvlbJ68LouQwK8HJ9x23gj6R4fy89e3+t2eJapn0wThpriw7q/HVFhhHwXVvgaRHh/O4Yo6t5ttjnh5kpyziJBAxqdG81WOJgir3PHyRkKCAnj2xmnEOtYBaxUZGnTC/h/hIYHce9EYdhZWcsmjX/D+1sK2kU0NTS2s2F3CEyv2abOg6jJLZ1L3JnER3V+PqehYHZEhgYSHHP9jbx3JlFdaw8jkyJPep9gHk+ScnT4snqdW5lDT0ERYsP4KedLe4kq2HTrGfReNIS0uzO3r5o5N4YGFE3lo2V5u/e86MpIjGZ4cwYpdJVQ6BlY0Nht+cO5wq0JXvZDWINwUHhxAcKCt253U7ZuX4Ou5ELluLtrni0lyzqYPjaex2ZCd2zf7IZbvKuaNDQWW3PudzYcRgQvG9+/SdSLCNycN5KM7zubBqzIBWLO/lPkT+vPUdVnMHp3EPz/d11b7VMod+uefm0SEuLDuzaYuPFZ3QvMSHF+DcIevE0TW4FgCbcKXOUc5e6R7S6wbY7Av6tvz/ebt7eQcqWZzfgW/mD/ao1u+vrv5MNPS446bTNkVgQE2FmSmsiAz9bjyIYnhzHlgBQ9+vIffXDLOE6GqPkBrEF0Q18l6THlHa07a/FR0rM7lf/yYsGCi+wW5PZIpr7SG4EBbW7+It4WHBDIxLcbtjuplO4rI+u3HPLRsj8WRWa+suoGcI9Wkx4fxzBf7+dGLGzzWtr+7qJI9xVVcOKFrtQd3DEuM4Jppg3hhTR77SnRvceUeTRBdEB8R7LKKbozhmqe+4orHv+xwIl1zi6G4sp6UaNd/9ad3YSTTugNlTBwY7dG/XLvq9KHxbCmo6HTiYEuL4YGPdnPTv7NpaG7hbx/ttqxpxls25pcD8PtLx/O/3xjFu1sOc90za6htcD9JGGP40/s7+fFLG2h2WibjnU2HsAnMG+f5BAHw49kj6BcUwB91zoRykyaILhiZHMnOwkoamo4fSlhQXkt+WS17i6u465VNLpfjOFpVT3OLcdnEBO7PhahtaGZrQQVZ6XGn9k14yPSh8TS3GNbmuh57X1HbyM3PZfPgsj1cNnkgq352HqcNiePuJZt79Hj9DXnl2MS+P8YtZw/jwasyWbO/lD9/cOKbbl1jM3uLK48rM8bwx/d28s9P9/HmxkM8sSKnrfydLYeZPjTesqbDhIgQvjdzGB9tL2LFbmu251W9iyaILpiaHkt9UwtbCo7fVa110tilk1J5b2th2396Z62T5DpqW06PD6OgrPaE5NPepvxymloMWYNjT+Vb8Jgpg2MJChC+ctHMlJ1byjceXMlnu0u4f8FY/nrFBCJDg3js2in0jwnllufWcdDN/hZ/syGvjIyUqLaRaAsyU7nu9MH864vc44b+1jU2c+O/1jL7byv4/vNff7//+GQvj6/I4drpg5g/vj9/+2gXWwsq2FlYSU5JNfMtaF5y9u0ZQ0iN6cd1z6xh0VNf8f7WQp07oTqkndRdMGWw/a/27NxSpji9QWfnlhEeHMCfL59AfVMLf3p/J+NSo5kxPKHtnLZlNlyMYgIYFB9Oi4H8shqGJkZ0GENrMpri4wTRLziASWmxfOn0ptjcYnj4k708uGw3A2PDeOXW09tWgAX7nhpPXz+VSx/9gnP+spygABs2EfoFB/Dct6cxLjXaF9+K21paDBsPlnPRxAHHlf/sglF8uquEu5ds5r0fn0VoUAB3vLyRL3OOcumkVJZuPcyyHcXMzEjkg21FXDo5lfsvHsexukayD5Ry+8sbOWdkor15aWyKpd9Dv+AA3vrhDF5ck8cLq/O49b/rGJ4UwVs/nKFDltUJtAbRBYmRIQxNCGdtu+Gd6w6UkTkohsAAG3+6fALDEiNO6Lws6mCZjVbprau6nuQv6+zcUoYnRbTtk+1L04fGsbWggtte3MA1T37FOX9ZzgMf72ZBZirv3nbmccmh1fCkCF64eTq3njOMG2ak863TB1Pb0MwLa/J88B10Tc6RKirrmpiUFnNceVhwIH+9YiIHy2r443s7+eWbW3lvayG/mD+avy3M5JM7ZzJvXAofbCti3tgU/nzZBGw2ISYsmP+7IpO9xVU8/fl+zhiWQLwXlk+Jjwjhh+eNYMXd5/Lnyyewt7iK19b37L4hZQ1NEF2UlR7LugOlbTNVq+qb2Fl4rK12ERESyN3zRlFa3cAmp+05C4/VEWCTDt8AWld1PdDJXIiWFsO6A2VMTfdt7aHV3HEpxIWHsCm/nPqmFsYOiOLBqzJ5YGEmkaFBHV43LjWau+eN4p4LRvO/3xjNnLHJLN1y+KTNa7623rHMuavEN21IHN+eMYT/fHWAF1bn8b2Zw/jOWUMBGBDTjwevmsTnPz2XRxZNJjDg6/92Z45I4NszhgBY3rzUXmCAjSumDGR8ajTPrsrt9lL2qvfROmUXZaXHsTg7n5wjVQxPimRDXhkthuP6BFqbf9bllXHa0HjAvsxGUmQIAR2MPEqICCY8OKDT7Uf3FFdxrK6pLRn52tgB0WT/wuW24l1ySWYqb248xGe7Szh/TLIHIrPGhrxyokIDGZoQ7vL4XXMz2HiwnPGp0dw9N+OE4wNjXc+M/ukFGWSkRHDJpFSXx60kItw4I52fLN7Eyj1H3J7XovoGrUF00VTH6KE1++3NTNm5ZdjEvl9zq7jwYIYmhrPOqSmqozkQrUTkpCOZsh2jf3zdQe1pZ45IID482OtDYOsam/l8zxG3d2TbkFdG5qDYDocXhwYF8Or3zuC+i8d2aVJgSGAAC6cOOm4BPm+aP6E/CREhPLsq1yfPV/7L0gQhIneIyDYR2SoiL4pIaLvjISLysojsFZHVIpLudOweR/kuEZlrZZxdkR4fRkJEMNmO4Z3rHaNa2jepTBkUy7q8srZqe0ezqJ1lpESy4/CxDo+vyy0jISK4beZ1bxEUYOPCCf35eEcRlXWNXnvukytyuPbp1Vz+2Cp2Fnb8cwd7U+LuosoT+h96g5DAABadNohPdhaz383lXlTfYFmCEJFU4DYgyxgzDggArmp32k1AmTFmOPAA8CfHtWMc544F5gGPiohv/rxqR0TIGhzH2gOlNLcYNuSVM2VwzAnnZaXHUl7TyL4S+3+4ooq6DkcwtRo7IIqiY/UUV9a5PL72QClZg+N6zZIVzhZMSqW+qYX3txZ67ZnLdhaTGtOP3KM1XPjQ5/zhvR0dzorenF9Oizm+ptibLJo+iKAA4d9ai1BOrG5iCgT6iUggEAYcand8AfBvx+dLgFlif/dbALxkjKk3xuwH9gLTLI7VbVnpsRwsrWXF7hKq6pvIctEn0NoPsf5AGdX1TVTWN510fZ3WYZ7bDp3412zxsToOltaS5Scd1J42KS2GwfFhvLHRO81MpdUNbMov54qsgSz7yTlcOjmVxz/L4c4OJjq27sOd2QtrEABJkaFcOGEAr2Qf9GotTvk3yxKEMaYA+CuQBxwGKowxH7Y7LRU46Di/CagA4p3LHfIdZScQkVtEJFtEsktKvDM7dNoQe0J47LN9gOs5CUMTIogJCyL7QOnXO8l1sMxGqzEDogDY1m4iHkC2n8x/sIqIsCAzlVX7jrYNCT5V9U3N1DR0vnf4it0lGAPnZiQRGx7Mny+fyF1zM3h382Fed9EXsiGvjKGJ4X4xvNgqN85Ip7qhuccvh6I8x8ompljsNYEhwAAgXESu9fRzjDFPGGOyjDFZiYneGYExpn8UYcEBrN5fSnJUCANj+51wjs0mTB4US/aBMooqOp9F3SoqNIj0+DC2FpxYg8jOLSMk0MbYAf49maw7LskcgDHw1sb2FU33GWO46dlsTvvdMp5ckdPh0NlPdxUTHx7MeKfJebeeM4yp6bH86s1tx830NsbelDgprXcm51YTBsaQHh/G8l26DIeys7KJaTaw3xhTYoxpBF4Dzmh3TgGQBuBohooGjjqXOwx0lPmFwABbW1t0Z30CUwbHklNSzY5C+3o8J+ukBhibGs3WQ65qEKVkpsUQHNh7B54NTYxg8qAYnliZ07akeVct3VLI53uP0D8mlN8t3cGcBz7j4+1Fx53T3GL4bHeJffay04ikAJvwtyszEeAnizfS3GI4WFrDz9/YytHqBia76Gvqbc4ckcBXOUf9fk6K8g4r323ygOkiEuboV5gF7Gh3zlvA9Y7PLwc+MfYG4LeAqxyjnIYAI4A1FsbaZa39DpM7afJpbQ56b8thoONlNpyNGxBNflkt5TVfLx1eWt3A1oKKtjkVvdnvLx3PsdpG7nh543ErnbqjtqGZ3y/dwej+Ubz347N59sapBAbY+M5z2W3/BmDvcC6raeScjBNrnGlxYdx/yVjW5pZx6T9XMfOvn/JK9kGunpbGZZMHdvv783dnDk+kpqGZDXl9czModTwr+yBWY+94Xg9scTzrCRG5X0Qudpz2NBAvInuBnwA/c1y7DVgMbAfeB35gjPGrDXXPHZVEcKCNs0ckdHjOxIExBNqE7ANlRIUGurXWzbhURz+EU0f1JzuLaTFw/mj/nUTmKaNSovj1xWP5fO8RHl2+97hj+0qqOu1beHzFPgrKa7nvojEE2ISZGUm89+OzGNM/it+8s73t2uW7SrAJnD3CdZPkJZmpfHNSKrsLK7nhjHRW3n0ef7h0AqFBfjGQzlKnD4vHJvD53iO+DkX5AUtnUhtj7gXubVf8K6fjdcAVHVz7O+B31kXXPZlpMWz/9dzjlk1or19wAGMHRLEpv8Kt2gPYaxAAWwsq2hb7+2h7ISlRoW3Jo7dbODWNL3OO8sDHu5k8OJbS6gaeXZXLugNlLMgcwINXTTrhmoLyWh77bB/zJ/Q/rqYVFGDj/gVjufyxL3n4k73cPW8Un+0qJjMththw1x3OIsL/XTGRP1w6vk8kBWfR/YKYmBbDyj1HuHPOibPBVd/Sexu0vaCz5NCqdVkMd7eQjA0PJjWmH1sdNYi6xmZW7D7C7DFJvXL+gysiwu++OZ70+HAWPbWaH724gSNV9Zw5PIG3Nh1ib/GJO6L9fqm99fJ/vzH6hGNZ6XFcOjmVJ1fmkJ1byqb8CmZmJHUag80mfS45tDpreAKb88upqNHhrn2dJgiLtfZDuNNB3WpcalTbUNdV+45Q29jM+WOsXQba30SEBPLYt6Zw6aRUnr4+i0/unMnfr8okNDDghKanT3YW8e7mw9x6zjBSY04cUQb2JblDAwP4znPZgH14q3LtzBGJtBj4Mkebmfo6TRAWa53Y1t/NJiawNzPlHKmmsq6Rj7YXERESyPSh/rFAnzeNTI7kbwszmTU6mQCbkBARwrXTB/HGxgJyHUtClFY3cPeSLYxKieR7M4d1eK+kyFBuP38k5TWNJEQEM3ZA32iuOxWTBsUQHhzAyj2aIPo6TRAWS44K5eFrJrFo+mC3r3GeUf3xjmLOGZnos4Xc/M3NZw8lKMDGI8v3YozhF29soaK2gb9dmXnSn9H1pw8mMy2GBZmpPt3P298FBdiYPjReO6qVLvftDRdOGHDyk5yMdXRGv7A6j5LKer9eAtvbkiJDuea0QTz35QEGx4exdEshd8/LaJuF3pnAABuvf/+MPtOX0x1njkhg2c5iDpbWkBbXuxaHVO7TGoQfSooMJSkyhLc3H3IM19Q1+p3des4wAmzCXz/czZTBsXz37I6bltrT5OCesxzDt7WZqW/TBOGnxqVGYwxMS4/r1ev/nIrkqFCumz6YyJBA/nblxA43YVKnblhiBClRoXy+V5fd6Ms0QfipcY4mE21ecu3n80fzxT3ntW3VqjxLRDh3VCLLdhSzzcXSL6pv0AThp84emUhCRDDzxvWt4a3uEhGiOtn3WnXfT87PIDYsmO/9d73OieijNEH4qaz0OLJ/cT4DOhjXr5TVEiNDePTayRyuqOX2lze4vTWr6j00QSilOjR5UCy/umgsy3eV8NAne3wdjvIyTRBKqU5de9ogLp2cyoPL9rBeV3ntUzRBKKU6JSL8ZsE4+gUF8Ep2vq/DUV6kCUIpdVLhIYHMHp3M+1sP09ismwn1FZoglFJuuXBCf8pqGvlCl+DoMzRBKKXcck5GIpGhgbyz+fDJT1a9giYIpZRbQgIDmDMmhQ+2FlLf5FcbPCqLaIJQSrntoon9qaxv4rNdugRHX6AJQinlthnDE4gNC9Jmpj7CrQQhIuEiYnN8PlJELhYRXedAqT4mKMDGvHH9+XhHEbUN2szU27lbg1gBhIpIKvAh8C3gWauCUkr5r4sm9qemoZlPdhb7OhRlMXcThBhjaoBLgUeNMVcAYzu9QCRDRDY6vY6JyO3tzrnL6fhWEWkWkTjHsVwR2eI4ln0K35tSygKnDYknMTKEtzcd8nUoymLu7ignInI6sAi4yVHW6f6OxphdQKbj4gCgAHi93Tl/Af7iOOci4A5jTKnTKecaY3TQtVJ+JMAmzBubwpJ1+dQ1NhMapNvh9lbu1iBuB+4BXjfGbBORocDyLjxnFrDPGHOgk3OuBl7swj2VUj4yZ2wytY3NuuNcL+dWgjDGfGaMudgY8ydHZ/URY8xtXXjOVXTy5i8iYcA84FXnxwIfisg6Ebmlk2tvEZFsEckuKdGhd0p5w/Sh8USGBvLBtkJfh6Is5O4ophdEJEpEwoGtwHYRucvNa4OBi4FXOjntIuCLds1LZxpjJgMXAD8QkbNdXWiMecIYk2WMyUpM1L2blfKGoAAbs0YlsWxHEU26NlOv5W4T0xhjzDHgEuA9YAj2kUzuuABYb4wp6uScE2oYxpgCx8di7H0X09x8nlLKC+aOTaGsppG1uboEeG/lboIIcsx7uAR4yxjTiL0JyB2d9i2ISDRwDvCmU1m4iES2fg7MwV5zUUr5iXMyEgkJtGkzUy/mboJ4HMgFwoEVIjIYOHayixxv7ucDrzmV3Soitzqd9k3gQ2NMtVNZMvC5iGwC1gDvGmPedzNWpZQXhAUHctaIBD7aXoQxuh1pb+TWMFdjzEPAQ05FB0TkXDeuqwbi25U91u7rZ2k36c4YkwNMdCc2pZTvzBmbwsc7itl26BjjUqN9HY7yMHc7qaNF5G+to4VE5P+w1yaUUn3YrFFJ2AQ+1GamXsndJqZngErgSsfrGPAvq4JSSvUM8REhTE2P44NtnY1BUT2VuwlimDHmXmNMjuP1a2ColYEppXqGuWNT2FVUydaCCl+HojzM3QRRKyJntn4hIjOAWmtCUkr1JJdNGUhsWBC/X7pDO6t7GXcTxK3AI44F9HKBh4HvWhaVUqrHiO4XxO2zR7Jq31E+3qErvPYm7i61sckYMxGYAEwwxkwCzrM0MqVUj3HNaYMYlhjO75fuoKFJZ1b3Fl3aUc4Yc8wxoxrgJxbEo5TqgYICbPx8/mj2H6nmv191tian6km6s+WoeCwKpVSPd25GEmcOT+DBZXsor2nwdTjKA7qTILQ3SinVRkT4xYWjqaxr5MmVOb4OR3lApwlCRCodO8G1f1UCA7wUo1KqhxiVEsXZIxN5c+MhHdHUC3SaIIwxkcaYKBevSGOMu7vRKaX6kPnj+5NfVsvmfJ0X0dN1p4lJKaVOMGdMCkEBwrtbDvs6FNVNmiCUUh4VHRbEmcMTeHfzYW1m6uE0QSilPG7+hAEUlNeySZuZejRNEEopjzt/TLK9mWnzIV+HorpBE4RSyuOi+wVx1ohElm4p1GamHkwThFLKEvPH96egvJaNB8t9HYo6RZoglFKWmN3WzKSjmXoqTRBKKUtE9wvi7BGJLN2io5l6Kk0QSinLzB6TzKGKOnKOVPs6FHUKNEEopSwzfWg8AF/lHPVxJOpUWJYgRCRDRDY6vY6JyO3tzpkpIhVO5/zK6dg8EdklIntF5GdWxamUsk56fBjJUSF8uU8TRE9k2XpKxphdQCaAiAQABcDrLk5daYy50LnAcf4jwPlAPrBWRN4yxmy3Kl6llOeJCNOHxvPF3qMYYxDxv10C6puaKaqoZ1B8mK9D8TveamKaBewzxri7k8g0YK8xJscY0wC8BCywLDqllGVOHxrPkap69pX4Zz/EUyv3M/Ovy/li7xFfh+J3vJUgrgJe7ODY6SKySUTeE5GxjrJU4KDTOfmOshOIyC0iki0i2SUlJZ6LWCnlEf7eD/FVzlFaDPzwhfUcLK3xdTh+xfIEISLBwMXAKy4OrwcGO/a7/gfwRlfvb4x5whiTZYzJSkxM7FasSinPGxwfRkpUqF8miKbmFtYfKOOckYk0tRhu+c86ahuafR2W3/BGDeICYL0xpqj9Acce11WOz5cCQSKSgL2/Is3p1IGOMqVUD2Pvh4jjq5xSv5sPsbOwkuqGZi6dnMpDV09iZ+Ex7n51s9/F6SveSBBX00HzkoikiKPXSkSmOeI5CqwFRojIEEcN5CrgLS/EqpSywPS2fogqX4dynHUHygCYMjiWczOS+J85Gby96RCvrde/R8HiBCEi4dhHIr3mVHariNzq+PJyYKuIbAIeAq4ydk3AD4EPgB3AYmPMNitjVUpZp7Uf4sucUh9HcrzsA2WkRIWSGtMPgO/PHMbQhHBeXZ/v48j8g6XbhhpjqoH4dmWPOX3+MPBwB9cuBZZaGZ9Syjuc+yG+NX2wr8Npsy63lCnpsW3Db0WECyf05+HleymprCcxMsTHEfqWzqRWSllORDh9WDyrc476Tfv+ofJaDlXUkTU49rjy+RMG0GLg/a26yKAmCKWUV0wfGseRqga/6YfIdvQ/ZA2OO648IyWSEUkRvK2r0GqCUEp5R2s/xIrd/jEhbV1uKWHBAYzuH3nCsfkT+rM2t5SiY3U+iMx/aIJQSnnF4PhwRqVE8u4W//jLPPtAGZlpMQQGnPg2eOGE/hgDS/0kVl/RBKGU8pqLJg5g3YEy8st8O2O5qr6JHYePndD/0Gp4UqQ9mfXxZiZNEEopr7lowgAA3vHyG291fROPLN/L4YpaADbmldNiICs9rsNr5o/vT/aBsrZr+iJNEEoprxkUH8bEtBje2njIq8/9aHsRf/lgF7P/7zOe+Xw/a/YfxSYwaVBMh9fMn9AfoE/XIjRBKKW86uKJA9h++Bh7i703mqmg3F4LmDw4lvvf2c4/lu8lIyWKyNCgDq8ZmhjBmP5RfXo0kyYIpZRXXTihPyLw9ibv1SLyy2qJDw/muW9P4+FrJtE/KpS5Y5NPet03J6Wy6WA5uworvRCl/9EEoZTyquSoUE4bEsfbmw95bdLcofJaBsT0c8yUHsCqe2Zx++yRJ73usikDCQ6w8eKaPC9E6X80QSilvO6iiQPIKalm26FjXnneofLatvWWuiIuPJi541J4fUMBdY19bxlwTRBKKa+7YFx/Am3C25utb2YyxlDgqEGciqunpVFR28h7fXDpDU0QSimviwsP5swRCby7+bDlzUwVtY3UNDSTGntqCeL0ofGkx4fx4uqDJz+5l9EEoZTyifPHJJNfVsvuImtHM+WX2UcwpcaEntL1IsJV0waxJreUvcV9q7NaE4RSyidmjbKPIlq284TNJj3qUHlrggg75XtcPmUgQQHCi2v6Vi1CE4RSyidSokMZlxrFsh3Flj6ndQ7EgFOsQQAkRIQwZ0wKr63P71Od1ZoglFI+M2tUMuvzyjhaVW/ZMw6V1xIaZCMuPLhb97l62iDKahp5fUPf2Y5UE4RSymdmj07GGFi+q8SyZxQ4zYHojhnD48lMi+GhZXv6TC1CE4RSymfGpUaRHBXCsh3W9UMUlNed0hyI9kSEu+dmcLiijudX942Jc5oglFI+IyKcNyqJFbtLqG+y5q/ygrJTmyTnyhnDE5gxPJ5Hlu+lqr7JI/f0Z5YlCBHJEJGNTq9jInJ7u3MWichmEdkiIqtEZKLTsVxH+UYRybYqTqWUb80alUx1QzNr9pd6/N51jc0cqao/5Ulyrtw1dxSl1Q088/l+j93TX1mWIIwxu4wxmcaYTGAKUAO83u60/cA5xpjxwG+AJ9odP9dxjyyr4lRK+daM4QmEBNosGc10uMK+ZainahAAmWkxzBmTzJMrciirbvDYff2Rt5qYZgH7jDEHnAuNMauMMWWOL78CBnopHqWUn+gXHMCZwxP4eEeRx2dVH2ob4uq5BAFw55wMqhqaeHj5Xo/e1994K0FcBbx4knNuAt5z+toAH4rIOhG5xbLIlFI+N2u0fVb1loIKj963wDGLeuApLrPRkYyUSK6eNohnvtjPit3WjcDyNcsThIgEAxcDr3RyzrnYE8RPnYrPNMZMBi4AfiAiZ3dw7S0iki0i2SUlvfcfSqne7BvjU4gJC+K37+7waC2ioLwWEfsS4572y/ljGJkUye0vb6TQ0ZTV23ijBnEBsN4Y43Icm4hMAJ4CFhhjjraWG2MKHB+LsfddTHN1vTHmCWNMljEmKzEx0ePBK6WsFxMWzE/njWLN/lLe2Oi5iWiHymtJjgwlONDzb3X9ggN4ZNFk6hqbue3FDTQ1t3j8Gb7mjQRxNR00L4nIIOA14FvGmN1O5eEiEtn6OTAH2OqFWJVSPrIwK42JaTH87t2dHKtr9Mg97ZPkPF97aDU8KYLff3M8a3JLeeDj3Se/oIexNEE43tzPx54EWstuFZFbHV/+CogHHm03nDUZ+FxENgFrgHeNMe9bGatSyrdsNuG3C8ZxtLqev33omTfbQ93YB8Jdl0xK5aqpaTyyfF+vW+3V0gRhjKk2xsQbYyqcyh4zxjzm+Pw7xpjY1uGwrcNZjTE5xpiJjtdYY8zvrIxTKeUfxg+MZtFpg3juy1y2Hepeh3VLi+FQed0p7wPRFf8zN4NAm/Dy2t612qvOpFZK+ZW75owiNiyYPyzd2a37HKmup6G5xaNzIDqSEBHCrNFJvLa+gIam3tMXoQlCKeVXosOC+O45Q/l87xE2Hiw/5fsUtG0UZH2CAFg4NY2j1Q18YvH+Ft6kCUIp5XeuOW0w0f2CeLQbE9EOlduHnlrdB9Hq7BGJJEeF9KpmJk0QSim/ExESyPVnpPPh9iL2FJ1ax29BeQ2AV/ogAAIDbFwxJY3Pdpf0mnkRmiCUUn7pxjPSCQsO4J+f7jul6w+V1xEZEkhUaJCHI+vYlVlptBhYsq531CI0QSil/FJseDBXTxvEm5sOcbC0psvX55fVeq320GpQfBinD41ncXY+LS2eXVfKFzRBKKX81s1nDcUm8MSKnC5fu6+kivT4cAui6tzCqWnkldbw1f6jJz/Zz2mCUEr5rZToUC6fMpCXsw9S2oWltWsbmsk9Wk1GSqSF0bk2b1wK0f2C+NcXuV5/tqdpglBK+bVvTU+noamFdzcfcvuaPcWVGAOjfJAgQoMCuHFGOh9tL2L7oWNef74naYJQSvm1MQOiGJUSyWsb3F/Eb2ehfeSTL2oQADfOGEJkSCAPLdvjk+d7iiYIpZTfu2RSKhvyysk9Uu3W+bsKKwkNsjHYB30QANH9grhxRjrvbytkZ2HPrUVoglBK+b0FmQMQgdfdrEXsKqxkRFIkATaxOLKOffvMIUSEBPKPZV9P9jPGsLPwGM09ZISTJgillN/rH92P04fG88bGArc2FNpZWOmz5qVWMWHB3HBGOku3HmZ3USXZuaVc+fiXzPv7Sv73tS0e317VCpoglFI9wjcnpXLgaA3r88o7Pe9oVT1Hqup90kHd3k1nDiEsKIBrnlzN5Y99Se7RGuaMSebl7IO8sCbP1+GdlCYIpVSPMG9cCqFBNt44STPTLh93UDuLDQ/m1nOGUd/UzF1zM/jsrpn889opzMxI5L63trHuQJmvQ+yUJgilVI8QGRrE+WNSeHvzoU6X1Pb1CKb2fnjecDbfO4cfnDucsOBAAmzCgwsnMSCmH9/77zqKj/nvuk2aIJRSPcalk1Ipr2nk013FHZ6zq7CSuPBgEiNCvBhZx0QEkeM7y6PDgnjiW1lU1Tdx5eNfsnjtQeqbmn0UYcc0QSileowzRySQEBHCknX5HZ6zs6iSjOTIE96U/U1GSiRPXZdFWHAgd7+6mbP+tJzHPttHU7P/bDikCUIp1WMEBdi4bHIqn+wsprjyxKaZlhbDniLfj2By1xnDE3j3tjP5z03TGJkcyR/f28nfP/afyXWaIJRSPcqVU9NoajG8tv7EzuqDZTXUNDT7xQgmd4kIZ41I5L/fOY0rswbyyKd7+WLvEV+HBWiCUEr1MMMSI5iWHsfLaw+eMJfA3zqou+q+i8cyLDGC21/eyJGqel+HY12CEJEMEdno9DomIre3O0dE5CER2Ssim0VkstOx60Vkj+N1vVVxKqV6noVT09h/pJo1+0uPK28d4joyuWcmiLDgQB6+ZhIVtY3cuXiTz/eUsCxBGGN2GWMyjTGZwBSgBni93WkXACMcr1uAfwKISBxwL3AaMA24V0RirYpVKdWzfGN8fyJDAo/f/3nzYhZ9OZ+c0EWEP5oJmxf7LL7uGJUSxa8uHMNnu0t4dlWuT2PxVhPTLGCfMeZAu/IFwHPG7isgRkT6A3OBj4wxpcaYMuAjYJ6XYlVK+bl+wQEsmDSAd7ccpqK20Z4M3r6N+KYibBioOAhv39Zjk8Si0wZx+tB4nlyZ49N1m7yVIK4CXnRRngo4b96a7yjrqFwppQBYmDWI+qYW3tp0iJaPfw2Ntcef0FgLy+73TXDdJCJ86/TBHK6oY+WeEp/FYXmCEJFg4GLgFYvuf4uIZItIdkmJ736QSinvGpcaxZj+Ufzm7e1Q0cG8iI7Ke4DZo5OJCw8+vhnNy7xRg7gAWG+MKXJxrABIc/p6oKOso/ITGGOeMMZkGWOyEhMTPRSyUsrfiQg/u2AU88alUBWa4vqk6IHeDcqDggNtXDoplY93FPlsRJM3EsTVuG5eAngLuM4xmmk6UGGMOQx8AMwRkVhH5/QcR5lSSrU5e2QiD109iaj5v4GgfscfDOoHs37lm8A8ZOHUNBqbDa+7mPPhDZYmCBEJB84HXnMqu1VEbnV8uRTIAfYCTwLfBzDGlAK/AdY6Xvc7ypRS6kQTroSLHoLoNEDsHy96yF7eg41IjmTyoBhezj5xzoc3SE/YtMJdWVlZJjs729dhKKWUx7y8No+fvrqFV793OlMGx3n8/iKyzhiT5eqYzqRWSik/duGEAYQHB/DSGu93VmuCUEopPxYeEshFEwfwzubDlFSe2FltjLFsroQmCKWU8nM3nz2U5hbDr9/edsKxp1bu5/pn1lDT0OTx52qCUEopPzcsMYLbZg3nnc2H+Xj71zMGlu8q5g/v7SC6XxD9ggI8/lxNEEop1QPccvYwRqVE8os3tnKsrpG9xVXc9sIGRqVE8ZcrJliyQZImCKWU6gGCA2388bIJFFfWcd+b27jluWyCA208eb19VzorWHNXpZRSHpeZFsONM4bw9Of7CQoQXrh5Oqkx/U5+4SnSBKGUUj3InXNGknukmosmDmBquufnRTjTBKGUUj1IWHAgT98w1SvP0j4IpZRSLmmCUEop5ZImCKWUUi5pglBKKeWSJgillFIuaYJQSinlkiYIpZRSLmmCUEop5VKv2lFOREqAA0A0UOEoPtnnrR8TgCNdfKTz/dw91r7c3ficy7oaa2dxdnS8szhPFqtVP9PuxtoT//17Uqz6u+q5WL357z/YGJPo8ogxpte9gCfc/dzpY3Z3nuPusfbl7sbXnVg7i7Oj453F6cbP0pKfaXdj7Yn//j0pVv1d7Zm/q529emsT09td+Ny5rDvPcfdY+/KuxHeqsZ7sOlfHO4uz/dftY7XqZ9rRcXdj7Yn//s6f+3us+rt68uP++LvaoV7VxNQdIpJtOti429/0lFh7SpygsVqlp8TaU+IE78baW2sQp+IJXwfQBT0l1p4SJ2isVukpsfaUOMGLsWoNQimllEtag1BKKeWSJgillFIuaYJQSinlkiYIN4jIWSLymIg8JSKrfB1PR0TEJiK/E5F/iMj1vo6nMyIyU0RWOn6uM30dz8mISLiIZIvIhb6OpSMiMtrx81wiIt/zdTydEZFLRORJEXlZROb4Op7OiMhQEXlaRJb4OhZXHL+b/3b8PBd58t69PkGIyDMiUiwiW9uVzxORXSKyV0R+1tk9jDErjTG3Au8A//bXOIEFwECgEci3Ik4PxmqAKiC0B8QK8FNgsTVReuz3dIfj9/RKYIafx/qGMeZm4FZgoZ/HmmOMucmqGF3pYtyXAkscP8+LPRrIqcyu60kv4GxgMrDVqSwA2AcMBYKBTcAYYDz2JOD8SnK6bjEQ6a9xAj8Dvuu4dok//0wBm+O6ZOB5P4/1fOAq4AbgQn+N03HNxcB7wDX+/DN1uu7/gMk9JFbL/k91M+57gEzHOS94Mo5AejljzAoRSW9XPA3Ya4zJARCRl4AFxpg/AC6bEERkEFBhjKn01zhFJB9ocHzZbEWcnorVSRkQYkmgeOznOhMIx/6fsVZElhpjWvwtTsd93gLeEpF3gRc8GaMnYxURAf4IvGeMWW9FnJ6K1Re6Ejf2GvhAYCMebhXq9QmiA6nAQaev84HTTnLNTcC/LIvIta7G+RrwDxE5C1hhZWAudClWEbkUmAvEAA9bGtmJuhSrMebnACJyA3DE08mhE139mc7E3twQAiy1MjAXuvq7+iNgNhAtIsONMY9ZGVw7Xf25xgO/AyaJyD2OROILHcX9EPCwiMyne8txnKCvJoguM8bc6+sYTsYYU4M9kfk9Y8xr2BNaj2GMedbXMXTGGPMp8KmPw3CLMeYh7G9sfs8YcxR7X4lfMsZUAzdace9e30ndgQIgzenrgY4yf9NT4gSN1Qo9JU7QWL3B63H31QSxFhghIkNEJBh7B+RbPo7JlZ4SJ2isVugpcYLG6g3ej9tbvfK+egEvAof5eujnTY7ybwC7sY8K+LnGqbFqnBqrv7z8JW5drE8ppZRLfbWJSSml1EloglBKKeWSJgillFIuaYJQSinlkiYIpZRSLmmCUEop5ZImCNWriUiVl5/nkf1CxL5fRoWIbBSRnSLyVzeuuURExnji+UqBJgilukREOl2/zBhzhgcft9IYkwlMAi4UkZPt8XAJ9hVnlfIITRCqzxGRYSLyvoisE/uudqMc5ReJyGoR2SAiH4tIsqP8PhH5j4h8AfzH8fUzIvKpiOSIyG1O965yfJzpOL7EUQN43rHENSLyDUfZOhF5SETe6SxeY0wt9qWcUx3X3ywia0Vkk4i8KiJhInIG9r0g/uKodQzr6PtUyl2aIFRf9ATwI2PMFOB/gEcd5Z8D040xk4CXgLudrhkDzDbGXO34ehT25cqnAfeKSJCL50wCbndcOxSYISKhwOPABY7nJ54sWBGJBUbw9RLurxljphpjJgI7sC/DsAr7ujx3GWMyjTH7Ovk+lXKLLvet+hQRiQDOAF5x/EEPX29YNBB4WUT6Y9+xa7/TpW85/pJv9a4xph6oF5Fi7Dvjtd86dY0xJt/x3I1AOvZtVnOMMa33fhG4pYNwzxKRTdiTw9+NMYWO8nEi8lvse2lEAB908ftUyi2aIFRfYwPKHW377f0D+Jsx5i3H5jv3OR2rbnduvdPnzbj+v+TOOZ1ZaYy5UESGAF+JyGJjzEbgWeASY8wmxyZGM11c29n3qZRbtIlJ9SnGmGPAfhG5AuxbX4rIRMfhaL5eX/96i0LYBQx12k5y4ckucNQ2/gj81FEUCRx2NGstcjq10nHsZN+nUm7RBKF6uzARyXd6/QT7m+pNjuabbdj39QV7jeEVEVkHHLEiGEcz1feB9x3PqQQq3Lj0MeBsR2L5JbAa+ALY6XTOS8Bdjk72YXT8fSrlFl3uWykvE5EIY0yVY1TTI8AeY8wDvo5Lqfa0BqGU993s6LTehr1Z63HfhqOUa1qDUEop5ZLWIJRSSrmkCUIppZRLmiCUUkq5pAlCKaWUS5oglFJKuaQJQimllEv/D2y1xXQI40BGAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "callbacks = [\n", - " ReduceLROnPlateau(factor=0.5, min_lr=1e-6),\n", - " EarlyStoppingCallback(patience=5),\n", - " # SaveModelCallback()\n", - "]\n", - "archs = {\n", - " 'LSTMPlus': {'n_layers':3, 'bidirectional': True}\n", - "}\n", - "model = create_model(TSTPlus, dls=dataloaders, verbose=True)\n", - "learner = Learner(\n", - " dataloaders, model, metrics=[mse, mae]\n", - ")\n", - "learner.lr_find()" - ] - }, - { - "cell_type": "code", - "execution_count": 55, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
epochtrain_lossvalid_lossmsemaetime
02893242.75000033424628.00000033424628.0000004935.94091801:31
12916043.75000035351892.00000035351892.0000005150.01074203:49
22981291.50000033067046.00000033067046.0000004932.22900403:51
32908789.75000033899084.00000033899084.0000005003.19335904:06
43033225.00000034537444.00000034537444.0000005049.70605504:20
52969528.75000033934708.00000033934708.0000004960.64013704:21
63303880.25000036272200.00000036272200.0000004989.99609403:13
73163850.00000036075752.00000036075752.0000005165.19189504:09
" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "No improvement since epoch 2: early stopping\n" - ] - }, - { - "data": { - "text/plain": [ - "30076" - ] - }, - "execution_count": 55, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "learner.fit(n_epoch=10, lr=1e-3, cbs=callbacks)\n", - "gc.collect()" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwEAAAIYCAYAAAA1seDyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAACcZUlEQVR4nOzdeXxU1f3/8dcnewJhCTsBBZRNUcKmuOO+obhrq7UulbpXu1rbn22tdvt2s3UrLlXrgpa6VXGtKFpxAQUEAUHFsihb2ElClvP749yQSUgghJm5dzLv5+Mxj5m5986ddybJ3Pu559xzzTmHiIiIiIikj4ywA4iIiIiISHKpCBARERERSTMqAkRERERE0oyKABERERGRNKMiQEREREQkzagIEBERERFJMyoCpFUws8VmdkzYOUREJLHMbKCZzTSzjWZ2rZndbWb/Lw7r7WNmzsyympiv7Yy0Ko3+oYuIiIhE1A+BKc65krCDiKQytQSIiIhIKtkTmBt2CJFUpyJAWhUzyzWzP5vZ8uD2ZzPLDeZ1NrPnzGydmZWa2ZtmlhHM+5GZLQualxeY2dHB9Awzu8HMPjWzNWb2hJkVBfPyzOzhYPo6M3vfzLqF99OLiLRuZvYacCRwu5ltMrMBZvaAmd0SzB9jZkvN7HtmttLMvjSzi2Nef7KZfWhmG8xsiZn9vIU5tK2RlKciQFqbnwCjgRJgKHAA8NNg3veApUAXoBtwI+DMbCBwNTDKOVcIHA8sDl5zDXAacATQE1gL3BHM+ybQHugNdAIuB8oS9YOJiKQ759xRwJvA1c65ts65TxpZrDv+u7kYuBS4w8w6BvM2AxcCHYCTgSvM7LQWRNG2RlJeqEWAmd0fVOpzmrHsn4ITgWaa2Sdmti4JESX1nA/c7Jxb6ZxbBfwC+EYwrxLoAezpnKt0zr3pnHNANZAL7GNm2c65xc65T4PXXA78xDm31DlXAfwcOCs4cawS/4W8t3Ou2jk3wzm3IWk/qUga0fZCdkElfjtQ6ZybDGwCBgI45153zn3knKtxzs0GHsPveO8qbWsk5YXdEvAAcEJzFnTOXe+cKwlOBPor8GQCc0nq6gl8EfP8i2AawP8Bi4CXzewzM7sBwDm3CLgO/6W70swmmlnta/YEngqaYNcB8/Bf5N2AfwAvAROD5uDfmVl2In84kTT2ANpeSPOscc5VxTzfArQFMLMDzWyKma0ys/X4ne/OLXgPbWsk5YVaBDjnpgKlsdPMbC8ze9HMZgT96AY18tKv4at3kYaW479Ma+0RTMM5t9E59z3nXD/gVOC7tf0xnXOPOucODV7rgN8Gr18CnOic6xBzy3POLQuO8PzCObcPcDAwFt/MLCJxpu2FxMmjwLNAb+dce+BuwFqwHm1rJOWF3RLQmAnANc65EcD3gTtjZ5rZnkBf4LUQskn0PQb81My6mFln4CbgYQAzG2tme5uZAevxR1lqzI85fVRwUlc5vq9lTbC+u4Fbg787gvWOCx4faWb7mVkmsAHfZFuDiCSLtheyqwqBUudcuZkdAHy9hevRtkZSXqSuE2BmbfFV7j/9/w7g+8/FOg+Y5JyrTmY2SRm3AO2A2cHzfwbTAPoDt+NP1loL3Omcm2Jm+wO/AQbjv1zfBsYHr7kNf5To5aDZdiXwOPAM/uSzu4Fe+D6nj+ObbUUkwbS9kBa6EviDmd0OvAE8gT9JeFdpWyMpz/y5KiEGMOsDPOecG2Jm7YAFzrkeO1j+Q+Aq59zbycooIiLh0/ZCRCR+ItUdKDjb/XMzOxvAvKG184P+nh2BaSFFFBGRCND2QkRk94Q9ROhj+C/ogeYv7nEpftitS81sFv6KgONiXnIeMNGF3XwhIiJJpe2FiEh8hd4dSEREREREkitS3YFERERERCTxVASIiIiIiKSZ0IYI7dSpk+vbt29Yb7/LqquryczMDDtGsyhr4qRS3lTKCqmVN1lZZ8yYsdo51yXhbxRxUdxeRO3vNWp5IHqZopYHlKk5opYHopmpJduL0IqA3r17M3369LDefpeVlpZSVFQUdoxmUdbESaW8qZQVUitvsrKa2RcJf5MUEMXtRdT+XqOWB6KXKWp5QJmaI2p5IJqZWrK9UHcgEREREZE0oyJARERERCTNqAgQEREREUkzoZ0TICKyI5WVlSxdupTy8vKwo2ynpqaGFStWxG19eXl59OrVi+zs7LitU0REZEdUBIhIJC1dupTCwkL69OmDmYUdp56qqiqysuLz9emcY82aNSxdupSojYAjIiKtV1y7A5lZBzObZGbzzWyemR0Uz/WLSPooLy+nU6dOkSsA4s3M6NSpUyRbPEREpPWKd0vAbcCLzrmzzCwHKGhqQaveGue3FpHWprUXALXS5ecUkRg11VjFBqhuB5nqmCHJF7e/OjNrDxwOXATgnNsKNLmnn7n2U1jzKXTaK14RRETiZt26dTz66KNceeWVu/S6k046iUcffZQOHTokJpiIpDbnYMFkePmndCz9zE/LzIXctpDTBnIKYx639bcmnxf6+4bPs/NBBxdkJ+JZevYFVgF/N7OhwAzgO865zU2+4pGz4NJXoE3nOMYQEdl969at484779yuCKiqqtrh6yZPnpzIWJFmZnnAVCAXv32Z5Jz7WYNlLgL+D1gWTLrdOXdvMnOKhOarOfDSj+HzqdB5IFsO+REF2ZmwdVNw2wwVwePy9bBhed3zrZugZsffP9tYZlAwtNm1giK3EGs3CIjWhbAkMeJZBGQBw4FrnHPvmtltwA3A/6tdwMzGA+MB+hZ3xa1fRvVDZ7DhjEchu8meQ5FQVlZGaWlp2DGaRVkTJ5XyplJW2D5vTU3NTne4E+lHP/oRn376KUOHDiU7O5u8vDw6duzIggULmDNnDuPGjWPJkiVUVFRw9dVXc9lllwGw9957884777Bp0yZOOeUUDj74YN555x169uzJk08+SX5+fqPvV1NTk1K/ryZUAEc55zaZWTbwlpm94Jx7p8Fyjzvnrm7uSq2mEmpqIEOjWkuK2rQKptwKHzwIee3hxP+DkRdTvn4jBc298qxzUL21flGwdTNUbPT3jT7fFCwfPN+wtP7zyi3bvU2HzFzY93QYfiHsebBaFFqxeBYBS4Glzrl3g+eT8EXANs65CcAEgJKSEmdn30rW4xdQ9J/vwbmPRLpPXBQvEd0UZU2cVMqbSllh+7wrVqzYNgLPL/49l4+Xb4jr++3Tsx0/O2XfJuf/9re/Ze7cucyaNYvXX3+dk08+mTlz5tC3b1+qqqr4+9//TlFREWVlZYwaNYpzzjmHTp06AZCVlUVWVhYLFy7kscce47777uOcc87hmWee4YILLmj0/TIyMlLq99UY55wDNgVPs4Ob2931Zq6eD7/uBV0GQJdB0Dm47zIQOvaBjMzdfQuRxKjaCu/9Dd74nd/xPmA8HPEjKGjB/7oZZOX6W5tO8clXUx0UBEFRsGklFTMeJW/BszB7InTa2xcDQ78ObbvE5z1Tndvtr7TIiNtet3PuKzNbYmYDnXMLgKOBj3f4okEnw0m/h+e/62+n3KaKU0Qi6YADDqg3hOdf/vIXnnrqKQCWLFnCwoULtxUBtfr27UtJSQkAI0aMYPHixcmKGxozy8R3B90buCPmwFCsM83scOAT4Hrn3JJG1rOt5bh/z46U73sOmWsWkrloChmzHtu2nMvMobrjXlQX7R3c+lNd1J+a9ntCZuKuuxC1lrbI5KmuJHPNJ2SumktVu70oZUTYibZJ6mfkHNmfv0rBm78ic/1itu45hi2H/YSaor2hHCgvTX6mHcqFjFxo14my0TeSf+iN5CyaTO7ciWS/chPuPzdT2fcYKoacR+Ueh4Elr1UuEp9R9Vayl/yXnIWTyf7sZQrzO7PhyF9S1fvgcHPtpngfer8GeCQYGegz4OKdvmLUpbBhGbz5B2jfC474YZwjiUiq29ER+2Rp06bNtsdvvPEGr776KtOmTaOgoIAxY8Y0OsRnbm7utseZmZmUlZUlJWuYnHPVQImZdQCeMrMhzrk5MYv8G3jMOVdhZt8GHgSOamQ99VqO8077c93M8vWweiGsmo+tmk/WqgVkrZwNn/y7bpmMLCjay7cW1LYadBkInfpDdt5u/5xRa2kLJY9zsO5/sGyGvy2dDl/Ogir/d94WYM9D4aCrYMAJoXfnStpntGIuvHQjfPa6b7U6fxI5/Y8lJ8xMu2Bbpm6XwSGXwaoF2AcPkTPrMXI+fRHa7wHDLvC39sXJy5NsVVvhsykw92lY8Lz/3sltBwOOx33xLu2eOh+GnAXH3wqF3ZOfLw7iWgQ452YCI3f5hUf9P3/yy5RboV1P/4clIhKiwsJCNm7c2Oi89evX07FjRwoKCpg/fz7vvNOwy7s459aZ2RTgBGBOzPQ1MYvdC/xul1ee1x56jfS3WFs3B8XBAlg1H1Z/Ais/hvnPgavxy1iG70LUsFtR5wH+BElpWtk6WP5BsMM/A5ZNh82r/LzMXOgxFEZeDMUjoNsQtsx+loKPHoKJX/MF2egroOTr/iTU1mjzar8fM+MBv7N44u9g5CUJbZFKii4D/Y7u0TfB/Ofhg4fg9V/BG7+BvY+B4d+EAcen/s8JUFUBn06Bj5+G+ZOhYj3ktodBJ8E+p8FeR0JWLutXLqdo7oPw1p/gk5fgqJ/AqMsi3a29MdFIawan/AU2fgXPXgttu0P/Y8JOJSJprFOnThxyyCEMGTKE/Px8unXrtm3e8ccfzz333MPgwYMZOHAgo0ePDjFpdJhZF6AyKADygWOB3zZYpodz7svg6anAvLgFyGkDPUv8LVZVBaxZ5AuDVZ8E9wtg4StQU1m3XPve9VsOOgetB/kd4hYxZVRXwoo59Xf4V39SN79Tf78DWDzCF2Nd94Ws+se6y4d3peDI78K8Z2Ha7TD5+/DaLX7H+IDx0K5Hkn+oBKnaCu9NCPr9b/I7g2NuaFm//yjLyoUhZ/jb2sXw4cP+9vj50LabL/CGXwhF/cJOumuqKuDT14Ij/i/E7PifDPueBv3G+J89VlYeHPlj2P8cmPwDePEG+PAROPkPsMeBIfwQLWMupBMcSkpK3MyZM+tPrNgIfz8R1nwGFz8PPYeFkq0xUWyya4qyJk4q5U2lrLB93nnz5jF48OAQEzWtqqpq20nL8dLYz2tmM5xzu966GhIz2x/fvScTf0X6J5xzN5vZzcB059yzZvZr/M5/FVAKXOGcm7+j9Ta6vYiH6kq/M7Nqfl1hsGq+b02oiune1bZ7XXeioNVgnWtLh14DI3NUe7f+352DdV/47jzLPvA7/F/OqvsMCjr7Hf3ikdBrBPQc3qzCqF4m52DJezDtrzDvOd9da8iZvqtQj/1blnsXxf070Tn45EV46SdQ+qkvio7/lf8bCStTHOxSpuoqWPSKbx345CVw1dDnMBhxEQwaG92ud5Xlfsf/46eDHf8NvoVx0Fh/xL/fmO2K2iYzOecL3RdugI3LYdg34JhfxO/k7WZqyfYiGi0BtXIL4fxJcO+x8Mg58K1XfLOtiIhEnnNuNrDd0Rvn3E0xj38M/DiZuZqUmQ2d+/vb4FPqptdU+77usd2KVs2HmY/6I71Ah9plcwp9f+DC7v5o6LbH3etPzy2MzsAXZevq+vHX9uXfstrPy8qDHiUw8lK/w188EjrssfvZzfwR0j0OhNLP4N2/wQf/8CPQ9D0cDroa9j429PMGmm3Fx0G//ynb+v3T/9iwUyVfZhYMPNHfNnwJMx/xBcG/LoX8jrD/eTDim9A1Agd0anf85z7ld/y3boS8DjD4VH/Ev+8RO9zxb5IZ7DMO9joa3vgtvHOn74J49M98V6kI/01HqwgA/4V5wSS47zh4+Ey45OWkV1MiIpLGMjKhqK+/DTyhbrpzfiCL1QvZ9NUi2tZshE0rfFfWjV/5I+gbV2w7Mbae7DZQ2G374qCwR/3pee3jWyxUbY3p1jPd369ZWDe/80Dof1zdDn+3fRPft7uoH5z4WxjzYz9u/rt/g0fP8V2MDrrS7zjmRPTaQZvX+P7w0+/3hd0Jv/UDnLSG/vC7q10POPz7cOh3YfFUmPEgTL8P3r0Leo3yO8T7np7cc28qy+HT/9R19and8d93HOxzui9AW7Lj35jctnDcL323qOe/B89d57tLnfyH7bsoRkT0igDwTWlffxwePBUeOw8ufCa6XwgiIpIezPwodu17sbXjUGisi4JzfhSR2OJg01e+ONj4pZ/+5SzfdaJy8/avz8prpEUhKBa2Te/hj7I2LBac892btu3wT4cvZ0N1hZ/fpqvv1jP0XL/DXzzcFx1hye8Ah3wHRl8JHz8Db/8Vnrse/vNLv2M96jL/s0dB1VZ4/15/MmzFJhj1LV/EtLZ+//GQkeG70/Qb44um2RN9QfDs1fDij2G/M/25Az2HJ6Z1rLIMFv0n6Orzot/xz+/oj/bXHvFPZNHWdTBc9DzMfhxe/incc6T/eznyJ5E7vyiaRQDAHqPhzHvhiQvhycvgnId0QRgREYk2M7+hz++w877hFRt9cbDpqwYFQ3BbOc+PVFLRyIXyMnPqioK23WhbvhlWzoYtweBLWfn+6OMBl9WdvNu+d3S6JMXKzIb9zvLnCPxvGky7A6b+Hv57G+x3ti8Sug8JJ5tzsPBl3/VnzSLY6yjf7z8K3VtSQZtO/ryP0Vf6c0I+eBBmPe5HUOq2n+8qtN/Zu79zXFkGi171R/w/edF328svgiGn+z7+fQ9PbmuNGQw9zw+N+9otvoCc+7QfZWm/syPzfxjdIgBgn1P9EFsv/ABe+KG/sFhEPjgREZHdklvob5333vFyW7c0KBQatDKs+ZSMmhoYcGLQrWcEdN0n9bqomMGeB/vbmk/h3bt9d4qZj/ijygddA3sfnbz9gJXz/M7/p6/5K+d+/QnfdUr7Ibsu9pyQE34NH03yBcHk7/uj5fuc5lsH9jy4+Z9vZZkf4evjp33L2rYd/zP9Ef8+h4X/P5DfAU7+PQw733cRevIyf87ESb+HroPCzUbUiwCAA8fDhqX+iEC7Yjjsu2EnEhERSZ6cAt+PfgdDL26I4Cgzu6XTXnDS//kuNzMe8ENwPnKmH7519JV+aMbs/MS895ZSmFLb778tnPAb350j7B3K1iKvfdDd61JYPtPvFH/0T99tqNPevhgY+nVo22X7127d4kcjmvt0XZe6gk6+JWmf04Id/wju2vYcBpe+6gufV38Odx/iW0gO/2Go1yeJ7inLsY7+Oex3DvznFzBrYthpRES207at/yJfvnw5Z511VqPLjBkzhunTpyczlkhqKyjyB/++MxtOn+C7Qf37WvjTEJjya9i0Kn7vVV0J79wFfynxBcDIS+CaD/1FzlQAJEbPEhj7R/jefDjtLj8c7Ss3wR8HwePf8F18tm72O/3/vAj+by/fTfzzqb4QvPAZ+N4ncMpt/kJeUSwAamVk+IvpXTPDn/z+39vgjgPh42d9t7MQRPjTipGRAePu8M2ez1wFbbv6fnkiIhHTs2dPJk2aFHYMkdYlK8ef0Lz/ObD4LX/ewBu/8Vds3f8cf1R1d/rpf1Lb738h9DvSd1lRv//kyWnjR9Up+bofmveDh/yQvPOepSMGOGjTxfez3+c02POQaO/w70ibznDaHTD8G76L0BPf8MPjnvS7pF9oLXU+wawcOPdh+PtJ8PiFcPHkpF1gRETSzw033EDv3r256qqrAPj5z39OVlYWU6ZMobS0lKqqKm655RbGjRtX73WLFy9m7NixzJkzh7KyMi6++GJmzZrFoEGDKCtrZOhIEWk+M+h7mL+tXuiP3M98FD78hx+n/aCr/EHC5vYrXzkfXv6JP+LcaW/42uMw4Hj1+w9Tl4H+BNqjb4L5z1P++bvkDznZ7/i3pgFi9hgN49/wXd2m3Ap3jPatXodcF5eLrDVH6hQB4PuRnf/P4GJiZ8G3XvUXMRGR1u2FG+Crj+K7zu77wYm/aXL2ueeey3XXXbetCHjiiSd46aWXuPbaaykoKGDdunWMHj2aU089FWtih+Guu+6ioKCAefPmMXv2bIYPHx7fn0EknXXu77uSHPVT333nvQnw8Bn+pOjRV/pRWJramdpSCq//xo/aktPWj/gz6rL4jRkvuy8rF4acQVnPMeS3pvNdYmVm+Wtj7Hu6L0Zf/7Xv9n7S76H/MQl/+9Q4JyBWu57+YmJV5f5iYmXrwk4kIq3QsGHDWLlyJcuXL2fWrFl07NiR7t27c+ONNzJs2DCOOeYYli1bxooVK5pcx9SpU7ngggsA2H///dl/f7VeisRdQZG/SNV1H8Fpd4Nl+DHp/zwE3vgdbF5dt2x1pb842V+Gwfv3+D7a137oWxBUAEhY2vWAs+735zhkZPmT4B//BqxfmtC3Ta2WgFpdB8N5j8GDp/ihpcbdHnYiEUmkHRyxT6Szzz6bSZMm8dVXX3HuuefyyCOPsGrVKt577z3y8/Pp06cP5eXloWQTkQaycqHka77f+OdTYdrtvpvFm3+AoeeR07kEZtwFqxf4IUeP/zV02yfs1CJ1+o2BK/7rL5w39ff+omdjfuRbthJwcnrqtQTU6hMMr/ThP2Dxf8NOIyKt0LnnnsvEiROZNGkSZ599NuvXr6dr165kZ2czZcoUvvjiix2+/vDDD+fRRx8FYM6cOcyePTsZsUXSmxn0O8J3H77qPV8UzJpI25eug5oq+NpE+MbTKgAkmrJyfcvWVe/6v+NXboK7D/UnxMdZ6hYBAGNu8OcE/Ps7UFURdhoRaWX23XdfNm7cSHFxMT169OD8889n+vTplJSU8NBDDzFo0I4v9nLFFVewadMmBg8ezE033cSIESOSlFxEAH+S6Sm3wfVz2XjqA3DlOzDwRJ34K9HXcU/42mO+aK3cAg+cDE9+GzatjNtbpGZ3oFo5beDkP/m+U2/9yRcFIiJx9NFHdSckd+7cmWnTplFVVUVWVv2vz02bNgHQp08f5syZA0B+fj4TJ+raJiKha9OZyj5HqN+/pJ6BJ0LfI3y3tv/eBgte8CfDj7p0t0dLSu2WAPBnTw850384qz4JO42IiIiISPzkFMDR/w+unAbFw+CFH8A9R8LS3bv4ZOoXAeAv6Z2dD89dH9pV10REREREEqZzf38+y1l/992C7j3Gd4nfUtqi1bWOIqBtVzj2ZvjiLfjw4bDTiIiIiIjEnxkMOQOuft8PkPPBP+D2kS1aVesoAgCGXQh7HOSHDN20Kuw0IhIHLk1a9tLl5xQRkTjJLfRXVv72VH+BvBZoPUVARoYfAWDrZnjpxrDTiMhuysvLY82aNa1+B9k5x5o1a8jLS85l4kVEpBXpPgQueq5FL03t0YEa6jIQDr0epv7OXzBkr6PCTiQiLdSrVy+WLl3KqlXRa9mrqakhIyN+x1Dy8vLo1atX3NYnIiKyM62rCAA47Hsw90l/kvAV0/wZ1SKScrKzs+nbt2/YMRpVWlpKUVFR2DFERERarPV0B6qVnQdj/wRrF/sWARERERERqaf1FQEAfQ+HkvPh7b/CirlhpxERSQtmlmdm75nZLDOba2a/aGSZXDN73MwWmdm7ZtYnhKgiImkvrkWAmS02s4/MbKaZ7d4VDHbXcbdAXns/fmpNTahRRETSRAVwlHNuKFACnGBmoxsscymw1jm3N/An4LfJjSgiIpCYloAjnXMlzrmWDVoaLwVFcPyvYOn7MP2+UKOIiKQD520KnmYHt4bDO40DHgweTwKONjNLUkQREQm0vhODY+1/Lsx8FP5zMwwaC+16hJ1IRKRVM7NMYAawN3CHc+7dBosUA0sAnHNVZrYe6ASsbrCe8cB4gOLiYkpLW3ZFzEQpKyuLVKao5YHoZYpaHlCm5ohaHohmppaIdxHggJfNzAF/c85NiPP6d42ZP0n4roPhhR/Cuf8INY6ISGvnnKsGSsysA/CUmQ1xzs1pwXomABMASkpKXNRGY4raCFFRywPRyxS1PKBMzRG1PBDNTC0R7yLgUOfcMjPrCrxiZvOdc1NrZ4ZyZMc6kjfqagqm/Z6N05+gst8xLVpNKlV9ypo4qZQ3lbJCauVNpaxhcc6tM7MpwAlAbBGwDOgNLDWzLKA9sCaEiCIiaS2uRYBzbllwv9LMngIOAKbGzA/nyM7RP4JFz1M49Rew30mQ23aXV5FKVZ+yJk4q5U2lrJBaeVMpazKZWRegMigA8oFj2f7E32eBbwLTgLOA11xrvyy0iEgExe3EYDNrY2aFtY+B46h/9Cc8WTlwym2wYSlMuTXsNCIirVUPYIqZzQbeB15xzj1nZjeb2anBMvcBncxsEfBd4IaQsoqIpLV4tgR0w/f/rF3vo865F+O4/t2zx4Ew8hJ4927Y/xzoOSzsRCIirYpzbjaw3Zerc+6mmMflwNnJzCUiItuLW0uAc+4z59zQ4Lavcy56h9yP/hm06QLPXgvVVWGnEREREREJReu8YnBT8jvAib+Fr2b7FgERERERkTSUXkUAwD6nQf/j/bkB6/4XdhoRERERkaRLvyLADE7+vX/8/PdBg1KIiIiISJpJvyIAoMMecORPYOFL8PHTYacREREREUmq9CwCAA68HHoMhRd+BGXrwk4jIiIiIpI06VsEZGb5awdsXgWTf6BuQSIiIiKSNtK3CAB/rYAxP4aPnoDXbgk7jYiIiIhIUsTzYmGp6fAfwPql8ObvoV0PGPWtsBOJiIiIiCSUigAzOPmPsGmlHy2obTcYfErYqUREREREEia9uwPVysyCs+6H4hEw6VL4YlrYiUREREREEkZFQK2cAvj6E9ChNzx2HqycH3YiEREREZGEUBEQq00nuOBfkJULD58JG5aHnUhEREREJO5UBDTUsQ+c/08oXw8Pn+XvRURERERaERUBjekxFM79B6xeABPPh6qKsBOJiIiIiMSNioCm7HUknHYXLH4Tnvo2uJqwE4mIiIiIxIWGCN2R/c+BjV/CKzdRkN0Rxv3RDykqIiIiIpLCVATszMHXwoYvyXv3LujaDw6+JuxEIiIiIiK7RUXAzpjB8b+iovR/5L78U2jbHfY/O+xUIiIiIiItpiKgOTIy2HzsH8jduh6evgLadoF+Y8JOJSIiIiLSIjoxuLmycuG8R6DzAJh4AXw5O+xEIiIiIiItoiJgV+R3gAsmQV57eOQsWPtF2IlERERERHaZioBd1a6nv6pwVYW/qvCW0rATiYhEgpn1NrMpZvaxmc01s+80sswYM1tvZjOD201hZBURSXcqAlqi6yD42kRY9z949BzYuiXsRCIiUVAFfM85tw8wGrjKzPZpZLk3nXMlwe3m5EYUERFQEdByex4EZ94LS6fDvy6F6qqwE4mIhMo596Vz7oPg8UZgHlAcbioREWmMRgfaHfucCif9H0z+Pjz5LdjnNOjcH4r6QXZ+2OlEREJjZn2AYcC7jcw+yMxmAcuB7zvn5jby+vHAeIDi4mJKS6PV9bKsrCxSmaKWB6KXKWp5QJmaI2p5IJqZWkJFwO464DLYsgZe/w3MfSqYaNC+N3TeGzr1h0571z1uVwwZaoARkdbLzNoC/wKuc85taDD7A2BP59wmMzsJeBro33AdzrkJwASAkpISV1RUlNjQu6i0tJQoZYpaHohepqjlAWVqjqjlgWhmagkVAfEw5gY46Goo/RRWL4Q1i/xt9UJY8ghs3VS3bFY+dNrLFwad9vYtB7WP8zs0/R7OQXUl1FRBTSXUVPvH26ZVbXuesaUCCjIht52/2JmISJKYWTa+AHjEOfdkw/mxRYFzbrKZ3WlmnZ1zq5OZU0Qk3cW1CDCzTGA6sMw5Nzae64683LbQY6i/xXIONq3Yvjj46iOY929w1XXL5hdBZk6wk1/lzzOo3el3Nc2O0qH2QWYOFHSGNp2hTZf69wW1j2und4acNrv7KYhIGjMzA+4D5jnn/tjEMt2BFc45Z2YH4M9NW5PEmCIiQvxbAr6DPxGsXZzXm7rMoLC7v/U9rP68qq2wdnFQHCyE0s/9zn5mNmRk1b9lZkNGJmRkN/48Iwsy/f2mtatoa+WweRVsXg1bVvvHaxbC5jVQubnxrNkFdUVDp73h6P8HHfZI+EckIq3GIcA3gI/MbGYw7UZgDwDn3N3AWcAVZlYFlAHnOedcCFlFRNJa3IoAM+sFnAzcCnw3Xutt1bJyoMsAf4ujraWlsKO+als3++IgtkCoLRg2B88XTIZPXoST/wD7na1uRSKyU865t4Adflk4524Hbk9OIhERaUo8WwL+DPwQKGxqgaiP9rAjqXQmePOyFkJBIRT0hc7bz81Y/z/avPw9sp+8jIqPnmHLkbfg8tqHlDU6UilvKmWF1MqbSllFREQaE5ciwMzGAiudczPMbExTy0V9tIcdSaUzweOStagILnsZ3voTua//mtyvPoDT74J+Y+KSsVYqfa6QWnlTKSukVt5UyioiItKYeI1VeQhwqpktBiYCR5nZw3Fat4QlIxMO/z5861V/0vBD4+DFG6GyPOxkIiIiIrIb4lIEOOd+7Jzr5ZzrA5wHvOacuyAe65YI6DkMvj0VRl0G79wB9xwJX80JO5WIiIiItJCuWiXNk1MAJ/8ezp/kL452z5Hw379ATfOHLhURERGRaIh7EeCcez3trhGQTvofC1dMg/7HwSv/Dx46FdYtCTuViIiIiOwCtQTIrmvTCc59GMbdAcs/hLsOgdn/DDuViIiIiDSTigBpGTMYdgFc/hZ0HQRPfgsmXQpla8NOJiIiIiI7oSJAdk9RX7hoMhz1U/j4ad8q8NkbYaeq4xxUbIKydWEnEREREYmMeF4sTNJVZhYc/gPY62h4crw/T+Cgq+HomyArN37vU1MD5et8a8OWNbClFMpKY+5rp62tP616K2Aw6GQ46CrY4yBdAVlERETSmooAiZ/i4X4o0Vf+H0y7HT59DfocBjVV4Kr9fU11cKuCmiraVpRDVkbMvODe1S3D1i1+Z758HbgmRiPKyIL8jlDQCfKLfAtF/nAoKPLTtpTCBw/B/Of8kKcHXQ37jIPM7KR+RCIiIiJRoCJA4iunAE7+A/Q/Hl74Icye6HfQa2+W6S9CFjzPcA6ycxuZXzstE7Lz63buC4rq7mMf57bb+dH9I37k80y7E/51KbxyExz4bRj+TcjvkJSPR0RERCQKVARIYgw4zt92YkNpKUVFRUkIhC9QRl4Cwy+CRa/41opXboLXfwvDvwEHXu5bEERERERaORUBkn4yMmDA8f725Wx45054/z54928weKzvKtT7QJ03ICIiEnGVlZUsXbqU8vLypL1nTU0NK1asSNr7xcrLy6NXr15kZ+9+d2YVAZLeeuwPp98NR/8M3r/HFwPz/g3FI2D0lTpvIFmcCzuBiIikoKVLl1JYWEifPn2wJB28q6qqIisr+bvQzjnWrFnD0qVL6dt393suqAgQAWjXw49mdNj3YNZjMecN/Cw4b+BCnTeQCNVV8Naf4M3f09E5yGsf3Nr58zy23bdv8Dxmel47yA3uVbCJiKSV8vLypBYAYTIzOnXqxKpVq+KyPhUBIrFy2sCob8GIS2DhSzDtDj/a0Ru/hWHfIKv4cCg4xO98yu5ZvQie+jYsmw6DxlKe35182wrlG6Big7/fsKzueeWWna+z1wFw6PUw4ATf7UtERFq9dCgAasXzZ1URINKYjAwYeKK/LZ8ZnDdwD+3evcvPL+oHPYbG3Er8KEWpYkspLHkPug6Gjnsm971rauD9e/1J2Vm5cNb9MORMykpLyd/RSeLVlVCxEcrX1xUJsfebV/vRnyZ+DbruA4dcB0PO9NexSJbKcpj3LHw6xV9Ru88hyXtvERGRXaAiQGRnepbAGRPg+F+zccFUCjd9Cl/OgmUzYO5Tdcu138OfY9CjxBcGPUugbdeQQjdQvgG+eBs+n+pvKz7y0zOy/YhJh/8A2nZJfI71S+GZq+Cz12HvY+HUv/quWM2RmV03NGxTjvgRzPmX72L01HiYcgscfK3fIc/Oj8uP0KiV8+GDB31XsrK1kJnjC5LDf+g/22QWIiIiklTr1q3j0Ucf5corr9yl15100kk8+uijdOjQITHBdkJbJpHmatOJyj5HQNHpddO2lMJXs31RsHymv5//XN38wh71Wwt6DIV2PRM/8tDWLWT970344EP4/E1Y/qG/AFtmLuxxIBz1U+g1yhcx798LMx+Bg6/xV1TOLYx/Hufgo3/C89+HmkoY+ycYcXH8P4fMLBh6Lux3tu/O9eYfYfL3fXeu0VfCqEvj15Wrsgw+fhZm/B3+N80XVINPgREX+QvSTf4BvPEbX3SdeQ+07xWf9xURkUhZt24dd95553ZFwM5OIJ48eXKio+2QigCR3VFQBP3G+Fut8g3w1Ufw5UxfFHw5Cxa+XHe149z20KG33yls3wvaB4877OHv23bf9f7sVRWwdDosftPvdC55j3Y1lf6Ca8Uj/QnPfQ/3O/7ZeXWv6zfGD4n62i/h9V/De/fAET/0O+hZObv32dTavAaevx4+fsYPvXraXdBpr/isuym13bkGnABf/NcXA//5hW8hGHWpLwha2kqzch4Fb/8N5j/lr2Jd1A+OvRlKzoc2neuWO+NvsNeR8Nx34a5D4LQ7YdDJcfnxREQkOm644QY+/fRTSkpKyM7OJi8vj44dOzJ//nw++eQTTjvtNJYsWUJ5eTnf+c53GD9+PAB9+vRh+vTpbNq0iRNPPJFDDz2Ut99+m+LiYp555hny8xPYgo2KAJH4y2vn+4LH9gffuhlWzPWtBas/8d1i1i/1R5DL19d/fUa2by1o3zumWIgpFNoV++4mX86Cz9/wO/3/eweqygDzrQ2jr2Bj5+EU7nss5Lbdcd7O/eGch2DpDHj1Z/5Kz9Pu8K0FQ87avRNsF7wIz17ju8gc83PfNScjs+Xr21Vm0OdQf1s+0xcBb/0Z3rnLdxE6+NrmnRNRWQZzn4YZD8CSd8jNyIZ9TvVH/fsc1nSLxtDzfOE16WKY+HUYdRkcd0v9QkxEROLiF/+ey8fLN8R1nfv0bMfPTtl3h8v85je/Yc6cOcycOZPXX3+dk08+mTlz5mwbxvP++++nqKiIsrIyRo0axZlnnkmnTp3qrWPhwoU89thj3HPPPZxzzjn861//4oILLojrz9KQigCRZMhpA70P8LeGakfBWbcE1tfegiLh8zdh4/K6VoRamTlQvdU/7roPjPimP9K/58GQ3xGAytLSnRcAsXqNgG/+Gz79D7z6c3jyMvjvX+CYn8Hex+xa152KjfDSjfDBQ9BtCHzjKeg+pPmvT4SeJXDOg35Uov/+GWY8CNP/Dvud5U8i7rbP9q9Z8XFdX//y9dBpbzjuFtbtcQIde/Vv3vt22gsufQX+c7O/SvX/pvmTobsMjOMPFw1m1ht4COgGOGCCc+62BssYcBtwErAFuMg590Gys4qIJMoBBxxQbxz/v/zlLzz1lD+HcMmSJSxcuHC7IqBv376UlJQAMGLECBYvXpzwnCoCRMKWF4x733Vw4/Orq3whUFsYrPufP7JePNwfhY7nycdmfoe/31Ew90nfTeiRs2DPQ+HYX0CvkTtfx+L/wtOX+6yHXg9jfuxHAYqKznvDuNt9rml3+KP7sx+HASfCYd/1RcvHTwdH/d/1Bdfg2qP+h4IZrrR0194zKxeOvxX6HuE/mwlj4EQ/7GwruzJ1FfA959wHZlYIzDCzV5xzH8cscyLQP7gdCNwV3IuI7JadHbFPljZt2mx7/Prrr/Pqq68ybdo0CgoKGDNmTKNXN87NrdtOZmZmUlZWlvCcKgJEoi4zy3cD6rBH8t4zI8MfIR98qj8S/sZv4d6j/YmvR90EXQZs/5rKcj8az9u3+y42F78Ae4xOXuZd1b4YTvgVHP59eG8CvHs33HcsZOX7rlWd+sNxt8LQr0GbTjtfX3MMOA4u/68fuejZa/xQoqf8udVcd8I59yXwZfB4o5nNA4qB2CJgHPCQc84B75hZBzPrEbxWRCTlFBYWsnHjxkbnrV+/no4dO1JQUMD8+fN55513kpyuaSoCRKRpWTlwwGV+R3jaHfD2X2D+874//Zgf+3MXwJ+f8OS3YdU8f1LxcbfsWlekMBUUwZgb/AnSHzwIqxf6AmjPQxJzlL5dD/jG075L0mu3+oulnXk/9B4V//cKkZn1AYYB7zaYVQwsiXm+NJhWrwgws/HAeIDi4mJKd7X1JcHKysoilSlqeSB6maKWB5SpOXaWp6amhqqqqiQm2v4927dvz8EHH8yQIUPIy8ujW7du2+Yfc8wx3HXXXQwePJgBAwZw4IEHUl1dvW1+VVVVvce169/Rz1VTUxOX35GKABHZudy2MOZHfmSdqb/3w4rOfgIO/DbkFPqWgoJOcP4k6H9s2GlbJretHyI1GTIy/YhNfQ6DSZfC30+AI3/iz01oBVc6NrO2wL+A65xzLTpLzzk3AZgAUFJS4op2dCG5EJSWlhKlTFHLA9HLFLU8oEzNsbM8K1as2OEwnInQ2NCfjz32WKPLZmVl8eKLLzY6L7bf/5w5c7Y9/uEPf7jD98/IyIjL7yj1tzYikjxtOsOJv4FrZsA+p/kTh6fc4rsJXTktdQuAsPQ+AC5/EwaN9UOYPnw6bFwRdqrdYmbZ+ALgEefck40ssgzoHfO8VzBNRESSSEWAiOy6jnv6cfCveBsu+Bec/fcdX8lXmpbfAc5+AE65Df73Ltx1MCx8NexULRKM/HMfMM8598cmFnsWuNC80cB6nQ8gIpJ86g4kIi3XbZ/Gh9aUXWPmRx/qPdpfU+CRM/05CqnnEOAbwEdmNjOYdiOwB4Bz7m5gMn540EX4IUIvTn5MERFRESAiEhVdB8Flr8FLP/HXFEgxzrm3gB2eTR2MCpSkky9ERKQp6g4kIhIl2fkw9o9wzj/CTiIiIq1Y3IoAM8szs/fMbJaZzTWzX8Rr3SIiaWefU8NOICIirVg8WwIqgKOcc0OBEuCE4KQvEREREREB2rb119FZvnw5Z511VqPLjBkzhunTpyc0R9zOCQj6eW4KnmYHNxev9YuIiIiItBY9e/Zk0qRJob1/XE8MNrNMYAawN3CHc+7dBvMjfQXIHYnaFfR2RFkTJ5XyplJWSK28qZRVREQS64YbbqB3795cdZUf8+DnP/85WVlZTJkyhbVr11JZWcktt9zCuHHj6r1u8eLFjB07ljlz5lBWVsbFF1/MrFmzGDRoEGVlZQnPHdciwDlXDZSYWQfgKTMb4pybEzM/0leA3JGoXUFvR5Q1cVIpbyplhdTKm0pZRUTSxgs3wFcfxXed3ffzF8ncgXPPPZfrrrtuWxHwxBNP8NJLL3HttdfSrl07Vq9ezejRozn11FPxl1PZ3l133UVBQQHz5s1j9uzZDB8+PL4/RyMSMkSoc26dmU0BTgDm7Gx5EREREZFUNGzYMFauXMny5ctZtWoVHTt2pHv37lx//fVMnTqVjIwMli1bxooVK+jevXuj65g6dSrXXnstAPvvvz/7779/wnPHrQgwsy5AZVAA5APHAr+N1/pFRERERJq0kyP2iXT22WczadIkvvrqK84991weeeQRVq1axYwZM8jOzqZPnz6Ul5eHlq8x8RwdqAcwxcxmA+8Drzjnnovj+kVEREREIufcc89l4sSJTJo0ibPPPpv169fTtWtXsrOzmTJlCl988cUOX3/44Yfz6KOPAjBnzhxmz56d8MzxHB1oNjAsXusTEREREUkF++67Lxs3bqS4uJgePXpw/vnnc8opp7DffvsxcuRIBg0atMPXX3HFFVx88cUMHjyYwYMHM2LEiIRnTsg5ASIiIiIi6eSjj+pOSu7cuTPTpk1rdLlNm/yI+n369GHOHH/qbH5+PhMnTkx8yBjx7A4kIiIiIiIpQEWAiIiIiEiaUREgIiIiIinLORd2hKSJ58+qIkBEREREUlJeXh5r1qxJi0LAOceaNWvIy8uLy/p0YrCIiIiIpKRevXqxdOlSVq1albT3rKmpISMjnOPoeXl59OrVKy7rUhEgIiIiIikpOzubvn37JvU9S0tLKSoqSup7JoK6A4mIiIiIpBkVASIiIiIiaUZFgIiIiIhImlERICIiIiKSZlQEiIiIiIikGRUBIiISF2Z2v5mtNLM5TcwfY2brzWxmcLsp2RlFRMTTEKEiIhIvDwC3Aw/tYJk3nXNjkxNHRESaopYAERGJC+fcVKA07BwiIrJzagkQEZFkOsjMZgHLge875+Y2tpCZjQfGAxQXF1NaGq3aoqysLFKZopYHopcpanlAmZojankgmplaQkWAiIgkywfAns65TWZ2EvA00L+xBZ1zE4AJACUlJS5qV+eM2hVDo5YHopcpanlAmZojankgmplaQt2BREQkKZxzG5xzm4LHk4FsM+scciwRkbSkIkBERJLCzLqbmQWPD8Bvg9aEm0pEJD2pO5CIiMSFmT0GjAE6m9lS4GdANoBz7m7gLOAKM6sCyoDznHMupLgiImlNRYCIiMSFc+5rO5l/O34IURERCZm6A4mIiIiIpBkVASIiIiIiaUZFgIiIiIhImlERICIiIiKSZlQEiIiIiIikmbgVAWbW28ymmNnHZjbXzL4Tr3WLiIiIiEj8xHOI0Crge865D8ysEJhhZq845z6O43uIiIiIiMhuiltLgHPuS+fcB8HjjcA8oDhe6xcRERERkfhIyDkBZtYHGAa8m4j1i4iIiIhIy8X9isFm1hb4F3Cdc25Dg3njgfEAxcXFlJaWxvvtE6asrCxl8ipr4qRS3lTKCqmVN5WyioiINCauRYCZZeMLgEecc082nO+cmwBMACgpKXFFRUXxfPuEKi0tJVXyKmvipFLeVMoKqZU3lbKKiIg0Jp6jAxlwHzDPOffHeK1XRERERETiK57nBBwCfAM4ysxmBreT4rh+ERERERGJg7h1B3LOvQVYvNYnIiIiIiKJoSsGi4iIiIikGRUBIiIiIiJpRkWAiIiIiEiaUREgIiIiIpJmVASIiIiIiKQZFQEiIiIiImlGRYCIiIiISJpRESAiIiIikmZUBIiISFyY2f1mttLM5jQx38zsL2a2yMxmm9nwZGcUERFPRYCIiMTLA8AJO5h/ItA/uI0H7kpCJhERaYSKABERiQvn3FSgdAeLjAMect47QAcz65GcdCIiEisr7AAiIpI2ioElMc+XBtO+bLigmY3HtxZQXFxMaemOaovkKysri1SmqOWB6GWKWh5QpuaIWh6IZqaWUBEgIiKR45ybAEwAKCkpcUVFRSEnqq+0tJQoZYpaHohepqjlAWVqjqjlgWhmagl1BxIRkWRZBvSOed4rmCYiIkmmIkBERJLlWeDCYJSg0cB659x2XYFERCTx1B1IRETiwsweA8YAnc1sKfAzIBvAOXc3MBk4CVgEbAEuDiepiIioCBARkbhwzn1tJ/MdcFWS4oiIyA6oO5CIiIiISJpRESAiIiIikmZUBIiIiIiIpBkVASIiIiIiaUZFgIiIiIhImlERICIiIiKSZlQEiIiIiIikGRUBIiIiIiJpRkWAiIiIiEiaUREgIiIiIpJm4lYEmNn9ZrbSzObEa50iIiIiIhJ/8WwJeAA4IY7rExERERGRBIhbEeCcmwqUxmt9IiIiIiKSGFnJfDMzGw+MByguLqa0NHVqhrKyspTJq6yJk0p5UykrpFbeVMoqIiLSmKQWAc65CcAEgJKSEldUVJTMt98tpaWlpEpeZU2cVMqbSlkhtfKmUlYREZHGaHQgEREREZE0oyJARERERCTNxHOI0MeAacBAM1tqZpfGa90iIiIiIhI/cTsnwDn3tXitS0REREREEkfdgURERERE0oyKABERERGRNBNaEbBy01be+WwNldU1YUUQEREREUlLSb1OQKzSzZWcN+EdCnOzOLR/Z8YM7MKYgV3p1i4vrEgiIrKbzOwE4DYgE7jXOfebBvMvAv4PWBZMut05d29SQ4qISHhFwICubfjdBSN445OVTJm/ihfmfAXA4B7tfEEwoAvD9+xIdqZ6LImIpAIzywTuAI4FlgLvm9mzzrmPGyz6uHPu6qQHFBGRbUIrAjIMThjSnROGdMc5x4IVG3l9wSqmzF/JPVM/467XP6UwL4vD+ndmzICuHDGwi1oJRESi7QBgkXPuMwAzmwiMAxoWASIiErLQioBYZsag7u0Y1L0dlx+xFxvKK/nvwtW8vmAVr3+ykskf+VaCfYJWgiMHdWVY7w5kqZVARCRKioElMc+XAgc2styZZnY48AlwvXNuScMFzGw8MB6guLiY0tLSBMRtubKyskhliloeiF6mqOUBZWqOqOWBaGZqiUgUAQ21y8vmxP16cOJ+PXDOMe/Ljbz+yUpen7+Kv039jDtf/5R2eVkc1r8Lo/sVMapvEQO6FpKRYWFHFxGRHfs38JhzrsLMvg08CBzVcCHn3ARgAkBJSYkrKipKbsqdKC0tJUqZopYHopcpanlAmZojankgmplaIpJFQCwzY5+e7dinZzuuHLM368sq+e+i1UyZv5KpC1fx/EdfAtA+P5uRe3ZkVN8iRvUpYr/i9uRkqaVARCSJlgG9Y573ou4EYACcc2tint4L/C4JuUREpIHIFwENtc/P5qT9enBS0Erwv9ItvL94Le9/Xsr7i0v5z/yVAORlZ1DSuwMH9PEtBcP36Eib3JT7cUVEUsn7QH8z64vf+T8P+HrsAmbWwzn3ZfD0VGBeciOKiAikYBEQy8zYs1Mb9uzUhrNG9AJg1cYKpi8u5b3Fvii4fcoial6DzAxj357tGNWnKLh1pFPb3JB/AhGR1sM5V2VmVwMv4YcIvd85N9fMbgamO+eeBa41s1OBKqAUuCi0wCIiaSyli4DGdCnM3XY+AcDG8ko++N863v/cFwb/eOcL7nvrcwD26tKGA/oWsVeXthS1yaFjQQ4d2+TQsSCbDgU5tMvLwkznGYiINJdzbjIwucG0m2Ie/xj4cbJziYhIfa2uCGioMC+bIwZ04YgBXQCoqKrmo6XrfUvB56U8N/tLNpZXNfrarAyjQ0E2HQtyaJtjdGlXQFGbHDoU5FDUxhcKHYPH7fKyycnKIDvT33IyM8jOMrIzM8jKMBUTIiIiIhIZrb4IaCg3K5ORfYoY2acIxoBzjg3lVazdvJW1W4Lb5sptj0s3V7Juy1ZWrd/CF2u2MHPJOtZu2UpltWv2e5pRVxhkWl2hkFX3vLaAqLdM1o5fU2/5mNdv3ryJNm3KcDicA+egxjkcgGPb9JqYx36e89OcIyszg+IO+fQuyqe4QwH5OZmJ+YWIiIiISNKlXRHQkJnRPj+b9vnZ9KFNk8vFDgflnGPz1upthUPp5q1sLK+isrqGyuoatlbVsLXa+edVwbTa59uWcWytN9+/rqyymg3l/rFf1tVbb+3zqprmFyHx0LltDr06FtCrYz69i/x9r44F9O6YT88O+eRlq0gQERERSRVpXwS0hJnRNjeLtrlZ9C4qCCVDTY2jsiYoChoUEuvWradDh/aYGQZkmFHbG8nM588wMPx0C36m2McVVdUsX1fGktIylq7dwtK1ZSxZu4WPlq3npblfbdcS0rUwN6Y4yKd3xwJ6dSyga7tcOgVdqDJ1HQcRERGRSFARkKIyMozcjExys4AGgxyVZm6lqKjtbr9Hj/b5jNhz++nVNY4VG8pZutYXCLGFwowv1vLc7C+pbtBSkWFQ1CaHzm1z6dQ2h05t/H1BRjW9u2zaNr32viBHf5qSfM45qmqczuMRwP891Dj/nVdd46iqqaGmBqqdY+2WSlxOhV+u3msarIPtJjT2cKev3X5efRs2VmC5W8nNziA3K1MHXSQpqmsc5ZXVVFTVUF5ZHdxqKK+qpmLbvZ9WURXMa2SZ8srgcey6guUrtlaRnZ1JphmZGUZWptU9zsggIwOyMjKC50ZGcJ8Zc8uq9ziDDAvWkxG7rvqvrXtNxnbzyrZspl1hBRnBazNi1lM7LTODuvnW9PTaXNsem5GRwXavywgO4saT9rRkl2VmGD07+G5AB/Td/op5VdU1fBUUCas2VrBmUwVrNm9l9aatrNlUwepNFcxau441m7ayqaIKWLLdOvKzM+lc6IuFwrwscrP8+RA5wXkR/nEm2VlGbr1pGeRkZdZ7nhucL1GrqQ2r2zbNbTet9sm6DRvIL9hKVU2N3ymo9jsGVcFOQmW1ozp47ufVf167M9FYlqbyNJwOvkUnLyuT/JwM8rMzycvOJD8nk/xsf8vLyaSqbDNdN2dsmx67TGM7CFVB97Ot1XXd0WrvK6rqd0nbWl3N1irfNa3GueDclJhzW7Js2+8ndl5Og3NdmvpC85+lb92qCrrAba3yn2Ndt7oGj6v8xqRsazVlldVs2Vr3uGxr8Lyyatvj8mCZ2MdlldXbMmQYZAUn9mdlWL3HhiM3OyvYIAXTM+s2ENmZwUYmZsOxbQNi9TdO225mZAYbt6xgwyPeVxu3cuNTH1Fd7ah2df9HdTvn/v+s2uHvG8yvdjH/f428vnYdNbXrctv/r6aa7EwjNyuT3KwM8rL9fW7tfcy0unkZ5GVlbisi8hrcN7aebfNils3JzGj0/7q2wK79f62qrt+9NfZ/uf68mMfB91Pt927td0JV0OW2KnYdNTVUVvkW89rvkPKKCvLzcsnKqB3Ew//f1p5jl5VhMdP985ysDLIyMsjK9N9l/v/cf8fVnx67fMw6M43sDD9QiH/fHR9gqP3uq92+VAZ/z5UxP3ft9qT2s9s2PdjmVFXXUBncx86v3T75zzh4XFP3edcWuhs2l+EysrbtsDfcwa+oqvE79lXVu3R+ZENZGVb/7zDmbzAvK5PCvCzysjNx1ZVkZufU+/+v+3/1n8/WqqqY74Lt//dr3Pbzaov6qpoaUuVfPcNopGCwFhf9KgIk7rIyM4LzB3beVWr5ilW43Las3ljBms0VQaFQVyys3uTPt1hTVdfdaWsjj6Ms9qhCVmbGdtV8w3/d+tsHa3S6c46Kyhq2VFa3aEclJ9NvwB1s29EP40swO2YDWrvhqqyu2a7o2R152RkU5GT5Aqm2UMrJpENBDj07ZNabXpCTSVZmxrYNRFWNo7p2wxqzkd1SVk5GVva2jU7sxra6xrGpqqpuh7Jmxzuf1c5t27jF7ohKnQ1lVbw8dwWZwRG/2iN/GVZ3BLDhLSsjg7xs21aMNVV41T8amEFmBvXvGzlimGFQVlZGmzZ133H1/23r/1fv6H/cGsxtuH9oO5xXN2H9xk1k5+ZRXlVDRcxR14qqup242Pt1ZZVUBI/9Dl3d/e4UP2YEhUZmUJCxbQc00bIzG+6Y19/Bd64GrLLJAqT2/zvRag8aZGdk4HBU1wQ7/zXx/e5rjtjPrHYblZ0BBbnZ9QrBDvnZ9YrAvOz6hWRe7X3DnfrYZYLlcoP7rMzmHeiIPSczUVxMcRH7vVxVE1NA1H5PV9dQum4dhYXtqXF1y9fUxD5mu2n+PbafXrPd632X7xoXO50mlq2b9mELfm4VARKqvOxMijrkU9whv8XrcC726LWLKQ78Bq52ev0Nb8zjRna6m1p286aNFHXoUPeFmZFBZqaRHbPjkbltnp+W6G4lldX+hPLy2qPewZHvlWvWkZ1fQNnWmm3Ty2OOkpdXVmPGttaV7Ab3OQ1HroppiakdiSonywDbdlSu4UnvW+udLB87zdWbVlldQ1XlVgrbFNQb/aq5j7OC+9oWj4Icv2Ofl5VJRgK6RSRjo1RT48j8bULfImUM6FrA9J8eE3aMepLxN7Ar4pmnKmj9qy0YGisiYguHhtNrn2/dWkHbgvzgiLj/TtnREfZt/88Z2/9vN3W0vnZec7rwNeczcs7VtSTsoMWi9kh83bJ1R9i3LbNdy0dNvXVXVtdQXl5O24L8mJbG2s/CF6XbdtJrWxszM7Ztb2pbGrY9Du7989id+7oWidrtU+1yLf2cWhuz2s+3ecuXZldSVNQusaF20a0teI2KAEl5ZrXN3okfoai01EXuH79249guL7ve9NK2LqW+yNNxw7MjiSheRJojK9hpb5O7e7sIqfg/bWa+OyPJ6Y6Xip+RtB7qdCoiIiIikmZUBIiIiIiIpBkVASIiIiIiaUZFgIiIiIhImlERICIiIiKSZlQEiIiIiIikGRUBIiIiIiJpRkWAiIiIiEiaMZfsa1TXvrHZRmBBKG/eMp2B1WGHaCZlTZxUyptKWSG18iYr657OuS5JeJ9Ii+j2Imp/r1HLA9HLFLU8oEzNEbU8EM1MA51zhbvygjCvGLzAOTcyxPffJWY2PVXyKmvipFLeVMoKqZU3lbK2EpHbXkTtbyBqeSB6maKWB5SpOaKWB6KbaVdfo+5AIiIiIiJpRkWAiIiIiEiaCbMImBDie7dEKuVV1sRJpbyplBVSK28qZW0Novh5Ry1T1PJA9DJFLQ8oU3NELQ+0kkyhnRgsIiIiIiLhUHcgEREREZE0oyJARERERCTNJLwIMLMTzGyBmS0ysxsamZ9rZo8H8981sz6JztQYM+ttZlPM7GMzm2tm32lkmTFmtt7MZga3m8LIGpNnsZl9FGTZbmgo8/4SfLazzWx4SDkHxnxmM81sg5ld12CZUD9bM7vfzFaa2ZyYaUVm9oqZLQzuOzbx2m8Gyyw0s2+GlPX/zGx+8Ht+ysw6NPHaHf7NJDHvz81sWczv+6QmXrvD748kZX08JudiM5vZxGuT/tm2Rrvzv5jEPGcH24kaM0v6MIG78x2Q5Ey/DPLMNLOXzaxnmHli5n3PzJyZdU5WnqYyNfe7MFl5gunXBH9Lc83sd8nK01Sm5n4HJzlTiZm9U/t9b2YHhJxnqJlNC7ZB/zazds1amXMuYTcgE/gU6AfkALOAfRoscyVwd/D4PODxRGbaQdYewPDgcSHwSSNZxwDPhZGvicyLgc47mH8S8AJgwGjg3QhkzgS+wl8EKTKfLXA4MByYEzPtd8ANweMbgN828roi4LPgvmPwuGMIWY8DsoLHv20sa3P+ZpKY9+fA95vxt7LD749kZG0w/w/ATVH5bFvjraX/i0nOMxgYCLwOjIzIZ9Ss74AkZ2oX8/ja2m19WHmC6b2Bl4AvUuW7MMl5jgReBXKD513DztRgfpPfwUn+nF4GTgwenwS8HnKe94EjgseXAL9szroS3RJwALDIOfeZc24rMBEY12CZccCDweNJwNFmZgnOtR3n3JfOuQ+CxxuBeUBxsnPE2TjgIee9A3Qwsx4hZzoa+NQ590XIOepxzk0FShtMjv3bfBA4rZGXHg+84pwrdc6tBV4BTkhUTmg8q3PuZedcVfD0HaBXIjPsiiY+2+ZozvdHXO0oa/C9dA7wWCIzpLvd+F9MWh7n3DznXGhXMI7id0ATmTbEPG0DJG0kkh38L/8J+GEys9Taje/ChGgizxXAb5xzFcEyKyOQCQjvO7iJTA6oPdreHlgecp4BwNTg8SvAmc1ZV6KLgGJgSczzpWy/Y71tmeALbD3QKcG5dsh8l6RhwLuNzD7IzGaZ2Qtmtm9yk23HAS+b2QwzG9/I/OZ8/sl2Hk3/A0fpswXo5pz7Mnj8FdCtkWWi+Blfgm8BaszO/maS6eqgq8D9TXTviNpnexiwwjm3sIn5UfpsW5vm/C9KnR19BySVmd1qZkuA84Gwu9COA5Y552aFmaMRO/suTKYBwGHmu2e/YWajQs4Ta2ffwcl0HfB/wd/274EfhxuHudQdJDsb3+K1UzoxuAEzawv8C7iuwVEMgA/w3ViGAn8Fnk5yvIYOdc4NB04ErjKzw0POs0NmlgOcCvyzkdlR+2zrcb6NLfLj6ZrZT4Aq4JEmFonK38xdwF5ACfAlvok36r7Gjo9AReWzbdVS5X8xLM34Dkgq59xPnHO98XmuDiuHmRUANxJyIdKIqH0XZuG7t44GfgA8EUbvjCbs7Ds4ma4Arg/+tq8H7gs5zyXAlWY2A9+lfWtzXpToImAZ9auRXsG0Rpcxsyx8s8qaBOdqlJll4wuAR5xzTzac75zb4JzbFDyeDGQn+8SiBnmWBfcrgafw3SdiNefzT6YTgQ+ccysazojaZxtYUdt9KrhvrFk0Mp+xmV0EjAXOD3aUttOMv5mkcM6tcM5VO+dqgHuayBGlzzYLOAN4vKllovLZtlLN+V9Me835DgjRIzSzi0KC7AX0BWaZ2WL898kHZtY9xEzN/S5MpqXAk0E34veAGiDsbXGzvoOT7JtA7X7iPwn59+acm++cO845NwJfKH3anNclugh4H+hvZn2Do8DnAc82WOZZ/IcJcBbwWhhfXkGlex8wzzn3xyaW6V5bEQdngmcQXsHSxswKax/jTwprOArCs8CF5o0G1sc0qYehySo+Sp9tjNi/zW8CzzSyzEvAcWbWMWjGPS6YllRmdgK+n+upzrktTSzTnL+ZpGhwbsrpTeRozvdHshwDzHfOLW1sZpQ+21aqOf+Laa053wHJZmb9Y56OA+aHlcU595Fzrqtzro9zrg9+Z3e4c+6rsDJBs78Lk+lp/MnBmNkA/KAMq8MMFNjhd3AIlgNHBI+PAkLtomRmXYP7DOCnwN3NeuGunJHckhv+rOlP8FXJT4JpN+O/qADy8FXUIuA9oF+iMzWR81B8E/NsYGZwOwm4HLg8WOZqfL+rWfgTrw4OI2uQpV+QY1aQqfazjc1rwB3BZ/8RIYxiEZO3DX6nvn3MtMh8tvji5EugEr9xuBR/bsp/8P/crwJFwbIjgXtjXntJ8Pe7CLg4pKyL8P3na/92a0fc6glM3tHfTEh5/xH8Tc7G7+D1aJg3eL7d90eyswbTH6j9W41ZNvTPtjXeduV/McQ8pwePK4AVwEsR+Iwa/Q4IOdO/8Du1s4F/A8Vh5mkwfzHJHx2o2d+FIebJAR4Ofm8fAEeF/RkF07f7Dg75czoUmBF8578LjAg5z3eCbeUnwG8Aa866LFihiIiIiIikCZ0YLCIiIiKSZlQEiIiIiIikGRUBIiIiIiJpRkWAiIiIiEiaUREgIiIiIpJmVASIiIiIiKQZFQEiIiIiImlGRYCIiIiISJpRESAiIiIikmZUBIiIiIiIpBkVASIiIiIiaUZFgIiIiIhImlERICIiIiKSZlQEiIiIiIikGRUBIiIiIiJpRkWAtFpmttjMfmBms81ss5ndZ2bdzOwFM9toZq+aWUczyzOzh81sjZmtM7P3zaxbsI72weu+NLNlZnaLmWWG/bOJiEj8NHd7ESz7TzP7yszWm9lUM9s3Zj25ZvZ7M/ufma0ws7vNLD+8n0ykaSoCpLU7EzgWGACcArwA3Ah0wf/9Xwt8E2gP9AY6AZcDZcHrHwCqgL2BYcBxwLeSll5ERJKlOdsLgun9ga7AB8AjMev4TfD6Evx2oxi4KfHRRXZdqEWAmd1vZivNbE4zlv2Tmc0Mbp+Y2bokRJTU91fn3Arn3DLgTeBd59yHzrly4Cn8jn0lfud/b+dctXNuhnNuQ9AacBJwnXNus3NuJfAn4LyQfhYREUmc5mwvcM7d75zb6JyrAH4ODA1ajQ0YD1zvnCt1zm0EfoW2GRJRWSG//wPA7cBDO1vQOXd97WMzu4bgn1FkJ1bEPC5r5Hlb4B/4VoCJZtYBeBj4CbAnkA186b/bAV84L0lsZBERCcFOtxdBd9BbgbPxLQQ1wfzOQC5QAMyI2WYYoC6kEkmhtgQ456YCpbHTzGwvM3vRzGaY2ZtmNqiRl34NeCwpIaXVc85VOud+4ZzbBzgYGAtciN/ZrwA6O+c6BLd2zrl9d7Q+ERFptb4OjAOOwXcj7RNMN2A1vljYN2ab0d451zaUpCI7EcVzAiYA1zjnRgDfB+6MnWlmewJ9gddCyCatkJkdaWb7BUd4NuC7B9U4574EXgb+YGbtzCwjKFKPCDWwiIiEpRB/cGgN/qj/r2pnOOdqgHuAP5lZVwAzKzaz48MIKrIzkSoCzKwt/kjsP81sJvA3oEeDxc4DJjnnqpMcT1qv7sAkfAEwD3gD30UIfItADvAxsDZYruHfpIiIpIeHgC+AZfjtwjsN5v8IWAS8Y2YbgFeBgUlNKNJM5pwLN4BZH+A559wQM2sHLHDONbmTZWYfAlc5595OVkYRERERkdYkUi0BzrkNwOdmdjaAeUNr5wfnB3QEpoUUUUREREQk5YU9ROhj+B36gWa21MwuBc4HLjWzWcBc/Ak4tc4DJrqwmy9ERERERFJY6N2BREREREQkuSLVHUhERERERBIvtIuFderUyfXt2zest99l1dXVZGamxvU+lDVxUilvKmWF1MqbrKwzZsxY7ZzrkvA3irhU2l6k0t8xpFZeZU2cVMqbSlkh2tuL0IqA3r17M3369LDefpeVlpZSVFQUdoxmUdbESaW8qZQVUitvsrKa2RcJf5MUkErbi1T6O4bUyqusiZNKeVMpK0R7e6HuQCIiIiIiaUZFgIiIiIhImlERICIiIiKSZkI7J0BEZEcqKytZunQp5eXlYUfZTk1NDStWrIjb+vLy8ujVqxfZ2dlxW6eISDqI8rYCor29UBEgIpG0dOlSCgsL6dOnD2YWdpx6qqqqyMqKz9enc441a9awdOlSUmUEHBGRqIjytgKivb1QdyARiaTy8nI6deoUyS/1eDIzOnXqFNmjWCIiUZYu2wqI//ZCRYCIRFY6fKlD+vycIiKJkE7fofH8WZtVBJhZBzObZGbzzWyemR3UYL6Z2V/MbJGZzTaz4TtdZ+XmlmYWEZE0YtUVYUcQEWl1mtsScBvwonNuEDAUmNdg/olA/+A2HrhrZyvMXPcFlK1rflIRkQhr27YtAMuXL+ess85qdJkxY8akzEWvoiRjU/xOqhMRCVtUthc7LQLMrD1wOHAfgHNuq3NuXYPFxgEPOe8doIOZ9djhil01fPBgi0KLiERVz549mTRpUtgxWhWr2AAbvgw7hohIXIW9vWjO6cp9gVXA381sKDAD+I5zLrY/TzGwJOb50mBavW9tMxuPbylgv575VE+7i/UDzoPM6A+LV1ZWRmlpadgxmkVZEyeV8qZSVtg+b01NDVVVVaHlufHGG+nVqxdXXnklADfffDNZWVm8/vrrrF27lsrKSm6++WZOPfXUba+pqqpi8eLFnHbaacycOZOysjK+9a1vMXv2bAYOHMiWLVuoqqpq9OeqqalJqd9Xcjn44CEY86Owg4iIbOeGG26gd+/eXHXVVQD8/Oc/JysriylTplBaWkpVVRW33HIL48aNq/e6xYsXM3bsWObMmUNZWRkXX3wxs2bNYtCgQZSVlSU8d3OKgCxgOHCNc+5dM7sNuAH4f7v6Zs65CcAEgOH77OUyN31J0ZdTYf+zd3VVSVdaWkpRUVHYMZpFWRMnlfKmUlbYPu+KFSvqhlV74Qb46qP4vmH3/eDE3zQ5+2tf+xrXXXcd1157LQCTJk3ipZde4rrrrqOgoIB169YxevRoTj/99G0namVlZW3LnJWVxT333EObNm2YN28es2fPZvjw4fWWiZWRkZFSv69kcjmFMOMBOOx7kKmRrUWkCSFsKwDOPfdcrrvuum1FwBNPPMFLL73EtddeW297ceqppzZ5Yu9dd91FQUFBve1FojXnnIClwFLn3LvB80n4oiDWMqB3zPNewbQm1eQUQucBMO2v4Fxz84qIJMWwYcNYuXIly5cvZ9asWXTs2JHu3btz4403MmzYMI455hiWLVu2w4vATJ06lQsuuACA/fffn/333z9Z8VsVl18EG5fDJy+EHUVEZDupur3Y6SEV59xXZrbEzAY65xYARwMfN1jsWeBqM5sIHAisd87tvAPn6Cvhuevgi/9Cn0N3Pb2IpIedHIVJlLPPPptJkybx1Vdfce655/LII4+watUq3nvvPfLz8+nTp4/G90+Cmtx20C4P3r8PBp8SdhwRiaqQthWQmtuL5o4OdA3wiJnNBkqAX5nZ5WZ2eTB/MvAZsAi4B7iyWWsdeh4UdIK3b9+l0CIiyXDuuecyceJEJk2axNlnn8369evp2rUr2dnZTJkyhS+++GKHrz/88MN59NFHAZgzZw6zZ89ORuzWacRF8NkUWPNp2ElERLaTituLZnWudM7NBEY2mHx3zHwHXLXL756dD6O+BW/8FlYvhM79d3kVIiKJsu+++7Jx40aKi4vp0aMH559/PqeccgolJSWMGjWKQYMG7fD1V1xxBRdffDGDBw9m8ODBjBgxIknJW6HhF/ptxfT74fhbw04jIlJPKm4vwj/DatRl8Naf4Z07Yeyfwk4jIlLPRx/VnWTWuXNnpk2bRlVV1XYn927atAmAPn36MGfOHADy8/OZOHFi8sK2ZoXdYdDJMPMROOqn/iCSiEiEpNr2orndgRKnbRcYei7MfBQ2rwk7jYiIRNXIS6FsLcx9OuwkIiIpL/wiAGD0VVBV7pt5RUREGtP3cOjUH6bfF3YSEZGUF40ioOsg2PtYeG8CVEbrzGkRCY9Lk+GD0+Xn3G1mMPISWPo+fKmTrEXES6fv0Hj+rNEoAgAOugo2r4SP/hl2EhGJgLy8PNasWdPqv9ydc6xZs4a8vLywo6SGkq9BVr5aA0QESJ9tBcR/exH+icG1+o2BbkNg2h0w7AJ/xEdE0lavXr1YunQpq1atCjvKdmpqasjIiN8xlLy8PHr16hW39bVq+R1hyJkw+wk49mbIax92IhEJUZS3FRDt7UV0igAzOOhqePpy+PQ/sPcxYScSkRBlZ2fTt2/fsGM0qrS0lKKiorBjpK9Rl8DMh2HW43Dg+LDTiEiIorytgGhvL6LTHQj80Z223X1rgIiISGOKR0DPYb5LUBp0ARARSYRoFQFZOf6ozqevwYq5YacREZGoGnkprJoPX7wddhIRkZQUrSIAYMTFkF2g1gARkQgxsw5mNsnM5pvZPDM7yMx+bmbLzGxmcDspZvkfm9kiM1tgZsfHTD8hmLbIzG5ocaAhZ/rzAXSCsIhIi0SvCCgogpLz/UlfG78KO42IiHi3AS865wYBQ4F5wfQ/OedKgttkADPbBzgP2Bc4AbjTzDLNLBO4AzgR2Af4WrDsrsspgKFfh4+fhU0rd+sHExFJR9ErAgBGXwE1VfDePWEnERFJe2bWHjgcuA/AObfVObduBy8ZB0x0zlU45z4HFgEHBLdFzrnPnHNbgYnBsi0z8hKoqYQP/9HiVYiIpKvojA4Uq9NeMOhk38x72Pf8ER8REQlLX2AV8HczGwrMAL4TzLvazC4EpgPfc86tBYqBd2JevzSYBrCkwfQDG3tDMxsPjAcoLi6mtLR0+4UyO1PY6yAy3ruP9YMvhIzMFv548VNWVtZ41ohKpbzKmjiplDeVskK080azCAA/XOj852DWozDqW2GnERFJZ1nAcOAa59y7ZnYbcANwO/BLwAX3fwAuiccbOucmABMASkpKXJND7B30bfjnRRSVfgADjm98mSSK8nCAjUmlvMqaOKmUN5WyQrTzRrM7EMAeo6HncJh2J9TUhJ1GRCSdLQWWOufeDZ5PAoY751Y456qdczXAPfjuPgDLgN4xr+8VTGtqessNGgttu8H79+7WakRE0k10iwAzOPhqKP0UPnkx7DQiImnLOfcVsMTMBgaTjgY+NrMeMYudDswJHj8LnGdmuWbWF+gPvAe8D/Q3s75mloM/efjZ3QqXmQ3DL4SFr8Daxbu1KhGRdBLdIgBg8Dho3xum3R52EhGRdHcN8IiZzQZKgF8BvzOzj4JpRwLXAzjn5gJPAB8DLwJXBS0GVcDVwEv40YWeCJbdPSMu8geOZjyw26sSEUkX0T0nACAzCw68HF7+CSz7AIqHh51IRCQtOedmAiMbTP7GDpa/Fbi1kemTgclxDde+Fww4ET74B4z5MWTlxnX1IiKtUbRbAsA38+YU6uJhIiLStFGXwJbVMO/fYScREUkJ0S8C8trBiG/C3Kdg3ZKdLy8iIumn31HQsS+8rysIi4g0R/SLAPBdggDe+1u4OUREJJoyMmDkxfC/t2HFx2GnERGJvNQoAjr0hn1PgxkPQvmGsNOIiEgUlVwAmbkw/f6wk4iIRF5qFAEAB10FFRvgw4fDTiIiIlHUppM/YDRrIlRsCjuNiEikpU4RUDwC9jgY3rkLqqvCTiMiIlE08lLYuhE+eiLsJCIikZY6RQD41oD1/4N5u3dtGRERaaV6HwDdhsD794NzYacREYms1CoCBp4IRf38xcP05S4iIg2ZwchLYMVHsPT9sNOIiERWs4oAM1scXBVypplNb2T+GDNbH8yfaWY3xT8qkJEJo6+EZTNgybsJeQsREUlx+5/jry+j4UJFRJq0Ky0BRzrnSpxzDa8YWevNYH6Jc+7meIRrVMnXIa+Dbw0QERFpKLcQhp7rry+zpTTsNCIikZRa3YEActrAqEth3nNQ+lnYaUREJIpGXgrVFRpRTkSkCVnNXM4BL5uZA/7mnJvQyDIHmdksYDnwfefc3IYLmNl4YDxAcXExpaUtO0Jj/c+mw3//QsXrf2LLmF+0aB27qqysrMV5k01ZEyeV8qZSVkitvKmUNW112wf2OMhfM+Cgq/3FxEREZJvmFgGHOueWmVlX4BUzm++cmxoz/wNgT+fcJjM7CXga6N9wJUHxMAGgpKTEFRUVtSx1URHsdzZ5H08i74RfQEEL17MLSktLaXHeJFPWxEmlvKmUFVIrbyplTWsjL4UnvwWfTYG9jw47jYhIpDTr0IhzbllwvxJ4CjigwfwNzrlNwePJQLaZdY5z1vpGXw6VW2Dukwl9GxERSVH7nAoFnXUFYRGRRuy0CDCzNmZWWPsYOA6Y02CZ7mZmweMDgvWuiX/cGN33h6K9/LkBIiIiDWXlwrALYMFkWL8s7DQiIpHSnJaAbsBbQX//94DnnXMvmtnlZnZ5sMxZwJxgmb8A5zmX4IH8zWDwWFj8JpStTehbiYhIihp5sb+uzIwHwk4iIhIpOy0CnHOfOeeGBrd9nXO3BtPvds7dHTy+PZg31Dk32jn3dqKDAzDoFKipgk9eTsrbiYhIiunYB/Y+Bj54CGpqwk4jIhIZqT1cQvEIaNsd5v877CQiIhJV+50Fm77yVxEWEREg1YuAjAzfJWjhq7B1S9hpREQkivoe4e8/ez3UGCIiUZLaRQDAoLFQVQafvhZ2EhERiaJ2PaDLIBUBIiIxUr8I6HMo5HWA+RolSEREmtDvSPjibagsDzuJiEgkpH4RkJkNA06ABS9AdWXYaUREJIr6jYGqcljybthJREQiIfWLAIDBp0D5Olj8VthJREQkivocApapLkEiIoHWUQTsdRRk5atLkIiINC63EHqNUhEgIhJoHUVATgHsfTTMf17jQIuISOP2OhKWfwhbSsNOIiISutZRBIDvErTxS1j+QdhJREQkivqNAZy/0ryISJprPUXAgOMhIwvmPRt2EhERiaLiEZBTqC5BIiK0piIgvyP0OQzmPQfOhZ1GRESiJjPbDyutIkBEpBUVAeCvHlz6KayaH3YSERGJon5joPQzWPtF2ElERELVuoqAgSf7+3kaJUhERBrRb4y/V2uAiKS51lUEtOsBvQ7QeQEiItK4LgOhsIeKABFJe62rCADfJeir2WrqFRGR7Zn51oDP39CQ0iKS1lpfETBorL+f/3y4OUREJJr6jYEta2DFR2EnEREJTesrAjrtBV330dWDRUSkcX2P8PfqEiQiaaz1FQHgLxz2xduwaVXYSUREJGra9YAug1QEiEhaa51FwKCxgIMFk8NOIiIiUdTvSH+wqLI87CQiIqFonUVA9/2gwx7qEiQiIo3rNwaqymHJu2EnEREJRessAsxg0Cm+qbd8Q9hpREQkavocApapLkEikrZaZxEA/ryA6q2w8OWwk4iISNTkFkKvUSoCRCRttd4ioPcB0KaLugSJiEjj9joSln8IW0rDTiIiknSttwjIyISBJ8HCV3Til4jIbjKzDmY2yczmm9k8MzvIzIrM7BUzWxjcdwyWNTP7i5ktMrPZZjY8Zj3fDJZfaGbfDO8nwp8XgIPFb4YaQ0QkDK23CADfJWjrJn9lSBER2R23AS865wYBQ4F5wA3Af5xz/YH/BM8BTgT6B7fxwF0AZlYE/Aw4EDgA+Flt4RCK4hGQU6guQSKSllp3EdD3cMhtB/OeDTuJiEjKMrP2wOHAfQDOua3OuXXAOODBYLEHgdOCx+OAh5z3DtDBzHoAxwOvOOdKnXNrgVeAE5L2gzSUmQ19DlURICJpKas5C5nZYmAjUA1UOedGNphv+KNEJwFbgIuccx/EN2oLZOVC/+NgwQtQXQWZzfpxRUSkvr7AKuDvZjYUmAF8B+jmnPsyWOYroFvwuBhYEvP6pcG0pqZvx8zG41sRKC4uprQ0Mf32c7uPos0nL7Bu8Wxq2vXa7fWVlZUlLGsipFJeZU2cVMqbSlkh2nl3Za/4SOfc6ibmxTb9Hohv+j1wN7PFx+CxMGcSLHnHH/EREZFdlQUMB65xzr1rZrdR1/UHAOecMzMXrzd0zk0AJgCUlJS4oqKieK26viEnw9Sb6bDmQ+iz/26vrrS0lIRlTYBUyqusiZNKeVMpK0Q7b7y6AzXV9Bu+vY+FzFyYp1GCRERaaCmw1DlXe2WtSfiiYEXtd31wvzKYvwzoHfP6XsG0pqaHp8tAKOyhLkEiknaa2xLggJeDozx/C47QxGqqiffL2IWS1bzbUNs9DiXz42dYf8AP/IXEWiDKzTkNKWvipFLeVMoKqZU3lbLGg3PuKzNbYmYDnXMLgKOBj4PbN4HfBPfPBC95FrjazCbiW4XXO+e+NLOXgF/FnAx8HPDjZP4s2zHzowQtfBlqaiCjdZ8qJyJSq7lFwKHOuWVm1hV4xczmO+em7uqbJa15t6H9z4BnrqKo4n/Qc1iLVhHl5pyGlDVxUilvKmWF1MqbSlnj6BrgETPLAT4DLsa3Jj9hZpcCXwDnBMtOxp8jtgh/ntjFAM65UjP7JfB+sNzNzrnwq6l+Y2DWY7DiI+gxNOw0IiJJ0awiwDm3LLhfaWZP4Yd2iy0CotfEG2vAiWAZvktQC4sAEZF05pybCYxsZNbRjSzrgKuaWM/9wP1xDbe7+h7h7z97XUWAiKSNnbZ7mlkbMyusfYxvvp3TYLFngQuDC8SMJmj6jXvalmrTCfY8RFcPFhGR7bXrAV0G6bwAEUkrzen82A14y8xmAe8BzzvnXjSzy83s8mCZyfjm4UXAPcCVCUm7OwafAqvmw+qFYScREZGo6XckfPG2rjAvImljp0WAc+4z59zQ4Lavc+7WYPrdzrm7g8fOOXeVc24v59x+zrnpiQ6+ywad7O/n/TvcHCIiEj39xkBVOSx5d6eLioi0BukzDEL7Xv58AHUJEhGRhvocAhlZ6hIkImkjfYoAgEFjYdkM2LA87CQiIhIluYXQa5SKABFJG+lVBAw+1d/Pfz7cHCIiEj39xsDyD2FL+KOWiogkWnoVAV0GQOcBMO/ZsJOIiEjU9BsDOFj8ZthJREQSLr2KAPBdghb/V0d6RESkvuIRkFOoLkEikhbSrwgYPBZcNXzyYthJREQkSjKzoc+hKgJEJC2kXxHQczi0K/ZXDxYREYnVbwyUfgZrvwg7iYhIQqVfEWDmuwR9+h/YujnsNCIiEiX9xvh7tQaISCuXfkUA+C5BVeWw6NWwk4iISJR0GQiFPVQEiEirl55FwB4HQ36RugSJiEh9Zr414PM3oKYm7DQiIgmTnkVAZhYMPAk+eQmqtoadRkREoqTfGNiyBlZ8FHYSEZGESc8iAHyXoIr1sHhq2ElERCRK+h7h79UlSERasfQtAvodCdlt1CVIRETqa9cDugxWESAirVr6FgHZedD/GFgwWf0+RUSkvn5j4Iu3obI87CQiIgmRvkUAwOBTYdMKWPp+2ElERCRK+o3xo8gteTfsJCIiCZHeRUD/YyEjGz5+JuwkIiISJX0OgYwsdQkSkVYrvYuAvPYw8ASY+YguHCYiInVyC6HXKBUBItJqpXcRAHDQNVC+Dj58JOwkIiISJf3GwPIPYUtp2ElEROJORcAeB/qjPe/cATXVYacREZGo6DcGcLD4zbCTiIjEnYoAgIOvgbWLYb6GCxURkUDxCMgpVJcgEWmVVAQADBoLHfvA27eHnURERKIiMxv6HAqfTgk7iYhI3KkIAMjIhNFXwdL34H8aDk5ERAL9xsDaz31rsYhIK6IioNaw8yGvA0z7a9hJREQkKvqN8fefvRFqDBGReFMRUCunDYy6FOY9B6WfhZ1GRESioMtAKOyh8wJEpNVRERDrgPG+D+i0O8NOIiIiUWDmWwM+fwNqasJOIyISNyoCYhV2h/3O8RcP07jQIiICvgjYsgZWfBR2EhGRuGl2EWBmmWb2oZltN46mmV1kZqvMbGZw+1Z8YybRQVdB5RaYfl/YSUREJAr6HuHv1SVIRFqRXWkJ+A4wbwfzH3fOlQS3e3czV3i67QN7HQ3vToCqirDTiIhI2Nr1gC6DVQSISKvSrCLAzHoBJwOpu3O/Kw6+BjavhNlPhJ1ERESioN8Y+OJtqCwPO4mISFxkNXO5PwM/BAp3sMyZZnY48AlwvXNuScMFzGw8MB6guLiY0tKI9rvvsD/tOg+Ct25jw54ngRllZWXRzduAsiZOKuVNpayQWnlTKavESb8x8O5d8L9psNeRYacREdltOy0CzGwssNI5N8PMxjSx2L+Bx5xzFWb2beBB4KiGCznnJgATAEpKSlxRUVFLcyfeYdfDU9+mqPQD6H8spaWlRDpvDGVNnFTKm0pZIbXyplJWiZO+h0F2Acx7VkWAiLQKzekOdAhwqpktBiYCR5nZw7ELOOfWOOdqO9DfC4yIa8ow7HuGHxv6bV08TEQk7eW0gQEnwMfPQHVV2GlERHbbTosA59yPnXO9nHN9gPOA15xzF8QuY2Y9Yp6eyo5PIE4NWTlw4OV+bOgvZ4edRkREwjbkDD9U6Oe6erCIpL4WXyfAzG42s1ODp9ea2VwzmwVcC1wUj3ChG3ER5LSFabeHnURERMK297GQUwhznww7iYjIbtulIsA597pzbmzw+Cbn3LPB4x875/Z1zg11zh3pnJufiLBJl98Bhl8Ic/6Fbfwy7DQiIhKm7DwYPBbm/RuqtoadRkRkt+iKwTtz4OXgasib9UDYSUREJGz7ngHl6+HT18JOIiKyW1QE7EzHPWGf08id8xiUbwg7jYiIhKnfGMjvCHP+FXYSEZHdoiKgOQ6+moytG+HDf4SdREQkFGa22Mw+MrOZZjY9mPZzM1sWTJtpZifFLP9jM1tkZgvM7PiY6ScE0xaZ2Q1h/Cy7JSsHBp8CCyZDZVnYaUREWkxFQHMUj6Cy5wHwzl0aGk5E0tmRzrkS59zImGl/CqaVOOcmA5jZPvjR5PYFTgDuNLNMM8sE7gBOBPYBvhYsm1qGnAlbN8HCl8NOIiLSYioCmql8+Ldg/RL4+Omwo4iIRN04YKJzrsI59zmwCDgguC1yzn3mnNuKv/bMuBBztsyeh0KbLuoSJCIpTUVAM1X2PRo67e2HC3Uu7DgiIsnmgJfNbIaZjY+ZfrWZzTaz+82sYzCtGFgSs8zSYFpT01NLZhbscxp88jJUbAw7jYhIi2SFHSBlWAYcdBU8dz188Tb0OSTsRCIiyXSoc26ZmXUFXjGz+cBdwC/xBcIvgT8Al8TjzYJCYzxAcXExpaWl8Vht3GTteSzt3r+HTR9MYuvAusaMsrKyyGXdkVTKq6yJk0p5UykrRDuvioBdMfRr8Not8PZfVQSISFpxzi0L7lea2VPAAc65qbXzzewe4Lng6TKgd8zLewXT2MH0hu83AZgAUFJS4oqKiuLxY8RPh2OhsCdtP38JDrp42+TS0lIil3UHUimvsiZOKuVNpawQ7bzqDrQrsvNh1GXwyQuwemHYaUREksLM2phZYe1j4Dhgjpn1iFnsdGBO8PhZ4DwzyzWzvkB/4D3gfaC/mfU1sxz8ycPPJuvniKuMDBhyBix6FcrWhp1GRGSXqQjYVaO+BZm5/twAEZH00A14y8xm4Xfmn3fOvQj8Lhg2dDZwJHA9gHNuLvAE8DHwInCVc67aOVcFXA28BMwDngiWTU37ngE1lTD/+bCTiIjsMnUH2lVtu0DJ12DWRDjyp/65iEgr5pz7DBjayPRv7OA1twK3NjJ9MjA5rgHDUjwcOuwJc56EYReEnUZEZJeoJaAlRl8FVeXw/r1hJxERkbCY+WsGfPY6bF4ddhoRkV2iIqAlugyAASfC+/foipEiIulsyBngqmFeap7aICLpS0VASx18NWxZA7MeCzuJiIiEpdsQ6DzAdwkSEUkhKgJaas9DoOcwmHYH1NSEnUZERMJg5k8QXvwWbPgy7DQiIs2mIqClzOCgq2HNIvjkxbDTiIhIWIacATj4+Jmwk4iINJuKgN2xz2nQvreGCxURSWddBkK3/WDOv8JOIiLSbCoCdkdmFoy+Ar74LyybEXYaEREJy5DTYel7ZGxYGnYSEZFmURGwu4Z9A3LbwZt/hKqtYacREZEw7HsGADkLdeEwEUkNKgJ2V147OOAymP8c/K4fPHEhzHxMY0aLiKSTor7Qc7iKABFJGbpicDwc+RPoNcqfIPzJS8HJYQa9D4ABJ/hb18H+ZGIREWmdhpxJ1ss/gTWfQqe9wk4jIrJDagmIh4xMGHginHIbfHcejH8dxtwAVRXwn1/AXQfBbfvD5B/Aov/46SIi0rrse5q/1zUDRCQFqCUg3sz89QN6DvOFwIYvYeFLsOBF+OAf8N4EyGkLex3przrc/zho2yXs1CIisrva96Ky50iy5z4JR/wg7DQiIjukIiDR2vWAERf5W2UZfD4VFrzguw3N+zdg0GskDDge9j4Wcgv9clUVUFUGleX+vqoimF6+0/ltLA967Q9dBkPXQdB+D8hQo4+ISKJt7X8K2W/8DFbO891ARUQiSkVAMmXn+539AceDc/DVbN9C8MmL8Not/rYrLNOvMyuv7j4rj+xNK2F+THN0dhvoMqCuKNhWHPTWeQoiInG0tf+JtJn6C98l6KifhB1HRKRJKgLCYgY9hvrbmB/Bxq/8ZedrqiErt9Gde7LzICu/7j6z8V/futJSivINVi3wR6NWzff3n/4HZj1at2BOW3+Rm4bFQbtiFQciIi3gCrpAn8P8hcOOvFHfpSISWc0uAswsE5gOLHPOjW0wLxd4CBgBrAHOdc4tjmPO1q+wO+x3VvzWl98R9hjtb7G2lPqiYNV8WDkfVs2DhS/DzIfrlsltB3scBGfe64dAFRGR5htyBvz7O761t8fQsNOIiDRqV1oCvgPMAxrbK7wUWOuc29vMzgN+C5wbh3wSbwVFsOfB/hZrS2nQajAPVnwMMx6Ayd+HMyaEElNEJGUNPhWe/55vDVARICIR1ayzRc2sF3AycG8Ti4wDHgweTwKONlMbaEopKII+h8Cob8HYP8IRP4LZj8Osx8NOJiKSWgqKYK+jYM5T/vwvEZEIam5LwJ+BHwKFTcwvBpYAOOeqzGw90Amod9lcMxsPjAcoLi6mtLS0BZHDUVZWljJ545J1yMUUfvIyWc99l/WFA6jpsGd8wjWQSp8rpFbeVMoKqZU3lbJKSPY9AxZeDkunQ+9RYacREdnOTosAMxsLrHTOzTCzMbvzZs65CcAEgJKSEldUVLQ7q0uq0tJSUiVv3LKe8wDcdQgd/vM9uOQlyMze/XU2kEqfK6RW3lTKCqmVN5WySkgGnQSZub5LkIoAEYmg5nQHOgQ41cwWAxOBo8zs4QbLLAN6A5hZFtAef4KwpLIOveHU22DZDHj912GnERFJHXntof+xMPcpP+qbiEjE7LQIcM792DnXyznXBzgPeM05d0GDxZ4Fvhk8PitYRh0hW4N9T4dh34A3/wifvxl2GhGR1DHkDNj0FfxvWthJRES20+LLyJrZzWZ2avD0PqCTmS0CvgvcEI9wEhEn/hY67QVPjvejCImIyM4NOAGyC3yXIBGRiNmlIsA593rtNQKcczc5554NHpc75852zu3tnDvAOfdZIsJKSHLawJn3weZV8Ow1Gu1CRKQ5ctr4QuDjZ6C6Kuw0sjNl63yr97t/CzuJSFK0uCVA0kzPEjjmZzD/OZjx97DTiIikhiFnwpY18PkbYSeRpmxeDa/+Av68H/znF/DCD+G9e8JOJZJwKgKk+UZf5ce+fvFGf7VhERHZsb2P8Vdhn/tk2EmkofXL4IUb4E9D4K0/wV5HwmVTfOvNCz+EBS+EnVAkoVQESPNlZMBpd/sm7n99CyrLw04kIhJt2Xkw6GSY92+o2hp2GgFY86nv2nrbUHhvgh8A46r34JyHoHg4nHU/dN8fJl0Cyz4IO61IwqgIkF1T2A1OuxNWfASv/jzsNCIi0bfvGVC+Hj59Lewk6W3Fx/4A1u0jYdbjMOKbcO2HcPpd0GVA3XI5beDrT0BBZ3j0XFj7RXiZRRJIRYDsugHHw4GXw7t3wScvh51GRCTa+o2B/I4aJSgsy2bAY1+Huw7yXXwOuhqumw0n/wE67tn4awq7wQWToLoCHjkLytYmN7NIEqgIkJY55hfQdV94+grYuCLsNCIi0ZWVA4NPhQWTobIs7DTpwTl/bZuHToN7joIv/gtH3ADXfQTH/RIKu+98HV0GwnmPwtrFMPECqKpIdGqRpFIRIC2Tnef7TW7dBE9fDjU1YScSEYmuIWf478uFaj1NKOd8C/X9x8ODY2HFXDj2Zrh+Dhz5Yygo2rX19TkUxt0JX7wFz1ylbZ20KioCpOW6DoLjf+X7ub57V9hpRESiq89h0KarugQl0vzJ8LfD4dGzYcNyOOn3vtvPId+B3MKWr3f/s+Go/wcf/ROm3BK/vCIhywo7gKS4kZf4IuCVn/kjJj2Ghp1IRCR6MjJhn3Hw4cNQsXH3dkple3OehEkXQ6e9/ZH7/c+BzOz4rf+w78G6/8Gbf4D2vWHkxfFbt0hI1BIgu8cMTv0rtOkMky6FrZvDTiQiEk1DzoSqMljwYthJWpevPvJddXqPhiumwbDz41sAgN/WnfxHf92H578HC1+J7/pFQqCWANl9BUVw+t/goXHw4g2+KBARkfp6H+iPIj//XVg2HUZeWn9oStl1W0ph4tchr4Mf5z8rJ3HvlZkFZz8Afz8R/nkRXDxZrd9RUVPjz7nZugkqNsHWjb7FraJ22sa6+4pNvkg89LvQplPYyUOlIkDio98RcOh1wVUXj4Z9Tws7kYhItGRkwPn/9F1K3r8P3r0b+h4Bo74FA0/yO5nSfNVVfmd841dw8Yt+WM9Eyy2Er/8T7j0GHjkHvvUqdOid+PcV79PXKJzyf1C9JWaHP7g1h2VCbltfDGxaAWfem9i8EadvHImfI38Cn70B/74Weo2E9r3CyVG21l8Rcs2iulvVVjj+FijqF04mERGAroP9jsfxv4YPH4Lpf4cnvgGFPX0/8+EXNm/4SoFXfwafvwHj7oBeI5L3vu16+GLu/uPh0XPgkhchr33y3j9dla2Df11GRkY29NgPcvr5HfqcQn+fWwg5De5z29aflp3vu3ZN+TW88RsoOR/2OjLsnyw0KgIkfjKz4az74O7D4Mnx8M1/+5PhEqGyDEo/D3byF9bf6d+ypm45y/QXg9myBu491l8FMpkbCxGRxrTt4k82PeQ6+OQleP8emHIrvPFbGHwKjLoM9jzY77DI9mY/AdNuhwO+DcMuSP77d9sHzv0HPHwmPP4NOH9SYrsiCbzxO9iyhk3nPUP7QUfs3roOvR4+esKf33HF237Y8zSkIkDiq6ifvwrjU9+GN/8IR/ygea+rrvJXZqyqvZVD9Vb/eNPK+kf113wK65cAru71bbv7USEGjfX3tbeOffwX8+pF8PAZ8MDJcPbfYeCJifjpRUR2TUYmDDrJ39Z8CtPvhw//AXOfgi6DYdSlMPQ8jSYUa/lMePYa2PNQOP7W8HL0G+PPgXv6Ct8CftpdKtoSZdUCeO9vMPxCqrvut/vry87z+yr/OB3+exuM+dHurzMFqQiQ+Nv/XFj0Krz+a3+Vxuqtfqe+Krivrtj2uGNVhX/umnEBltx20Gkv2ONA6HSBf9xpb3+/sw1k5719381Hz/EnkZ38Bz+8qYhIVHTay+/UHvkTfz2B9++Byd+HV3/uC4FR3/LdidLZplUw8Xwo6OxP0o33KEC7quTrfujQ138NHfb0FyST+HLODzqS3QaOvgnideHmvY7yI3a9+QfY7yz//5dmVARI/Jn5neyqcn/CVmauH7khK9ffMnP90fmsPMora8hv2x6y8iDTTyMrJ1gmuOUXQef+0KbL7h1ladsVLnoe/nkxPHc9rF/qLwCjIzciEiU5BTD8G76by7IZ8P698ME//P2eh/rWgUFjwTKCUVBiRz/Z0OD5xrpb7OgoFRugcgvtstpAh2L//dimq79v2y24dfH3OW3C/kS86kr45yWwZbXvh9+2S9iJvCN+5AuBN34DHfbwQ5RK/CyY7K9HdMJv/XDkFaXxW/fxv/LDvT7/PfjGU2m3P6AiQBIjrz2c+/BOFysrLSW/aBcv4747ctrAeY/6Ifre/IMvBE69XX05RZrBzBYDG4FqoMo5N9LMioDHgT7AYuAc59xaMzPgNuAkYAtwkXPug2A93wR+Gqz2Fufcg8n8OVKGmR9koddIOO5W301o+n3+olgZWVBT1bz1ZOU1OFmyENr1hOwCajauhnVLYOl02LyKet0sa+W0baRIiHncc3hSRuYpeOtW+OItOH0C9ByW8PdrNjM45TbYsMx3C2rXEzpq6NC4qCyHF39c1zUu3gq7+9aFyd/3rW/7nRX/94gwFQGSfjKz/Bd2h97w2i2+teLcf2h0B5HmOdI5tzrm+Q3Af5xzvzGzG4LnPwJOBPoHtwOBu4ADg6LhZ8BI/B7nDDN71jm3Npk/RMpp08kPw3zwNb675Rf/9d0jakdAyS0MRklpZKSUHXSZ2VRaSlHtgZiaaj+IwqYVwW1lcL+qbtqqBfD5VChfV7eSnLZw3C9hxMWJO5L64SPkzXoQRl8FQ89NzHvsjsxsf52C+0+EJy4k88zHoejgsFOlvml/hXVfwIXPJK7r18hLYOYj8NKN/mJw+R0S8z4RpCJA0pMZHP4DaNcLnr3af3Gf/09oXxx2MpFUMw4YEzx+EHgdXwSMAx5yzjngHTPrYGY9gmVfcc6VApjZK8AJwGPJjZ2iMjJhwPH+loh11x7hZycnX1ZV+JaD9Uthyq98F8u5T/kTZTv2iW+upTPgueup7HUw2cfeHN91x1Nee78dufdoCp/5Jhw4HgacCN32TbtuJnGxfqkfYGTwqf4k7ETJyISxf4J7jvIHBk/+feLeq6H374MFL8B5j/juz0mmIkDSW8nXfDP24xfCfcf6Yd667RN2KpGocsDLZuaAvznnJgDdnHNfBvO/Amr7hRQDS2JeuzSY1tT0esxsPDAeoLi4mNLSOPYDTqCysrKUyQq7m7cA2g6AsX8nd+5jFLz5a7jzILYc/CMq9r/An7Owm2zzKtpP/DquTRdWH/l/5K7fsNvrTKx8Mk+5j/xXfkTOa7fAa7dQ3bYnlX2PorLv0VT2OiiUnb2dieLfbZsXbyCnppr1B/6AmphsCcmatycF+3+D3PfvZUO/sVR32z9uq24qb86CZ2n70ncB2PzfCVTsl/xzSVQEiOx1FFzyAjxyNtx/Apz3MPQ9POxUIlF0qHNumZl1BV4xs/mxM51zLigQdltQYEwAKCkpcUXJPHdoN5TGdq9JAXHLe/jVsP84+Pe1tHnjZ7RZ/AqM++vuXaDx/7d33+FRldkDx78nhd6riCgiCmIBRCkivQgoggoINrCBurZd+xYLumt3f+suVkQRFFCaiCCiIqwNFYQAgrSVJkWIgHSSvL8/zh2JYSaZZOpNzud55slk5s6dk8tw33nv+77nZB2CKYPg4C64fjalSx3rj2NbrS2ZNaZRLf0wrPqQ1JUfkLpiCmWWjIX0cnpV+5QeOpqTJIXhQn4OsrNg+bu6eLbzX+NXBPTHz2Hle9DhPqrU//0X8pj9H+v5CKyZReV5D8ANc6JW5yhovGvnwuy7dKF/1gHKL3iR8m2HxX19YuTddGOKg2POgOtmayXIMZdAxjuJjsiYpOOc2+T93AZMAVoCW71pPng/t3mbbwLq5Xr5cd5joR43flelHlw5WZMtbMmAF9rCVy9CThgpoIP54F7Y8BX0HaHnaL+pWFuzPA18E+5ZC1dM0gq1W5boAuJnGsFLHeDTx+Gn7zQVZrI4tA++fgX+fRZMvBYWj4Ox/bRqb6xlZ8HMe3S6btvbY/9+AWUqQ4/HYPNizcQVK1uWaJrbGifrZ6PT/bB7IywqOJlKtFknwJiAKvU07Vy9VjD5evjsn8l1UjYmgUSkvIhUDNwHugNLgWnAYG+zwcC73v1pwNWiWgO7vGlDs4DuIlJVRKp6+5kVxz/FxJKIfvG9+Suof55+kX/dK4RWGAte18JpbW/XXO5+l14GTu6q883vWKJVars8oKmxP30cXu4Iz54K026DFTP0S3gi7N0Bcx6Df56mGXMq1NJMf1dOhh2rYMKVuh4klha+DluXwvmParrceDrtYjipC3z8COzeXPD2hfXLOu1Mlamk04/LVtH3q3u2rn/IOhT998yHTQcyJreyVeGqyVoB8qOHdGFSzyejNixogti9GT55lHKHs6BWQx1uDtwq1bX0rcmjNjBFM3+SBrzlnPtARL4B3haR64B1wABv+xloetDVaIrQawCcc5ki8gjwjbfd8MAiYVOMVK4Ll7+tV5A/uA9eOFe/9La6seDz6fr58P5dOlWzy4PxiTeeRHSxcO3ToN2dsHc7rPoQVn4ASyfDwtGa1vXE9nByd2jYJbJpVWFI2bUevvwHfDcWsvbrgua2t8PxrY8sau4zAqYM0/bxkpGQEoPryPsydXFu/XbQpG/0918QEe2ojWgNs+7XgnTRsi8Txl6qx/faWUcSkYhAx/vhzUs1S9HZ10TvPQtgnQBj8korrSe4SnXhi+f0S+qlI+N/RaIkyMmGyTfAhvmUKlURlo3Ps4FoHvLKdb2OQb08nYTjtHiMZd6IOefcWuCo5OfOuR1AlyCPO+APIfY1ChgV7RhNkhHRiroNOsH0OzQF47Kp0Pd5nQoRzO7N8PZV+v/70ldLxgWY8jX0ODW7XK8Er/scVs6ClTO1cwDaCWjYVa8an9guegXcfvoOPn+Oyt9PBUnV9KttboVajY/etulArYXw8XBtH7s/Ep0YcpvzDziwC3o+kbjzerUGmj1wzqNasK9h18j3eXg/jBuoReWunnp05e+GXaBuCx0NaHZF3C5+FdgJEJEywDygtLf9ROfcg3m2GQI8xZF5nf9xzsVwQpUxMZaSoie4yvV0buLo3nD5BGwGXZT99xn48b/Q53l2ntCTahXLwu6fYNcGHYXZtdG7vwm2fg8rP9SrKLmlldEGqfJx0GJw8Zg6YExxUqkODBoPGW/r+fSFttD5L9Dmlt9/yc86qNNNDu6Bq6ZCOR8sAo62tFJwUie99Xxcp1Gt/lhrQ3w3Fr5+WacQHd9Gv5w27KpfKAvzhdk5WPMxfP4vrflQuhIHzrqBsh3+qP9W+TnvT3o+/uI5Pee2GhbZ35vblqVaDO+c63WUJJHa3gYZE3RE6uYvIb1s0feVkwUTr4MNX2stiROC1I/4bTSgHyx+C1oMKfr7FUI4IwEHgc7OuT0ikg58JiIznXNf5dlugnPuluiHaEwCtRqqJ8VJ18Or3Ujr/DhU7W5XnqNh3Rfw6WNwxgC9AvbLL3qirX6S3oJxTodUd23QK1K/dRI2agMy8VotcNT6pvj+LcaY/InoVeYGHbVi++wH4PtpOsWkVmP9v/3+nbDpW/2iZKmaVeB82GqoVs9d/6V2CNZ8ArP/preKx0LDztohaNBRp7UGk31Yazl8/i+dc1+xDnR7BFoMZv++bMpWCqPTJQK9noJfN8PMe3UfTS6K/O90TvdXpop+GU60tNJwwTPwxkV6db7zX4q2H+co9+mD8MP70Ovp/I9Vw65affu/z0DTy+MyGlBgJ8Ab0t3j/Zru3Wy1pCk5Tu0Ng9+DcQOpNHEAzDkZzroamg6CCjUTHZ0/7cvUjlXV+nDhs+F3qkS0cmr56nBss98/l3UQJl2n848P7IIO9yaus7Yvs2RexTSmIBVr60LTpZNgxt3wUjvoeJ9WHf5uDLS7C5r0SXSUySm9zJFRAtAr8mu8UYLv39ORAkmB487xRgm6QJ3mcHgfLHwDvnpeL5rUbAx9nocz+h/5ormvEMtyUlJ1qtbo3jqds0ItXTsQiWVTYN1nWrQrWc6dDTrAmZdpkpAz+kPNUwq/j3lPU2bpWzqC0vKG/LcNjAa81V/X0rQYnP/2USAujOwnIpIKLAAaAiOcc/fmeX4I8BjwM7AS+KNzbkOQ/eQu/tIiIyMj0vjjZv/+/ZQtG8FwUBxZrDFyaC/y/RQqrJpK+uYFuJQ0Dp/YlYOnDeDw8e2Tbu5qvsc26yClVr1PVp3m5FQ5Mb6BOUeF6UNJXzeX3QMmkV1LU/9F5bOQk0X5j++n9PKJHGh2Dfva/TUqBYvyChlrTjZlv3yasgteZG+HhzjYNLKTePXq1Rc4586OaCfFQLNmzdyiRYsSHUZYSmydgKLYs02v/i+fpr+ffD4MGhfyXOqnYxv3WLOzdBQlMHXop+8AB2WrgcuBAzvhhLa62Ldht6MW9RYp3r3btcjm/l80xXaodR4FObQX/tMSylWFoXMLbEvjemz3bIP/nA3HnKkXAwtzYWnhGJh2CwcbX0Lpy0aF91rn4JVOsG8H3LoQUtPDfjsRKXR7EVYnINcbVEFzQ9/qnFua6/HqwB7n3EERGQZc5pzrnN++/HRSBzv5xIqfYoVc8f78g15ZWTwe9m3XOenNLtdFRFXrJzpMIMSxzTqk2QfmPa15icvX0rSooabfxML8l3Re8PmPQZub84+3KHJydAHi/Bd0gVXv5yA1ujkQgsa6LxMmXgNrP9Xh+f2ZMGwe1GxU5Pcpykm9OPJTe+Hbc1oiLZsCP3ygi0HLVgm5WVLEGqaEx7p3O6yZox0ClwMth0K9c0JuXuR4M9fCyG6aOOO6j3Skp7A++TvMexKu+QBOaFPwW8b72H47Cqb/ES5+Wae0hWPlLBg3CBp0JLPHC1SrWYjjsnIWvDUALvq3zjoIU1Hai0JdInPO7QTmAD3yPL7DORdIHDsSaFGY/RrjOzUbwfl/hz8t1/mrtU7VL9b/agpv9IElE3X+ZrLIztKh4v+00CwdFY/R4WCXDW/01Tn18bA5Az78q1bLjNW8/ZQULfjS8X7t8EwcEvu81j8t0qI/677UQklDP9XKoJOH6jxcY0xop10Ml7yUbwfAFFL5GnBmfz2ul76SbwcgItUawBVva6fjrf66qLswfvlR1yic0T+sDkBCnDVEp1jN+rOOehRk4wJ4Z4gWuBvwRqGu5gOaFvbY5vqdIsbtR4GdABGp6Y0AICJlgW7Aijzb5F5OfhGwPIoxGpO80krp/NUrJ2kBmI5/hh1rdW76s411odOWpQXvJ1ZysnW0YsQ58O4foFx1LVBy/UfQ/AotAHNgp3YE9vwc21gO7tEr5eWqawcklvP1RXSecY/HYfl78NZlhW+cwrVoHIw6X6+2XTtTCyVVrA29/w82L4J5T8XmfY0xJhnUbaH59LcsgXcGF+6L64d/1ek/XR+OWXgRS0mBC57V0d2PCohzxxrtDFWoBVe8A6UrFP79RKDDfbBznbbfMRTOSEAdYI6IZKDFXWY756aLyHARCSxzvk1ElonIYuA2YEhswjUmiVWpBx3vhdsXw1VTNDf2t6Pgxbbwcif49jU4sDs+seTkUGrlezCilRZ3SS8PA8fBDXPg5G5HvoAf20wL+uzaCGMvjm1J+Bl369DxpSN1YW88tL5JOxz/mwtj+oZ3FSdcWYc0fdzUG/Uq0bC52hgGNOkDZw7UqzkbF0TvfY0xJtmccr4u6l39kY42hzPVfM0cvUjT7s4jhbOSVZ0zodVNWsl6wzfBt9mzDcZcrPevnKwdgaI65Xyo00wvIsVwNKDAToBzLsM519w5d6Zz7nTn3HDv8Qecc9O8+/c7505zzjV1znVyzq3If6/GFGMpKVrpsv9rcOcPejU664CeGJ9ppMOEX7+iIwQ5OdF975wc+P5deOFcKnxwG6Sk6XDksHnQuFfwq+8ntIGBY2HbCp2HeGhvdGMCvZqx+C1ofw/UPy/6+89P8yug/2jYvBhevxB+3RrxLmXvNs2M8c0rcO6tmtO8fI2jN+z1pKbQmzIUDu2L+H2NMSZptRii5/jvxsKnj+e/bfZhzeRWtb7Wi/CDTvfr+Xz6H3WKbW4Hf9Uc/3t/hsvfiXydXWA0e+c6rVcQI1b5yJhYKldNr0bf9AVc/4nOe1z/Fcy4S0cInqgPb/bXvMDrvij6OgLnYMUMeKk9vH015GSxp8dz+r5N+hRc3r1hV+j3Kmz8BsZfEd059NtXw/Q/wfHnahXGRGhykY54ZK6F13po1caiWj+fyuN7w5YM6DcKuj8aeuFxmcpaHXXHas2LbowxxVmnP2tChrmPa/KMUL4ZCT+v0AQR6WXiF18kSlfUxetbl8DXLx15PPswvD1YL+z1Hw3HRWlZ7Ck9oE7TmI4GRDdlhjEmOBE9MRzXQr+w71ynnYH1X+pi0kBp+NRSWizk+NZaEfL4VqELv4Dua/VHMOfvmhKu6olw8UtwRn8O7dxV8Jf/3Jr00cI9U2/Solv9R0eeVSfroK4DSCuli9OinKWnUE7qBFe/q1drXj1fS7cXJnOPc9pwfXA/rmJd3Vc4BY0adIDWN2uO7kY9NXe3McYURyLQ+1/w6xZ47w6ocAyc0v332+z5GeY8Bid10XOin5zaW1PZfvJ3bTMr1YVpt2q9hj4jjv5bIxFYGzB+kFbbbn5F9PbtsZEAY+JNRIdAmw7Uk+UtX8Pda3XOfqsbNWPPlyNg3GU6UvB8Gx1+zHgHdnrlN5zTipGvdtMvtft26Anolm91v0WtWdDscuj5JKyYrguJI52uNPtBvWLe53ktMZ9o9VrCkBlaxv21nl4u7TAc3q/HY8ZdcFJndg8MswMQ0OUBqNFI91GYojzGGOM3qekwYDTUPk0XCm9a+PvnPxkOh/fqVNlEFXQsKhGd5ulydDrTx8O1sFenv2qK8Ghr1FNrFMx76ugpSFFgIwHGJIPy1XXOfuNe+vuhffDTQh0lWP+ldgC+HaXPVTpOpxltydD7F/6fDr9Gq8R4q2G6gHnOo5rZoNfTRTtR/zBTc/W3uvHI35UMjjldayO80Rde7w2XT4D6bUNvv3M9TLhKM/10uA863IvbubNw75leVlP1jeyqHYl+oyL4A4wxJsmVrqjZcUZ207Vm182Gaidqh2DhGGjzh6JV4E0GVetDh7u1AwBw9rXQ/q7YvFdgbcD4y2HJ23qhLoqsE2BMMipVThfQBhbR5mTD1mXaIVj/JWT+T7+cn3U1pJWO/vu3vwsO7oYvnoPSlaDrg4V7/a5NMPVmvYLRbXj044tU9ZO0IzCmL4y9BAaMCT6Mu2aOTo3KyYZBE6BRj6O3CdexzaHDvTp1q1EvOKNf0fdljDHJruIxmj47MGJ97SxNm12+pp4L/azNrXqhq8oJRb9QFq5GvbTmwNwn4YwBUZ1Wa9OBjPGDlFQvRdkwzcc8bC60vCE2HQDQE1q34dDiGvjsWfjvs+G/NicbJt+g6wH6vRa7GCNVuS5cM1PXBYwfpAXeApzTAjZjL4EKtWHonMg6AAHn/Qnqng3v3wm7f4p8f8YYk8xqnqKjrTs3wIvtYOPX0PUhKFMp0ZFFJq2Ujm70e7Xo02/DFVgb8Mv/YMk7Ud21dQKMMcGJwAXPaEajjx/WtKbhmPcUrPscLnwWajSMbYyRKl8DBr8H9VrBpOu1lsPBPZrGdfYDuvDr+o8iT/cWkJqmC7ezDsK7t4SXS9sYY/zs+NaaGOLXzVpLpemgREcUHfFcz9D4Ah0NmPdkVNcGWCfAGBNaSir0fQFO6alz2RcXkK/4x89g7hN6km86MD4xRqpMZR2yPrm71nIY0RKWT4Nuj+hIRlEqPuanRkPo/ohmk/hmZHT3bYwxyahJH50ONHBc4bLWGSWiU6gy18LSiQVvHyb7lzDG5C81Xacg1W+n6UOXTw++3d4dMOkGTVPa6+m4hhix9LIw8E048zLIPqTFv9reFrsrPedcr+nxPvyb1lEwxpji7vhWULF2oqPwr0YXQG1vbUCURgOsE2CMKVh6GRg0The3TrxGF8zm5pyX/nK7Zr6J9tXzeEhNh0te1irPDTrE9r1ENKVrWmmtJhyD1G/GGGOKkZQU6HAPZK6BpZOis8uo7MUYU/wFUr5VP1nTla2ff+S5+S/Bypk6hebYZgkLMSpivcgroFIdXTexaYEuvjbGGGPy0/hCqH26rg3IyY54d9YJMMaEr1w1uGqKpn57sz9szoCfFsHsv+m6gVbDEh2hv5x+KZzeT9dRhFu4zBhjTMkUGA3YsToqowHWCTDGFE7F2nD1uzoyMOZizaRTrgb0fd5/1R+TwQVPQ/laMHmoViY2xhhjQmncG2qdpmsDIhwNsE6AMabwqhyvHQER2LkOLh2powSm8MpWhb4jYPtK+OjhREdjjDEmmf02GrAKlk6ObFdRCskYU9LUaKjFUq6eBvXbJjoafzupM7QcCvNfgLVzEx2NMcaYZHbqRVCriU4ljWA0wDoBxpiiq3YinNgu0VEUD10f1kXXU2+G/TsTHY0xxphklXs0YNmUou8miiEZY4wpqlLl4JKXtKrmzHsTHY0xxphkdmofqHlqRKMB1gkwxphkUbcFtL8bMsbD9+8mOhpjjDHJKjAasH1lkUcDrBNgjDHJpP1dWpTtvTsSHYkxxphk1qQv1GysmYKKwDoBxhiTTFLT4eKX4fC+REdijDEmmf02GvBD0V4e5XCMMcZEquYpcNG/Ex2FMcaYZNekL3S8v0gvtU6AMcYkozMHJDoCY4wxyS4lFTreV7SXRjkUY4wxxhhjTJKzToAxxhhjjDEljHUCjDHGGGOMKWGsE2CMMcYYY0wJU2AnQETKiMjXIrJYRJaJyMNBtiktIhNEZLWIzBeR+jGJ1hhjjDHGGBOxcEYCDgKdnXNNgWZADxFpnWeb64BfnHMNgX8CT0Q1SmOMMcYYY0zUFNgJcGqP92u6d3N5NusDjPbuTwS6iIhELUpjjDHGGGNM1KSFs5GIpAILgIbACOfc/Dyb1AU2ADjnskRkF1Ad2J5nP0OBoQB169YlMzMzsujjaP/+/b6J12KNHT/F66dYwV/x+ilWY4wxJhhxLu9F/Xw2FqkCTAFudc4tzfX4UqCHc26j9/saoJVzbnvQHek2vwJFq3OcGDXI06lJYhZr7PgpXj/FCv6KN16xnuCcqxmH90lqPmsv/PQ5Bn/Fa7HGjp/i9VOskMTtRVgjAQHOuZ0iMgfoASzN9dQmoB6wUUTSgMrAjgJ294Nz7uzCvH8iici3fonXYo0dP8Xrp1jBX/H6KdZiwjfthd8+G36K12KNHT/F66dYIbnjDSc7UE1vBAARKQt0A1bk2WwaMNi73w/4xBVmiMEYY4wxxhgTN+GMBNQBRnvrAlKAt51z00VkOPCtc24a8CowRkRWA5nAwJhFbIwxxhhjjIlIgZ0A51wG0DzI4w/kun8A6F/I9365kNsnmp/itVhjx0/x+ilW8Fe8foq1OPDT8fZTrOCveC3W2PFTvH6KFZI43kItDDbGGGOMMcb4XzjFwowxxhhjjDHFSMw7ASLSQ0R+EJHVInJfkOdLi8gE7/n5IlI/1jEFIyL1RGSOiHwvIstE5PYg23QUkV0issi7PRBsX/EiIj+KyBIvlm+DPC8i8px3bDNE5KwExdko1zFbJCK7ReSOPNsk9NiKyCgR2ealuw08Vk1EZovIKu9n1RCvHexts0pEBgfbJg6xPiUiK7x/5ymBxfxBXpvvZyaO8T4kIpty/Xv3CvHafM8fcYp1Qq44fxSRRSFeG/djW9xYexE71l5ELT7ftBX5xJuU7YWf2op84vVXe+Gci9kNSAXWAA2AUsBioEmebW4GXvTuDwQmxDKmfGKtA5zl3a8IrAwSa0dgeiLiCxHzj0CNfJ7vBcwEBGgNzE+CmFOBLWg+26Q5tkB74Cxgaa7HngTu8+7fBzwR5HXVgLXez6re/aoJiLU7kObdfyJYrOF8ZuIY70PAXWF8VvI9f8Qj1jzPPwM8kCzHtjjdrL2IeczWXkQnJt+0FfnEm5TthZ/ailDx5nk+6duLWI8EtARWO+fWOucOAeOBPnm26QOM9u5PBLqIiMQ4rqM45zY75xZ6938FlqOVkP2sD/CGU18BVUSkToJj6gKscc6tS3Acv+Ocm4dmtsot92dzNNA3yEvPB2Y75zKdc78As9E6GjETLFbn3IfOuSzv16+A42IZQ2GEOLbhCOf8EVX5xeqdlwYA42IZQwlm7UViWXsRBj+1FeCv9sJPbQUUj/Yi1p2AusCGXL9v5OgT5W/beB/KXUD1GMeVL2+IuTkwP8jTbURksYjMFJHT4hvZURzwoYgsEJGhQZ4P5/jH20BC/6dIpmMLUNs5t9m7vwWoHWSbZDzG16JX9IIp6DMTT7d4w9GjQgyfJ9uxbQdsdc6tCvF8Mh1bP7L2IrasvYgdv7YV4I/2wm9tBfikvbCFwXmISAVgEnCHc253nqcXosOSTYF/A1PjHF5e5znnzgJ6An8QkfYJjidfIlIKuAh4J8jTyXZsf8fp+F3Sp9ISkb8AWcCbITZJls/MC8BJQDNgMzpsmuwGkf9VnWQ5tiZOrL2IHb+2F35pK8A37YUf2wrwSXsR607AJqBert+P8x4Luo2IpAGVgR0xjisoEUlHT+hvOucm533eObfbObfHuz8DSBeRGnEOM3c8m7yf24Ap6JBYbuEc/3jqCSx0zm3N+0SyHVvP1sBwuPdzW5BtkuYYi8gQ4ELgCq8hOkoYn5m4cM5tdc5lO+dygFdCxJFMxzYNuASYEGqbZDm2PmbtRQxZexFTvmorwD/thd/aCvBXexHrTsA3wMkicqLXqx8ITMuzzTQgsEq+H/BJqA9kLHnzt14Fljvnng2xzTGB+aci0hI9folqgMqLSMXAfXShz9I8m00DrhbVGtiVa8gyEUL2jJPp2OaS+7M5GHg3yDazgO4iUtUbpuzuPRZXItIDuAe4yDm3L8Q24Xxm4iLPXOOLQ8QRzvkjXroCK5xzG4M9mUzH1sesvYgRay9izjdtBfirvfBhWwF+ai/CXUFc1BuacWAlunL7L95jw9EPH0AZdLhvNfA10CDWMYWI8zx0CC8DWOTdegE3Ajd629wCLENXnn8FnJuIWL1YGnhxLPZiChzb3PEKMMI79kuAsxMYb3n0JF0512NJc2zRxmYzcBidT3gdOtf4Y2AV8BFQzdv2bGBkrtde631+VwPXJCjW1eicyMBnN5BB5VhgRn6fmQTFO8b7TGagJ+s6eeP1fj/q/BHvWL3HXw98VnNtm/BjW9xuwf69sfYiGvFaexG92HzTVuQTb1K2FyFiTcq2IlS83uOv45P2wioGG2OMMcYYU8LYwmBjjDHGGGNKGOsEGGOMMcYYU8JYJ8AYY4wxxpgSxjoBxhhjjDHGlDDWCTDGGGOMMaaEsU6AMcYYY4wxJYx1AowxxhhjjClhrBNgjDHGGGNMCfP/OP+vEamibB8AAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "learner.plot_metrics()" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [], - "source": [ - "X_train, y_train, X_val, y_val, X_test, y_test = split_Xy(X, y, splits)" - ] - }, - { - "cell_type": "code", - "execution_count": 299, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - "
\n", - " \n", - " 0.00% [0/573 00:00<?]\n", - "
\n", - " " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "preds = learner.get_preds(0, with_decoded=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 300, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([[[ 24., 22., 22., ..., 56., 66., 52.]],\n", - "\n", - " [[ 22., 22., 20., ..., 66., 52., 20.]],\n", - "\n", - " [[ 22., 20., 22., ..., 52., 20., 23.]],\n", - "\n", - " ...,\n", - "\n", - " [[132., 122., 133., ..., 99., 142., 147.]],\n", - "\n", - " [[122., 133., 129., ..., 142., 147., 123.]],\n", - "\n", - " [[133., 129., 123., ..., 147., 123., 123.]]], dtype=float32)" - ] - }, - "execution_count": 300, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "y_train, preds[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "_, _, preds = learner.get_X_preds(X_test)" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": {}, - "outputs": [], - "source": [ - "y_preds = np.reshape(preds, y_test.shape)" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(array([[2586.70288086, 2968.31591797, 3197.52416992, 3346.09301758,\n", - " 3476.81274414, 3597.73413086, 3561.25561523, 3394.43505859,\n", - " 3087.15185547, 2682.01269531, 2252.69238281, 1835.6607666 ,\n", - " 1400.54528809, 1097.96679688, 926.84667969, 816.6517334 ,\n", - " 701.20263672, 601.66210938, 547.01208496, 628.43530273,\n", - " 918.94256592, 1302.46643066, 1699.14001465, 2150.11206055]]),\n", - " array([[7366., 7730., 8074., 8118., 8041., 8313., 8051., 8610., 8490.,\n", - " 9038., 9291., 9104., 9117., 5682., 2629., 2349., 2255., 2184.,\n", - " 2180., 2162., 2423., 2718., 2761., 3229.]], dtype=float32))" - ] - }, - "execution_count": 52, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "y_preds[0], y_test[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "15735" - ] - }, - "execution_count": 49, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "len(preds)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.11" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -}